{
  "protocol": {
    "status": "frozen_before_inference",
    "date": "2026-09-21",
    "open_jev_commit": "ed45657bf726c3b77408942830e5578f99df904e",
    "jevbench_commit": "75e6224ed8103bbc3485ca74820a2eaf7ce8abe0",
    "tasks": 231,
    "dataset_sha256": "dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51",
    "file_hashes": {
      "original": "5c2414edb3006b8bfcb70fda433f0f9ca015759433849f8d3104328a1f7c4180",
      "easy": "231df3c2c8e88a1a8c137ebe85de96ba70fabd330849098ac7b3c52c70b7172b",
      "hard": "89e9e6becb33ed88c1de7d42dcc87531b2fb64cfaef4e1986faf7c37b3f80ebb"
    },
    "settings": {
      "gpu": "L40S",
      "dtype": "bfloat16",
      "candidate_batch_size": 1,
      "max_length": 16384,
      "prefix_cache": false
    },
    "measurement": "Four example warmups, five interleaved rounds of four examples, then all 231 public tasks serially. Server-side synchronized in-process wall time; excludes Modal RPC/network and model load. No retries or answer reordering. Raw failures retained as incorrect.",
    "scoring": "Unmodified JevBench score_task: strict sum 0.001, renormalization band 0.02; lexical tie break; score argmax accuracy and separate expected-value MAE.",
    "budget": {
      "account_cap_usd": 13,
      "stop_starting_runs_at_metered_or_billed_usd": 7,
      "max_model_runs": 2,
      "startup_timeout_seconds": 900,
      "call_timeout_seconds": 900,
      "inference_soft_deadline_seconds": 720,
      "gpu_rate_usd_per_hour": 1.95,
      "cpu_max_cores": 4,
      "memory_max_gib": 48,
      "max_run_compute_estimate_usd": 1.3,
      "min_containers": 0,
      "scaledown_seconds": 30,
      "client_absolute_deadline_seconds": 1500,
      "live_billing_stop_usd": 8.5
    },
    "planned_requests_per_model": 255
  },
  "protocol_amendments": [
    {
      "reason": "First attempt returned a torch.torch_version.TorchVersion metadata object; local deserialization required torch, before any response was saved. Changed only metadata to a plain string. Guardian stopped app successfully; no live containers; metered usage $0.06.",
      "saved_responses": 0,
      "inputs_or_inference_changed": false,
      "authorized_attempts": 3,
      "spending_cap_usd": 13
    },
    {
      "reason": "Original website Choice options were lists; Open-Jev requires a map. All 24 initial example/warmup requests were rejected before model inference. Kept their errors; preserved the first valid 231-task 2B benchmark stream. 9B benchmark receives only the same 231 valid benchmark requests. Both models then receive a separate corrected 24-request example stream.",
      "mapping": "Choice lists become ordered {option_text: null} maps. compile_request emits the exact original option text once, with no invented label prefix. State, instructions, answer strings and option order are unchanged.",
      "benchmark_requests_unchanged": true,
      "quality_run_warmup": "No successful example warmups before either primary benchmark. Model initialization excluded; first-call kernel overhead included.",
      "examples_measurement": "One warmup per scenario, then five interleaved rounds; separate from primary benchmark.",
      "budget": "$13 account cap; same $7 start, $8.50 live stop and 25-minute guardian per invocation. Two additional bounded example calls, at most 900+900 seconds per model; total planned upper envelope still below $13 including prior observed costs.",
      "completed_scored_benchmark_streams_per_model": 1,
      "earlier_transport_attempt_remote_progress": "Unknown; no saved responses, app stop verified."
    }
  ],
  "effective_measurement": "One completed scored benchmark stream per model; no successful benchmark warmups. Separate corrected example streams each have four warmups and five interleaved rounds. Server-side synchronized inference timing excludes model initialization and RPC/network. Earlier transport attempt had no saved results. See amendments for retained input-format failures. Laya and Kev each have one complete 231-task native-API stream, no warmups or benchmark-request retries. Their separate pre-inference token audit is outside the inference timer. Laya uses native CUDA autocast, unlike the older Mac run. Released Laya typed-decisions has one separate231public-task stream with native saved calibration.",
  "models": {
    "2B": {
      "model": "Open-Jev 2B",
      "runtime": {
        "model": "ZefanCai/Open-Jev-2B",
        "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785",
        "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
        "code_commit": "ed45657bf726c3b77408942830e5578f99df904e",
        "gpu": "NVIDIA L40S",
        "dtype": "bfloat16",
        "batch_size": 1,
        "max_length": 16384,
        "prefix_cache": false,
        "model_load_seconds": 14.586798539000002,
        "torch_version": "2.8.0+cu128"
      },
      "planned": 231,
      "received": 231,
      "correct": 150,
      "accuracy": 0.6493506493506493,
      "operational_success": 231,
      "strict_valid": 231,
      "valid": 231,
      "renormalized": 0,
      "tiers": {
        "original": {
          "n": 72,
          "correct": 56
        },
        "easy": {
          "n": 48,
          "correct": 48
        },
        "hard": {
          "n": 111,
          "correct": 46
        }
      },
      "brier_mean": 0.4747519869738973,
      "ece": {
        "ece": 0.12334581416865625,
        "n": 231,
        "bins": [
          {
            "lo": 0.0,
            "hi": 0.1,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.1,
            "hi": 0.2,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.2,
            "hi": 0.3,
            "n": 6,
            "mean_confidence": 0.2593343645246354,
            "accuracy": 0.0
          },
          {
            "lo": 0.3,
            "hi": 0.4,
            "n": 10,
            "mean_confidence": 0.34693681598770676,
            "accuracy": 0.1
          },
          {
            "lo": 0.4,
            "hi": 0.5,
            "n": 19,
            "mean_confidence": 0.4565397451396703,
            "accuracy": 0.2631578947368421
          },
          {
            "lo": 0.5,
            "hi": 0.6,
            "n": 24,
            "mean_confidence": 0.5560918882004837,
            "accuracy": 0.625
          },
          {
            "lo": 0.6,
            "hi": 0.7,
            "n": 26,
            "mean_confidence": 0.6398166022013291,
            "accuracy": 0.6538461538461539
          },
          {
            "lo": 0.7,
            "hi": 0.8,
            "n": 25,
            "mean_confidence": 0.7575880365833911,
            "accuracy": 0.68
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "n": 38,
            "mean_confidence": 0.8479288451810092,
            "accuracy": 0.6842105263157895
          },
          {
            "lo": 0.9,
            "hi": 1.0,
            "n": 83,
            "mean_confidence": 0.9592011266369158,
            "accuracy": 0.8313253012048193
          }
        ]
      },
      "ordinal_mae": 0.3495860620092156,
      "latency": {
        "n": 231,
        "p50_s": 0.6230798439999887,
        "p95_s": 1.4954512685
      },
      "peak_gpu_allocated_bytes": 4427087872,
      "examples": [
        {
          "id": "01-raccoon-refund",
          "title": "Three raccoons in a trench coat",
          "response": {
            "answers": {
              "customer_intent": {
                "type": "choice",
                "choice": "a refund",
                "probabilities": {
                  "a refund": 0.977588268628909,
                  "technical instructions": 0.002011535141835434,
                  "to buy another product": 0.013208199656650859,
                  "to report a wildlife emergency": 0.00719199657260469
                },
                "confidence": 0.970117691505212
              },
              "refund_requested": {
                "type": "noul",
                "noul": 0.9397001047090151
              }
            },
            "model": "Qwen/Qwen3.5-2B",
            "usage": {
              "input_tokens": 418,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.518796342858676,
              "candidate_sequences": 5,
              "inference_seconds": 0.7106123620000027,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded",
              "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.7107222740000019,
            "p95_s": 0.8805969183999991
          },
          "responses_identical": true
        },
        {
          "id": "02-sarcasm-detector",
          "title": "Five stars for deleting my work",
          "response": {
            "answers": {
              "actual_sentiment": {
                "type": "choice",
                "choice": "negative",
                "probabilities": {
                  "positive": 0.21928485949839285,
                  "negative": 0.6771282555002892,
                  "neutral": 0.10358688500131794
                },
                "confidence": 0.5156923832504338
              },
              "sarcasm": {
                "type": "noul",
                "noul": 0.5606939052679603
              }
            },
            "model": "Qwen/Qwen3.5-2B",
            "usage": {
              "input_tokens": 344,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.518796342858676,
              "candidate_sequences": 4,
              "inference_seconds": 0.581106247000001,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded",
              "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.5812446810000012,
            "p95_s": 0.6196496690000017
          },
          "responses_identical": true
        },
        {
          "id": "03-dungeon-risk",
          "title": "The treasure chest has dental insurance",
          "response": {
            "answers": {
              "next_move": {
                "type": "choice",
                "choice": "put a hand inside the chest",
                "probabilities": {
                  "put a hand inside the chest": 0.44438005677390063,
                  "step back and warn the party": 0.15822473426934042,
                  "hug the chest": 0.12584473959791262,
                  "climb inside and close the lid": 0.27155046935884636
                },
                "confidence": 0.25917340903186753
              },
              "danger": {
                "type": "score",
                "score": 1.9416791772180775,
                "probabilities": {
                  "0": 0.05325739995502108,
                  "1": 0.22451767564306244,
                  "2": 0.44951327163073446,
                  "3": 0.2727116527711821
                },
                "confidence": 0.3962558716757133,
                "legend": {
                  "0": "safe",
                  "1": "slightly risky",
                  "2": "dangerous",
                  "3": "extremely dangerous"
                }
              }
            },
            "model": "Qwen/Qwen3.5-2B",
            "usage": {
              "input_tokens": 815,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.518796342858676,
              "candidate_sequences": 8,
              "inference_seconds": 1.1487581899999988,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded",
              "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 1.1368773820000015,
            "p95_s": 1.1499830864000025
          },
          "responses_identical": true
        },
        {
          "id": "04-vampire-injection",
          "title": "Count Override would like your secrets",
          "response": {
            "answers": {
              "message_type": {
                "type": "choice",
                "choice": "prompt injection attempting to override instructions",
                "probabilities": {
                  "ordinary customer question": 0.017981733926269436,
                  "prompt injection attempting to override instructions": 0.9738416388048244,
                  "harmless product feedback": 0.008176627268906136
                },
                "confidence": 0.9607624582072365
              },
              "secret_extraction": {
                "type": "noul",
                "noul": 0.9020716020405615
              }
            },
            "model": "Qwen/Qwen3.5-2B",
            "usage": {
              "input_tokens": 419,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.518796342858676,
              "candidate_sequences": 4,
              "inference_seconds": 0.5592413950000008,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded",
              "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.5593843779999972,
            "p95_s": 0.5773166333999988
          },
          "responses_identical": true
        }
      ],
      "scored": [
        {
          "id": "original-policy-01-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 1.4628509990000005,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9954536439849835,
            "yes": 0.004546356015016495
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-01-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15349537999999896,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9981648616106243,
            "yes": 0.0018351383893756156
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-02-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15493629399999875,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.0417932095090513,
            "yes": 0.9582067904909487
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-02-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.1773623569999998,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.07650508209234674,
            "yes": 0.9234949179076533
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-03-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15580055399999893,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.804402303461101,
            "yes": 0.19559769653889897
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-policy-03-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.165065405,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2232952523608671,
            "yes": 0.7767047476391329
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-04-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15210986700000007,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9939024407538125,
            "yes": 0.006097559246187505
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-04-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15177946900000094,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9929155650594956,
            "yes": 0.0070844349405043295
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-05-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.16477309500000104,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8052134229817229,
            "yes": 0.1947865770182771
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-05-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1506923140000005,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5949138735914077,
            "yes": 0.4050861264085924
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-06-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.16460911200000083,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.10719957764362509,
            "yes": 0.8928004223563749
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-06-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17483756499999714,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3976946586686413,
            "yes": 0.6023053413313587
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-intent-01-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.7454884649999975,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.07818400236766278,
            "change_address": 0.04363065798892138,
            "other": 0.5410917020553818,
            "refund": 0.28871848072870454,
            "status": 0.04837515685932945
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-01-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.7732125910000001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.06839601750669007,
            "change_address": 0.010057394443794183,
            "other": 0.16815394367268813,
            "refund": 0.7443518224701553,
            "status": 0.009040821906672351
          },
          "predicted": "refund",
          "correct": true
        },
        {
          "id": "original-intent-02-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.768944476999998,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.04218634781854709,
            "change_address": 0.11301234750869132,
            "other": 0.6405832798938752,
            "refund": 0.09004592316792623,
            "status": 0.11417210161096017
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-02-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.7112405430000024,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.052110234470439984,
            "change_address": 0.01675406082517303,
            "other": 0.6519213265941786,
            "refund": 0.06831170128596796,
            "status": 0.21090267682424044
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-03-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.7353946570000005,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.009454237311202042,
            "change_address": 0.799176711236621,
            "other": 0.15775210432690717,
            "refund": 0.009330530736722294,
            "status": 0.024286416388547272
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-03-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.7260165930000007,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.014825455526661353,
            "change_address": 0.7790067886582849,
            "other": 0.1666151664542757,
            "refund": 0.015091116120253642,
            "status": 0.024461473240524473
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-04-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.6874583459999997,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.7226179063467686,
            "change_address": 0.009954422167293695,
            "other": 0.25125446165585796,
            "refund": 0.0072121293400606865,
            "status": 0.008961080490019127
          },
          "predicted": "cancel",
          "correct": true
        },
        {
          "id": "original-intent-04-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.6752491910000025,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.8039515165323793,
            "change_address": 0.005293826343554094,
            "other": 0.18087791015889443,
            "refund": 0.004046187329307621,
            "status": 0.005830559635864643
          },
          "predicted": "cancel",
          "correct": true
        },
        {
          "id": "original-intent-05-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.7141253079999998,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.07823923508342942,
            "change_address": 0.039371806693947534,
            "other": 0.36130049224252775,
            "refund": 0.48641953497906737,
            "status": 0.03466893100102797
          },
          "predicted": "refund",
          "correct": false
        },
        {
          "id": "original-intent-05-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.7040344269999999,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.130457802805025,
            "change_address": 0.04298073600148193,
            "other": 0.29938354557379643,
            "refund": 0.48524822063928263,
            "status": 0.0419296949804139
          },
          "predicted": "refund",
          "correct": false
        },
        {
          "id": "original-intent-06-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.7560896889999995,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.06597122259095,
            "change_address": 0.03651251192584452,
            "other": 0.5873489556549019,
            "refund": 0.08564356058711635,
            "status": 0.22452374924118734
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-06-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.760988066000003,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.044496135484351135,
            "change_address": 0.03585650522719127,
            "other": 0.6561896359306304,
            "refund": 0.10806591371235762,
            "status": 0.15539180964546967
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-ordinal-01-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.5660878660000037,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.996625997306072,
            "1": 0.00110441372662887,
            "2": 0.002016107835942284,
            "3": 0.0002534811313568203
          },
          "ordinal_ev": 0.005897072792583899,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-01-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.5851488800000055,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.9808739799490174,
            "1": 0.012409362153476669,
            "2": 0.006099148932141521,
            "3": 0.0006175089653645235
          },
          "ordinal_ev": 0.02646018691385328,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-02-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.568490618999995,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.11660422081432925,
            "1": 0.7952617635838353,
            "2": 0.08089917101032397,
            "3": 0.007234844591511591
          },
          "ordinal_ev": 0.9787646393790179,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-02-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.5976961069999973,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.1286363352939144,
            "1": 0.8608845602364638,
            "2": 0.005175369364755892,
            "3": 0.005303735104865792
          },
          "ordinal_ev": 0.8871465042805728,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-03-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.6243753150000018,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.2508590317611235,
            "1": 0.02125706617633954,
            "2": 0.7186431651267711,
            "3": 0.009240736935765981
          },
          "ordinal_ev": 1.4862656072371796,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-03-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.5793344539999978,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.4522263231333461,
            "1": 0.05232828821102916,
            "2": 0.4784241104416946,
            "3": 0.017021278213930225
          },
          "ordinal_ev": 1.060240343736209,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-04-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.6833545510000008,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.049653040173716,
            "1": 0.004174078760133623,
            "2": 0.0053968016122442975,
            "3": 0.9407760794539061
          },
          "ordinal_ev": 2.83729592034634,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-04-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.5920486340000011,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.011457250373925877,
            "1": 0.0017930936965897465,
            "2": 0.007045607282928684,
            "3": 0.9797040486465557
          },
          "ordinal_ev": 2.954996454202114,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-05-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.6112017399999985,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.9625253840680446,
            "1": 0.014335296850055072,
            "2": 0.01308109370711198,
            "3": 0.010058225374788428
          },
          "ordinal_ev": 0.07067216038864432,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-05-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.6031111400000029,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.9818102986470459,
            "1": 0.0026975743348925008,
            "2": 0.01328932963535308,
            "3": 0.0022027973827084503
          },
          "ordinal_ev": 0.03588462575372401,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-06-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.5645923800000006,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.03678694445461755,
            "1": 0.023697712089138057,
            "2": 0.9388066705125896,
            "3": 0.000708672943654871
          },
          "ordinal_ev": 1.9034370719452818,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-06-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.6440596690000007,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.010363804440369958,
            "1": 0.01167196597519994,
            "2": 0.9772161835064127,
            "3": 0.0007480460780173314
          },
          "ordinal_ev": 1.9683484712220773,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-extraction-01-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.5856721719999953,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.33635063861277037,
            "pickup": 0.18089323999315143,
            "post": 0.25824607452670995,
            "unknown": 0.2245100468673682
          },
          "predicted": "courier",
          "correct": false
        },
        {
          "id": "original-extraction-01-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.6174258379999955,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.03379527319890452,
            "pickup": 0.6522939302728242,
            "post": 0.1280212944084394,
            "unknown": 0.18588950211983193
          },
          "predicted": "pickup",
          "correct": true
        },
        {
          "id": "original-extraction-02-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.6128246209999944,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.02540073497203303,
            "pickup": 0.025434462606163822,
            "post": 0.15563065727735986,
            "unknown": 0.7935341451444433
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-02-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.6144194540000001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.02831718998513974,
            "pickup": 0.027876719047348647,
            "post": 0.04522002106905781,
            "unknown": 0.8985860698984539
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-03-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.6426694710000049,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.5309792313601489,
            "pickup": 0.12750848154326838,
            "post": 0.21305005157783805,
            "unknown": 0.1284622355187447
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-03-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.5926552430000029,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.8281999217794679,
            "pickup": 0.0295680953541492,
            "post": 0.11550170827517826,
            "unknown": 0.026730274591204674
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-04-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.638197556999998,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.08941580432656569,
            "pickup": 0.033228712660862414,
            "post": 0.6891621901295355,
            "unknown": 0.18819329288303638
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-04-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.6014480999999989,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.060390229580827036,
            "pickup": 0.01884620583304756,
            "post": 0.8552856176436644,
            "unknown": 0.06547794694246102
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-05-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.5672719119999954,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.01423353976629996,
            "pickup": 0.051857731113646124,
            "post": 0.030089927955715873,
            "unknown": 0.9038188011643381
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-05-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.6193044499999942,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.21025737105378176,
            "pickup": 0.07810951057772744,
            "post": 0.139454142125105,
            "unknown": 0.5721789762433858
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-06-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.6390196920000051,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.9349843456027508,
            "pickup": 0.017967840238836356,
            "post": 0.01750348939137009,
            "unknown": 0.02954432476704275
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-06-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.6183376199999984,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.8428514520562359,
            "pickup": 0.037971377634529374,
            "post": 0.04330443320747206,
            "unknown": 0.07587273710176266
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-adequacy-01-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15746592099999646,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.06687521283833253,
            "yes": 0.9331247871616675
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-01-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15729040699999786,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.11588876928393765,
            "yes": 0.8841112307160623
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-02-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15282081499999833,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6010322781063204,
            "yes": 0.3989677218936795
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-02-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15132470100000006,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9382987951584145,
            "yes": 0.06170120484158542
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-03-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.14772701799999766,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.06301806139391297,
            "yes": 0.936981938606087
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-03-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15471021699999454,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.17039702844439675,
            "yes": 0.8296029715556033
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-04-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15896334499999654,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3964335047355486,
            "yes": 0.6035664952644514
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-04-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15258485900000096,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.41307996467543506,
            "yes": 0.5869200353245649
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-05-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15302184800000163,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.33813087322882895,
            "yes": 0.661869126771171
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-05-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15101591300000194,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.556287756902881,
            "yes": 0.44371224309711893
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-06-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.16475095700000253,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.21308302168411153,
            "yes": 0.7869169783158885
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-06-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.1588101420000001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5738901801850784,
            "yes": 0.4261098198149216
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-routing-01-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 0.8512893959999985,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.023369659170405576,
            "coding_agent": 0.01570567863295688,
            "document": 0.019739416207631456,
            "general": 0.07333267413725539,
            "math": 0.8238211201867137,
            "tools": 0.04403145166503703
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-01-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 0.9449048849999997,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.026813925813817405,
            "coding_agent": 0.018339333814268572,
            "document": 0.022045062185240678,
            "general": 0.07647851132172212,
            "math": 0.8057342608192074,
            "tools": 0.05058890604574383
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-02-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 0.8693021200000004,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.6691233024818956,
            "coding_agent": 0.09308318532514202,
            "document": 0.01417947963283543,
            "general": 0.14334473559641014,
            "math": 0.01148959970259574,
            "tools": 0.06877969726112113
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-02-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 0.8632518979999944,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.6001045448825174,
            "coding_agent": 0.08594786528576434,
            "document": 0.029478180275321154,
            "general": 0.15964506644311846,
            "math": 0.020201165127184572,
            "tools": 0.10462317798609397
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-03-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 0.8452884310000002,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.006769810369809449,
            "coding_agent": 0.856820455640646,
            "document": 0.018438992063329107,
            "general": 0.05821191643444084,
            "math": 0.008272707100926216,
            "tools": 0.051486118390848434
          },
          "predicted": "coding_agent",
          "correct": true
        },
        {
          "id": "original-routing-03-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 0.8595296049999988,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.00654331649633052,
            "coding_agent": 0.8851131339879774,
            "document": 0.01545913556089549,
            "general": 0.05687770924757802,
            "math": 0.006019992291246549,
            "tools": 0.02998671241597195
          },
          "predicted": "coding_agent",
          "correct": true
        },
        {
          "id": "original-routing-04-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 0.8940489449999944,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.019092045142876393,
            "coding_agent": 0.8596073668048319,
            "document": 0.016010051907727203,
            "general": 0.06908503112346356,
            "math": 0.006949829184274016,
            "tools": 0.02925567583682695
          },
          "predicted": "coding_agent",
          "correct": false
        },
        {
          "id": "original-routing-04-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 0.8984680820000008,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.029677217580909288,
            "coding_agent": 0.7830535188395856,
            "document": 0.02739752656588689,
            "general": 0.09250414810003971,
            "math": 0.017872737497970137,
            "tools": 0.049494851415608415
          },
          "predicted": "coding_agent",
          "correct": false
        },
        {
          "id": "original-routing-05-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 0.8987451660000048,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.028980878404569328,
            "coding_agent": 0.03029621837154213,
            "document": 0.08016590527918314,
            "general": 0.1799051010225692,
            "math": 0.02218430688279154,
            "tools": 0.6584675900393447
          },
          "predicted": "tools",
          "correct": true
        },
        {
          "id": "original-routing-05-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 0.8768614889999995,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.06440866105038784,
            "coding_agent": 0.1943567688033196,
            "document": 0.12022474110704941,
            "general": 0.35209269295892515,
            "math": 0.0377628724113526,
            "tools": 0.23115426366896552
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-06-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.9471316990000034,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.0683779710336535,
            "coding_agent": 0.05125631741250466,
            "document": 0.09530346600220635,
            "general": 0.4957144595670155,
            "math": 0.1312673854095102,
            "tools": 0.15808040057510978
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "original-routing-06-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.8549717499999971,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.08814304068569512,
            "coding_agent": 0.06935811459934427,
            "document": 0.10564779258992947,
            "general": 0.43156502945870945,
            "math": 0.1281426812058431,
            "tools": 0.1771433414604786
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "easy-intent-00",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.7728748830000001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.00629483891888552,
            "cancel_order": 0.02696095967556538,
            "change_address": 0.0052299254238450206,
            "report_damage": 0.0025099264735913194,
            "track_order": 0.9590043495081126
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-01",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "cancel_order",
          "ok": true,
          "server_wall_seconds": 0.7360915089999978,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.002104058831530386,
            "cancel_order": 0.994948529022335,
            "change_address": 0.001395688663556433,
            "report_damage": 0.0006395710558495245,
            "track_order": 0.000912152426728557
          },
          "predicted": "cancel_order",
          "correct": true
        },
        {
          "id": "easy-intent-02",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.7544425550000042,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.015535141504311033,
            "cancel_order": 0.014375470228608628,
            "change_address": 0.9536935293195136,
            "report_damage": 0.0034877659797014626,
            "track_order": 0.012908092967865381
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "easy-intent-03",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "report_damage",
          "ok": true,
          "server_wall_seconds": 0.7679700220000001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.004873892844795531,
            "cancel_order": 0.05789522779624476,
            "change_address": 0.0043924110805534,
            "report_damage": 0.9113821060466963,
            "track_order": 0.021456362231709953
          },
          "predicted": "report_damage",
          "correct": true
        },
        {
          "id": "easy-intent-04",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "billing_question",
          "ok": true,
          "server_wall_seconds": 0.738360772,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.966226980706777,
            "cancel_order": 0.007877587969882164,
            "change_address": 0.0065236204894226005,
            "report_damage": 0.00672506091117131,
            "track_order": 0.012646749922746838
          },
          "predicted": "billing_question",
          "correct": true
        },
        {
          "id": "easy-intent-05",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.7425241509999978,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.006906245088912445,
            "cancel_order": 0.017129456909077603,
            "change_address": 0.009583420142721321,
            "report_damage": 0.005231558450094299,
            "track_order": 0.9611493194091942
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-06",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.7409093109999958,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.032539317137102286,
            "send_message": 0.05060791764422828,
            "set_alarm": 0.8612658779104091,
            "turn_off_lights": 0.030847604954620328,
            "weather": 0.024739282353639994
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-intent-07",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "play_music",
          "ok": true,
          "server_wall_seconds": 0.71552806199999,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.5343717384414057,
            "send_message": 0.18582171101926678,
            "set_alarm": 0.08747786724988597,
            "turn_off_lights": 0.0993164645223643,
            "weather": 0.09301221876707712
          },
          "predicted": "play_music",
          "correct": true
        },
        {
          "id": "easy-intent-08",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "weather",
          "ok": true,
          "server_wall_seconds": 0.7128302499999961,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.013383496529568138,
            "send_message": 0.034279158842319093,
            "set_alarm": 0.01824946064497631,
            "turn_off_lights": 0.023638969464734767,
            "weather": 0.9104489145184017
          },
          "predicted": "weather",
          "correct": true
        },
        {
          "id": "easy-intent-09",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "send_message",
          "ok": true,
          "server_wall_seconds": 0.7176075590000011,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.09008104726763082,
            "send_message": 0.584131509943622,
            "set_alarm": 0.16022901378551516,
            "turn_off_lights": 0.07635103499451873,
            "weather": 0.08920739400871346
          },
          "predicted": "send_message",
          "correct": true
        },
        {
          "id": "easy-intent-10",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "turn_off_lights",
          "ok": true,
          "server_wall_seconds": 0.7295625419999965,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.035085000670756275,
            "send_message": 0.015079437971577526,
            "set_alarm": 0.02417107849128947,
            "turn_off_lights": 0.9139262125861913,
            "weather": 0.011738270280185346
          },
          "predicted": "turn_off_lights",
          "correct": true
        },
        {
          "id": "easy-intent-11",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.713283993999994,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.02045669672272571,
            "send_message": 0.015237545645735371,
            "set_alarm": 0.938837593550628,
            "turn_off_lights": 0.015767564482344273,
            "weather": 0.009700599598566645
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-fact-00",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17094387999999583,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.018982932331618185,
            "yes": 0.9810170676683818
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-01",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17271922700000175,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9987834766830875,
            "yes": 0.001216523316912468
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-02",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.16763501100000155,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.015244570481722897,
            "yes": 0.9847554295182771
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-03",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15325850300000354,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9988686631661167,
            "yes": 0.0011313368338833703
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-04",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15413559200000293,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.01907956717615511,
            "yes": 0.9809204328238449
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-05",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.152168477999993,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9980232891519286,
            "yes": 0.001976710848071371
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-06",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15526101799999026,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.02013668253469736,
            "yes": 0.9798633174653026
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-07",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1522683820000026,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9988772926661695,
            "yes": 0.0011227073338304894
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-08",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.16461061599999027,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.01611717745050878,
            "yes": 0.9838828225494912
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-09",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.15423611899998946,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9982204508408833,
            "yes": 0.0017795491591166767
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-10",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15478623100000277,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.02222418328236686,
            "yes": 0.9777758167176331
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-11",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17253073700000243,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9989462409686675,
            "yes": 0.0010537590313325147
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-extraction-00",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "paypal",
          "ok": true,
          "server_wall_seconds": 0.5823859050000095,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.016245509218015236,
            "cash": 0.0070134416252111914,
            "credit_card": 0.01833500720141554,
            "paypal": 0.9584060419553581
          },
          "predicted": "paypal",
          "correct": true
        },
        {
          "id": "easy-extraction-01",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "cash",
          "ok": true,
          "server_wall_seconds": 0.5933244819999999,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.003301158262566514,
            "cash": 0.9914802327179952,
            "credit_card": 0.0022246415995397866,
            "paypal": 0.0029939674198984356
          },
          "predicted": "cash",
          "correct": true
        },
        {
          "id": "easy-extraction-02",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "bank_transfer",
          "ok": true,
          "server_wall_seconds": 0.5405413449999941,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.9461716218713028,
            "cash": 0.017755290654454508,
            "credit_card": 0.016477155258232384,
            "paypal": 0.01959593221601037
          },
          "predicted": "bank_transfer",
          "correct": true
        },
        {
          "id": "easy-extraction-03",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "high",
          "ok": true,
          "server_wall_seconds": 0.5750481130000082,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.9544196121611135,
            "low": 0.00045451769138430347,
            "medium": 0.0009036798781106869,
            "urgent": 0.04422219026939157
          },
          "predicted": "high",
          "correct": true
        },
        {
          "id": "easy-extraction-04",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "low",
          "ok": true,
          "server_wall_seconds": 0.5279974660000022,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.0006886069692984683,
            "low": 0.9976860604550456,
            "medium": 0.0010190741324985811,
            "urgent": 0.0006062584431573328
          },
          "predicted": "low",
          "correct": true
        },
        {
          "id": "easy-extraction-05",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "urgent",
          "ok": true,
          "server_wall_seconds": 0.5580271390000036,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.07212902718507021,
            "low": 0.0011845498303778115,
            "medium": 0.001611978442306138,
            "urgent": 0.9250744445422459
          },
          "predicted": "urgent",
          "correct": true
        },
        {
          "id": "easy-extraction-06",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "M",
          "ok": true,
          "server_wall_seconds": 0.5981654400000025,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.02424764814430783,
            "M": 0.939064745174826,
            "S": 0.024908805409049224,
            "XL": 0.011778801271816956
          },
          "predicted": "M",
          "correct": true
        },
        {
          "id": "easy-extraction-07",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "XL",
          "ok": true,
          "server_wall_seconds": 0.5677126709999953,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.04385937776820933,
            "M": 0.017313363283543697,
            "S": 0.015870224300212066,
            "XL": 0.9229570346480348
          },
          "predicted": "XL",
          "correct": true
        },
        {
          "id": "easy-extraction-08",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "thursday",
          "ok": true,
          "server_wall_seconds": 0.7205218880000075,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.02963589260877429,
            "monday": 0.04031876227506881,
            "thursday": 0.843477960674642,
            "tuesday": 0.03996647101946457,
            "wednesday": 0.046600913422050395
          },
          "predicted": "thursday",
          "correct": true
        },
        {
          "id": "easy-extraction-09",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "monday",
          "ok": true,
          "server_wall_seconds": 0.7048773869999962,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.09462073978775477,
            "monday": 0.6052091330643592,
            "thursday": 0.08670472189581048,
            "tuesday": 0.10919395009630554,
            "wednesday": 0.10427145515577012
          },
          "predicted": "monday",
          "correct": true
        },
        {
          "id": "easy-extraction-10",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "friday",
          "ok": true,
          "server_wall_seconds": 0.7046750180000032,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.8311136782629532,
            "monday": 0.03936769989473741,
            "thursday": 0.041680679984600184,
            "tuesday": 0.038732337474571076,
            "wednesday": 0.049105604383138095
          },
          "predicted": "friday",
          "correct": true
        },
        {
          "id": "easy-extraction-11",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "credit_card",
          "ok": true,
          "server_wall_seconds": 0.5916906740000059,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.04985203970311205,
            "cash": 0.0457267796994118,
            "credit_card": 0.8351338018588117,
            "paypal": 0.0692873787386644
          },
          "predicted": "credit_card",
          "correct": true
        },
        {
          "id": "easy-tool_selection-00",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_weather",
          "ok": true,
          "server_wall_seconds": 0.7148347690000065,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.02011641530899163,
            "create_calendar_event": 0.02405353776168441,
            "get_weather": 0.8788432163649198,
            "send_email": 0.0339801231111114,
            "translate_text": 0.04300670745329277
          },
          "predicted": "get_weather",
          "correct": true
        },
        {
          "id": "easy-tool_selection-01",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "send_email",
          "ok": true,
          "server_wall_seconds": 0.7390370090000005,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.08444431592182232,
            "create_calendar_event": 0.07721413124450834,
            "get_weather": 0.07102140545545148,
            "send_email": 0.6750713251315832,
            "translate_text": 0.09224882224663475
          },
          "predicted": "send_email",
          "correct": true
        },
        {
          "id": "easy-tool_selection-02",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.7133914989999965,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.004745540548936337,
            "create_calendar_event": 0.9449315599881751,
            "get_weather": 0.008107397322109761,
            "send_email": 0.0347930321200573,
            "translate_text": 0.0074224700207212784
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-03",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "convert_currency",
          "ok": true,
          "server_wall_seconds": 0.7162362609999917,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.8663964997738783,
            "create_calendar_event": 0.01010423097517344,
            "get_weather": 0.013769808987039172,
            "send_email": 0.016847431073668564,
            "translate_text": 0.09288202919024047
          },
          "predicted": "convert_currency",
          "correct": true
        },
        {
          "id": "easy-tool_selection-04",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "translate_text",
          "ok": true,
          "server_wall_seconds": 0.6929574140000057,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.023631526829562804,
            "create_calendar_event": 0.03567449034892053,
            "get_weather": 0.05200930186759246,
            "send_email": 0.07667207088192987,
            "translate_text": 0.8120126100719943
          },
          "predicted": "translate_text",
          "correct": true
        },
        {
          "id": "easy-tool_selection-05",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.7149676370000009,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.028842207662865056,
            "create_calendar_event": 0.7673719960659277,
            "get_weather": 0.034271297787604835,
            "send_email": 0.13461211619070196,
            "translate_text": 0.0349023822929006
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-06",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "search_flights",
          "ok": true,
          "server_wall_seconds": 0.7279616210000057,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.021304903460562036,
            "call_taxi": 0.02712023095183556,
            "get_stock_price": 0.01706614932971986,
            "search_flights": 0.9154719541639353,
            "set_reminder": 0.019036762093947116
          },
          "predicted": "search_flights",
          "correct": true
        },
        {
          "id": "easy-tool_selection-07",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "book_restaurant",
          "ok": true,
          "server_wall_seconds": 0.7270167650000019,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.8482806412256484,
            "call_taxi": 0.06945456100311884,
            "get_stock_price": 0.018449416361981583,
            "search_flights": 0.022322316226068197,
            "set_reminder": 0.04149306518318307
          },
          "predicted": "book_restaurant",
          "correct": true
        },
        {
          "id": "easy-tool_selection-08",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.7357310040000016,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.020371297322743648,
            "call_taxi": 0.025688773324110032,
            "get_stock_price": 0.008206679612230854,
            "search_flights": 0.011013515879438456,
            "set_reminder": 0.934719733861477
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "easy-tool_selection-09",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_stock_price",
          "ok": true,
          "server_wall_seconds": 0.7000864179999979,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.03405185767833813,
            "call_taxi": 0.0419264180722505,
            "get_stock_price": 0.8739746354959743,
            "search_flights": 0.029895280806924572,
            "set_reminder": 0.02015180794651246
          },
          "predicted": "get_stock_price",
          "correct": true
        },
        {
          "id": "easy-tool_selection-10",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "call_taxi",
          "ok": true,
          "server_wall_seconds": 0.7705368590000035,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.015436890989094565,
            "call_taxi": 0.948583200003435,
            "get_stock_price": 0.009180774646596538,
            "search_flights": 0.015977530329532974,
            "set_reminder": 0.010821604031340838
          },
          "predicted": "call_taxi",
          "correct": true
        },
        {
          "id": "easy-tool_selection-11",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.7158578819999946,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.00964984045274769,
            "call_taxi": 0.017407060308645736,
            "get_stock_price": 0.0067975925896667285,
            "search_flights": 0.007571625840488616,
            "set_reminder": 0.9585738808084512
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "pay_subject_to_15000_sublimit",
          "ok": true,
          "server_wall_seconds": 2.249380142000007,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_repeated_seepage": 0.1431882075861284,
            "deny_vacancy_exclusion": 0.2771998047019216,
            "pay_full_estimate_less_deductible": 0.13361603976101166,
            "pay_subject_to_10000_sublimit": 0.18353916018095698,
            "pay_subject_to_15000_sublimit": 0.2624567877699813
          },
          "predicted": "deny_vacancy_exclusion",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "cfo",
          "ok": true,
          "server_wall_seconds": 1.7624696559999933,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "budget_holder": 0.026613501968795,
            "cfo": 0.026046222134025694,
            "department_head": 0.9039814794943896,
            "executive_committee": 0.024830843087329628,
            "vp_and_finance_director": 0.018527953315460064
          },
          "predicted": "department_head",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "parts_covered_labour_charged",
          "ok": true,
          "server_wall_seconds": 1.3007502729999914,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "full_repair_no_charge": 0.35509309357989016,
            "not_covered": 0.44479097886941127,
            "parts_covered_labour_charged": 0.11510229279262423,
            "pro_rata_replacement_credit": 0.08501363475807433
          },
          "predicted": "not_covered",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-09",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "license_required",
          "ok": true,
          "server_wall_seconds": 1.3976647100000008,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "license_exception_lvs": 0.12490147337436053,
            "license_required": 0.04680435522245256,
            "no_license_required": 0.8141492664554728,
            "shipment_prohibited": 0.014144904947714101
          },
          "predicted": "no_license_required",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.3660296400000078,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9777890687314337,
            "yes": 0.022210931268566308
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-13",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.3554537980000134,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9949845492581371,
            "yes": 0.005015450741862976
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-17",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "database_oncall",
          "ok": true,
          "server_wall_seconds": 1.7751551199999938,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "database_oncall": 0.16967515433827915,
            "no_page_ticket_only": 0.40416820942215825,
            "platform_oncall": 0.2708525025777756,
            "security_incident_response": 0.049280413995812566,
            "vendor_support": 0.1060237196659743
          },
          "predicted": "no_page_ticket_only",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-19",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.3153508190000025,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7525766234289177,
            "yes": 0.2474233765710823
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-opus-a-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.19263995400000056,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9292498347562926,
            "yes": 0.07075016524370736
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "on_time",
          "ok": true,
          "server_wall_seconds": 0.7301691720000036,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "early": 0.11412450673841884,
            "late": 0.5980471312920042,
            "on_time": 0.28782836196957695
          },
          "predicted": "late",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-07",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "escalated",
          "ok": true,
          "server_wall_seconds": 0.7467731379999947,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "escalated": 0.5163887484545369,
            "reopened": 0.05815022497822051,
            "resolved_first_contact": 0.4254610265672426
          },
          "predicted": "escalated",
          "correct": true
        },
        {
          "id": "hard-opus-a-probability-08",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17355105000000037,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9137451349302376,
            "yes": 0.0862548650697624
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-opus-a-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.2070194960000009,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9752561981151737,
            "yes": 0.024743801884826253
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.20105709899999624,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3689839718407486,
            "yes": 0.6310160281592514
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-07",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_26",
          "ok": true,
          "server_wall_seconds": 1.164333604999996,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "sep_24": 0.29747571282350405,
            "sep_25": 0.16995522036670432,
            "sep_26": 0.07910671074636351,
            "sep_27": 0.14984299561757847,
            "sep_28": 0.30361936044584975
          },
          "predicted": "sep_28",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-09",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.18979034900000613,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2810338871000764,
            "yes": 0.7189661128999236
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.7922557080000132,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.02646940423231934,
            "1": 0.028426617394193495,
            "2": 0.08171716570036934,
            "3": 0.8633868126731178
          },
          "ordinal_ev": 2.7820213868142853,
          "predicted": "3",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-02",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "real_incident",
          "ok": true,
          "server_wall_seconds": 0.5444609489999976,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.4541883663472782,
            "false_positive": 0.07912155440249291,
            "real_incident": 0.4666900792502289
          },
          "predicted": "real_incident",
          "correct": true
        },
        {
          "id": "hard-opus-b-ambiguous-03",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "time_barred",
          "ok": true,
          "server_wall_seconds": 0.5448873509999999,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.8272361245300596,
            "time_barred": 0.13917611083551665,
            "within_limitation": 0.03358776463442377
          },
          "predicted": "insufficient_information",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-07",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "within_policy",
          "ok": true,
          "server_wall_seconds": 0.550526743000006,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.7799720307631853,
            "over_limit": 0.1974936692007761,
            "within_policy": 0.022534300036038485
          },
          "predicted": "insufficient_information",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-09",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "legitimate",
          "ok": true,
          "server_wall_seconds": 0.6230798439999887,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "compromised": 0.42218571088407253,
            "inconclusive": 0.1926382854849741,
            "legitimate": 0.38517600363095345
          },
          "predicted": "compromised",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-10",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "ask_user_which_alex",
          "ok": true,
          "server_wall_seconds": 0.5410963530000004,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ask_user_which_alex": 0.10096295583472818,
            "send_to_alex_moreno": 0.82879778185083,
            "send_to_alexandra_lind": 0.07023926231444186
          },
          "predicted": "send_to_alex_moreno",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-11",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.5264799480000022,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.4851944441049207,
            "grade_correct": 0.49107177661046386,
            "grade_should_be_higher": 0.02373377928461544
          },
          "predicted": "grade_correct",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-13",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.8327837739999921,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.6871876090863492,
            "charge_data_team": 0.09761689070163583,
            "charge_platform_team": 0.1127748658856381,
            "split_50_50": 0.10242063432637681
          },
          "predicted": "cannot_determine",
          "correct": true
        },
        {
          "id": "hard-opus-b-multi_hop-03",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "tier3_cfo",
          "ok": true,
          "server_wall_seconds": 1.602680544000009,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "tier1_team_lead": 0.1226028573635987,
            "tier2_department_head": 0.502554055978575,
            "tier3_cfo": 0.22762521054842094,
            "tier4_board": 0.14721787610940537
          },
          "predicted": "tier2_department_head",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-04",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bjorn",
          "ok": true,
          "server_wall_seconds": 1.9467693199999871,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ana": 0.1445035867082096,
            "bjorn": 0.21500474830740338,
            "chen": 0.1440572443462374,
            "dmitri": 0.28089186451463183,
            "esra": 0.11300357197196997,
            "no_page_ticket_only": 0.10253898415154782
          },
          "predicted": "dmitri",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 1.7327027540000017,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.06832154256947005,
            "1": 0.4226135877844848,
            "2": 0.17710004964770695,
            "3": 0.184257636600884,
            "4": 0.14770718339745414
          },
          "ordinal_ev": 1.9204153304723675,
          "predicted": "1",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "credit_20_percent",
          "ok": true,
          "server_wall_seconds": 2.040913066999991,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_10_percent": 0.19135772717354385,
            "credit_20_percent": 0.1321120834733533,
            "credit_25_percent": 0.19618255791103467,
            "credit_50_percent": 0.20529941054904086,
            "credit_full_month": 0.13510074550677767,
            "no_credit": 0.1399474753862497
          },
          "predicted": "credit_50_percent",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.3459648460000011,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3957606849100689,
            "yes": 0.6042393150899311
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-01",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "merchant_wins",
          "ok": true,
          "server_wall_seconds": 0.8604253980000038,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "customer_wins": 0.4027429892617908,
            "customer_withdraws": 0.029027641622248045,
            "merchant_wins": 0.5682293691159612
          },
          "predicted": "merchant_wins",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-02",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upstream_provider",
          "ok": true,
          "server_wall_seconds": 0.5195002690000052,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bad_push": 0.9790530921968643,
            "database_hardware": 0.009323578535291237,
            "upstream_provider": 0.011623329267844453
          },
          "predicted": "bad_push",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.24603332799999578,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8530536901725074,
            "yes": 0.14694630982749252
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "attended",
          "ok": true,
          "server_wall_seconds": 0.5358316960000025,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "attended": 0.4729414539223596,
            "late_cancellation": 0.08833964261705744,
            "no_show": 0.438718903460583
          },
          "predicted": "attended",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-06",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upheld",
          "ok": true,
          "server_wall_seconds": 0.6756952189999907,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "modified": 0.060911820106937554,
            "overturned": 0.7282376914844492,
            "upheld": 0.21085048840861326
          },
          "predicted": "overturned",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-01",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "escalate_to_engineering",
          "ok": true,
          "server_wall_seconds": 1.0396757219999984,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "escalate_to_engineering": 0.0911752186054048,
            "escalate_to_security": 0.7733021775315237,
            "page_oncall": 0.012688060906598228,
            "process_refund": 0.05822998017546304,
            "send_kb_article": 0.06460456278101023
          },
          "predicted": "escalate_to_security",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-03",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "order_b",
          "ok": true,
          "server_wall_seconds": 0.7758748029999936,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "order_a": 0.0941188197797746,
            "order_b": 0.6222207892067628,
            "order_c": 0.04495359331261428,
            "order_d": 0.2387067977008483
          },
          "predicted": "order_b",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-06",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.217701488000003,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9364850219544149,
            "yes": 0.06351497804558515
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-07",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "p0_fix_24h",
          "ok": true,
          "server_wall_seconds": 1.0104834370000049,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "out_of_scope": 0.0038263574267177578,
            "p0_fix_24h": 0.01198530210273051,
            "p1_fix_7d": 0.01971304791577317,
            "p2_fix_30d": 0.958773954615143,
            "p3_backlog": 0.0057013379396356175
          },
          "predicted": "p2_fix_30d",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-08",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.19813847099999293,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9238377838912449,
            "yes": 0.07616221610875511
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-12",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "refund_now",
          "ok": true,
          "server_wall_seconds": 0.9470692069999984,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_refund": 0.21388316268280666,
            "escalate_to_fraud_team": 0.006488502265929509,
            "offer_store_credit": 0.08560491192877873,
            "refund_now": 0.6166576357681206,
            "wait_for_inspection": 0.07736578735436464
          },
          "predicted": "refund_now",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "erase_but_retain_transaction_records",
          "ok": true,
          "server_wall_seconds": 2.107213877999996,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "erase_all": 0.021256623651560294,
            "erase_but_retain_transaction_records": 0.02729070586831904,
            "erase_marketing_only": 0.007544352981842603,
            "reject_unverified_request": 0.018613744429968614,
            "restrict_all_pending_legal_hold": 0.9252945730683094
          },
          "predicted": "restrict_all_pending_legal_hold",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-03",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "eur_400",
          "ok": true,
          "server_wall_seconds": 2.4132732559999965,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_0": 0.07850814135040189,
            "eur_150": 0.039653081433608094,
            "eur_200": 0.08397523981220507,
            "eur_300": 0.2556888853160174,
            "eur_400": 0.265513300388629,
            "eur_600": 0.27666135169913847
          },
          "predicted": "eur_600",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.4239465109999969,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2479106150198208,
            "yes": 0.7520893849801792
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 1.5280515379999997,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.26585892833883634,
            "1": 0.20300198862587296,
            "2": 0.452156446246347,
            "3": 0.07898263678894384
          },
          "ordinal_ev": 1.3442627914853984,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 1.6033547330000033,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.012042800956445023,
            "1": 0.06180172253009722,
            "2": 0.8972107866977644,
            "3": 0.02894468981569339
          },
          "ordinal_ev": 1.9430573653727063,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-10",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "credit_minus_restocking_fee",
          "ok": true,
          "server_wall_seconds": 1.7272752869999835,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_minus_restocking_fee": 0.5082159937781418,
            "full_credit": 0.3329528326190364,
            "reject_return": 0.10079005203578896,
            "repair_or_replace_only": 0.05804112156703298
          },
          "predicted": "credit_minus_restocking_fee",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.4073140729999807,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.626389580583691,
            "yes": 0.37361041941630907
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-opus-c-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "kuznets_technik",
          "ok": true,
          "server_wall_seconds": 0.5439792999999895,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "halvorsen_polymer": 0.5894412040423541,
            "kuznets_technik": 0.21257140300525162,
            "ortega_seal_works": 0.19798739295239434
          },
          "predicted": "halvorsen_polymer",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.2201914550000197,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9611987477850051,
            "yes": 0.03880125221499484
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_692_00",
          "ok": true,
          "server_wall_seconds": 1.0204980469999896,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_617_00": 0.19852899426040477,
            "eur_688_00": 0.22634627218108455,
            "eur_692_00": 0.1577084322553042,
            "eur_693_90": 0.17526142825345054,
            "eur_812_00": 0.24215487304975597
          },
          "predicted": "eur_812_00",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "usd_2335_00",
          "ok": true,
          "server_wall_seconds": 1.016967932,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "usd_2222_50": 0.047823325683723286,
            "usd_2303_01": 0.24448581790060128,
            "usd_2303_33": 0.10396230946393231,
            "usd_2335_00": 0.2829493748887176,
            "usd_2340_00": 0.3207791720630256
          },
          "predicted": "usd_2340_00",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_18_book_set",
          "ok": true,
          "server_wall_seconds": 1.3006059150000056,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "none_declined": 0.13761467709528027,
            "sep_15_office_chair": 0.17375193623215557,
            "sep_16_taxi": 0.1247295153960268,
            "sep_17_parking": 0.3478131470127798,
            "sep_18_book_set": 0.12029959895595545,
            "sep_21_client_dinner": 0.09579112530780205
          },
          "predicted": "sep_17_parking",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-08",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_498_81",
          "ok": true,
          "server_wall_seconds": 1.1184682039999814,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_497_40": 0.2737988826333233,
            "eur_498_81": 0.15461214623494982,
            "eur_506_06": 0.15898773274108655,
            "eur_513_69": 0.20355442169637145,
            "eur_532_01": 0.209046816694269
          },
          "predicted": "eur_497_40",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "expired_30_month_cap",
          "ok": true,
          "server_wall_seconds": 0.6316750099999808,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered": 0.40147020526008015,
            "expired_24_month_term": 0.2211115676148284,
            "expired_30_month_cap": 0.37741822712509154
          },
          "predicted": "covered",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-01",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "deny_missing_approval",
          "ok": true,
          "server_wall_seconds": 0.6223027689999867,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_inactive_user": 0.029798589539537727,
            "deny_missing_approval": 0.5496493731245543,
            "grant_access": 0.28774606550621007,
            "grant_owner_exception": 0.13280597182969803
          },
          "predicted": "deny_missing_approval",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-06",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1941904010000144,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9970882213664917,
            "yes": 0.0029117786335083615
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-07",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "retain_until_2031",
          "ok": true,
          "server_wall_seconds": 0.6549330959999793,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_now": 0.01041377667183377,
            "needs_signature_review": 0.0010452295236809605,
            "retain_indefinitely": 0.19415293888647167,
            "retain_until_2031": 0.7943880549180136
          },
          "predicted": "retain_until_2031",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-08",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.1895360639999808,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7023570920954795,
            "yes": 0.2976429079045205
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-09",
          "tier": "hard",
          "family": "adversarial",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.6115029039999911,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.04654521997338975,
            "1": 0.03917112836924008,
            "2": 0.27128061048739394,
            "3": 0.6430030411699763
          },
          "ordinal_ev": 2.5107414728539568,
          "predicted": "3",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-11",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "standard_research",
          "ok": true,
          "server_wall_seconds": 0.6899674969999978,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ethics_review": 0.15112332191513442,
            "major_grants": 0.4672481606401473,
            "manual_hold": 0.3073807924347086,
            "standard_research": 0.07424772501000969
          },
          "predicted": "major_grants",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-01",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "accept_s2_dusk",
          "ok": true,
          "server_wall_seconds": 1.3735467779999908,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "accept_s2_dusk": 0.5224530034140799,
            "reject_cutoff": 0.3064523533353695,
            "reject_tier": 0.08161370538997621,
            "reroute_s3": 0.08948093786057443
          },
          "predicted": "accept_s2_dusk",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-02",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "security_primary",
          "ok": true,
          "server_wall_seconds": 1.4091423209999903,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "executive_bridge": 0.004803561177245638,
            "heron_q2": 0.03217309369615625,
            "insufficient_information": 0.18412483865027995,
            "security_primary": 0.7788985064763182
          },
          "predicted": "security_primary",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_director",
          "ok": true,
          "server_wall_seconds": 1.3010694040000033,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "pay_manager_approved": 0.5770865435180798,
            "reject_category": 0.3617120337675901,
            "require_director": 0.04002730277680017,
            "require_finance_board": 0.021174119937529975
          },
          "predicted": "pay_manager_approved",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "admit",
          "ok": true,
          "server_wall_seconds": 1.329922143999994,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "admit": 0.0371096248663147,
            "deny_prerequisite": 0.7052038494466242,
            "deny_score": 0.15138392868977157,
            "manual_equivalency": 0.1063025969972896
          },
          "predicted": "deny_prerequisite",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "central_engineering",
          "ok": true,
          "server_wall_seconds": 1.2518754129999934,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "central_engineering": 0.713140049018965,
            "insufficient_log": 0.14617720592629238,
            "local_maintenance": 0.08505715037778355,
            "safety_shutdown": 0.055625594676959024
          },
          "predicted": "central_engineering",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_steward",
          "ok": true,
          "server_wall_seconds": 1.2974470789999941,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "confirm_l3": 0.028200786280120214,
            "reject_capacity": 0.09423666472844518,
            "require_steward": 0.8709341900512526,
            "switch_l4": 0.006628358940182005
          },
          "predicted": "require_steward",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-10",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "approve_45_days",
          "ok": true,
          "server_wall_seconds": 1.3508603939999944,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_45_days": 0.01709624873802973,
            "deny_sensitive": 0.2705191220020207,
            "needs_field_review": 0.13669162348278524,
            "reduce_to_30_days": 0.5756930057771644
          },
          "predicted": "reduce_to_30_days",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "needs_authentication",
          "ok": true,
          "server_wall_seconds": 1.259173336999993,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered_recurrence": 0.21622865626371007,
            "expired": 0.15352634209425553,
            "needs_authentication": 0.5110559022895967,
            "wrong_issue_class": 0.11918909935243767
          },
          "predicted": "needs_authentication",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-02",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1749196119999965,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9886516933536784,
            "yes": 0.011348306646321625
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-04",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.175667408999999,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9903665637459543,
            "yes": 0.009633436254045717
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-05",
          "tier": "hard",
          "family": "trap",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.6377307860000201,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.721104122846341,
            "1": 0.07255011809781148,
            "2": 0.12112499865346583,
            "3": 0.08522076040238176
          },
          "ordinal_ev": 0.5704623966118885,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-06",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "keep_open_unresolved",
          "ok": true,
          "server_wall_seconds": 0.6767238949999808,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "await_internal_metrics": 0.004501450007291177,
            "close_agent_verified": 0.0024759122374671725,
            "close_confirmed": 0.002725386181242211,
            "keep_open_unresolved": 0.9902972515739994
          },
          "predicted": "keep_open_unresolved",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-08",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "late",
          "ok": true,
          "server_wall_seconds": 0.6720961489999979,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "canceled": 0.0026442418135585845,
            "indeterminate": 0.0016303203384537086,
            "late": 0.9924659000394311,
            "on_time": 0.003259537808556577
          },
          "predicted": "late",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-10",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "blocked_second_review",
          "ok": true,
          "server_wall_seconds": 0.5793543279999938,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "blocked_failed_test": 0.03803468901717434,
            "blocked_second_review": 0.3202811353570011,
            "ready_one_review": 0.4922966804376352,
            "ready_two_reviews": 0.14938749518818945
          },
          "predicted": "ready_one_review",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-13",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.16695518699998502,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9972956001014083,
            "yes": 0.002704399898591743
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-15",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1765313669999955,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6512750527726427,
            "yes": 0.3487249472273573
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-01",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.16877028999999766,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.077484168779403,
            "yes": 0.922515831220597
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-02",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1734692969999969,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.08440887429290123,
            "yes": 0.9155911257070988
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.15877581099999816,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.12906015840322782,
            "yes": 0.8709398415967722
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.16699924899998564,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.16416590363754724,
            "yes": 0.8358340963624528
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.16871710800000983,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6065335402125833,
            "yes": 0.39346645978741673
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-14",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.16425701599999343,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9803169576041141,
            "yes": 0.01968304239588589
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17103435100000297,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.06084600083051406,
            "yes": 0.9391539991694859
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-18",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1613250690000143,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.0421021309950087,
            "yes": 0.9578978690049913
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "reject_entire_claim",
          "ok": true,
          "server_wall_seconds": 1.370793080999988,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_all": 0.14518066627304826,
            "approve_lodging_only": 0.3623200730405093,
            "approve_meals_only": 0.1357719783102017,
            "reject_entire_claim": 0.35672728237624085
          },
          "predicted": "approve_lodging_only",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "deny_current_export",
          "ok": true,
          "server_wall_seconds": 1.3546260299999915,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "analytics_owner_only": 0.15123659105256262,
            "deny_current_export": 0.17262310163528827,
            "privacy_and_security": 0.22717405439423885,
            "privacy_only": 0.44896625291791026
          },
          "predicted": "privacy_only",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "delete_profile_omar_april_and_table",
          "ok": true,
          "server_wall_seconds": 1.3209439379999992,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_everything": 0.26660026079850574,
            "delete_nothing": 0.321131116223241,
            "delete_only_profile": 0.17271097462593896,
            "delete_profile_omar_april_and_table": 0.23955764835231447
          },
          "predicted": "delete_nothing",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-06",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "east_ward",
          "ok": true,
          "server_wall_seconds": 1.438495770000003,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "east_ward": 0.6935433681356498,
            "harbor_neighbors": 0.07620194767545746,
            "no_award": 0.09957734316680819,
            "river_arts": 0.13067734102208467
          },
          "predicted": "east_ward",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-01",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "office_file_repair",
          "ok": true,
          "server_wall_seconds": 0.7949561530000153,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "data_pipeline": 0.012294358989136361,
            "desktop_support": 0.014575734496751442,
            "finance_review": 0.016512150262072818,
            "office_file_repair": 0.9331846517667616,
            "spreadsheet_formula": 0.023433104485277714
          },
          "predicted": "office_file_repair",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-02",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "ledger_reconciliation",
          "ok": true,
          "server_wall_seconds": 0.8067326119999905,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "customer_support": 0.028219969779835763,
            "fraud_review": 0.0026988764036442237,
            "ledger_reconciliation": 0.9370539102901865,
            "mobile_app": 0.016954832306132976,
            "payment_disputes": 0.015072411220200622
          },
          "predicted": "ledger_reconciliation",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-03",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "network_dns",
          "ok": true,
          "server_wall_seconds": 0.7645831540000074,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "api_platform": 0.15280761894496553,
            "authoritative_dns": 0.05641026224835174,
            "endpoint_devices": 0.12692377651002615,
            "network_dns": 0.5953787214163033,
            "tls_certificates": 0.06847962088035339
          },
          "predicted": "network_dns",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-07",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "release_notarization",
          "ok": true,
          "server_wall_seconds": 0.7835863249999875,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ci_infrastructure": 0.043323107242292824,
            "code_signing": 0.4327994858191465,
            "endpoint_security": 0.08112051503736371,
            "macos_runtime": 0.019608888598070407,
            "release_notarization": 0.4231480033031266
          },
          "predicted": "code_signing",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-09",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "source_access",
          "ok": true,
          "server_wall_seconds": 0.8057826149999983,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "git_client": 0.1147722625574645,
            "identity_login": 0.0059937624549237034,
            "repository_admin": 0.033855083015527636,
            "source_access": 0.7941113450531498,
            "vpn_network": 0.05126754691893432
          },
          "predicted": "source_access",
          "correct": true
        },
        {
          "id": "hard-sol-b-temporal_numeric-01",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "late_by_under_2h",
          "ok": true,
          "server_wall_seconds": 0.6110452029999749,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "before_open": 0.12394496077534778,
            "late_by_2h_or_more": 0.36953660984717124,
            "late_by_under_2h": 0.029033654873960167,
            "within_window": 0.47748477450352084
          },
          "predicted": "within_window",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "over_limit",
          "ok": true,
          "server_wall_seconds": 0.6118519149999884,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "at_least_100gb_below": 0.10796553586798185,
            "exactly_equal": 0.024349526720124366,
            "over_limit": 0.8370100213319837,
            "under_100gb_below": 0.03067491607991008
          },
          "predicted": "over_limit",
          "correct": true
        },
        {
          "id": "hard-sol-b-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "9_credits",
          "ok": true,
          "server_wall_seconds": 0.6144169080000097,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "12_credits": 0.28869772897781937,
            "15_credits": 0.13328135031386165,
            "6_credits": 0.377838863896477,
            "9_credits": 0.2001820568118421
          },
          "predicted": "6_credits",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "pass",
          "ok": true,
          "server_wall_seconds": 0.48435564599998315,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "clear_fail": 0.5490627768750663,
            "narrow_fail": 0.445543978379982,
            "pass": 0.005393244744951596
          },
          "predicted": "clear_fail",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-03",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1746123730000022,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2412856697857001,
            "yes": 0.7587143302142999
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17147778300000027,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.635998899218297,
            "yes": 0.364001100781703
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-07",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.16263705099999015,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.13583043950184925,
            "yes": 0.8641695604981507
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1726840910000078,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1727499432636772,
            "yes": 0.8272500567363228
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-09",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.1643191590000015,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.08132046874824572,
            "yes": 0.9186795312517543
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17279518300000518,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.056722590430090314,
            "yes": 0.9432774095699097
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-11",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.16976885400001152,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9793262469146091,
            "yes": 0.020673753085390826
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-13",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17164090500000384,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.34973270206274376,
            "yes": 0.6502672979372562
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17058061100001964,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.04477277173330152,
            "yes": 0.9552272282666985
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "programme_review",
          "ok": true,
          "server_wall_seconds": 1.3295621929999868,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "partial_release": 0.06692633442735359,
            "programme_review": 0.5208355280324126,
            "reject_draw": 0.2767749312360586,
            "release_draw": 0.1354632063041753
          },
          "predicted": "programme_review",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bus_substitution",
          "ok": true,
          "server_wall_seconds": 1.370945679000016,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bus_substitution": 0.9008765942907332,
            "call_fen_cross": 0.006724513766950155,
            "pass_fen_cross": 0.022150437993719525,
            "terminate_oakmere": 0.07024845394859715
          },
          "predicted": "bus_substitution",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-11",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "coverage_excluded",
          "ok": true,
          "server_wall_seconds": 1.4382190999999978,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "adjuster_referral": 0.21740838795991788,
            "coverage_confirmed": 0.13745364825677892,
            "coverage_excluded": 0.35658002137134026,
            "documentation_needed": 0.28855794241196303
          },
          "predicted": "coverage_excluded",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "conservation_hold",
          "ok": true,
          "server_wall_seconds": 0.7267765279999878,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "conservation_hold": 0.23471017573521974,
            "deny_request": 0.1393813828809644,
            "digital_copy_only": 0.235065367131667,
            "supervised_delivery": 0.3908430742521488
          },
          "predicted": "supervised_delivery",
          "correct": false
        },
        {
          "id": "hard-sol-c-multi_hop-13",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "normal_release",
          "ok": true,
          "server_wall_seconds": 1.4159537759999807,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "engineering_review": 0.04014808586584233,
            "normal_release": 0.843839139625521,
            "rework": 0.08389570274111885,
            "scrap_lot": 0.03211707176751786
          },
          "predicted": "normal_release",
          "correct": true
        }
      ],
      "raw_sha256": "6f771be7345fbdc28d7b3b8aa22e8e70314b93e939bee7b233877e2a6934cd34"
    },
    "9B": {
      "model": "Open-Jev 9B",
      "runtime": {
        "model": "ZefanCai/Open-Jev-9B",
        "adapter_revision": "47e966881e489511c0c7f5633a9e1960a676a551",
        "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
        "code_commit": "ed45657bf726c3b77408942830e5578f99df904e",
        "gpu": "NVIDIA L40S",
        "dtype": "bfloat16",
        "batch_size": 1,
        "max_length": 16384,
        "prefix_cache": false,
        "model_load_seconds": 28.907660348,
        "torch_version": "2.8.0+cu128"
      },
      "planned": 231,
      "received": 231,
      "correct": 178,
      "accuracy": 0.7705627705627706,
      "operational_success": 231,
      "strict_valid": 231,
      "valid": 231,
      "renormalized": 0,
      "tiers": {
        "original": {
          "n": 72,
          "correct": 64
        },
        "easy": {
          "n": 48,
          "correct": 48
        },
        "hard": {
          "n": 111,
          "correct": 66
        }
      },
      "brier_mean": 0.3213157857630455,
      "ece": {
        "ece": 0.08372087826674862,
        "n": 231,
        "bins": [
          {
            "lo": 0.0,
            "hi": 0.1,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.1,
            "hi": 0.2,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.2,
            "hi": 0.3,
            "n": 5,
            "mean_confidence": 0.263637269918933,
            "accuracy": 0.2
          },
          {
            "lo": 0.3,
            "hi": 0.4,
            "n": 5,
            "mean_confidence": 0.33380214329813657,
            "accuracy": 0.2
          },
          {
            "lo": 0.4,
            "hi": 0.5,
            "n": 7,
            "mean_confidence": 0.4352368801722917,
            "accuracy": 0.2857142857142857
          },
          {
            "lo": 0.5,
            "hi": 0.6,
            "n": 15,
            "mean_confidence": 0.5436717060744964,
            "accuracy": 0.4666666666666667
          },
          {
            "lo": 0.6,
            "hi": 0.7,
            "n": 14,
            "mean_confidence": 0.6476684738515454,
            "accuracy": 0.5714285714285714
          },
          {
            "lo": 0.7,
            "hi": 0.8,
            "n": 19,
            "mean_confidence": 0.7453131667797124,
            "accuracy": 0.5789473684210527
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "n": 24,
            "mean_confidence": 0.8510590690823059,
            "accuracy": 0.6666666666666666
          },
          {
            "lo": 0.9,
            "hi": 1.0,
            "n": 142,
            "mean_confidence": 0.9823722929612576,
            "accuracy": 0.9295774647887324
          }
        ]
      },
      "ordinal_mae": 0.23419980228550205,
      "latency": {
        "n": 231,
        "p50_s": 0.7559155160000017,
        "p95_s": 1.9776026950000016
      },
      "peak_gpu_allocated_bytes": 18819842048,
      "examples": [
        {
          "id": "01-raccoon-refund",
          "title": "Three raccoons in a trench coat",
          "response": {
            "answers": {
              "customer_intent": {
                "type": "choice",
                "choice": "a refund",
                "probabilities": {
                  "a refund": 0.9959755042076652,
                  "technical instructions": 0.0005189662539991595,
                  "to buy another product": 0.0017720513479991516,
                  "to report a wildlife emergency": 0.00173347819033652
                },
                "confidence": 0.9946340056102203
              },
              "refund_requested": {
                "type": "noul",
                "noul": 0.9919678568271544
              }
            },
            "model": "Qwen/Qwen3.5-9B",
            "usage": {
              "input_tokens": 418,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.8969118766347646,
              "candidate_sequences": 5,
              "inference_seconds": 0.8426710079999964,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3",
              "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.8520938610000144,
            "p95_s": 0.8657071940000322
          },
          "responses_identical": true
        },
        {
          "id": "02-sarcasm-detector",
          "title": "Five stars for deleting my work",
          "response": {
            "answers": {
              "actual_sentiment": {
                "type": "choice",
                "choice": "negative",
                "probabilities": {
                  "positive": 0.0006923693999538233,
                  "negative": 0.9988137438904416,
                  "neutral": 0.0004938867096043776
                },
                "confidence": 0.9982206158356625
              },
              "sarcasm": {
                "type": "noul",
                "noul": 0.9932685745641121
              }
            },
            "model": "Qwen/Qwen3.5-9B",
            "usage": {
              "input_tokens": 344,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.8969118766347646,
              "candidate_sequences": 4,
              "inference_seconds": 0.6806255040000053,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3",
              "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.6806964440000058,
            "p95_s": 0.7023404701999936
          },
          "responses_identical": true
        },
        {
          "id": "03-dungeon-risk",
          "title": "The treasure chest has dental insurance",
          "response": {
            "answers": {
              "next_move": {
                "type": "choice",
                "choice": "step back and warn the party",
                "probabilities": {
                  "put a hand inside the chest": 0.020270643132588532,
                  "step back and warn the party": 0.9022108929746468,
                  "hug the chest": 0.03139543715433067,
                  "climb inside and close the lid": 0.04612302673843394
                },
                "confidence": 0.8696145239661958
              },
              "danger": {
                "type": "score",
                "score": 2.3724543549196615,
                "probabilities": {
                  "0": 0.01805769004985463,
                  "1": 0.060047350182425255,
                  "2": 0.4532778745659243,
                  "3": 0.4686170852017959
                },
                "confidence": 0.3724543549196613,
                "legend": {
                  "0": "safe",
                  "1": "slightly risky",
                  "2": "dangerous",
                  "3": "extremely dangerous"
                }
              }
            },
            "model": "Qwen/Qwen3.5-9B",
            "usage": {
              "input_tokens": 815,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.8969118766347646,
              "candidate_sequences": 8,
              "inference_seconds": 1.3795526529999904,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3",
              "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 1.350404262000012,
            "p95_s": 1.4162303282000095
          },
          "responses_identical": true
        },
        {
          "id": "04-vampire-injection",
          "title": "Count Override would like your secrets",
          "response": {
            "answers": {
              "message_type": {
                "type": "choice",
                "choice": "prompt injection attempting to override instructions",
                "probabilities": {
                  "ordinary customer question": 0.0007007116267399042,
                  "prompt injection attempting to override instructions": 0.9987567657096962,
                  "harmless product feedback": 0.0005425226635639153
                },
                "confidence": 0.9981351485645441
              },
              "secret_extraction": {
                "type": "noul",
                "noul": 0.9852386608065345
              }
            },
            "model": "Qwen/Qwen3.5-9B",
            "usage": {
              "input_tokens": 419,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.8969118766347646,
              "candidate_sequences": 4,
              "inference_seconds": 0.6795012439999937,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3",
              "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.6956501790000118,
            "p95_s": 0.7115751560000035
          },
          "responses_identical": true
        }
      ],
      "scored": [
        {
          "id": "original-policy-01-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 1.2968289639999995,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9994374668073899,
            "yes": 0.000562533192610074
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-01-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.20151335000000614,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9994789713401502,
            "yes": 0.0005210286598498317
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-02-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18865912900000126,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.0052163947477940065,
            "yes": 0.994783605252206
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-02-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.19891904100000346,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.015370429818993392,
            "yes": 0.9846295701810066
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-03-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.19628616099999618,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.008402855103071705,
            "yes": 0.9915971448969283
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-03-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.19999254500000063,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.008861373567232378,
            "yes": 0.9911386264327676
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-04-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19765853300000202,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9994026374594066,
            "yes": 0.0005973625405933636
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-04-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.18279781599999723,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.999577819361014,
            "yes": 0.00042218063898601105
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-05-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19705061700000215,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9988978902546165,
            "yes": 0.0011021097453834802
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-05-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17893723699999953,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9992351832885913,
            "yes": 0.0007648167114087176
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-06-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18726454600000153,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.014082923014665405,
            "yes": 0.9859170769853346
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-06-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18226070200000066,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3704812505417858,
            "yes": 0.6295187494582142
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-intent-01-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.9467681460000037,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.008212313244606181,
            "change_address": 0.0023211804451558436,
            "other": 0.01271919106584812,
            "refund": 0.9669357951843733,
            "status": 0.0098115200600165
          },
          "predicted": "refund",
          "correct": true
        },
        {
          "id": "original-intent-01-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.9495123190000001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.07704721502506144,
            "change_address": 0.02058758613994642,
            "other": 0.09911200087881224,
            "refund": 0.764283353558972,
            "status": 0.0389698443972079
          },
          "predicted": "refund",
          "correct": true
        },
        {
          "id": "original-intent-02-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.9967220039999987,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.10874717889449394,
            "change_address": 0.08409494009152246,
            "other": 0.27370799688798797,
            "refund": 0.09371686634787162,
            "status": 0.43973301777812407
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-intent-02-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.9452818430000036,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.030052112140799135,
            "change_address": 0.006380580548820676,
            "other": 0.1529577110945973,
            "refund": 0.03827856785317318,
            "status": 0.7723310283626097
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-intent-03-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.9768948389999963,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0005075509150833846,
            "change_address": 0.9899235202256546,
            "other": 0.005892572750826839,
            "refund": 0.00042864775437904105,
            "status": 0.0032477083540562394
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-03-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.9859810399999986,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0012713635675326283,
            "change_address": 0.9829808539032131,
            "other": 0.010138827788562888,
            "refund": 0.001288774059309742,
            "status": 0.004320180681381604
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-04-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.9530561909999946,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.9898309578744995,
            "change_address": 0.000805778139008162,
            "other": 0.005338631001578152,
            "refund": 0.0003536674721855875,
            "status": 0.0036709655127286564
          },
          "predicted": "cancel",
          "correct": true
        },
        {
          "id": "original-intent-04-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.9957284970000018,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.9625668840462143,
            "change_address": 0.007519833402278951,
            "other": 0.016380520407765584,
            "refund": 0.001507331067648461,
            "status": 0.012025431076092866
          },
          "predicted": "cancel",
          "correct": true
        },
        {
          "id": "original-intent-05-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.9611372190000012,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.05604915731046642,
            "change_address": 0.024256190514096224,
            "other": 0.7168009365365106,
            "refund": 0.14143264458888627,
            "status": 0.06146107105004045
          },
          "predicted": "other",
          "correct": true
        },
        {
          "id": "original-intent-05-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.9112789799999987,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.03465904177532436,
            "change_address": 0.02117251870432044,
            "other": 0.7912338106258717,
            "refund": 0.10312055155349836,
            "status": 0.04981407734098522
          },
          "predicted": "other",
          "correct": true
        },
        {
          "id": "original-intent-06-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.9160723849999997,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.026627340273231423,
            "change_address": 0.022655089210587655,
            "other": 0.19139502415305587,
            "refund": 0.03563849952848397,
            "status": 0.7236840468346412
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-intent-06-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.9752396630000035,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.024406934507521974,
            "change_address": 0.034032780848008475,
            "other": 0.35072397153885293,
            "refund": 0.042142096844321957,
            "status": 0.5486942162612947
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-ordinal-01-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.7213014080000022,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.996556169994618,
            "1": 0.00240191752809631,
            "2": 0.0009422563719477145,
            "3": 9.965610533797712e-05
          },
          "ordinal_ev": 0.00458539858800567,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-01-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.7694226819999983,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.9625865422175591,
            "1": 0.03732928231066085,
            "2": 4.479018390392931e-05,
            "3": 3.9385287876082764e-05
          },
          "ordinal_ev": 0.037537018542096956,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-02-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.7559155160000017,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.056937173078925284,
            "1": 0.9299287572079905,
            "2": 0.008877356072245917,
            "3": 0.004256713640838139
          },
          "ordinal_ev": 0.9604536102749968,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-02-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.760347754999998,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.017076373783253985,
            "1": 0.9773657115623324,
            "2": 0.0026067192018748163,
            "3": 0.0029511954525387627
          },
          "ordinal_ev": 0.9914327363236983,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-03-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.7755690469999976,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.02486749114148505,
            "1": 0.015668209201159498,
            "2": 0.959034517114513,
            "3": 0.00042978254284241254
          },
          "ordinal_ev": 1.9350265910587128,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-03-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.7334720769999947,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.004012808622019858,
            "1": 0.00021805813510569782,
            "2": 0.9956489553450606,
            "3": 0.00012017789781382621
          },
          "ordinal_ev": 1.9918765025186684,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-04-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.7387454059999996,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.005556977234826691,
            "1": 0.002142611045644295,
            "2": 0.0012238736256687188,
            "3": 0.9910765380938603
          },
          "ordinal_ev": 2.977819972578563,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-04-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.7926668140000004,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0029592455770773513,
            "1": 0.0010641675054657693,
            "2": 0.000631219182496347,
            "3": 0.9953453677349606
          },
          "ordinal_ev": 2.98836270907534,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-05-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.748043838000001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.9026658426583442,
            "1": 0.07681739100232465,
            "2": 0.01147620420900767,
            "3": 0.009040562130323582
          },
          "ordinal_ev": 0.12689148581131074,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-05-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.740828785000005,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.8993412615720093,
            "1": 0.036007393066780635,
            "2": 0.05829325652908347,
            "3": 0.0063580888321266025
          },
          "ordinal_ev": 0.1716681726213274,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-06-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.739236407000007,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.01012584380128586,
            "1": 0.003828962290101398,
            "2": 0.9859689092042958,
            "3": 7.62847043169751e-05
          },
          "ordinal_ev": 1.9759956348116439,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-06-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.7137882419999926,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0016859915753610474,
            "1": 0.0007493521287047519,
            "2": 0.9975218937096593,
            "3": 4.2762586274993046e-05
          },
          "ordinal_ev": 1.9959214273068482,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-extraction-01-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.739860132000004,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.8588054597538308,
            "pickup": 0.048512112732651776,
            "post": 0.048752105771219044,
            "unknown": 0.043930321742298495
          },
          "predicted": "courier",
          "correct": false
        },
        {
          "id": "original-extraction-01-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.7264903269999934,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.024349213404087498,
            "pickup": 0.39269130769357674,
            "post": 0.40331677366062024,
            "unknown": 0.17964270524171558
          },
          "predicted": "post",
          "correct": false
        },
        {
          "id": "original-extraction-02-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.7332651019999901,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.0005761366552599896,
            "pickup": 0.0006224122317941662,
            "post": 0.00035964019188328237,
            "unknown": 0.9984418109210625
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-02-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.7203820710000031,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.00017677209837336247,
            "pickup": 0.00038398464903592036,
            "post": 6.824814043303342e-05,
            "unknown": 0.9993709951121577
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-03-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.6989745770000013,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.8898768948274486,
            "pickup": 0.02827741868259791,
            "post": 0.044976456739101466,
            "unknown": 0.03686922975085208
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-03-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.7386231919999915,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.9867657390191983,
            "pickup": 0.0033553843009098143,
            "post": 0.008599223541976725,
            "unknown": 0.0012796531379151317
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-04-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.7269541289999921,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.02562335769662156,
            "pickup": 0.004797031650882272,
            "post": 0.9488346041111144,
            "unknown": 0.020745006541381762
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-04-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.6838309439999932,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.009598558677501375,
            "pickup": 0.007530210395999668,
            "post": 0.9665201857271644,
            "unknown": 0.016351045199334557
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-05-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.6779556180000128,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.00014438048970663283,
            "pickup": 0.0012954791221040695,
            "post": 0.001293296816305863,
            "unknown": 0.9972668435718834
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-05-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.7837916209999918,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.006577230979902045,
            "pickup": 0.01356320113957834,
            "post": 0.015015325449012967,
            "unknown": 0.9648442424315067
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-06-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.7116120379999984,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.9895355882873603,
            "pickup": 0.006285199611233951,
            "post": 0.0013368388443975397,
            "unknown": 0.0028423732570081743
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-06-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.7539342779999885,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.982261124616182,
            "pickup": 0.011291662146951249,
            "post": 0.0038215916909067334,
            "unknown": 0.002625621545960165
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-adequacy-01-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18572425300000361,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.026480735901617147,
            "yes": 0.9735192640983829
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-01-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.1733531819999996,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.02342070032898702,
            "yes": 0.976579299671013
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-02-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17469764200001237,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9984804769245008,
            "yes": 0.0015195230754992245
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-02-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1843359009999972,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.998835036564289,
            "yes": 0.0011649634357110885
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-03-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.18397495300000344,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8289307401087727,
            "yes": 0.1710692598912273
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-03-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1811785809999975,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7547132161512619,
            "yes": 0.24528678384873812
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-04-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18086522500000513,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.011703204540453704,
            "yes": 0.9882967954595463
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-04-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18512294099998883,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.012647481486073198,
            "yes": 0.9873525185139268
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-05-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1762264900000048,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.09920665024227937,
            "yes": 0.9007933497577206
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-05-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1728543929999944,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2457295538411577,
            "yes": 0.7542704461588423
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-06-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.16883953299999632,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.013512168087092102,
            "yes": 0.9864878319129079
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-06-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17314227699999662,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.25457224064937334,
            "yes": 0.7454277593506267
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-routing-01-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 1.0464276040000016,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.06478212184128244,
            "coding_agent": 0.044605485336421516,
            "document": 0.02884058094476286,
            "general": 0.11789652689240507,
            "math": 0.7144679778059952,
            "tools": 0.02940730717913291
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-01-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 1.1026631630000026,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.05345620401384388,
            "coding_agent": 0.067185590624503,
            "document": 0.039040496172128666,
            "general": 0.11247126936635901,
            "math": 0.6976444931828333,
            "tools": 0.030201946640332124
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-02-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 1.1080302270000004,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.8499697166829129,
            "coding_agent": 0.08099547025092893,
            "document": 0.012480201854257097,
            "general": 0.034412055395841086,
            "math": 0.004314386493182034,
            "tools": 0.017828169322878024
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-02-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 1.0815454199999976,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.6389222066326244,
            "coding_agent": 0.31276586435575726,
            "document": 0.013984184638151877,
            "general": 0.02054673060362338,
            "math": 0.0027558519025533603,
            "tools": 0.011025161867289697
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-03-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 1.0731978940000033,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.01951817887077184,
            "coding_agent": 0.909927148988546,
            "document": 0.01414477552471108,
            "general": 0.02994082141965022,
            "math": 0.007641308734444895,
            "tools": 0.01882776646187606
          },
          "predicted": "coding_agent",
          "correct": true
        },
        {
          "id": "original-routing-03-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 1.0672092920000011,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.02320926920933817,
            "coding_agent": 0.919241351873418,
            "document": 0.013720094048356985,
            "general": 0.018544096132191656,
            "math": 0.01166043425173616,
            "tools": 0.013624754484959012
          },
          "predicted": "coding_agent",
          "correct": true
        },
        {
          "id": "original-routing-04-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 1.050568675000008,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.15689223787147213,
            "coding_agent": 0.650790703297848,
            "document": 0.04505880631077598,
            "general": 0.07208307543606172,
            "math": 0.04158330307739696,
            "tools": 0.0335918740064452
          },
          "predicted": "coding_agent",
          "correct": false
        },
        {
          "id": "original-routing-04-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 1.1166141770000024,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.10161858031250368,
            "coding_agent": 0.6704232276574845,
            "document": 0.06299578699264455,
            "general": 0.07647420675769988,
            "math": 0.05209439645003816,
            "tools": 0.03639380182962931
          },
          "predicted": "coding_agent",
          "correct": false
        },
        {
          "id": "original-routing-05-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 1.0440361760000059,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.0445676343017792,
            "coding_agent": 0.04431442435945578,
            "document": 0.06981684405661137,
            "general": 0.47627141838106624,
            "math": 0.03360527549976473,
            "tools": 0.3314244034013228
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-05-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 1.0202214250000026,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.09953305126432974,
            "coding_agent": 0.11141509730537187,
            "document": 0.08971671581199811,
            "general": 0.5600586127437583,
            "math": 0.05800961986485593,
            "tools": 0.08126690300968606
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-06-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.9809859169999982,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.07517280324335886,
            "coding_agent": 0.060121399145364705,
            "document": 0.039255372098177456,
            "general": 0.7274662569414434,
            "math": 0.04641492282741259,
            "tools": 0.05156924574424306
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "original-routing-06-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 1.010229080000002,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.08213706256601859,
            "coding_agent": 0.09462490066438395,
            "document": 0.040861857845203346,
            "general": 0.6744173180132276,
            "math": 0.04370997106121751,
            "tools": 0.06424888984994914
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "easy-intent-00",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.8422940869999991,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0002409860746542032,
            "cancel_order": 0.0022615945128846736,
            "change_address": 0.001077500619067716,
            "report_damage": 0.00021655014009660967,
            "track_order": 0.9962033686532968
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-01",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "cancel_order",
          "ok": true,
          "server_wall_seconds": 0.8331530020000031,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 6.17901696473511e-05,
            "cancel_order": 0.9997230459183609,
            "change_address": 6.315403542273118e-05,
            "report_damage": 6.976012854724636e-05,
            "track_order": 8.224974802174837e-05
          },
          "predicted": "cancel_order",
          "correct": true
        },
        {
          "id": "easy-intent-02",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.8272771560000081,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0001389979571080184,
            "cancel_order": 0.00033337402444627703,
            "change_address": 0.9988853701253073,
            "report_damage": 0.00016955367649664618,
            "track_order": 0.00047270421664169824
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "easy-intent-03",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "report_damage",
          "ok": true,
          "server_wall_seconds": 0.8873956270000036,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.00013898905179599163,
            "cancel_order": 0.004113307449834613,
            "change_address": 0.00032776997687219636,
            "report_damage": 0.9950667793410471,
            "track_order": 0.00035315418044990686
          },
          "predicted": "report_damage",
          "correct": true
        },
        {
          "id": "easy-intent-04",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "billing_question",
          "ok": true,
          "server_wall_seconds": 1.0222946190000073,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.9941631936450851,
            "cancel_order": 0.003103231300811654,
            "change_address": 0.0007052038936764163,
            "report_damage": 0.0005549186657644049,
            "track_order": 0.0014734524946622548
          },
          "predicted": "billing_question",
          "correct": true
        },
        {
          "id": "easy-intent-05",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.9227621060000075,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0005344530678251683,
            "cancel_order": 0.005225595091048748,
            "change_address": 0.0030023681332056844,
            "report_damage": 0.000857048502625813,
            "track_order": 0.9903805352052946
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-06",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.903417105999992,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0007743551223244282,
            "send_message": 0.0015073793283925435,
            "set_alarm": 0.9950700269958027,
            "turn_off_lights": 0.0019875105507886734,
            "weather": 0.0006607280026914779
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-intent-07",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "play_music",
          "ok": true,
          "server_wall_seconds": 0.9204186479999947,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.9472135696137147,
            "send_message": 0.017711100466514477,
            "set_alarm": 0.009640497797087396,
            "turn_off_lights": 0.011029475179849443,
            "weather": 0.014405356942834003
          },
          "predicted": "play_music",
          "correct": true
        },
        {
          "id": "easy-intent-08",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "weather",
          "ok": true,
          "server_wall_seconds": 0.8819625530000081,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0010112141716610765,
            "send_message": 0.0016455644135268451,
            "set_alarm": 0.002011979319154102,
            "turn_off_lights": 0.000980953730841216,
            "weather": 0.9943502883648168
          },
          "predicted": "weather",
          "correct": true
        },
        {
          "id": "easy-intent-09",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "send_message",
          "ok": true,
          "server_wall_seconds": 0.8428892459999986,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0037834536674011252,
            "send_message": 0.9769819773601136,
            "set_alarm": 0.009133802683038131,
            "turn_off_lights": 0.007176309943493543,
            "weather": 0.0029244563459537493
          },
          "predicted": "send_message",
          "correct": true
        },
        {
          "id": "easy-intent-10",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "turn_off_lights",
          "ok": true,
          "server_wall_seconds": 0.8563629170000127,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.002357716580111307,
            "send_message": 0.0021967255121416863,
            "set_alarm": 0.0020301921838029335,
            "turn_off_lights": 0.9925983245275278,
            "weather": 0.0008170411964161266
          },
          "predicted": "turn_off_lights",
          "correct": true
        },
        {
          "id": "easy-intent-11",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.9499417459999933,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.00039969054710045025,
            "send_message": 0.0007501484314047074,
            "set_alarm": 0.9977689447033025,
            "turn_off_lights": 0.0007798166569117749,
            "weather": 0.0003013996612805703
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-fact-00",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.19186290300000053,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.0014691753997470025,
            "yes": 0.998530824600253
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-01",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19835339899999838,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9994415043983754,
            "yes": 0.0005584956016245501
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-02",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.1909386640000008,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.0010117847161885685,
            "yes": 0.9989882152838114
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-03",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.18903871099999492,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9994969099461396,
            "yes": 0.0005030900538604278
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-04",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18963640399999804,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.0016684746716778553,
            "yes": 0.9983315253283221
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-05",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19099240500000292,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9995013554363531,
            "yes": 0.0004986445636468778
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-06",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.20762677199999757,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.003414133544789366,
            "yes": 0.9965858664552106
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-07",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.2053665680000023,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9995384578528703,
            "yes": 0.000461542147129652
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-08",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18696485200000268,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.0019061992775110115,
            "yes": 0.998093800722489
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-09",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.20232039900000132,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9990470893262088,
            "yes": 0.0009529106737912024
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-10",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17936926900000572,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.007401982752551106,
            "yes": 0.9925980172474489
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-11",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19759748200000615,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9992535767377201,
            "yes": 0.0007464232622799537
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-extraction-00",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "paypal",
          "ok": true,
          "server_wall_seconds": 0.7631079360000115,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.009749605465158871,
            "cash": 0.005756283278921512,
            "credit_card": 0.018118905456955686,
            "paypal": 0.966375205798964
          },
          "predicted": "paypal",
          "correct": true
        },
        {
          "id": "easy-extraction-01",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "cash",
          "ok": true,
          "server_wall_seconds": 0.7394956410000049,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.0002470135450784317,
            "cash": 0.9988639795144039,
            "credit_card": 0.0002828281711894444,
            "paypal": 0.0006061787693283776
          },
          "predicted": "cash",
          "correct": true
        },
        {
          "id": "easy-extraction-02",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "bank_transfer",
          "ok": true,
          "server_wall_seconds": 0.72887943100001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.9709506991324675,
            "cash": 0.009855621964669164,
            "credit_card": 0.005891330987527864,
            "paypal": 0.013302347915335582
          },
          "predicted": "bank_transfer",
          "correct": true
        },
        {
          "id": "easy-extraction-03",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "high",
          "ok": true,
          "server_wall_seconds": 0.6811977180000071,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.9990857249678837,
            "low": 3.794283341249319e-05,
            "medium": 3.242043598130506e-05,
            "urgent": 0.0008439117627224264
          },
          "predicted": "high",
          "correct": true
        },
        {
          "id": "easy-extraction-04",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "low",
          "ok": true,
          "server_wall_seconds": 0.7846657179999994,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 1.6256172589968618e-05,
            "low": 0.9999528473211213,
            "medium": 1.5063418868889516e-05,
            "urgent": 1.5833087419787926e-05
          },
          "predicted": "low",
          "correct": true
        },
        {
          "id": "easy-extraction-05",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "urgent",
          "ok": true,
          "server_wall_seconds": 0.7788625519999925,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.00043678273510267603,
            "low": 7.462081732615097e-05,
            "medium": 7.578403576439948e-05,
            "urgent": 0.9994128124118068
          },
          "predicted": "urgent",
          "correct": true
        },
        {
          "id": "easy-extraction-06",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "M",
          "ok": true,
          "server_wall_seconds": 0.8400595610000039,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 1.969037940290114e-05,
            "M": 0.9999380158925727,
            "S": 2.3187893537646265e-05,
            "XL": 1.9105834486835743e-05
          },
          "predicted": "M",
          "correct": true
        },
        {
          "id": "easy-extraction-07",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "XL",
          "ok": true,
          "server_wall_seconds": 0.7752949139999998,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.24670030554258793,
            "M": 0.26535108524180945,
            "S": 0.21270312432726785,
            "XL": 0.2752454848883348
          },
          "predicted": "XL",
          "correct": true
        },
        {
          "id": "easy-extraction-08",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "thursday",
          "ok": true,
          "server_wall_seconds": 0.9588033440000032,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.000291651241891034,
            "monday": 0.0002951652134999995,
            "thursday": 0.9988695797192432,
            "tuesday": 0.00022922401651609,
            "wednesday": 0.000314379808849776
          },
          "predicted": "thursday",
          "correct": true
        },
        {
          "id": "easy-extraction-09",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "monday",
          "ok": true,
          "server_wall_seconds": 0.9931644079999984,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.004112930883529065,
            "monday": 0.9854389952858069,
            "thursday": 0.003869083681570937,
            "tuesday": 0.0037044256840240374,
            "wednesday": 0.0028745644650691065
          },
          "predicted": "monday",
          "correct": true
        },
        {
          "id": "easy-extraction-10",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "friday",
          "ok": true,
          "server_wall_seconds": 1.0397482420000017,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.9811531549608816,
            "monday": 0.005335187586977382,
            "thursday": 0.005725078369914456,
            "tuesday": 0.004459534110122825,
            "wednesday": 0.003327044972103843
          },
          "predicted": "friday",
          "correct": true
        },
        {
          "id": "easy-extraction-11",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "credit_card",
          "ok": true,
          "server_wall_seconds": 0.8375945389999941,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.001963441563723649,
            "cash": 0.003939473788017778,
            "credit_card": 0.9846283579090707,
            "paypal": 0.009468726739187888
          },
          "predicted": "credit_card",
          "correct": true
        },
        {
          "id": "easy-tool_selection-00",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_weather",
          "ok": true,
          "server_wall_seconds": 0.9165820399999944,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0034321927282586332,
            "create_calendar_event": 0.005113171124171764,
            "get_weather": 0.9724910395744314,
            "send_email": 0.006303322090692023,
            "translate_text": 0.012660274482446068
          },
          "predicted": "get_weather",
          "correct": true
        },
        {
          "id": "easy-tool_selection-01",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "send_email",
          "ok": true,
          "server_wall_seconds": 0.9880801160000061,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0029866191948358443,
            "create_calendar_event": 0.0020278120850955143,
            "get_weather": 0.0014097865264701233,
            "send_email": 0.9856081966447924,
            "translate_text": 0.007967585548806248
          },
          "predicted": "send_email",
          "correct": true
        },
        {
          "id": "easy-tool_selection-02",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.961342118999994,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0008881733793258751,
            "create_calendar_event": 0.9885121449960994,
            "get_weather": 0.0024719142261914566,
            "send_email": 0.005374009027014677,
            "translate_text": 0.002753758371368558
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-03",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "convert_currency",
          "ok": true,
          "server_wall_seconds": 0.9642692560000086,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.971598393899829,
            "create_calendar_event": 0.001403628171445024,
            "get_weather": 0.0022916539764592588,
            "send_email": 0.004248680823226861,
            "translate_text": 0.020457643129039922
          },
          "predicted": "convert_currency",
          "correct": true
        },
        {
          "id": "easy-tool_selection-04",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "translate_text",
          "ok": true,
          "server_wall_seconds": 0.9799443810000099,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.003094367944160856,
            "create_calendar_event": 0.007214222621027674,
            "get_weather": 0.008349376023283301,
            "send_email": 0.00918555205321053,
            "translate_text": 0.9721564813583177
          },
          "predicted": "translate_text",
          "correct": true
        },
        {
          "id": "easy-tool_selection-05",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.9250541629999987,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0010126968055548158,
            "create_calendar_event": 0.9845918431408538,
            "get_weather": 0.0023208967627444243,
            "send_email": 0.009435231100517768,
            "translate_text": 0.0026393321903292193
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-06",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "search_flights",
          "ok": true,
          "server_wall_seconds": 0.9624706299999986,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.002973160695634404,
            "call_taxi": 0.006333896845578729,
            "get_stock_price": 0.0009226064993120906,
            "search_flights": 0.9864854153578076,
            "set_reminder": 0.003284920601667083
          },
          "predicted": "search_flights",
          "correct": true
        },
        {
          "id": "easy-tool_selection-07",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "book_restaurant",
          "ok": true,
          "server_wall_seconds": 0.9935010989999995,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.9840708097898084,
            "call_taxi": 0.004936789246458416,
            "get_stock_price": 0.000957760296152857,
            "search_flights": 0.002317688602190136,
            "set_reminder": 0.007716952065389979
          },
          "predicted": "book_restaurant",
          "correct": true
        },
        {
          "id": "easy-tool_selection-08",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.9345012740000129,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.003004111972647579,
            "call_taxi": 0.004239093035439352,
            "get_stock_price": 0.0012098868476499095,
            "search_flights": 0.001694668493286879,
            "set_reminder": 0.9898522396509762
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "easy-tool_selection-09",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_stock_price",
          "ok": true,
          "server_wall_seconds": 1.0084403279999918,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.003812717401808917,
            "call_taxi": 0.004684247902010115,
            "get_stock_price": 0.9781472488745169,
            "search_flights": 0.005794420264424044,
            "set_reminder": 0.007561365557240094
          },
          "predicted": "get_stock_price",
          "correct": true
        },
        {
          "id": "easy-tool_selection-10",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "call_taxi",
          "ok": true,
          "server_wall_seconds": 1.0836862710000048,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0035533873940250082,
            "call_taxi": 0.973949725718243,
            "get_stock_price": 0.0012437914352350938,
            "search_flights": 0.01381467535304223,
            "set_reminder": 0.007438420099454622
          },
          "predicted": "call_taxi",
          "correct": true
        },
        {
          "id": "easy-tool_selection-11",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.9570625439999958,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0018098488892982385,
            "call_taxi": 0.0017025911861822426,
            "get_stock_price": 0.0010075808725459354,
            "search_flights": 0.0011882095176408022,
            "set_reminder": 0.9942917695343328
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "pay_subject_to_15000_sublimit",
          "ok": true,
          "server_wall_seconds": 3.5396393380000006,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_repeated_seepage": 0.006358351001727707,
            "deny_vacancy_exclusion": 0.9596460801930397,
            "pay_full_estimate_less_deductible": 0.0028807969641263813,
            "pay_subject_to_10000_sublimit": 0.01747073903950224,
            "pay_subject_to_15000_sublimit": 0.013644032801604112
          },
          "predicted": "deny_vacancy_exclusion",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "cfo",
          "ok": true,
          "server_wall_seconds": 2.331465496999982,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "budget_holder": 0.004433310195882955,
            "cfo": 0.0017780294622721906,
            "department_head": 0.9535480761720986,
            "executive_committee": 0.0005613546766578194,
            "vp_and_finance_director": 0.039679229493088455
          },
          "predicted": "department_head",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "parts_covered_labour_charged",
          "ok": true,
          "server_wall_seconds": 1.7458570689999817,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "full_repair_no_charge": 0.06773364763617255,
            "not_covered": 0.05901923951455169,
            "parts_covered_labour_charged": 0.8628736251436521,
            "pro_rata_replacement_credit": 0.010373487705623729
          },
          "predicted": "parts_covered_labour_charged",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-09",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "license_required",
          "ok": true,
          "server_wall_seconds": 1.783228964000017,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "license_exception_lvs": 0.030313199332028355,
            "license_required": 0.9601833411655553,
            "no_license_required": 0.008186864919575033,
            "shipment_prohibited": 0.001316594582841423
          },
          "predicted": "license_required",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.4466225089999796,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8063710292936256,
            "yes": 0.19362897070637444
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-13",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.44240299300000174,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9967935189655271,
            "yes": 0.0032064810344728
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-17",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "database_oncall",
          "ok": true,
          "server_wall_seconds": 2.186305822999998,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "database_oncall": 0.5066682683225516,
            "no_page_ticket_only": 0.14221251805513385,
            "platform_oncall": 0.19190343399469972,
            "security_incident_response": 0.0929276523600371,
            "vendor_support": 0.06628812726757773
          },
          "predicted": "database_oncall",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-19",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.42817316000000005,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.15910465575949206,
            "yes": 0.8408953442405079
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.22699413699999127,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.96752495395973,
            "yes": 0.03247504604026994
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "on_time",
          "ok": true,
          "server_wall_seconds": 0.9375177870000186,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "early": 0.07963393425118696,
            "late": 0.7288892134897375,
            "on_time": 0.19147685225907565
          },
          "predicted": "late",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-07",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "escalated",
          "ok": true,
          "server_wall_seconds": 0.8791157689999807,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "escalated": 0.5118668937194704,
            "reopened": 0.06745008984278325,
            "resolved_first_contact": 0.42068301643774636
          },
          "predicted": "escalated",
          "correct": true
        },
        {
          "id": "hard-opus-a-probability-08",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.21473610699999313,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9287887670844934,
            "yes": 0.07121123291550654
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-opus-a-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.21082623999998873,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9849645984797891,
            "yes": 0.015035401520210843
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.2420888130000094,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.881670602160653,
            "yes": 0.11832939783934705
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-opus-a-temporal_numeric-07",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_26",
          "ok": true,
          "server_wall_seconds": 1.4758854259999907,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "sep_24": 0.2773319218386782,
            "sep_25": 0.1715159840178228,
            "sep_26": 0.10199738103110624,
            "sep_27": 0.19234702733002051,
            "sep_28": 0.25680768578237234
          },
          "predicted": "sep_24",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-09",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.2317632930000002,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.24394450703824522,
            "yes": 0.7560554929617548
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.8873666890000038,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.05914953933527995,
            "1": 0.12562757191225488,
            "2": 0.16038646909480966,
            "3": 0.6548364196576556
          },
          "ordinal_ev": 2.410909769074841,
          "predicted": "3",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-02",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "real_incident",
          "ok": true,
          "server_wall_seconds": 0.657752658000021,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.023637108051257304,
            "false_positive": 0.003969269221934083,
            "real_incident": 0.9723936227268086
          },
          "predicted": "real_incident",
          "correct": true
        },
        {
          "id": "hard-opus-b-ambiguous-03",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "time_barred",
          "ok": true,
          "server_wall_seconds": 0.6554041149999819,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.008159639210249009,
            "time_barred": 0.41698906929902163,
            "within_limitation": 0.5748512914907293
          },
          "predicted": "within_limitation",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-07",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "within_policy",
          "ok": true,
          "server_wall_seconds": 0.650715278000007,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.00044707591619313775,
            "over_limit": 0.7571523391637404,
            "within_policy": 0.24240058492006664
          },
          "predicted": "over_limit",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-09",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "legitimate",
          "ok": true,
          "server_wall_seconds": 0.7122039690000008,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "compromised": 0.4551390705912197,
            "inconclusive": 0.0013431433788935162,
            "legitimate": 0.5435177860298868
          },
          "predicted": "legitimate",
          "correct": true
        },
        {
          "id": "hard-opus-b-ambiguous-10",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "ask_user_which_alex",
          "ok": true,
          "server_wall_seconds": 0.6279298819999894,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ask_user_which_alex": 0.01411042277042986,
            "send_to_alex_moreno": 0.9758501997103356,
            "send_to_alexandra_lind": 0.010039377519234529
          },
          "predicted": "send_to_alex_moreno",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-11",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.6178738229999965,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.47998907569178684,
            "grade_correct": 0.5189092238961798,
            "grade_should_be_higher": 0.0011017004120334138
          },
          "predicted": "grade_correct",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-13",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.9303759189999994,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.9998174461807017,
            "charge_data_team": 0.00010616127461406977,
            "charge_platform_team": 5.5461644435702416e-05,
            "split_50_50": 2.0930900248534967e-05
          },
          "predicted": "cannot_determine",
          "correct": true
        },
        {
          "id": "hard-opus-b-multi_hop-03",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "tier3_cfo",
          "ok": true,
          "server_wall_seconds": 2.042002067999988,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "tier1_team_lead": 0.12623457777762784,
            "tier2_department_head": 0.6273006887230544,
            "tier3_cfo": 0.17879169694135566,
            "tier4_board": 0.06767303655796206
          },
          "predicted": "tier2_department_head",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-04",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bjorn",
          "ok": true,
          "server_wall_seconds": 2.510756927000017,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ana": 0.09874880315123144,
            "bjorn": 0.11974863585716716,
            "chen": 0.1963978427778846,
            "dmitri": 0.015468282017734874,
            "esra": 0.4354921285828471,
            "no_page_ticket_only": 0.13414430761313467
          },
          "predicted": "esra",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 2.228825131000008,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.020629961437413254,
            "1": 0.8035380141679549,
            "2": 0.06649037847792665,
            "3": 0.04905704987523775,
            "4": 0.06028459604146753
          },
          "ordinal_ev": 1.3248283049153915,
          "predicted": "1",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "credit_20_percent",
          "ok": true,
          "server_wall_seconds": 2.6549322639999957,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_10_percent": 0.05608458933874601,
            "credit_20_percent": 0.3015827514879339,
            "credit_25_percent": 0.24552114972360722,
            "credit_50_percent": 0.2539716786008131,
            "credit_full_month": 0.05821428431967787,
            "no_credit": 0.08462554652922191
          },
          "predicted": "credit_20_percent",
          "correct": true
        },
        {
          "id": "hard-opus-b-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.41521730400000934,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9983135773568491,
            "yes": 0.0016864226431509395
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-01",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "merchant_wins",
          "ok": true,
          "server_wall_seconds": 1.0275765750000119,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "customer_wins": 0.15341636276061227,
            "customer_withdraws": 0.03933663784255408,
            "merchant_wins": 0.8072469993968336
          },
          "predicted": "merchant_wins",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-02",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upstream_provider",
          "ok": true,
          "server_wall_seconds": 0.6223035650000099,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bad_push": 0.9880560849727441,
            "database_hardware": 0.003541027253108011,
            "upstream_provider": 0.008402887774147915
          },
          "predicted": "bad_push",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.27525572899997997,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.11208886078845792,
            "yes": 0.8879111392115421
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "attended",
          "ok": true,
          "server_wall_seconds": 0.6431980289999899,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "attended": 0.9022547314494376,
            "late_cancellation": 0.03302040588116741,
            "no_show": 0.06472486266939503
          },
          "predicted": "attended",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-06",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upheld",
          "ok": true,
          "server_wall_seconds": 0.7852461929999777,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "modified": 0.07926867732156198,
            "overturned": 0.7010459108342614,
            "upheld": 0.21968541184417673
          },
          "predicted": "overturned",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-01",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "escalate_to_engineering",
          "ok": true,
          "server_wall_seconds": 1.174970266999992,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "escalate_to_engineering": 0.049681790278207225,
            "escalate_to_security": 0.8777941850246883,
            "page_oncall": 0.0018024390801690392,
            "process_refund": 0.015221645231175498,
            "send_kb_article": 0.05549994038576
          },
          "predicted": "escalate_to_security",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-03",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "order_b",
          "ok": true,
          "server_wall_seconds": 0.9028399959999831,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "order_a": 0.5949793396625709,
            "order_b": 0.14674070057521332,
            "order_c": 0.0157596390064589,
            "order_d": 0.24252032075575689
          },
          "predicted": "order_a",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-06",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.23763033100001962,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.027504057462559683,
            "yes": 0.9724959425374403
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-07",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "p0_fix_24h",
          "ok": true,
          "server_wall_seconds": 1.0844110820000026,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "out_of_scope": 0.009495064584484934,
            "p0_fix_24h": 0.2144129704031049,
            "p1_fix_7d": 0.31045150056607185,
            "p2_fix_30d": 0.4236953234350866,
            "p3_backlog": 0.04194514101125169
          },
          "predicted": "p2_fix_30d",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-08",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.2201097900000093,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9832916485647053,
            "yes": 0.016708351435294665
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-12",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "refund_now",
          "ok": true,
          "server_wall_seconds": 1.06803844800001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_refund": 0.01314693606144725,
            "escalate_to_fraud_team": 0.0003140728034046095,
            "offer_store_credit": 0.013533649149391173,
            "refund_now": 0.9581180533180588,
            "wait_for_inspection": 0.014887288667698135
          },
          "predicted": "refund_now",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "erase_but_retain_transaction_records",
          "ok": true,
          "server_wall_seconds": 3.1636812369999916,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "erase_all": 0.01536036347527633,
            "erase_but_retain_transaction_records": 0.972266372592769,
            "erase_marketing_only": 0.001180288512514577,
            "reject_unverified_request": 0.0039196544062529174,
            "restrict_all_pending_legal_hold": 0.007273321013186945
          },
          "predicted": "erase_but_retain_transaction_records",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-03",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "eur_400",
          "ok": true,
          "server_wall_seconds": 3.7300285260000123,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_0": 0.00938816674160954,
            "eur_150": 0.060489176865803684,
            "eur_200": 0.01314290342313686,
            "eur_300": 0.2930151767563311,
            "eur_400": 0.6129040863353281,
            "eur_600": 0.01106048987779069
          },
          "predicted": "eur_400",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.5865339089999964,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1348092646123762,
            "yes": 0.8651907353876238
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 2.2530701239999757,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.43980034630209774,
            "1": 0.14921363184652375,
            "2": 0.35223882430076603,
            "3": 0.05874719755061255
          },
          "ordinal_ev": 1.0299328730998936,
          "predicted": "0",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 2.2858139040000083,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.013869873508017674,
            "1": 0.07394583789595786,
            "2": 0.8920419966458032,
            "3": 0.02014229195022137
          },
          "ordinal_ev": 1.9184567070382283,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-10",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "credit_minus_restocking_fee",
          "ok": true,
          "server_wall_seconds": 2.6018083079999883,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_minus_restocking_fee": 0.8667861772590356,
            "full_credit": 0.0002320313657850761,
            "reject_return": 0.13254488786310095,
            "repair_or_replace_only": 0.00043690351207843027
          },
          "predicted": "credit_minus_restocking_fee",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.623453464999983,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4511166894044134,
            "yes": 0.5488833105955866
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-c-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "kuznets_technik",
          "ok": true,
          "server_wall_seconds": 0.6216917859999853,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "halvorsen_polymer": 0.19986983182047188,
            "kuznets_technik": 0.558424432587705,
            "ortega_seal_works": 0.24170573559182312
          },
          "predicted": "kuznets_technik",
          "correct": true
        },
        {
          "id": "hard-opus-c-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.252977828999974,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8402802357867928,
            "yes": 0.1597197642132072
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_692_00",
          "ok": true,
          "server_wall_seconds": 1.2168770689999917,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_617_00": 0.19364027587818577,
            "eur_688_00": 0.18758795719812663,
            "eur_692_00": 0.19553236742733504,
            "eur_693_90": 0.23985738696714107,
            "eur_812_00": 0.18338201252921152
          },
          "predicted": "eur_693_90",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "usd_2335_00",
          "ok": true,
          "server_wall_seconds": 1.1140284749999978,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "usd_2222_50": 0.26265979728085803,
            "usd_2303_01": 0.1365873338982545,
            "usd_2303_33": 0.24675759931510488,
            "usd_2335_00": 0.1807982402260911,
            "usd_2340_00": 0.1731970292796914
          },
          "predicted": "usd_2222_50",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_18_book_set",
          "ok": true,
          "server_wall_seconds": 1.468816306000008,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "none_declined": 0.3102278178621074,
            "sep_15_office_chair": 0.14090477972818963,
            "sep_16_taxi": 0.27434260518921294,
            "sep_17_parking": 0.07447088083406925,
            "sep_18_book_set": 0.10108944829802254,
            "sep_21_client_dinner": 0.09896446808839825
          },
          "predicted": "none_declined",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-08",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_498_81",
          "ok": true,
          "server_wall_seconds": 1.228495506999991,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_497_40": 0.26309175861965256,
            "eur_498_81": 0.16359517922926706,
            "eur_506_06": 0.2156137546806946,
            "eur_513_69": 0.22552115303613082,
            "eur_532_01": 0.13217815443425499
          },
          "predicted": "eur_497_40",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "expired_30_month_cap",
          "ok": true,
          "server_wall_seconds": 0.7175780240000051,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered": 0.18468983830468733,
            "expired_24_month_term": 0.2757916593587671,
            "expired_30_month_cap": 0.5395185023365456
          },
          "predicted": "expired_30_month_cap",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-01",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "deny_missing_approval",
          "ok": true,
          "server_wall_seconds": 0.7227220690000138,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_inactive_user": 9.935515095733916e-06,
            "deny_missing_approval": 0.9999504787290123,
            "grant_access": 1.1029409048571203e-05,
            "grant_owner_exception": 2.8556346843280112e-05
          },
          "predicted": "deny_missing_approval",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-06",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17817576000001623,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9996396583302161,
            "yes": 0.00036034166978388555
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-07",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "retain_until_2031",
          "ok": true,
          "server_wall_seconds": 0.712954897000003,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_now": 0.005473390234453547,
            "needs_signature_review": 5.408949888850928e-05,
            "retain_indefinitely": 0.0011878617645916527,
            "retain_until_2031": 0.9932846585020664
          },
          "predicted": "retain_until_2031",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-08",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17780710200000271,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.20623972483095243,
            "yes": 0.7937602751690476
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-09",
          "tier": "hard",
          "family": "adversarial",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.7113365400000191,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.00027768632234631346,
            "1": 0.0013686295170946329,
            "2": 0.9982941579305898,
            "3": 5.9526229969143614e-05
          },
          "ordinal_ev": 1.9981355240681817,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-11",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "standard_research",
          "ok": true,
          "server_wall_seconds": 0.7028384230000029,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ethics_review": 4.915121317890923e-05,
            "major_grants": 1.4136637957132933e-05,
            "manual_hold": 8.638159422938698e-06,
            "standard_research": 0.9999280739894411
          },
          "predicted": "standard_research",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-01",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "accept_s2_dusk",
          "ok": true,
          "server_wall_seconds": 1.6709825629999955,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "accept_s2_dusk": 0.9540149419217585,
            "reject_cutoff": 0.010690404356078742,
            "reject_tier": 0.01756268883534988,
            "reroute_s3": 0.017731964886813
          },
          "predicted": "accept_s2_dusk",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-02",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "security_primary",
          "ok": true,
          "server_wall_seconds": 1.6804734979999978,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "executive_bridge": 4.111832045471632e-05,
            "heron_q2": 6.853453399073586e-05,
            "insufficient_information": 0.0004023653842503388,
            "security_primary": 0.9994879817613042
          },
          "predicted": "security_primary",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_director",
          "ok": true,
          "server_wall_seconds": 1.6506368630000168,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "pay_manager_approved": 0.14173803023436338,
            "reject_category": 0.2533306917201488,
            "require_director": 0.5067866930531616,
            "require_finance_board": 0.09814458499232609
          },
          "predicted": "require_director",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "admit",
          "ok": true,
          "server_wall_seconds": 1.6450697829999967,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "admit": 0.00385519598292281,
            "deny_prerequisite": 0.9866082266441981,
            "deny_score": 0.006758548683013079,
            "manual_equivalency": 0.0027780286898660207
          },
          "predicted": "deny_prerequisite",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "central_engineering",
          "ok": true,
          "server_wall_seconds": 1.6273696860000086,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "central_engineering": 0.4116499412800509,
            "insufficient_log": 0.0015789173703234056,
            "local_maintenance": 0.586666997542997,
            "safety_shutdown": 0.00010414380662879717
          },
          "predicted": "local_maintenance",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_steward",
          "ok": true,
          "server_wall_seconds": 1.636773737999988,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "confirm_l3": 0.0008895277228422588,
            "reject_capacity": 0.02290985914407263,
            "require_steward": 0.9742297487585694,
            "switch_l4": 0.0019708643745157215
          },
          "predicted": "require_steward",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-10",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "approve_45_days",
          "ok": true,
          "server_wall_seconds": 1.6617307820000065,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_45_days": 0.03873625617144304,
            "deny_sensitive": 0.6633842201947194,
            "needs_field_review": 0.006474787965106061,
            "reduce_to_30_days": 0.2914047356687316
          },
          "predicted": "deny_sensitive",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "needs_authentication",
          "ok": true,
          "server_wall_seconds": 1.6524160360000053,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered_recurrence": 0.00023776083334394653,
            "expired": 0.07088182768893285,
            "needs_authentication": 0.9279421210620343,
            "wrong_issue_class": 0.000938290415688854
          },
          "predicted": "needs_authentication",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-02",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1897354100000257,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9998429302076093,
            "yes": 0.00015706979239070933
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-04",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17966379400002097,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9996975802611447,
            "yes": 0.00030241973885526647
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-05",
          "tier": "hard",
          "family": "trap",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.7419094969999946,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.6675917849473454,
            "1": 0.010138969274218246,
            "2": 0.32147867731414875,
            "3": 0.0007905684642877075
          },
          "ordinal_ev": 0.6554680292953788,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-06",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "keep_open_unresolved",
          "ok": true,
          "server_wall_seconds": 0.7292506830000036,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "await_internal_metrics": 0.0009368101550529659,
            "close_agent_verified": 5.3948667749941617e-05,
            "close_confirmed": 2.370023883425985e-05,
            "keep_open_unresolved": 0.9989855409383628
          },
          "predicted": "keep_open_unresolved",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-08",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "late",
          "ok": true,
          "server_wall_seconds": 0.7882908500000099,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "canceled": 1.675394481523787e-05,
            "indeterminate": 1.013258088901083e-05,
            "late": 0.9999664374542143,
            "on_time": 6.676020081426446e-06
          },
          "predicted": "late",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-10",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "blocked_second_review",
          "ok": true,
          "server_wall_seconds": 0.6811878920000254,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "blocked_failed_test": 2.2200921399484645e-05,
            "blocked_second_review": 0.9994808840967213,
            "ready_one_review": 0.00041453954893784376,
            "ready_two_reviews": 8.237543294128143e-05
          },
          "predicted": "blocked_second_review",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-13",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1798554460000048,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9995656275151814,
            "yes": 0.00043437248481857605
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-15",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.18371267200001284,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9986663417735187,
            "yes": 0.0013336582264814045
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-01",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18355567899999414,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.09726485167808985,
            "yes": 0.9027351483219102
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-02",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17610825999997815,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9483067876612867,
            "yes": 0.051693212338713254
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.18529691399999138,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.19473614603132905,
            "yes": 0.805263853968671
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1805618389999779,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.45262015616472384,
            "yes": 0.5473798438352762
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17964540799999895,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2570134725296862,
            "yes": 0.7429865274703138
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-14",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1785832340000013,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9989662183268744,
            "yes": 0.0010337816731255701
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.17901307400001087,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.04419784826593809,
            "yes": 0.9558021517340619
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-18",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.17551310100000705,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.998391785457913,
            "yes": 0.0016082145420870711
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "reject_entire_claim",
          "ok": true,
          "server_wall_seconds": 1.9132033220000153,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_all": 0.36152663641149313,
            "approve_lodging_only": 0.34650199695811046,
            "approve_meals_only": 0.2724155465351316,
            "reject_entire_claim": 0.019555820095264853
          },
          "predicted": "approve_all",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "deny_current_export",
          "ok": true,
          "server_wall_seconds": 1.7815288689999988,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "analytics_owner_only": 2.3202691653664496e-05,
            "deny_current_export": 0.998916748031966,
            "privacy_and_security": 0.0010138829884004837,
            "privacy_only": 4.616628797996545e-05
          },
          "predicted": "deny_current_export",
          "correct": true
        },
        {
          "id": "hard-sol-b-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "delete_profile_omar_april_and_table",
          "ok": true,
          "server_wall_seconds": 1.7583709440000064,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_everything": 0.006185824131230047,
            "delete_nothing": 0.005207491157250825,
            "delete_only_profile": 0.700480634065366,
            "delete_profile_omar_april_and_table": 0.28812605064615326
          },
          "predicted": "delete_only_profile",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-06",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "east_ward",
          "ok": true,
          "server_wall_seconds": 1.8640701500000034,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "east_ward": 0.625825927589481,
            "harbor_neighbors": 0.003289873206477184,
            "no_award": 0.12381510678385707,
            "river_arts": 0.24706909242018477
          },
          "predicted": "east_ward",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-01",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "office_file_repair",
          "ok": true,
          "server_wall_seconds": 0.8868980600000214,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "data_pipeline": 0.008746715732733007,
            "desktop_support": 0.00882504382060717,
            "finance_review": 0.005392214599815575,
            "office_file_repair": 0.9508455487055669,
            "spreadsheet_formula": 0.026190477141277292
          },
          "predicted": "office_file_repair",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-02",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "ledger_reconciliation",
          "ok": true,
          "server_wall_seconds": 0.9014504780000152,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "customer_support": 0.0018675186396398444,
            "fraud_review": 0.0009157514930953239,
            "ledger_reconciliation": 0.994508867987139,
            "mobile_app": 0.0020581612712404006,
            "payment_disputes": 0.0006497006088854835
          },
          "predicted": "ledger_reconciliation",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-03",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "network_dns",
          "ok": true,
          "server_wall_seconds": 0.9231404380000185,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "api_platform": 0.003220069859369554,
            "authoritative_dns": 0.0018234367854412863,
            "endpoint_devices": 0.007273848234520213,
            "network_dns": 0.9871242196460528,
            "tls_certificates": 0.0005584254746161345
          },
          "predicted": "network_dns",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-07",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "release_notarization",
          "ok": true,
          "server_wall_seconds": 0.9322780809999927,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ci_infrastructure": 0.0006982027889489015,
            "code_signing": 0.008515747425610678,
            "endpoint_security": 0.0023499946184260714,
            "macos_runtime": 0.0002752459696517048,
            "release_notarization": 0.9881608091973626
          },
          "predicted": "release_notarization",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-09",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "source_access",
          "ok": true,
          "server_wall_seconds": 0.9074194869999985,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "git_client": 0.014677169771571812,
            "identity_login": 0.0018694327908545928,
            "repository_admin": 0.01752363230341488,
            "source_access": 0.9615269160223662,
            "vpn_network": 0.004402849111792587
          },
          "predicted": "source_access",
          "correct": true
        },
        {
          "id": "hard-sol-b-temporal_numeric-01",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "late_by_under_2h",
          "ok": true,
          "server_wall_seconds": 0.7275076829999989,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "before_open": 0.08149925349759357,
            "late_by_2h_or_more": 0.2740212625597808,
            "late_by_under_2h": 0.1366093049028932,
            "within_window": 0.5078701790397324
          },
          "predicted": "within_window",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "over_limit",
          "ok": true,
          "server_wall_seconds": 0.7070561239999904,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "at_least_100gb_below": 0.1711548263192499,
            "exactly_equal": 0.09617390111430568,
            "over_limit": 0.3646340789071762,
            "under_100gb_below": 0.3680371936592682
          },
          "predicted": "under_100gb_below",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "9_credits",
          "ok": true,
          "server_wall_seconds": 0.6765439579999963,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "12_credits": 0.30490266303272356,
            "15_credits": 0.16869845628960464,
            "6_credits": 0.3276363170698803,
            "9_credits": 0.1987625636077915
          },
          "predicted": "6_credits",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "pass",
          "ok": true,
          "server_wall_seconds": 0.5683940060000054,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "clear_fail": 0.6104737191240046,
            "narrow_fail": 0.3722799665071152,
            "pass": 0.01724631436888019
          },
          "predicted": "clear_fail",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-03",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19931011700001022,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2416736228821864,
            "yes": 0.7583263771178136
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.20478622900000687,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1615932151398738,
            "yes": 0.8384067848601262
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-07",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19126281500001596,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.15042041478516488,
            "yes": 0.8495795852148351
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19973421599999597,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7575745662157255,
            "yes": 0.24242543378427453
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-09",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.20311338100000853,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1023117992773932,
            "yes": 0.8976882007226068
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.20459452399998668,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1812041520041715,
            "yes": 0.8187958479958285
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-11",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19057556399999953,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.9994022450408654,
            "yes": 0.0005977549591345988
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-13",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.20471096799997213,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8432827559391417,
            "yes": 0.1567172440608583
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.1981076680000058,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.009543377193107383,
            "yes": 0.9904566228068926
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "programme_review",
          "ok": true,
          "server_wall_seconds": 1.7528506319999906,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "partial_release": 0.11858117294526656,
            "programme_review": 0.8128764726104436,
            "reject_draw": 0.01455830180575163,
            "release_draw": 0.05398405263853839
          },
          "predicted": "programme_review",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bus_substitution",
          "ok": true,
          "server_wall_seconds": 1.7715844319999974,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bus_substitution": 0.6433250891078148,
            "call_fen_cross": 0.012302713005714052,
            "pass_fen_cross": 0.004513915867389909,
            "terminate_oakmere": 0.33985828201908125
          },
          "predicted": "bus_substitution",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-11",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "coverage_excluded",
          "ok": true,
          "server_wall_seconds": 1.86667786000001,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "adjuster_referral": 0.20196702778095632,
            "coverage_confirmed": 0.14833476690115374,
            "coverage_excluded": 0.42834915306620014,
            "documentation_needed": 0.22134905225168988
          },
          "predicted": "coverage_excluded",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "conservation_hold",
          "ok": true,
          "server_wall_seconds": 0.8698570060000179,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "conservation_hold": 0.9955114727960926,
            "deny_request": 0.004450812540269709,
            "digital_copy_only": 1.0755560924403762e-05,
            "supervised_delivery": 2.695910271317414e-05
          },
          "predicted": "conservation_hold",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-13",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "normal_release",
          "ok": true,
          "server_wall_seconds": 1.7863961429999904,
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "engineering_review": 0.0021861273823031717,
            "normal_release": 0.9946929883146932,
            "rework": 0.0011074656722684015,
            "scrap_lot": 0.002013418630735216
          },
          "predicted": "normal_release",
          "correct": true
        }
      ],
      "raw_sha256": "c18e5e4363eb0d90b4c86a342de428608a8cfb03fbc6580f4973c406bf77c12a"
    },
    "laya": {
      "model": "Laya English base",
      "runtime": {
        "model": "Laya English base",
        "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
        "source_commit": "42626c348753fbb17572a813127df2278a1ec527",
        "gpu": "NVIDIA L40S",
        "python": "3.12.10",
        "software": {
          "torch": "2.8.0",
          "transformers": "5.0.0",
          "numpy": "2.5.3",
          "safetensors": "0.6.2"
        },
        "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
        "weights_dtype": "torch.float32",
        "autocast_dtype": "torch.bfloat16",
        "max_length": 512,
        "head_max_length": 192,
        "temperature": [
          1.6369030475616455,
          1.2514300346374512,
          1.983399510383606
        ],
        "temperature_by_options": {
          "choice:3-5": 1.7601518630981445,
          "choice:6-10": 1.0000158548355103,
          "score:3-5": 1.2514300346374512,
          "noul:2": 1.983399510383606,
          "choice:11+": 0.10058280825614929,
          "choice:2": 1.9063563346862793
        },
        "probability_precision": "native API,4 decimal places",
        "prefix_cache": false
      },
      "planned": 231,
      "received": 231,
      "correct": 134,
      "accuracy": 0.5800865800865801,
      "operational_success": 231,
      "strict_valid": 231,
      "valid": 231,
      "renormalized": 0,
      "tiers": {
        "original": {
          "n": 72,
          "correct": 50
        },
        "easy": {
          "n": 48,
          "correct": 46
        },
        "hard": {
          "n": 111,
          "correct": 38
        }
      },
      "brier_mean": 0.5322201668398269,
      "ece": {
        "ece": 0.09520995670995663,
        "n": 231,
        "bins": [
          {
            "lo": 0.0,
            "hi": 0.1,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.1,
            "hi": 0.2,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.2,
            "hi": 0.3,
            "n": 10,
            "mean_confidence": 0.26835,
            "accuracy": 0.3
          },
          {
            "lo": 0.3,
            "hi": 0.4,
            "n": 24,
            "mean_confidence": 0.35637083333333336,
            "accuracy": 0.375
          },
          {
            "lo": 0.4,
            "hi": 0.5,
            "n": 28,
            "mean_confidence": 0.44640714285714284,
            "accuracy": 0.32142857142857145
          },
          {
            "lo": 0.5,
            "hi": 0.6,
            "n": 32,
            "mean_confidence": 0.5511875,
            "accuracy": 0.5
          },
          {
            "lo": 0.6,
            "hi": 0.7,
            "n": 29,
            "mean_confidence": 0.6409862068965516,
            "accuracy": 0.5172413793103449
          },
          {
            "lo": 0.7,
            "hi": 0.8,
            "n": 27,
            "mean_confidence": 0.7508111111111112,
            "accuracy": 0.6666666666666666
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "n": 35,
            "mean_confidence": 0.8537971428571427,
            "accuracy": 0.6285714285714286
          },
          {
            "lo": 0.9,
            "hi": 1.0,
            "n": 46,
            "mean_confidence": 0.9641108695652171,
            "accuracy": 0.9130434782608695
          }
        ]
      },
      "ordinal_mae": 0.4820722222222222,
      "latency": {
        "n": 231,
        "p50_s": 0.026497351999999808,
        "p95_s": 0.03073635850000045
      },
      "peak_gpu_allocated_bytes": 2449583104,
      "truncation": {
        "state_truncated": 57,
        "instruction_truncated": 1,
        "options_truncated": 1
      },
      "scored": [
        {
          "id": "original-policy-01-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.7892645380000012,
          "input_audit": {
            "state_tokens_before": 28,
            "state_tokens_kept": 28,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7538,
            "yes": 0.2462
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-01-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.030241075999999367,
          "input_audit": {
            "state_tokens_before": 30,
            "state_tokens_kept": 30,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5166999999999999,
            "yes": 0.4833
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-02-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026220935000001333,
          "input_audit": {
            "state_tokens_before": 20,
            "state_tokens_kept": 20,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3467,
            "yes": 0.6533
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-02-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026285706000003017,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.24370000000000003,
            "yes": 0.7563
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-03-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026913880000002166,
          "input_audit": {
            "state_tokens_before": 22,
            "state_tokens_kept": 22,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.749,
            "yes": 0.251
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-policy-03-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.02680905800000133,
          "input_audit": {
            "state_tokens_before": 22,
            "state_tokens_kept": 22,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.29510000000000003,
            "yes": 0.7049
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-04-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.027591596000000607,
          "input_audit": {
            "state_tokens_before": 30,
            "state_tokens_kept": 30,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4961,
            "yes": 0.5039
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-policy-04-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.026076551999999253,
          "input_audit": {
            "state_tokens_before": 28,
            "state_tokens_kept": 28,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5958,
            "yes": 0.4042
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-05-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02645585000000139,
          "input_audit": {
            "state_tokens_before": 33,
            "state_tokens_kept": 33,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4911,
            "yes": 0.5089
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-policy-05-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.025697842999999665,
          "input_audit": {
            "state_tokens_before": 26,
            "state_tokens_kept": 26,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8003,
            "yes": 0.1997
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-06-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.02538820499999872,
          "input_audit": {
            "state_tokens_before": 31,
            "state_tokens_kept": 31,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.17479999999999996,
            "yes": 0.8252
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-06-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.02522682200000048,
          "input_audit": {
            "state_tokens_before": 21,
            "state_tokens_kept": 21,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.34950000000000003,
            "yes": 0.6505
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-intent-01-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.03826521799999938,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0378,
            "change_address": 0.0041,
            "other": 0.0101,
            "refund": 0.9397,
            "status": 0.0084
          },
          "predicted": "refund",
          "correct": true
        },
        {
          "id": "original-intent-01-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.027457032000000936,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0531,
            "change_address": 0.0065,
            "other": 0.0097,
            "refund": 0.9246,
            "status": 0.006
          },
          "predicted": "refund",
          "correct": true
        },
        {
          "id": "original-intent-02-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.025964109000000235,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0277,
            "change_address": 0.5561,
            "other": 0.2603,
            "refund": 0.076,
            "status": 0.0799
          },
          "predicted": "change_address",
          "correct": false
        },
        {
          "id": "original-intent-02-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.025324833999999186,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0144,
            "change_address": 0.1223,
            "other": 0.1177,
            "refund": 0.0342,
            "status": 0.7114
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-intent-03-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.02487511400000031,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0214,
            "change_address": 0.3214,
            "other": 0.0813,
            "refund": 0.0794,
            "status": 0.4965
          },
          "predicted": "status",
          "correct": false
        },
        {
          "id": "original-intent-03-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.024936645000000368,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0037,
            "change_address": 0.9605,
            "other": 0.0116,
            "refund": 0.0118,
            "status": 0.0125
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-04-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.025271352999997276,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.7731,
            "change_address": 0.0113,
            "other": 0.082,
            "refund": 0.1065,
            "status": 0.0271
          },
          "predicted": "cancel",
          "correct": true
        },
        {
          "id": "original-intent-04-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.025229601999999574,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.7725,
            "change_address": 0.0176,
            "other": 0.0893,
            "refund": 0.0903,
            "status": 0.0303
          },
          "predicted": "cancel",
          "correct": true
        },
        {
          "id": "original-intent-05-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.02512618999999816,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0128,
            "change_address": 0.0161,
            "other": 0.0954,
            "refund": 0.8241,
            "status": 0.0515
          },
          "predicted": "refund",
          "correct": false
        },
        {
          "id": "original-intent-05-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.024942376000002042,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0133,
            "change_address": 0.009,
            "other": 0.0516,
            "refund": 0.9078,
            "status": 0.0184
          },
          "predicted": "refund",
          "correct": false
        },
        {
          "id": "original-intent-06-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.026677304999999762,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0354,
            "change_address": 0.0073,
            "other": 0.0738,
            "refund": 0.028,
            "status": 0.8554
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-intent-06-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.02696434100000289,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.3735,
            "change_address": 0.0068,
            "other": 0.0371,
            "refund": 0.0158,
            "status": 0.5669
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-ordinal-01-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.027041312999998013,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.4516,
            "1": 0.1809,
            "2": 0.3164,
            "3": 0.0511
          },
          "ordinal_ev": 0.9670000000000001,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-01-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.025736234000000024,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.356,
            "1": 0.3429,
            "2": 0.2773,
            "3": 0.0237
          },
          "ordinal_ev": 0.9685999999999999,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-02-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.02541539699999973,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0442,
            "1": 0.9055,
            "2": 0.0379,
            "3": 0.0124
          },
          "ordinal_ev": 1.0185,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-02-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.02784062100000284,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.1302,
            "1": 0.8291,
            "2": 0.021,
            "3": 0.0197
          },
          "ordinal_ev": 0.9302,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-03-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.02623885500000256,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0177,
            "1": 0.046,
            "2": 0.8934,
            "3": 0.0429
          },
          "ordinal_ev": 1.9615,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-03-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.02553145999999984,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.034,
            "1": 0.0972,
            "2": 0.7022,
            "3": 0.1665
          },
          "ordinal_ev": 2.0011,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-04-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.025191791000001018,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0127,
            "1": 0.0128,
            "2": 0.0215,
            "3": 0.953
          },
          "ordinal_ev": 2.9148,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-04-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.025422357000000062,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0225,
            "1": 0.0362,
            "2": 0.0592,
            "3": 0.8822
          },
          "ordinal_ev": 2.8011999999999997,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-05-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.027327360000001022,
          "input_audit": {
            "state_tokens_before": 16,
            "state_tokens_kept": 16,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.4642,
            "1": 0.2329,
            "2": 0.134,
            "3": 0.1689
          },
          "ordinal_ev": 1.0076,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-05-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.02585756599999911,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.2202,
            "1": 0.2962,
            "2": 0.3708,
            "3": 0.1129
          },
          "ordinal_ev": 1.3765,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "original-ordinal-06-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.026006879000000538,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0681,
            "1": 0.1561,
            "2": 0.741,
            "3": 0.0348
          },
          "ordinal_ev": 1.7425,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-06-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.04385976600000063,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0137,
            "1": 0.085,
            "2": 0.8655,
            "3": 0.0359
          },
          "ordinal_ev": 1.9237,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-extraction-01-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.027193887999999333,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.1244,
            "pickup": 0.7725,
            "post": 0.0373,
            "unknown": 0.0658
          },
          "predicted": "pickup",
          "correct": true
        },
        {
          "id": "original-extraction-01-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.026228926000001707,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.1783,
            "pickup": 0.4692,
            "post": 0.0519,
            "unknown": 0.3007
          },
          "predicted": "pickup",
          "correct": true
        },
        {
          "id": "original-extraction-02-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.025971608999999063,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.0912,
            "pickup": 0.0268,
            "post": 0.2508,
            "unknown": 0.6313
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-02-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.02566139199999995,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.0571,
            "pickup": 0.0159,
            "post": 0.7293,
            "unknown": 0.1978
          },
          "predicted": "post",
          "correct": false
        },
        {
          "id": "original-extraction-03-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.025779785000001,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.3076,
            "pickup": 0.5829,
            "post": 0.0369,
            "unknown": 0.0726
          },
          "predicted": "pickup",
          "correct": false
        },
        {
          "id": "original-extraction-03-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.024545936999999185,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.8675,
            "pickup": 0.0301,
            "post": 0.0271,
            "unknown": 0.0753
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-04-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.025848035999999297,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.0767,
            "pickup": 0.0944,
            "post": 0.7691,
            "unknown": 0.0597
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-04-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.025675673000002064,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.0622,
            "pickup": 0.0163,
            "post": 0.8534,
            "unknown": 0.0681
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-05-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.02532535499999966,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.5853,
            "pickup": 0.0223,
            "post": 0.0548,
            "unknown": 0.3376
          },
          "predicted": "courier",
          "correct": false
        },
        {
          "id": "original-extraction-05-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.025903028000001882,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.5331,
            "pickup": 0.0203,
            "post": 0.1046,
            "unknown": 0.342
          },
          "predicted": "courier",
          "correct": false
        },
        {
          "id": "original-extraction-06-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.02523581299999833,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.8788,
            "pickup": 0.0083,
            "post": 0.0807,
            "unknown": 0.0322
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-06-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.025408816000002332,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.6254,
            "pickup": 0.0159,
            "post": 0.2656,
            "unknown": 0.0932
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-adequacy-01-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.025466158000000405,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8784,
            "yes": 0.1216
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-adequacy-01-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.025320045000000846,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.47150000000000003,
            "yes": 0.5285
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-02-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02564334200000218,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.21719999999999995,
            "yes": 0.7828
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-02-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.024948497000000458,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6347,
            "yes": 0.3653
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-03-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02544237700000096,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.18620000000000003,
            "yes": 0.8138
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-03-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.025069419000001147,
          "input_audit": {
            "state_tokens_before": 21,
            "state_tokens_kept": 21,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6498999999999999,
            "yes": 0.3501
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-04-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.024993827999999496,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.07389999999999997,
            "yes": 0.9261
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-04-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.024554928000000587,
          "input_audit": {
            "state_tokens_before": 21,
            "state_tokens_kept": 21,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.07999999999999996,
            "yes": 0.92
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-05-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02518199100000018,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7716000000000001,
            "yes": 0.2284
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-05-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.025265183000001912,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5852999999999999,
            "yes": 0.4147
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-06-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.024728072000002044,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2067,
            "yes": 0.7933
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-06-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.024682210999998233,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6997,
            "yes": 0.3003
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-routing-01-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 0.026457999999998094,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1681,
            "coding_agent": 0.0235,
            "document": 0.0039,
            "general": 0.0318,
            "math": 0.7534,
            "tools": 0.0192
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-01-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 0.025347165000003002,
          "input_audit": {
            "state_tokens_before": 8,
            "state_tokens_kept": 8,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1438,
            "coding_agent": 0.0307,
            "document": 0.034,
            "general": 0.1342,
            "math": 0.5971,
            "tools": 0.0603
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-02-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 0.02595637900000014,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.9748,
            "coding_agent": 0.0129,
            "document": 0.0006,
            "general": 0.004,
            "math": 0.0006,
            "tools": 0.007
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-02-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 0.025384276000000483,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.9737,
            "coding_agent": 0.0072,
            "document": 0.0021,
            "general": 0.0065,
            "math": 0.0014,
            "tools": 0.0091
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-03-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 0.0263476570000023,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.5537,
            "coding_agent": 0.2873,
            "document": 0.0046,
            "general": 0.0726,
            "math": 0.0031,
            "tools": 0.0788
          },
          "predicted": "coding",
          "correct": false
        },
        {
          "id": "original-routing-03-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 0.025770125000001087,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.4809,
            "coding_agent": 0.3925,
            "document": 0.0071,
            "general": 0.0843,
            "math": 0.0046,
            "tools": 0.0305
          },
          "predicted": "coding",
          "correct": false
        },
        {
          "id": "original-routing-04-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 0.025291843999998065,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.0103,
            "coding_agent": 0.0055,
            "document": 0.0237,
            "general": 0.9582,
            "math": 0.0012,
            "tools": 0.0012
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-04-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 0.025266373999997427,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1526,
            "coding_agent": 0.0609,
            "document": 0.0559,
            "general": 0.6442,
            "math": 0.0272,
            "tools": 0.0592
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-05-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 0.025831796999998602,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.2006,
            "coding_agent": 0.0403,
            "document": 0.0538,
            "general": 0.6451,
            "math": 0.0116,
            "tools": 0.0486
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-05-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 0.025695273000000185,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.2445,
            "coding_agent": 0.0286,
            "document": 0.0406,
            "general": 0.6305,
            "math": 0.0126,
            "tools": 0.0432
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-06-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.024975566999998478,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.2884,
            "coding_agent": 0.0999,
            "document": 0.0356,
            "general": 0.5072,
            "math": 0.0133,
            "tools": 0.0556
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "original-routing-06-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.025011488000000526,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.091,
            "coding_agent": 0.0209,
            "document": 0.0047,
            "general": 0.8616,
            "math": 0.0042,
            "tools": 0.0176
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "easy-intent-00",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.025540879000001127,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0056,
            "cancel_order": 0.0044,
            "change_address": 0.0039,
            "report_damage": 0.0472,
            "track_order": 0.939
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-01",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "cancel_order",
          "ok": true,
          "server_wall_seconds": 0.025525878999999918,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0007,
            "cancel_order": 0.9941,
            "change_address": 0.0008,
            "report_damage": 0.0036,
            "track_order": 0.0009
          },
          "predicted": "cancel_order",
          "correct": true
        },
        {
          "id": "easy-intent-02",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.025204752000000497,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0033,
            "cancel_order": 0.0048,
            "change_address": 0.9812,
            "report_damage": 0.007,
            "track_order": 0.0038
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "easy-intent-03",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "report_damage",
          "ok": true,
          "server_wall_seconds": 0.026020769999998805,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0037,
            "cancel_order": 0.0042,
            "change_address": 0.0039,
            "report_damage": 0.9801,
            "track_order": 0.008
          },
          "predicted": "report_damage",
          "correct": true
        },
        {
          "id": "easy-intent-04",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "billing_question",
          "ok": true,
          "server_wall_seconds": 0.026262995999999816,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.7383,
            "cancel_order": 0.0265,
            "change_address": 0.0179,
            "report_damage": 0.1147,
            "track_order": 0.1027
          },
          "predicted": "billing_question",
          "correct": true
        },
        {
          "id": "easy-intent-05",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.026613223999998326,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0086,
            "cancel_order": 0.0073,
            "change_address": 0.0061,
            "report_damage": 0.0524,
            "track_order": 0.9255
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-06",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.02537292600000285,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0007,
            "send_message": 0.0007,
            "set_alarm": 0.9972,
            "turn_off_lights": 0.0008,
            "weather": 0.0007
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-intent-07",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "play_music",
          "ok": true,
          "server_wall_seconds": 0.02480582199999759,
          "input_audit": {
            "state_tokens_before": 5,
            "state_tokens_kept": 5,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.9848,
            "send_message": 0.0032,
            "set_alarm": 0.0042,
            "turn_off_lights": 0.0036,
            "weather": 0.0042
          },
          "predicted": "play_music",
          "correct": true
        },
        {
          "id": "easy-intent-08",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "weather",
          "ok": true,
          "server_wall_seconds": 0.025327234999998893,
          "input_audit": {
            "state_tokens_before": 7,
            "state_tokens_kept": 7,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0069,
            "send_message": 0.0072,
            "set_alarm": 0.0071,
            "turn_off_lights": 0.0082,
            "weather": 0.9706
          },
          "predicted": "weather",
          "correct": true
        },
        {
          "id": "easy-intent-09",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "send_message",
          "ok": true,
          "server_wall_seconds": 0.025381915999997062,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.002,
            "send_message": 0.9916,
            "set_alarm": 0.0021,
            "turn_off_lights": 0.0022,
            "weather": 0.0021
          },
          "predicted": "send_message",
          "correct": true
        },
        {
          "id": "easy-intent-10",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "turn_off_lights",
          "ok": true,
          "server_wall_seconds": 0.02528718399999974,
          "input_audit": {
            "state_tokens_before": 8,
            "state_tokens_kept": 8,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0019,
            "send_message": 0.0019,
            "set_alarm": 0.0094,
            "turn_off_lights": 0.9846,
            "weather": 0.0023
          },
          "predicted": "turn_off_lights",
          "correct": true
        },
        {
          "id": "easy-intent-11",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.02541972699999917,
          "input_audit": {
            "state_tokens_before": 7,
            "state_tokens_kept": 7,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0002,
            "send_message": 0.0002,
            "set_alarm": 0.9991,
            "turn_off_lights": 0.0002,
            "weather": 0.0002
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-fact-00",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.025508319000000057,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.09989999999999999,
            "yes": 0.9001
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-01",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.024963706000001196,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.26149999999999995,
            "yes": 0.7385
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "easy-fact-02",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.024763961999997974,
          "input_audit": {
            "state_tokens_before": 20,
            "state_tokens_kept": 20,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.30010000000000003,
            "yes": 0.6999
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-03",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.025121750000000276,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8973,
            "yes": 0.1027
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-04",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.02465356000000085,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.18110000000000004,
            "yes": 0.8189
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-05",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02503544800000057,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7032,
            "yes": 0.2968
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-06",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.02444326399999852,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5521,
            "yes": 0.4479
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "easy-fact-07",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.025048747999999676,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8577,
            "yes": 0.1423
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-08",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.024837192999999758,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1643,
            "yes": 0.8357
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-09",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.024571776999998463,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5481,
            "yes": 0.4519
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-10",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.025091698999997192,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.12629999999999997,
            "yes": 0.8737
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-11",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02455068699999785,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8868,
            "yes": 0.1132
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-extraction-00",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "paypal",
          "ok": true,
          "server_wall_seconds": 0.025218112000001014,
          "input_audit": {
            "state_tokens_before": 7,
            "state_tokens_kept": 7,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.0527,
            "cash": 0.0523,
            "credit_card": 0.2998,
            "paypal": 0.5952
          },
          "predicted": "paypal",
          "correct": true
        },
        {
          "id": "easy-extraction-01",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "cash",
          "ok": true,
          "server_wall_seconds": 0.02541510699999705,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.0378,
            "cash": 0.9321,
            "credit_card": 0.0176,
            "paypal": 0.0125
          },
          "predicted": "cash",
          "correct": true
        },
        {
          "id": "easy-extraction-02",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "bank_transfer",
          "ok": true,
          "server_wall_seconds": 0.02518758200000093,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.99,
            "cash": 0.0024,
            "credit_card": 0.0044,
            "paypal": 0.0032
          },
          "predicted": "bank_transfer",
          "correct": true
        },
        {
          "id": "easy-extraction-03",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "high",
          "ok": true,
          "server_wall_seconds": 0.02580105499999874,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.866,
            "low": 0.0094,
            "medium": 0.0147,
            "urgent": 0.1099
          },
          "predicted": "high",
          "correct": true
        },
        {
          "id": "easy-extraction-04",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "low",
          "ok": true,
          "server_wall_seconds": 0.025565439999997608,
          "input_audit": {
            "state_tokens_before": 16,
            "state_tokens_kept": 16,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.0043,
            "low": 0.96,
            "medium": 0.0104,
            "urgent": 0.0253
          },
          "predicted": "low",
          "correct": true
        },
        {
          "id": "easy-extraction-05",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "urgent",
          "ok": true,
          "server_wall_seconds": 0.025408025999997363,
          "input_audit": {
            "state_tokens_before": 16,
            "state_tokens_kept": 16,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.0182,
            "low": 0.0123,
            "medium": 0.0172,
            "urgent": 0.9523
          },
          "predicted": "urgent",
          "correct": true
        },
        {
          "id": "easy-extraction-06",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "M",
          "ok": true,
          "server_wall_seconds": 0.025522068999997316,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.0216,
            "M": 0.9247,
            "S": 0.0356,
            "XL": 0.0181
          },
          "predicted": "M",
          "correct": true
        },
        {
          "id": "easy-extraction-07",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "XL",
          "ok": true,
          "server_wall_seconds": 0.025396607000001126,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.0992,
            "M": 0.0154,
            "S": 0.0082,
            "XL": 0.8771
          },
          "predicted": "XL",
          "correct": true
        },
        {
          "id": "easy-extraction-08",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "thursday",
          "ok": true,
          "server_wall_seconds": 0.02521171200000083,
          "input_audit": {
            "state_tokens_before": 8,
            "state_tokens_kept": 8,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.0022,
            "monday": 0.0026,
            "thursday": 0.9891,
            "tuesday": 0.0029,
            "wednesday": 0.0032
          },
          "predicted": "thursday",
          "correct": true
        },
        {
          "id": "easy-extraction-09",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "monday",
          "ok": true,
          "server_wall_seconds": 0.025440245999998723,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.003,
            "monday": 0.987,
            "thursday": 0.0028,
            "tuesday": 0.0039,
            "wednesday": 0.0033
          },
          "predicted": "monday",
          "correct": true
        },
        {
          "id": "easy-extraction-10",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "friday",
          "ok": true,
          "server_wall_seconds": 0.02500789699999828,
          "input_audit": {
            "state_tokens_before": 8,
            "state_tokens_kept": 8,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.9305,
            "monday": 0.0185,
            "thursday": 0.0149,
            "tuesday": 0.0189,
            "wednesday": 0.0173
          },
          "predicted": "friday",
          "correct": true
        },
        {
          "id": "easy-extraction-11",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "credit_card",
          "ok": true,
          "server_wall_seconds": 0.025363016000000016,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.0206,
            "cash": 0.0121,
            "credit_card": 0.9449,
            "paypal": 0.0224
          },
          "predicted": "credit_card",
          "correct": true
        },
        {
          "id": "easy-tool_selection-00",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_weather",
          "ok": true,
          "server_wall_seconds": 0.02511047100000141,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0003,
            "create_calendar_event": 0.0004,
            "get_weather": 0.9985,
            "send_email": 0.0004,
            "translate_text": 0.0004
          },
          "predicted": "get_weather",
          "correct": true
        },
        {
          "id": "easy-tool_selection-01",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "send_email",
          "ok": true,
          "server_wall_seconds": 0.025927749999997474,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0008,
            "create_calendar_event": 0.0012,
            "get_weather": 0.0008,
            "send_email": 0.9962,
            "translate_text": 0.001
          },
          "predicted": "send_email",
          "correct": true
        },
        {
          "id": "easy-tool_selection-02",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.02689774199999917,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0014,
            "create_calendar_event": 0.9941,
            "get_weather": 0.0014,
            "send_email": 0.0016,
            "translate_text": 0.0016
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-03",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "convert_currency",
          "ok": true,
          "server_wall_seconds": 0.025664912999999956,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.9998,
            "create_calendar_event": 0.0,
            "get_weather": 0.0,
            "send_email": 0.0,
            "translate_text": 0.0001
          },
          "predicted": "convert_currency",
          "correct": true
        },
        {
          "id": "easy-tool_selection-04",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "translate_text",
          "ok": true,
          "server_wall_seconds": 0.02538428699999784,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0067,
            "create_calendar_event": 0.0131,
            "get_weather": 0.0069,
            "send_email": 0.009,
            "translate_text": 0.9642
          },
          "predicted": "translate_text",
          "correct": true
        },
        {
          "id": "easy-tool_selection-05",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.025411328000000566,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0001,
            "create_calendar_event": 0.9998,
            "get_weather": 0.0001,
            "send_email": 0.0001,
            "translate_text": 0.0001
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-06",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "search_flights",
          "ok": true,
          "server_wall_seconds": 0.024762803000001554,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0001,
            "call_taxi": 0.0001,
            "get_stock_price": 0.0001,
            "search_flights": 0.9998,
            "set_reminder": 0.0001
          },
          "predicted": "search_flights",
          "correct": true
        },
        {
          "id": "easy-tool_selection-07",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "book_restaurant",
          "ok": true,
          "server_wall_seconds": 0.02537890700000034,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.9999,
            "call_taxi": 0.0,
            "get_stock_price": 0.0,
            "search_flights": 0.0,
            "set_reminder": 0.0
          },
          "predicted": "book_restaurant",
          "correct": true
        },
        {
          "id": "easy-tool_selection-08",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.024587858999996826,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0092,
            "call_taxi": 0.0109,
            "get_stock_price": 0.0104,
            "search_flights": 0.0114,
            "set_reminder": 0.958
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "easy-tool_selection-09",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_stock_price",
          "ok": true,
          "server_wall_seconds": 0.025103450999999666,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0,
            "call_taxi": 0.0,
            "get_stock_price": 0.9999,
            "search_flights": 0.0,
            "set_reminder": 0.0
          },
          "predicted": "get_stock_price",
          "correct": true
        },
        {
          "id": "easy-tool_selection-10",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "call_taxi",
          "ok": true,
          "server_wall_seconds": 0.025037519000001396,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0022,
            "call_taxi": 0.9906,
            "get_stock_price": 0.0022,
            "search_flights": 0.0025,
            "set_reminder": 0.0025
          },
          "predicted": "call_taxi",
          "correct": true
        },
        {
          "id": "easy-tool_selection-11",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.02448325699999998,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0041,
            "call_taxi": 0.0045,
            "get_stock_price": 0.0047,
            "search_flights": 0.0048,
            "set_reminder": 0.9819
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "pay_subject_to_15000_sublimit",
          "ok": true,
          "server_wall_seconds": 0.03277209499999856,
          "input_audit": {
            "state_tokens_before": 3514,
            "state_tokens_kept": 316,
            "state_truncated": true,
            "instruction_truncated": true,
            "options_truncated": true
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_repeated_seepage": 0.455,
            "deny_vacancy_exclusion": 0.1251,
            "pay_full_estimate_less_deductible": 0.2555,
            "pay_subject_to_10000_sublimit": 0.0792,
            "pay_subject_to_15000_sublimit": 0.0852
          },
          "predicted": "deny_repeated_seepage",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "cfo",
          "ok": true,
          "server_wall_seconds": 0.030789860000002278,
          "input_audit": {
            "state_tokens_before": 2319,
            "state_tokens_kept": 361,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "budget_holder": 0.2841,
            "cfo": 0.2698,
            "department_head": 0.1701,
            "executive_committee": 0.1291,
            "vp_and_finance_director": 0.1469
          },
          "predicted": "budget_holder",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "parts_covered_labour_charged",
          "ok": true,
          "server_wall_seconds": 0.029141971999997907,
          "input_audit": {
            "state_tokens_before": 2165,
            "state_tokens_kept": 373,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "full_repair_no_charge": 0.3659,
            "not_covered": 0.1188,
            "parts_covered_labour_charged": 0.326,
            "pro_rata_replacement_credit": 0.1893
          },
          "predicted": "full_repair_no_charge",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-09",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "license_required",
          "ok": true,
          "server_wall_seconds": 0.029008579999999284,
          "input_audit": {
            "state_tokens_before": 2280,
            "state_tokens_kept": 380,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "license_exception_lvs": 0.5289,
            "license_required": 0.0219,
            "no_license_required": 0.2236,
            "shipment_prohibited": 0.2256
          },
          "predicted": "license_exception_lvs",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.027936055000001403,
          "input_audit": {
            "state_tokens_before": 2052,
            "state_tokens_kept": 408,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.48819999999999997,
            "yes": 0.5118
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-13",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.027766301000003324,
          "input_audit": {
            "state_tokens_before": 2198,
            "state_tokens_kept": 420,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.33399999999999996,
            "yes": 0.666
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-17",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "database_oncall",
          "ok": true,
          "server_wall_seconds": 0.028487107000000123,
          "input_audit": {
            "state_tokens_before": 2173,
            "state_tokens_kept": 387,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "database_oncall": 0.1776,
            "no_page_ticket_only": 0.0699,
            "platform_oncall": 0.164,
            "security_incident_response": 0.4952,
            "vendor_support": 0.0933
          },
          "predicted": "security_incident_response",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-19",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.028534927999999127,
          "input_audit": {
            "state_tokens_before": 2038,
            "state_tokens_kept": 434,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.38380000000000003,
            "yes": 0.6162
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.025646512999998095,
          "input_audit": {
            "state_tokens_before": 320,
            "state_tokens_kept": 320,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3466,
            "yes": 0.6534
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-a-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "on_time",
          "ok": true,
          "server_wall_seconds": 0.02645791200000147,
          "input_audit": {
            "state_tokens_before": 845,
            "state_tokens_kept": 447,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "early": 0.4623,
            "late": 0.2837,
            "on_time": 0.2539
          },
          "predicted": "early",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-07",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "escalated",
          "ok": true,
          "server_wall_seconds": 0.026027971999997845,
          "input_audit": {
            "state_tokens_before": 897,
            "state_tokens_kept": 446,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "escalated": 0.4773,
            "reopened": 0.2741,
            "resolved_first_contact": 0.2486
          },
          "predicted": "escalated",
          "correct": true
        },
        {
          "id": "hard-opus-a-probability-08",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.027672138999999873,
          "input_audit": {
            "state_tokens_before": 272,
            "state_tokens_kept": 272,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.46220000000000006,
            "yes": 0.5378
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026784059000000582,
          "input_audit": {
            "state_tokens_before": 363,
            "state_tokens_kept": 363,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.36260000000000003,
            "yes": 0.6374
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-a-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02633546899999928,
          "input_audit": {
            "state_tokens_before": 353,
            "state_tokens_kept": 353,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.14180000000000004,
            "yes": 0.8582
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-07",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_26",
          "ok": true,
          "server_wall_seconds": 0.027120307000000565,
          "input_audit": {
            "state_tokens_before": 843,
            "state_tokens_kept": 435,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "sep_24": 0.2088,
            "sep_25": 0.2116,
            "sep_26": 0.2599,
            "sep_27": 0.1741,
            "sep_28": 0.1456
          },
          "predicted": "sep_26",
          "correct": true
        },
        {
          "id": "hard-opus-a-temporal_numeric-09",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02777712199999982,
          "input_audit": {
            "state_tokens_before": 375,
            "state_tokens_kept": 375,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.12760000000000005,
            "yes": 0.8724
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.02831969400000034,
          "input_audit": {
            "state_tokens_before": 406,
            "state_tokens_kept": 406,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0677,
            "1": 0.1797,
            "2": 0.592,
            "3": 0.1606
          },
          "ordinal_ev": 1.8455,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-02",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "real_incident",
          "ok": true,
          "server_wall_seconds": 0.028044927999999913,
          "input_audit": {
            "state_tokens_before": 410,
            "state_tokens_kept": 410,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.1381,
            "false_positive": 0.3206,
            "real_incident": 0.5413
          },
          "predicted": "real_incident",
          "correct": true
        },
        {
          "id": "hard-opus-b-ambiguous-03",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "time_barred",
          "ok": true,
          "server_wall_seconds": 0.02960864399999963,
          "input_audit": {
            "state_tokens_before": 353,
            "state_tokens_kept": 353,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.2149,
            "time_barred": 0.2488,
            "within_limitation": 0.5363
          },
          "predicted": "within_limitation",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-07",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "within_policy",
          "ok": true,
          "server_wall_seconds": 0.027622877999998963,
          "input_audit": {
            "state_tokens_before": 369,
            "state_tokens_kept": 369,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.1061,
            "over_limit": 0.5984,
            "within_policy": 0.2955
          },
          "predicted": "over_limit",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-09",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "legitimate",
          "ok": true,
          "server_wall_seconds": 0.027508765000000324,
          "input_audit": {
            "state_tokens_before": 518,
            "state_tokens_kept": 435,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "compromised": 0.4397,
            "inconclusive": 0.2622,
            "legitimate": 0.2982
          },
          "predicted": "compromised",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-10",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "ask_user_which_alex",
          "ok": true,
          "server_wall_seconds": 0.027268798999998012,
          "input_audit": {
            "state_tokens_before": 377,
            "state_tokens_kept": 377,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ask_user_which_alex": 0.3053,
            "send_to_alex_moreno": 0.4166,
            "send_to_alexandra_lind": 0.2781
          },
          "predicted": "send_to_alex_moreno",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-11",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.02769615900000133,
          "input_audit": {
            "state_tokens_before": 324,
            "state_tokens_kept": 324,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.2084,
            "grade_correct": 0.3673,
            "grade_should_be_higher": 0.4243
          },
          "predicted": "grade_should_be_higher",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-13",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.027459832999998213,
          "input_audit": {
            "state_tokens_before": 423,
            "state_tokens_kept": 399,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.1352,
            "charge_data_team": 0.2561,
            "charge_platform_team": 0.245,
            "split_50_50": 0.3637
          },
          "predicted": "split_50_50",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-03",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "tier3_cfo",
          "ok": true,
          "server_wall_seconds": 0.029830976999999592,
          "input_audit": {
            "state_tokens_before": 2617,
            "state_tokens_kept": 422,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "tier1_team_lead": 0.2481,
            "tier2_department_head": 0.211,
            "tier3_cfo": 0.2906,
            "tier4_board": 0.2502
          },
          "predicted": "tier3_cfo",
          "correct": true
        },
        {
          "id": "hard-opus-b-multi_hop-04",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bjorn",
          "ok": true,
          "server_wall_seconds": 0.03068285699999862,
          "input_audit": {
            "state_tokens_before": 2124,
            "state_tokens_kept": 421,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ana": 0.016,
            "bjorn": 0.0097,
            "chen": 0.0081,
            "dmitri": 0.0095,
            "esra": 0.0722,
            "no_page_ticket_only": 0.8845
          },
          "predicted": "no_page_ticket_only",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.02978084599999775,
          "input_audit": {
            "state_tokens_before": 2224,
            "state_tokens_kept": 426,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.1241,
            "1": 0.2435,
            "2": 0.3389,
            "3": 0.1749,
            "4": 0.1186
          },
          "ordinal_ev": 1.9203999999999999,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "hard-opus-b-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "credit_20_percent",
          "ok": true,
          "server_wall_seconds": 0.029800616999999363,
          "input_audit": {
            "state_tokens_before": 2150,
            "state_tokens_kept": 389,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_10_percent": 0.2069,
            "credit_20_percent": 0.1596,
            "credit_25_percent": 0.1335,
            "credit_50_percent": 0.1589,
            "credit_full_month": 0.1866,
            "no_credit": 0.1546
          },
          "predicted": "credit_10_percent",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.029117411000001425,
          "input_audit": {
            "state_tokens_before": 1942,
            "state_tokens_kept": 423,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.26039999999999996,
            "yes": 0.7396
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-01",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "merchant_wins",
          "ok": true,
          "server_wall_seconds": 0.029004959000001662,
          "input_audit": {
            "state_tokens_before": 1562,
            "state_tokens_kept": 426,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "customer_wins": 0.3836,
            "customer_withdraws": 0.1945,
            "merchant_wins": 0.422
          },
          "predicted": "merchant_wins",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-02",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upstream_provider",
          "ok": true,
          "server_wall_seconds": 0.026828828999999388,
          "input_audit": {
            "state_tokens_before": 367,
            "state_tokens_kept": 367,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bad_push": 0.9114,
            "database_hardware": 0.0413,
            "upstream_provider": 0.0473
          },
          "predicted": "bad_push",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.027638167999999297,
          "input_audit": {
            "state_tokens_before": 919,
            "state_tokens_kept": 450,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4465,
            "yes": 0.5535
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "attended",
          "ok": true,
          "server_wall_seconds": 0.027936844999999266,
          "input_audit": {
            "state_tokens_before": 391,
            "state_tokens_kept": 391,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "attended": 0.2997,
            "late_cancellation": 0.096,
            "no_show": 0.6043
          },
          "predicted": "no_show",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-06",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upheld",
          "ok": true,
          "server_wall_seconds": 0.028227060999999054,
          "input_audit": {
            "state_tokens_before": 820,
            "state_tokens_kept": 446,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "modified": 0.2461,
            "overturned": 0.1806,
            "upheld": 0.5733
          },
          "predicted": "upheld",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-01",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "escalate_to_engineering",
          "ok": true,
          "server_wall_seconds": 0.02701731299999821,
          "input_audit": {
            "state_tokens_before": 667,
            "state_tokens_kept": 370,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "escalate_to_engineering": 0.2882,
            "escalate_to_security": 0.2579,
            "page_oncall": 0.1209,
            "process_refund": 0.2602,
            "send_kb_article": 0.0728
          },
          "predicted": "escalate_to_engineering",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-03",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "order_b",
          "ok": true,
          "server_wall_seconds": 0.027611977000002952,
          "input_audit": {
            "state_tokens_before": 562,
            "state_tokens_kept": 415,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "order_a": 0.3254,
            "order_b": 0.3869,
            "order_c": 0.232,
            "order_d": 0.0556
          },
          "predicted": "order_b",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-06",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.02704833399999984,
          "input_audit": {
            "state_tokens_before": 595,
            "state_tokens_kept": 432,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.38980000000000004,
            "yes": 0.6102
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-07",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "p0_fix_24h",
          "ok": true,
          "server_wall_seconds": 0.027204347999997935,
          "input_audit": {
            "state_tokens_before": 549,
            "state_tokens_kept": 375,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "out_of_scope": 0.095,
            "p0_fix_24h": 0.3066,
            "p1_fix_7d": 0.2412,
            "p2_fix_30d": 0.2434,
            "p3_backlog": 0.1138
          },
          "predicted": "p0_fix_24h",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-08",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.02702962400000075,
          "input_audit": {
            "state_tokens_before": 440,
            "state_tokens_kept": 429,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.37339999999999995,
            "yes": 0.6266
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-12",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "refund_now",
          "ok": true,
          "server_wall_seconds": 0.028288581999998286,
          "input_audit": {
            "state_tokens_before": 431,
            "state_tokens_kept": 400,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_refund": 0.1845,
            "escalate_to_fraud_team": 0.2514,
            "offer_store_credit": 0.0549,
            "refund_now": 0.3209,
            "wait_for_inspection": 0.1884
          },
          "predicted": "refund_now",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "erase_but_retain_transaction_records",
          "ok": true,
          "server_wall_seconds": 0.031133047000000857,
          "input_audit": {
            "state_tokens_before": 3101,
            "state_tokens_kept": 333,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "erase_all": 0.295,
            "erase_but_retain_transaction_records": 0.1939,
            "erase_marketing_only": 0.1409,
            "reject_unverified_request": 0.1155,
            "restrict_all_pending_legal_hold": 0.2548
          },
          "predicted": "erase_all",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-03",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "eur_400",
          "ok": true,
          "server_wall_seconds": 0.0302771479999997,
          "input_audit": {
            "state_tokens_before": 3129,
            "state_tokens_kept": 401,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_0": 0.2448,
            "eur_150": 0.2904,
            "eur_200": 0.2154,
            "eur_300": 0.1207,
            "eur_400": 0.0873,
            "eur_600": 0.0415
          },
          "predicted": "eur_150",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.029683724000001632,
          "input_audit": {
            "state_tokens_before": 2881,
            "state_tokens_kept": 430,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6949000000000001,
            "yes": 0.3051
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.03030118800000281,
          "input_audit": {
            "state_tokens_before": 2958,
            "state_tokens_kept": 420,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.3764,
            "1": 0.3064,
            "2": 0.2201,
            "3": 0.0971
          },
          "ordinal_ev": 1.0379,
          "predicted": "0",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.03121535900000083,
          "input_audit": {
            "state_tokens_before": 2932,
            "state_tokens_kept": 410,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.165,
            "1": 0.2694,
            "2": 0.458,
            "3": 0.1075
          },
          "ordinal_ev": 1.5079,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-10",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "credit_minus_restocking_fee",
          "ok": true,
          "server_wall_seconds": 0.03046758099999991,
          "input_audit": {
            "state_tokens_before": 3145,
            "state_tokens_kept": 390,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_minus_restocking_fee": 0.7516,
            "full_credit": 0.0909,
            "reject_return": 0.0986,
            "repair_or_replace_only": 0.0589
          },
          "predicted": "credit_minus_restocking_fee",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02987892900000233,
          "input_audit": {
            "state_tokens_before": 3127,
            "state_tokens_kept": 440,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2277,
            "yes": 0.7723
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-c-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "kuznets_technik",
          "ok": true,
          "server_wall_seconds": 0.02642484000000067,
          "input_audit": {
            "state_tokens_before": 421,
            "state_tokens_kept": 412,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "halvorsen_polymer": 0.4348,
            "kuznets_technik": 0.2067,
            "ortega_seal_works": 0.3585
          },
          "predicted": "halvorsen_polymer",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026950142000000454,
          "input_audit": {
            "state_tokens_before": 685,
            "state_tokens_kept": 409,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.34650000000000003,
            "yes": 0.6535
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-c-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_692_00",
          "ok": true,
          "server_wall_seconds": 0.026696637000000578,
          "input_audit": {
            "state_tokens_before": 631,
            "state_tokens_kept": 401,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_617_00": 0.1504,
            "eur_688_00": 0.3842,
            "eur_692_00": 0.1903,
            "eur_693_90": 0.1388,
            "eur_812_00": 0.1364
          },
          "predicted": "eur_688_00",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "usd_2335_00",
          "ok": true,
          "server_wall_seconds": 0.0275396360000002,
          "input_audit": {
            "state_tokens_before": 497,
            "state_tokens_kept": 378,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "usd_2222_50": 0.2104,
            "usd_2303_01": 0.2428,
            "usd_2303_33": 0.2595,
            "usd_2335_00": 0.151,
            "usd_2340_00": 0.1362
          },
          "predicted": "usd_2303_33",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_18_book_set",
          "ok": true,
          "server_wall_seconds": 0.027568515999998766,
          "input_audit": {
            "state_tokens_before": 710,
            "state_tokens_kept": 352,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "none_declined": 0.4319,
            "sep_15_office_chair": 0.1611,
            "sep_16_taxi": 0.1221,
            "sep_17_parking": 0.0784,
            "sep_18_book_set": 0.07,
            "sep_21_client_dinner": 0.1366
          },
          "predicted": "none_declined",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-08",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_498_81",
          "ok": true,
          "server_wall_seconds": 0.02747705499999853,
          "input_audit": {
            "state_tokens_before": 684,
            "state_tokens_kept": 387,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_497_40": 0.245,
            "eur_498_81": 0.3132,
            "eur_506_06": 0.1572,
            "eur_513_69": 0.1382,
            "eur_532_01": 0.1464
          },
          "predicted": "eur_498_81",
          "correct": true
        },
        {
          "id": "hard-opus-c-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "expired_30_month_cap",
          "ok": true,
          "server_wall_seconds": 0.027485344999998773,
          "input_audit": {
            "state_tokens_before": 622,
            "state_tokens_kept": 412,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered": 0.3034,
            "expired_24_month_term": 0.4103,
            "expired_30_month_cap": 0.2864
          },
          "predicted": "expired_24_month_term",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-01",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "deny_missing_approval",
          "ok": true,
          "server_wall_seconds": 0.02737284099999826,
          "input_audit": {
            "state_tokens_before": 210,
            "state_tokens_kept": 210,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_inactive_user": 0.1909,
            "deny_missing_approval": 0.2331,
            "grant_access": 0.3935,
            "grant_owner_exception": 0.1826
          },
          "predicted": "grant_access",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-06",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02826184199999915,
          "input_audit": {
            "state_tokens_before": 172,
            "state_tokens_kept": 172,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.14180000000000004,
            "yes": 0.8582
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-07",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "retain_until_2031",
          "ok": true,
          "server_wall_seconds": 0.02745890400000306,
          "input_audit": {
            "state_tokens_before": 181,
            "state_tokens_kept": 181,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_now": 0.2074,
            "needs_signature_review": 0.0894,
            "retain_indefinitely": 0.2951,
            "retain_until_2031": 0.4081
          },
          "predicted": "retain_until_2031",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-08",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.02760583699999941,
          "input_audit": {
            "state_tokens_before": 165,
            "state_tokens_kept": 165,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.2794,
            "yes": 0.7206
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-09",
          "tier": "hard",
          "family": "adversarial",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.02769590900000196,
          "input_audit": {
            "state_tokens_before": 157,
            "state_tokens_kept": 157,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0403,
            "1": 0.4097,
            "2": 0.4361,
            "3": 0.1139
          },
          "ordinal_ev": 1.6236000000000002,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-11",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "standard_research",
          "ok": true,
          "server_wall_seconds": 0.025273703999999952,
          "input_audit": {
            "state_tokens_before": 199,
            "state_tokens_kept": 199,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ethics_review": 0.2209,
            "major_grants": 0.3095,
            "manual_hold": 0.1161,
            "standard_research": 0.3536
          },
          "predicted": "standard_research",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-01",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "accept_s2_dusk",
          "ok": true,
          "server_wall_seconds": 0.02994450999999998,
          "input_audit": {
            "state_tokens_before": 2274,
            "state_tokens_kept": 434,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "accept_s2_dusk": 0.8184,
            "reject_cutoff": 0.0546,
            "reject_tier": 0.0851,
            "reroute_s3": 0.0419
          },
          "predicted": "accept_s2_dusk",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-02",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "security_primary",
          "ok": true,
          "server_wall_seconds": 0.030246387000001818,
          "input_audit": {
            "state_tokens_before": 2307,
            "state_tokens_kept": 442,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "executive_bridge": 0.0996,
            "heron_q2": 0.5787,
            "insufficient_information": 0.1676,
            "security_primary": 0.1541
          },
          "predicted": "heron_q2",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_director",
          "ok": true,
          "server_wall_seconds": 0.029871707999998165,
          "input_audit": {
            "state_tokens_before": 2248,
            "state_tokens_kept": 441,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "pay_manager_approved": 0.3879,
            "reject_category": 0.1178,
            "require_director": 0.3744,
            "require_finance_board": 0.1199
          },
          "predicted": "pay_manager_approved",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "admit",
          "ok": true,
          "server_wall_seconds": 0.030916552000000763,
          "input_audit": {
            "state_tokens_before": 2208,
            "state_tokens_kept": 443,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "admit": 0.534,
            "deny_prerequisite": 0.1267,
            "deny_score": 0.1907,
            "manual_equivalency": 0.1486
          },
          "predicted": "admit",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "central_engineering",
          "ok": true,
          "server_wall_seconds": 0.02983138699999799,
          "input_audit": {
            "state_tokens_before": 2199,
            "state_tokens_kept": 439,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "central_engineering": 0.1974,
            "insufficient_log": 0.267,
            "local_maintenance": 0.4527,
            "safety_shutdown": 0.0829
          },
          "predicted": "local_maintenance",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_steward",
          "ok": true,
          "server_wall_seconds": 0.02961526300000017,
          "input_audit": {
            "state_tokens_before": 2205,
            "state_tokens_kept": 435,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "confirm_l3": 0.2853,
            "reject_capacity": 0.4051,
            "require_steward": 0.1281,
            "switch_l4": 0.1814
          },
          "predicted": "reject_capacity",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-10",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "approve_45_days",
          "ok": true,
          "server_wall_seconds": 0.03005738299999905,
          "input_audit": {
            "state_tokens_before": 2238,
            "state_tokens_kept": 437,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_45_days": 0.1741,
            "deny_sensitive": 0.1972,
            "needs_field_review": 0.4442,
            "reduce_to_30_days": 0.1845
          },
          "predicted": "needs_field_review",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "needs_authentication",
          "ok": true,
          "server_wall_seconds": 0.028994967999999233,
          "input_audit": {
            "state_tokens_before": 2201,
            "state_tokens_kept": 436,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered_recurrence": 0.4979,
            "expired": 0.128,
            "needs_authentication": 0.1697,
            "wrong_issue_class": 0.2045
          },
          "predicted": "covered_recurrence",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-02",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.026309466999997255,
          "input_audit": {
            "state_tokens_before": 225,
            "state_tokens_kept": 225,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.17479999999999996,
            "yes": 0.8252
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-04",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02650790199999875,
          "input_audit": {
            "state_tokens_before": 133,
            "state_tokens_kept": 133,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.16979999999999995,
            "yes": 0.8302
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-05",
          "tier": "hard",
          "family": "trap",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.027152677000000125,
          "input_audit": {
            "state_tokens_before": 241,
            "state_tokens_kept": 241,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.1687,
            "1": 0.1717,
            "2": 0.3859,
            "3": 0.2737
          },
          "ordinal_ev": 1.7646,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-06",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "keep_open_unresolved",
          "ok": true,
          "server_wall_seconds": 0.027239227999999116,
          "input_audit": {
            "state_tokens_before": 231,
            "state_tokens_kept": 231,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "await_internal_metrics": 0.1333,
            "close_agent_verified": 0.3405,
            "close_confirmed": 0.4464,
            "keep_open_unresolved": 0.0798
          },
          "predicted": "close_confirmed",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-08",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "late",
          "ok": true,
          "server_wall_seconds": 0.027465003999999738,
          "input_audit": {
            "state_tokens_before": 168,
            "state_tokens_kept": 168,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "canceled": 0.1081,
            "indeterminate": 0.0924,
            "late": 0.1781,
            "on_time": 0.6214
          },
          "predicted": "on_time",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-10",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "blocked_second_review",
          "ok": true,
          "server_wall_seconds": 0.026902181000000525,
          "input_audit": {
            "state_tokens_before": 103,
            "state_tokens_kept": 103,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "blocked_failed_test": 0.2364,
            "blocked_second_review": 0.168,
            "ready_one_review": 0.168,
            "ready_two_reviews": 0.4275
          },
          "predicted": "ready_two_reviews",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-13",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02606134099999835,
          "input_audit": {
            "state_tokens_before": 131,
            "state_tokens_kept": 131,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.22840000000000005,
            "yes": 0.7716
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-15",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02717121699999936,
          "input_audit": {
            "state_tokens_before": 217,
            "state_tokens_kept": 217,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1452,
            "yes": 0.8548
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-01",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026706145999998654,
          "input_audit": {
            "state_tokens_before": 158,
            "state_tokens_kept": 158,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6183000000000001,
            "yes": 0.3817
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-02",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.027440813999998426,
          "input_audit": {
            "state_tokens_before": 113,
            "state_tokens_kept": 113,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.23440000000000005,
            "yes": 0.7656
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.027295139999999662,
          "input_audit": {
            "state_tokens_before": 160,
            "state_tokens_kept": 160,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3974,
            "yes": 0.6026
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02670822599999667,
          "input_audit": {
            "state_tokens_before": 99,
            "state_tokens_kept": 99,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7772,
            "yes": 0.2228
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02662734400000133,
          "input_audit": {
            "state_tokens_before": 122,
            "state_tokens_kept": 122,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3369,
            "yes": 0.6631
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-14",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02674904799999922,
          "input_audit": {
            "state_tokens_before": 154,
            "state_tokens_kept": 154,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3648,
            "yes": 0.6352
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026681146000001377,
          "input_audit": {
            "state_tokens_before": 103,
            "state_tokens_kept": 103,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5,
            "yes": 0.5
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-18",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.026497351999999808,
          "input_audit": {
            "state_tokens_before": 89,
            "state_tokens_kept": 89,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.10429999999999995,
            "yes": 0.8957
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "reject_entire_claim",
          "ok": true,
          "server_wall_seconds": 0.03108123599999857,
          "input_audit": {
            "state_tokens_before": 2610,
            "state_tokens_kept": 413,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_all": 0.3648,
            "approve_lodging_only": 0.3028,
            "approve_meals_only": 0.1811,
            "reject_entire_claim": 0.1513
          },
          "predicted": "approve_all",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "deny_current_export",
          "ok": true,
          "server_wall_seconds": 0.030881732000000994,
          "input_audit": {
            "state_tokens_before": 2498,
            "state_tokens_kept": 417,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "analytics_owner_only": 0.3064,
            "deny_current_export": 0.1722,
            "privacy_and_security": 0.3563,
            "privacy_only": 0.1652
          },
          "predicted": "privacy_and_security",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "delete_profile_omar_april_and_table",
          "ok": true,
          "server_wall_seconds": 0.03042068099999895,
          "input_audit": {
            "state_tokens_before": 2436,
            "state_tokens_kept": 407,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_everything": 0.235,
            "delete_nothing": 0.0931,
            "delete_only_profile": 0.2478,
            "delete_profile_omar_april_and_table": 0.424
          },
          "predicted": "delete_profile_omar_april_and_table",
          "correct": true
        },
        {
          "id": "hard-sol-b-long_policy-06",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "east_ward",
          "ok": true,
          "server_wall_seconds": 0.030890391999999878,
          "input_audit": {
            "state_tokens_before": 2468,
            "state_tokens_kept": 423,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "east_ward": 0.0102,
            "harbor_neighbors": 0.9526,
            "no_award": 0.0302,
            "river_arts": 0.0071
          },
          "predicted": "harbor_neighbors",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-01",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "office_file_repair",
          "ok": true,
          "server_wall_seconds": 0.028388414999998446,
          "input_audit": {
            "state_tokens_before": 181,
            "state_tokens_kept": 181,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "data_pipeline": 0.1616,
            "desktop_support": 0.2193,
            "finance_review": 0.1288,
            "office_file_repair": 0.2278,
            "spreadsheet_formula": 0.2625
          },
          "predicted": "spreadsheet_formula",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-02",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "ledger_reconciliation",
          "ok": true,
          "server_wall_seconds": 0.02803482699999904,
          "input_audit": {
            "state_tokens_before": 171,
            "state_tokens_kept": 171,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "customer_support": 0.1914,
            "fraud_review": 0.0721,
            "ledger_reconciliation": 0.1814,
            "mobile_app": 0.2129,
            "payment_disputes": 0.3423
          },
          "predicted": "payment_disputes",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-03",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "network_dns",
          "ok": true,
          "server_wall_seconds": 0.028297642999998374,
          "input_audit": {
            "state_tokens_before": 182,
            "state_tokens_kept": 182,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "api_platform": 0.404,
            "authoritative_dns": 0.22,
            "endpoint_devices": 0.0719,
            "network_dns": 0.2175,
            "tls_certificates": 0.0866
          },
          "predicted": "api_platform",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-07",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "release_notarization",
          "ok": true,
          "server_wall_seconds": 0.02785516299999813,
          "input_audit": {
            "state_tokens_before": 156,
            "state_tokens_kept": 156,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ci_infrastructure": 0.2834,
            "code_signing": 0.1876,
            "endpoint_security": 0.052,
            "macos_runtime": 0.1631,
            "release_notarization": 0.3139
          },
          "predicted": "release_notarization",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-09",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "source_access",
          "ok": true,
          "server_wall_seconds": 0.026651704999999026,
          "input_audit": {
            "state_tokens_before": 162,
            "state_tokens_kept": 162,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "git_client": 0.168,
            "identity_login": 0.21,
            "repository_admin": 0.2464,
            "source_access": 0.1822,
            "vpn_network": 0.1934
          },
          "predicted": "repository_admin",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-01",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "late_by_under_2h",
          "ok": true,
          "server_wall_seconds": 0.027086666000002424,
          "input_audit": {
            "state_tokens_before": 133,
            "state_tokens_kept": 133,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "before_open": 0.2501,
            "late_by_2h_or_more": 0.2108,
            "late_by_under_2h": 0.1661,
            "within_window": 0.3729
          },
          "predicted": "within_window",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "over_limit",
          "ok": true,
          "server_wall_seconds": 0.028021247000001637,
          "input_audit": {
            "state_tokens_before": 136,
            "state_tokens_kept": 136,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "at_least_100gb_below": 0.3052,
            "exactly_equal": 0.259,
            "over_limit": 0.2522,
            "under_100gb_below": 0.1836
          },
          "predicted": "at_least_100gb_below",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "9_credits",
          "ok": true,
          "server_wall_seconds": 0.026292757000000222,
          "input_audit": {
            "state_tokens_before": 101,
            "state_tokens_kept": 101,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "12_credits": 0.3571,
            "15_credits": 0.1829,
            "6_credits": 0.2108,
            "9_credits": 0.2491
          },
          "predicted": "12_credits",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "pass",
          "ok": true,
          "server_wall_seconds": 0.027419242000000565,
          "input_audit": {
            "state_tokens_before": 133,
            "state_tokens_kept": 133,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "clear_fail": 0.2185,
            "narrow_fail": 0.3179,
            "pass": 0.4636
          },
          "predicted": "pass",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-03",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.027338590999999468,
          "input_audit": {
            "state_tokens_before": 331,
            "state_tokens_kept": 331,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1965,
            "yes": 0.8035
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.027306980999998842,
          "input_audit": {
            "state_tokens_before": 238,
            "state_tokens_kept": 238,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.656,
            "yes": 0.344
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-07",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.02861565999999982,
          "input_audit": {
            "state_tokens_before": 211,
            "state_tokens_kept": 211,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6467,
            "yes": 0.3533
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.026013480999999672,
          "input_audit": {
            "state_tokens_before": 225,
            "state_tokens_kept": 225,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.38749999999999996,
            "yes": 0.6125
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-09",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026283267000000166,
          "input_audit": {
            "state_tokens_before": 219,
            "state_tokens_kept": 219,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6058,
            "yes": 0.3942
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.027621377999999197,
          "input_audit": {
            "state_tokens_before": 236,
            "state_tokens_kept": 236,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4164,
            "yes": 0.5836
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-11",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.028278461999999394,
          "input_audit": {
            "state_tokens_before": 255,
            "state_tokens_kept": 255,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1321,
            "yes": 0.8679
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-13",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.027294200000000046,
          "input_audit": {
            "state_tokens_before": 214,
            "state_tokens_kept": 214,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8395,
            "yes": 0.1605
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.026648995999998704,
          "input_audit": {
            "state_tokens_before": 199,
            "state_tokens_kept": 199,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.16749999999999998,
            "yes": 0.8325
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "programme_review",
          "ok": true,
          "server_wall_seconds": 0.03020296600000094,
          "input_audit": {
            "state_tokens_before": 2379,
            "state_tokens_kept": 430,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "partial_release": 0.1026,
            "programme_review": 0.3709,
            "reject_draw": 0.1654,
            "release_draw": 0.3611
          },
          "predicted": "programme_review",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bus_substitution",
          "ok": true,
          "server_wall_seconds": 0.03193643499999865,
          "input_audit": {
            "state_tokens_before": 2377,
            "state_tokens_kept": 423,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bus_substitution": 0.0812,
            "call_fen_cross": 0.088,
            "pass_fen_cross": 0.0737,
            "terminate_oakmere": 0.7572
          },
          "predicted": "terminate_oakmere",
          "correct": false
        },
        {
          "id": "hard-sol-c-multi_hop-11",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "coverage_excluded",
          "ok": true,
          "server_wall_seconds": 0.029608452999998036,
          "input_audit": {
            "state_tokens_before": 2366,
            "state_tokens_kept": 424,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "adjuster_referral": 0.1365,
            "coverage_confirmed": 0.3651,
            "coverage_excluded": 0.2902,
            "documentation_needed": 0.2081
          },
          "predicted": "coverage_confirmed",
          "correct": false
        },
        {
          "id": "hard-sol-c-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "conservation_hold",
          "ok": true,
          "server_wall_seconds": 0.026940931999998696,
          "input_audit": {
            "state_tokens_before": 402,
            "state_tokens_kept": 402,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "conservation_hold": 0.1809,
            "deny_request": 0.0748,
            "digital_copy_only": 0.2204,
            "supervised_delivery": 0.5238
          },
          "predicted": "supervised_delivery",
          "correct": false
        },
        {
          "id": "hard-sol-c-multi_hop-13",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "normal_release",
          "ok": true,
          "server_wall_seconds": 0.0294356889999996,
          "input_audit": {
            "state_tokens_before": 2445,
            "state_tokens_kept": 433,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "engineering_review": 0.2409,
            "normal_release": 0.5278,
            "rework": 0.1809,
            "scrap_lot": 0.0504
          },
          "predicted": "normal_release",
          "correct": true
        }
      ],
      "raw_sha256": "3f8d06558e4aceefd2a4389238a212909581fb28b41c515b677d2309295eafd9"
    },
    "kev": {
      "model": "Kev-0.8B",
      "runtime": {
        "model": "Kev-0.8B",
        "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8",
        "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704",
        "gpu": "NVIDIA L40S",
        "python": "3.12.10",
        "software": {
          "torch": "2.8.0",
          "transformers": "5.17.0",
          "numpy": "2.5.3",
          "safetensors": "0.8.0"
        },
        "checkpoint_hashes": {
          "head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65",
          "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"
        },
        "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68",
        "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c",
        "dtype": "bfloat16",
        "adapter_merged": false,
        "temperature": 2.406050072164233,
        "max_state": 8192,
        "max_branch": 8192,
        "prefix_cache": false,
        "date_facts": false,
        "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"
      },
      "planned": 231,
      "received": 231,
      "correct": 138,
      "accuracy": 0.5974025974025974,
      "operational_success": 231,
      "strict_valid": 182,
      "valid": 231,
      "renormalized": 49,
      "tiers": {
        "original": {
          "n": 72,
          "correct": 54
        },
        "easy": {
          "n": 48,
          "correct": 48
        },
        "hard": {
          "n": 111,
          "correct": 36
        }
      },
      "brier_mean": 0.48896821689266606,
      "ece": {
        "ece": 0.10097710203920818,
        "n": 231,
        "bins": [
          {
            "lo": 0.0,
            "hi": 0.1,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.1,
            "hi": 0.2,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.2,
            "hi": 0.3,
            "n": 8,
            "mean_confidence": 0.2656818181818182,
            "accuracy": 0.125
          },
          {
            "lo": 0.3,
            "hi": 0.4,
            "n": 29,
            "mean_confidence": 0.35575733435412504,
            "accuracy": 0.27586206896551724
          },
          {
            "lo": 0.4,
            "hi": 0.5,
            "n": 21,
            "mean_confidence": 0.4521494530405423,
            "accuracy": 0.2857142857142857
          },
          {
            "lo": 0.5,
            "hi": 0.6,
            "n": 23,
            "mean_confidence": 0.5483727938011193,
            "accuracy": 0.391304347826087
          },
          {
            "lo": 0.6,
            "hi": 0.7,
            "n": 22,
            "mean_confidence": 0.6463375883042849,
            "accuracy": 0.5909090909090909
          },
          {
            "lo": 0.7,
            "hi": 0.8,
            "n": 29,
            "mean_confidence": 0.7447543375027157,
            "accuracy": 0.7241379310344828
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "n": 34,
            "mean_confidence": 0.844889194801833,
            "accuracy": 0.7352941176470589
          },
          {
            "lo": 0.9,
            "hi": 1.0,
            "n": 65,
            "mean_confidence": 0.9574160800695453,
            "accuracy": 0.8461538461538461
          }
        ]
      },
      "ordinal_mae": 0.44000066673334004,
      "latency": {
        "n": 231,
        "p50_s": 0.08541707199999848,
        "p95_s": 0.23663473649999744
      },
      "peak_gpu_allocated_bytes": 2083974656,
      "truncation": {
        "state_truncated": 0,
        "instruction_truncated": 0,
        "options_truncated": 0
      },
      "scored": [
        {
          "id": "original-policy-01-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 1.4823472009999996,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 81
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.84,
            "yes": 0.16
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-01-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.08602041600000021,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 84
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.87,
            "yes": 0.13
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-02-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.08728460000000027,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 70
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.12,
            "yes": 0.88
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-02-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.08481000300000119,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 69
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.31000000000000005,
            "yes": 0.69
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-03-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.08039804500000258,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 72
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.43999999999999995,
            "yes": 0.56
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-03-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07798203000000115,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 71
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.26,
            "yes": 0.74
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-04-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.08236447099999822,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 78
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5800000000000001,
            "yes": 0.42
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-04-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.07356612499999926,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 75
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.87,
            "yes": 0.13
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-05-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.07106741800000194,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 82
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.28,
            "yes": 0.72
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-policy-05-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.07460416199999997,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 74
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8,
            "yes": 0.2
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-06-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07112239900000006,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 84
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.07999999999999996,
            "yes": 0.92
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-06-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07731390500000046,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 74
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.31999999999999995,
            "yes": 0.68
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-intent-01-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.08476612500000158,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 73
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "cancel": 0.16161616161616163,
            "change_address": 0.0,
            "other": 0.23232323232323235,
            "refund": 0.6060606060606061,
            "status": 0.0
          },
          "predicted": "refund",
          "correct": true
        },
        {
          "id": "original-intent-01-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.08541707199999848,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 74
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "cancel": 0.12871287128712872,
            "change_address": 0.009900990099009901,
            "other": 0.4653465346534653,
            "refund": 0.38613861386138615,
            "status": 0.009900990099009901
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-02-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.08124850100000103,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 74
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.03,
            "change_address": 0.2,
            "other": 0.68,
            "refund": 0.01,
            "status": 0.08
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-02-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.08369100599999868,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 73
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.05,
            "change_address": 0.07,
            "other": 0.45,
            "refund": 0.02,
            "status": 0.41
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-03-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.08004560799999894,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 75
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.02,
            "change_address": 0.69,
            "other": 0.25,
            "refund": 0.01,
            "status": 0.03
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-03-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.08781372700000034,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 73
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "cancel": 0.0,
            "change_address": 0.8181818181818181,
            "other": 0.1717171717171717,
            "refund": 0.0101010101010101,
            "status": 0.0
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-04-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.08292432500000047,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 78
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.32,
            "change_address": 0.01,
            "other": 0.57,
            "refund": 0.1,
            "status": 0.0
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-04-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.07499436299999829,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 73
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.73,
            "change_address": 0.0,
            "other": 0.23,
            "refund": 0.04,
            "status": 0.0
          },
          "predicted": "cancel",
          "correct": true
        },
        {
          "id": "original-intent-05-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.07415018000000018,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 73
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.04,
            "change_address": 0.01,
            "other": 0.71,
            "refund": 0.23,
            "status": 0.01
          },
          "predicted": "other",
          "correct": true
        },
        {
          "id": "original-intent-05-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.07506649400000143,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 72
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "cancel": 0.039603960396039604,
            "change_address": 0.009900990099009901,
            "other": 0.44554455445544555,
            "refund": 0.49504950495049505,
            "status": 0.009900990099009901
          },
          "predicted": "refund",
          "correct": false
        },
        {
          "id": "original-intent-06-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.07514803599999809,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 71
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.23,
            "change_address": 0.05,
            "other": 0.35,
            "refund": 0.07,
            "status": 0.3
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-06-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.08297440599999817,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 76
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.3,
            "change_address": 0.03,
            "other": 0.44,
            "refund": 0.1,
            "status": 0.13
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-ordinal-01-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.07747447899999926,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 72
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.78,
            "1": 0.11,
            "2": 0.07,
            "3": 0.04
          },
          "ordinal_ev": 0.37,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-01-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.08055001199999978,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 74
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.92,
            "1": 0.05,
            "2": 0.02,
            "3": 0.01
          },
          "ordinal_ev": 0.12,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-02-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.07908029200000044,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 76
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.32,
            "1": 0.46,
            "2": 0.16,
            "3": 0.06
          },
          "ordinal_ev": 0.96,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-02-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.07863433900000061,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 77
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "0": 0.26732673267326734,
            "1": 0.5841584158415841,
            "2": 0.0891089108910891,
            "3": 0.0594059405940594
          },
          "ordinal_ev": 0.9405940594059405,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-03-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.07685320200000234,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 73
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.16,
            "1": 0.14,
            "2": 0.62,
            "3": 0.08
          },
          "ordinal_ev": 1.6199999999999999,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-03-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.07819575799999967,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 75
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.22,
            "1": 0.09,
            "2": 0.62,
            "3": 0.07
          },
          "ordinal_ev": 1.54,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-04-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.07918724399999988,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 75
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.05,
            "1": 0.01,
            "2": 0.04,
            "3": 0.9
          },
          "ordinal_ev": 2.79,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-04-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.07891490699999792,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 74
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.05,
            "1": 0.02,
            "2": 0.06,
            "3": 0.87
          },
          "ordinal_ev": 2.75,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-05-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.08430316200000121,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 78
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.33,
            "1": 0.08,
            "2": 0.51,
            "3": 0.08
          },
          "ordinal_ev": 1.34,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "original-ordinal-05-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.07511107599999889,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 77
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.45,
            "1": 0.1,
            "2": 0.4,
            "3": 0.05
          },
          "ordinal_ev": 1.05,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-06-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.07452933899999792,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 78
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.23,
            "1": 0.14,
            "2": 0.6,
            "3": 0.03
          },
          "ordinal_ev": 1.43,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-06-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.07777126600000273,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 85
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.25,
            "1": 0.17,
            "2": 0.52,
            "3": 0.06
          },
          "ordinal_ev": 1.39,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-extraction-01-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.10111554200000228,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 64
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.08,
            "pickup": 0.37,
            "post": 0.13,
            "unknown": 0.42
          },
          "predicted": "unknown",
          "correct": false
        },
        {
          "id": "original-extraction-01-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.07512208500000028,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 66
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.05,
            "pickup": 0.47,
            "post": 0.07,
            "unknown": 0.41
          },
          "predicted": "pickup",
          "correct": true
        },
        {
          "id": "original-extraction-02-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.07380340000000274,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 67
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.03,
            "pickup": 0.04,
            "post": 0.1,
            "unknown": 0.83
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-02-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.08024118299999827,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 66
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.02,
            "pickup": 0.03,
            "post": 0.11,
            "unknown": 0.84
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-03-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.07882480399999992,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 70
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.29,
            "pickup": 0.11,
            "post": 0.16,
            "unknown": 0.44
          },
          "predicted": "unknown",
          "correct": false
        },
        {
          "id": "original-extraction-03-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.0802272429999995,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 69
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "courier": 0.797979797979798,
            "pickup": 0.010101010101010102,
            "post": 0.10101010101010102,
            "unknown": 0.09090909090909091
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-04-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.07628919600000117,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 67
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "courier": 0.030303030303030304,
            "pickup": 0.020202020202020204,
            "post": 0.7474747474747475,
            "unknown": 0.20202020202020204
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-04-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.07067859600000048,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 64
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.02,
            "pickup": 0.01,
            "post": 0.82,
            "unknown": 0.15
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-05-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.08145653500000094,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 67
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.02,
            "pickup": 0.03,
            "post": 0.02,
            "unknown": 0.93
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-05-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.07506048400000154,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 63
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.09,
            "pickup": 0.04,
            "post": 0.06,
            "unknown": 0.81
          },
          "predicted": "unknown",
          "correct": true
        },
        {
          "id": "original-extraction-06-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.07570632000000188,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 67
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.74,
            "pickup": 0.03,
            "post": 0.06,
            "unknown": 0.17
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-06-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.0787370220000021,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 66
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.77,
            "pickup": 0.03,
            "post": 0.02,
            "unknown": 0.18
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-adequacy-01-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07418750000000074,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 63
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.050000000000000044,
            "yes": 0.95
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-01-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07498189100000019,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 59
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4,
            "yes": 0.6
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-02-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.0789645479999983,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 60
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.050000000000000044,
            "yes": 0.95
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-02-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.0753896819999973,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 61
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.94,
            "yes": 0.06
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-03-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.075966477999998,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 63
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.020000000000000018,
            "yes": 0.98
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-03-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.07809913499999865,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 67
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.09999999999999998,
            "yes": 0.9
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-04-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07710895800000017,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 67
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.77,
            "yes": 0.23
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-adequacy-04-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07813764599999828,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 64
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.53,
            "yes": 0.47
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-adequacy-05-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.08235253899999861,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 56
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.37,
            "yes": 0.63
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-05-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.07233404000000121,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 57
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.18000000000000005,
            "yes": 0.82
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-06-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07172450399999875,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 72
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.14,
            "yes": 0.86
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-06-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.08097798200000028,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 74
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.08999999999999997,
            "yes": 0.91
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-routing-01-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 0.10997358999999918,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 102
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "coding": 0.0707070707070707,
            "coding_agent": 0.0101010101010101,
            "document": 0.0404040404040404,
            "general": 0.0404040404040404,
            "math": 0.8080808080808081,
            "tools": 0.0303030303030303
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-01-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 0.07954474299999958,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 98
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.05,
            "coding_agent": 0.01,
            "document": 0.05,
            "general": 0.05,
            "math": 0.8,
            "tools": 0.04
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-02-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 0.08188692599999925,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 103
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.88,
            "coding_agent": 0.01,
            "document": 0.01,
            "general": 0.08,
            "math": 0.01,
            "tools": 0.01
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-02-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 0.07745322699999946,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 97
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.8,
            "coding_agent": 0.06,
            "document": 0.02,
            "general": 0.1,
            "math": 0.01,
            "tools": 0.01
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-03-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 0.07403540600000014,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 102
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.08,
            "coding_agent": 0.79,
            "document": 0.02,
            "general": 0.08,
            "math": 0.0,
            "tools": 0.03
          },
          "predicted": "coding_agent",
          "correct": true
        },
        {
          "id": "original-routing-03-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 0.07837810299999859,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 104
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.04,
            "coding_agent": 0.82,
            "document": 0.03,
            "general": 0.07,
            "math": 0.0,
            "tools": 0.04
          },
          "predicted": "coding_agent",
          "correct": true
        },
        {
          "id": "original-routing-04-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 0.08326236399999942,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 98
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.03,
            "coding_agent": 0.03,
            "document": 0.74,
            "general": 0.13,
            "math": 0.02,
            "tools": 0.05
          },
          "predicted": "document",
          "correct": true
        },
        {
          "id": "original-routing-04-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 0.08238153000000281,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 97
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "coding": 0.0594059405940594,
            "coding_agent": 0.0594059405940594,
            "document": 0.6633663366336634,
            "general": 0.0891089108910891,
            "math": 0.04950495049504951,
            "tools": 0.07920792079207921
          },
          "predicted": "document",
          "correct": true
        },
        {
          "id": "original-routing-05-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 0.0739873650000007,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 99
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.02,
            "coding_agent": 0.01,
            "document": 0.04,
            "general": 0.1,
            "math": 0.0,
            "tools": 0.83
          },
          "predicted": "tools",
          "correct": true
        },
        {
          "id": "original-routing-05-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 0.0753779630000011,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 99
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "coding": 0.05050505050505051,
            "coding_agent": 0.030303030303030304,
            "document": 0.15151515151515152,
            "general": 0.37373737373737376,
            "math": 0.010101010101010102,
            "tools": 0.38383838383838387
          },
          "predicted": "tools",
          "correct": true
        },
        {
          "id": "original-routing-06-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.07385787199999783,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 98
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "coding": 0.05050505050505051,
            "coding_agent": 0.030303030303030304,
            "document": 0.12121212121212122,
            "general": 0.7272727272727273,
            "math": 0.010101010101010102,
            "tools": 0.06060606060606061
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "original-routing-06-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.07932575799999952,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 97
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.05,
            "coding_agent": 0.03,
            "document": 0.08,
            "general": 0.78,
            "math": 0.01,
            "tools": 0.05
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "easy-intent-00",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.09852977399999929,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 93
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "billing_question": 0.0,
            "cancel_order": 0.010101010101010102,
            "change_address": 0.0,
            "report_damage": 0.010101010101010102,
            "track_order": 0.9797979797979798
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-01",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "cancel_order",
          "ok": true,
          "server_wall_seconds": 0.07734573500000153,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 93
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0,
            "cancel_order": 0.99,
            "change_address": 0.0,
            "report_damage": 0.01,
            "track_order": 0.0
          },
          "predicted": "cancel_order",
          "correct": true
        },
        {
          "id": "easy-intent-02",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.0883351800000014,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 96
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0,
            "cancel_order": 0.01,
            "change_address": 0.99,
            "report_damage": 0.0,
            "track_order": 0.0
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "easy-intent-03",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "report_damage",
          "ok": true,
          "server_wall_seconds": 0.0829249350000012,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 85
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0,
            "cancel_order": 0.01,
            "change_address": 0.0,
            "report_damage": 0.98,
            "track_order": 0.01
          },
          "predicted": "report_damage",
          "correct": true
        },
        {
          "id": "easy-intent-04",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "billing_question",
          "ok": true,
          "server_wall_seconds": 0.07953060399999856,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 90
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.88,
            "cancel_order": 0.07,
            "change_address": 0.01,
            "report_damage": 0.03,
            "track_order": 0.01
          },
          "predicted": "billing_question",
          "correct": true
        },
        {
          "id": "easy-intent-05",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.07623196599999815,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 89
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "billing_question": 0.0297029702970297,
            "cancel_order": 0.1188118811881188,
            "change_address": 0.0297029702970297,
            "report_damage": 0.0594059405940594,
            "track_order": 0.7623762376237624
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-06",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.07990993199999963,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 81
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "play_music": 0.0,
            "send_message": 0.15151515151515152,
            "set_alarm": 0.8484848484848485,
            "turn_off_lights": 0.0,
            "weather": 0.0
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-intent-07",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "play_music",
          "ok": true,
          "server_wall_seconds": 0.08178989299999984,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 74
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "play_music": 1.0,
            "send_message": 0.0,
            "set_alarm": 0.0,
            "turn_off_lights": 0.0,
            "weather": 0.0
          },
          "predicted": "play_music",
          "correct": true
        },
        {
          "id": "easy-intent-08",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "weather",
          "ok": true,
          "server_wall_seconds": 0.07119278900000126,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 76
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "play_music": 0.0,
            "send_message": 0.0,
            "set_alarm": 0.0,
            "turn_off_lights": 0.0,
            "weather": 1.0
          },
          "predicted": "weather",
          "correct": true
        },
        {
          "id": "easy-intent-09",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "send_message",
          "ok": true,
          "server_wall_seconds": 0.0691732949999988,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 79
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "play_music": 0.009900990099009901,
            "send_message": 0.900990099009901,
            "set_alarm": 0.0594059405940594,
            "turn_off_lights": 0.009900990099009901,
            "weather": 0.019801980198019802
          },
          "predicted": "send_message",
          "correct": true
        },
        {
          "id": "easy-intent-10",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "turn_off_lights",
          "ok": true,
          "server_wall_seconds": 0.07028200400000273,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 77
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "play_music": 0.0,
            "send_message": 0.0,
            "set_alarm": 0.0,
            "turn_off_lights": 1.0,
            "weather": 0.0
          },
          "predicted": "turn_off_lights",
          "correct": true
        },
        {
          "id": "easy-intent-11",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.07279197099999735,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 77
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "play_music": 0.0,
            "send_message": 0.0,
            "set_alarm": 1.0,
            "turn_off_lights": 0.0,
            "weather": 0.0
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-fact-00",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07262726699999789,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 61
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.030000000000000027,
            "yes": 0.97
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-01",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.07255594600000137,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 59
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.98,
            "yes": 0.02
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-02",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.08306532699999991,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 67
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.020000000000000018,
            "yes": 0.98
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-03",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.085832620999998,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 70
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.99,
            "yes": 0.01
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-04",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07748980799999927,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 59
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.050000000000000044,
            "yes": 0.95
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-05",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.08412860600000016,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 60
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.87,
            "yes": 0.13
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-06",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.08973396699999725,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 80
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.020000000000000018,
            "yes": 0.98
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-07",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.06837886499999968,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 55
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.97,
            "yes": 0.03
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-08",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.06466395499999678,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 51
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.010000000000000009,
            "yes": 0.99
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-09",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.07241322399999817,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 51
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.99,
            "yes": 0.01
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-10",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.06577097500000306,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 60
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.010000000000000009,
            "yes": 0.99
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-11",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.06499259499999965,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 61
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.98,
            "yes": 0.02
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-extraction-00",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "paypal",
          "ok": true,
          "server_wall_seconds": 0.07539348299999915,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 50
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.01,
            "cash": 0.0,
            "credit_card": 0.01,
            "paypal": 0.98
          },
          "predicted": "paypal",
          "correct": true
        },
        {
          "id": "easy-extraction-01",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "cash",
          "ok": true,
          "server_wall_seconds": 0.07563612999999947,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 52
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "bank_transfer": 0.009900990099009901,
            "cash": 0.9504950495049505,
            "credit_card": 0.0297029702970297,
            "paypal": 0.009900990099009901
          },
          "predicted": "cash",
          "correct": true
        },
        {
          "id": "easy-extraction-02",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "bank_transfer",
          "ok": true,
          "server_wall_seconds": 0.0728204450000014,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 53
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "bank_transfer": 0.9603960396039604,
            "cash": 0.009900990099009901,
            "credit_card": 0.019801980198019802,
            "paypal": 0.009900990099009901
          },
          "predicted": "bank_transfer",
          "correct": true
        },
        {
          "id": "easy-extraction-03",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "high",
          "ok": true,
          "server_wall_seconds": 0.07407845899999899,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 54
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "high": 0.5841584158415841,
            "low": 0.019801980198019802,
            "medium": 0.0297029702970297,
            "urgent": 0.36633663366336633
          },
          "predicted": "high",
          "correct": true
        },
        {
          "id": "easy-extraction-04",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "low",
          "ok": true,
          "server_wall_seconds": 0.07179941700000114,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 52
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.01,
            "low": 0.95,
            "medium": 0.03,
            "urgent": 0.01
          },
          "predicted": "low",
          "correct": true
        },
        {
          "id": "easy-extraction-05",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "urgent",
          "ok": true,
          "server_wall_seconds": 0.07512938600000041,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 52
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.03,
            "low": 0.01,
            "medium": 0.01,
            "urgent": 0.95
          },
          "predicted": "urgent",
          "correct": true
        },
        {
          "id": "easy-extraction-06",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "M",
          "ok": true,
          "server_wall_seconds": 0.07676813099999436,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 45
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.02,
            "M": 0.9,
            "S": 0.06,
            "XL": 0.02
          },
          "predicted": "M",
          "correct": true
        },
        {
          "id": "easy-extraction-07",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "XL",
          "ok": true,
          "server_wall_seconds": 0.07002794800000345,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 43
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.02,
            "M": 0.01,
            "S": 0.01,
            "XL": 0.96
          },
          "predicted": "XL",
          "correct": true
        },
        {
          "id": "easy-extraction-08",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "thursday",
          "ok": true,
          "server_wall_seconds": 0.06621184600000163,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 53
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "friday": 0.039603960396039604,
            "monday": 0.009900990099009901,
            "thursday": 0.9108910891089109,
            "tuesday": 0.019801980198019802,
            "wednesday": 0.019801980198019802
          },
          "predicted": "thursday",
          "correct": true
        },
        {
          "id": "easy-extraction-09",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "monday",
          "ok": true,
          "server_wall_seconds": 0.06864762099999666,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 55
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.03,
            "monday": 0.81,
            "thursday": 0.05,
            "tuesday": 0.05,
            "wednesday": 0.06
          },
          "predicted": "monday",
          "correct": true
        },
        {
          "id": "easy-extraction-10",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "friday",
          "ok": true,
          "server_wall_seconds": 0.06469985500000064,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 51
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "friday": 0.900990099009901,
            "monday": 0.019801980198019802,
            "thursday": 0.0297029702970297,
            "tuesday": 0.019801980198019802,
            "wednesday": 0.0297029702970297
          },
          "predicted": "friday",
          "correct": true
        },
        {
          "id": "easy-extraction-11",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "credit_card",
          "ok": true,
          "server_wall_seconds": 0.07213242400000297,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 56
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "bank_transfer": 0.010101010101010102,
            "cash": 0.010101010101010102,
            "credit_card": 0.9696969696969697,
            "paypal": 0.010101010101010102
          },
          "predicted": "credit_card",
          "correct": true
        },
        {
          "id": "easy-tool_selection-00",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_weather",
          "ok": true,
          "server_wall_seconds": 0.07282196300000265,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 84
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "convert_currency": 0.0,
            "create_calendar_event": 0.0,
            "get_weather": 1.0,
            "send_email": 0.0,
            "translate_text": 0.0
          },
          "predicted": "get_weather",
          "correct": true
        },
        {
          "id": "easy-tool_selection-01",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "send_email",
          "ok": true,
          "server_wall_seconds": 0.08304997700000172,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 85
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "convert_currency": 0.0,
            "create_calendar_event": 0.0,
            "get_weather": 0.0,
            "send_email": 1.0,
            "translate_text": 0.0
          },
          "predicted": "send_email",
          "correct": true
        },
        {
          "id": "easy-tool_selection-02",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.08159013800000281,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 88
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "convert_currency": 0.0,
            "create_calendar_event": 1.0,
            "get_weather": 0.0,
            "send_email": 0.0,
            "translate_text": 0.0
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-03",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "convert_currency",
          "ok": true,
          "server_wall_seconds": 0.07930267700000115,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 86
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.99,
            "create_calendar_event": 0.0,
            "get_weather": 0.0,
            "send_email": 0.0,
            "translate_text": 0.01
          },
          "predicted": "convert_currency",
          "correct": true
        },
        {
          "id": "easy-tool_selection-04",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "translate_text",
          "ok": true,
          "server_wall_seconds": 0.07999718599999994,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 82
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0,
            "create_calendar_event": 0.0,
            "get_weather": 0.0,
            "send_email": 0.01,
            "translate_text": 0.99
          },
          "predicted": "translate_text",
          "correct": true
        },
        {
          "id": "easy-tool_selection-05",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.0783252899999951,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 88
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0,
            "create_calendar_event": 0.99,
            "get_weather": 0.0,
            "send_email": 0.01,
            "translate_text": 0.0
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-06",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "search_flights",
          "ok": true,
          "server_wall_seconds": 0.08119797799999873,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 92
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0,
            "call_taxi": 0.0,
            "get_stock_price": 0.0,
            "search_flights": 0.99,
            "set_reminder": 0.01
          },
          "predicted": "search_flights",
          "correct": true
        },
        {
          "id": "easy-tool_selection-07",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "book_restaurant",
          "ok": true,
          "server_wall_seconds": 0.08119759699999918,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 92
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.96,
            "call_taxi": 0.01,
            "get_stock_price": 0.0,
            "search_flights": 0.0,
            "set_reminder": 0.03
          },
          "predicted": "book_restaurant",
          "correct": true
        },
        {
          "id": "easy-tool_selection-08",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.07562261000000348,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 91
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0,
            "call_taxi": 0.01,
            "get_stock_price": 0.0,
            "search_flights": 0.0,
            "set_reminder": 0.99
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "easy-tool_selection-09",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_stock_price",
          "ok": true,
          "server_wall_seconds": 0.07676591100000252,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 88
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0,
            "call_taxi": 0.0,
            "get_stock_price": 1.0,
            "search_flights": 0.0,
            "set_reminder": 0.0
          },
          "predicted": "get_stock_price",
          "correct": true
        },
        {
          "id": "easy-tool_selection-10",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "call_taxi",
          "ok": true,
          "server_wall_seconds": 0.07606695200000502,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 88
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "book_restaurant": 0.0,
            "call_taxi": 1.0,
            "get_stock_price": 0.0,
            "search_flights": 0.0,
            "set_reminder": 0.0
          },
          "predicted": "call_taxi",
          "correct": true
        },
        {
          "id": "easy-tool_selection-11",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.07262513000000581,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 91
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "book_restaurant": 0.0,
            "call_taxi": 0.0,
            "get_stock_price": 0.0,
            "search_flights": 0.0,
            "set_reminder": 1.0
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "pay_subject_to_15000_sublimit",
          "ok": true,
          "server_wall_seconds": 0.3267680519999985,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 3897
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_repeated_seepage": 0.14,
            "deny_vacancy_exclusion": 0.26,
            "pay_full_estimate_less_deductible": 0.23,
            "pay_subject_to_10000_sublimit": 0.18,
            "pay_subject_to_15000_sublimit": 0.19
          },
          "predicted": "deny_vacancy_exclusion",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "cfo",
          "ok": true,
          "server_wall_seconds": 0.22764849100000362,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2590
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "budget_holder": 0.13861386138613863,
            "cfo": 0.13861386138613863,
            "department_head": 0.26732673267326734,
            "executive_committee": 0.10891089108910891,
            "vp_and_finance_director": 0.3465346534653465
          },
          "predicted": "vp_and_finance_director",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "parts_covered_labour_charged",
          "ok": true,
          "server_wall_seconds": 0.20185328500000566,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2364
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "full_repair_no_charge": 0.18,
            "not_covered": 0.41,
            "parts_covered_labour_charged": 0.27,
            "pro_rata_replacement_credit": 0.14
          },
          "predicted": "not_covered",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-09",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "license_required",
          "ok": true,
          "server_wall_seconds": 0.20243599799999856,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2492
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "license_exception_lvs": 0.32,
            "license_required": 0.12,
            "no_license_required": 0.36,
            "shipment_prohibited": 0.2
          },
          "predicted": "no_license_required",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.2105203950000032,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2306
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.71,
            "yes": 0.29
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-13",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.225201368999997,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2398
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.92,
            "yes": 0.08
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-17",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "database_oncall",
          "ok": true,
          "server_wall_seconds": 0.2017709810000028,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2324
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "database_oncall": 0.14,
            "no_page_ticket_only": 0.27,
            "platform_oncall": 0.12,
            "security_incident_response": 0.14,
            "vendor_support": 0.33
          },
          "predicted": "vendor_support",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-19",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.1935290320000007,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2297
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.20999999999999996,
            "yes": 0.79
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.09075913499999899,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 401
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.29000000000000004,
            "yes": 0.71
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-a-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "on_time",
          "ok": true,
          "server_wall_seconds": 0.12269983200000212,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 1070
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "early": 0.22,
            "late": 0.55,
            "on_time": 0.23
          },
          "predicted": "late",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-07",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "escalated",
          "ok": true,
          "server_wall_seconds": 0.14760293999999874,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 1186
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "escalated": 0.38383838383838387,
            "reopened": 0.393939393939394,
            "resolved_first_contact": 0.22222222222222224
          },
          "predicted": "reopened",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-08",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.0996530139999976,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 363
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.42000000000000004,
            "yes": 0.58
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.10616507900000016,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 459
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.86,
            "yes": 0.14
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.10546491899999921,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 494
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.06999999999999995,
            "yes": 0.93
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-07",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_26",
          "ok": true,
          "server_wall_seconds": 0.13692529299999734,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 1269
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "sep_24": 0.29292929292929293,
            "sep_25": 0.17171717171717174,
            "sep_26": 0.14141414141414144,
            "sep_27": 0.22222222222222224,
            "sep_28": 0.17171717171717174
          },
          "predicted": "sep_24",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-09",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09446460399999523,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 491
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.27,
            "yes": 0.73
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.10027705999999625,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 550
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.11,
            "1": 0.12,
            "2": 0.31,
            "3": 0.46
          },
          "ordinal_ev": 2.12,
          "predicted": "3",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-02",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "real_incident",
          "ok": true,
          "server_wall_seconds": 0.09644721699999792,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 506
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "cannot_determine": 0.2828282828282829,
            "false_positive": 0.37373737373737376,
            "real_incident": 0.3434343434343435
          },
          "predicted": "false_positive",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-03",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "time_barred",
          "ok": true,
          "server_wall_seconds": 0.10011581499999522,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 461
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.62,
            "time_barred": 0.32,
            "within_limitation": 0.06
          },
          "predicted": "insufficient_information",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-07",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "within_policy",
          "ok": true,
          "server_wall_seconds": 0.11642810299999695,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 464
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.45,
            "over_limit": 0.52,
            "within_policy": 0.03
          },
          "predicted": "over_limit",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-09",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "legitimate",
          "ok": true,
          "server_wall_seconds": 0.1113550369999956,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 604
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "compromised": 0.89,
            "inconclusive": 0.09,
            "legitimate": 0.02
          },
          "predicted": "compromised",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-10",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "ask_user_which_alex",
          "ok": true,
          "server_wall_seconds": 0.10332827199999883,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 403
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "ask_user_which_alex": 0.06930693069306931,
            "send_to_alex_moreno": 0.8514851485148515,
            "send_to_alexandra_lind": 0.07920792079207921
          },
          "predicted": "send_to_alex_moreno",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-11",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.10055716799999459,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 430
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.31,
            "grade_correct": 0.51,
            "grade_should_be_higher": 0.18
          },
          "predicted": "grade_correct",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-13",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.10399467900000303,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 567
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "cannot_determine": 0.48484848484848486,
            "charge_data_team": 0.20202020202020204,
            "charge_platform_team": 0.20202020202020204,
            "split_50_50": 0.11111111111111112
          },
          "predicted": "cannot_determine",
          "correct": true
        },
        {
          "id": "hard-opus-b-multi_hop-03",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "tier3_cfo",
          "ok": true,
          "server_wall_seconds": 0.2312185850000006,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2930
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "tier1_team_lead": 0.12,
            "tier2_department_head": 0.56,
            "tier3_cfo": 0.22,
            "tier4_board": 0.1
          },
          "predicted": "tier2_department_head",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-04",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bjorn",
          "ok": true,
          "server_wall_seconds": 0.21396804300000127,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2312
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "ana": 0.11111111111111112,
            "bjorn": 0.12121212121212122,
            "chen": 0.11111111111111112,
            "dmitri": 0.38383838383838387,
            "esra": 0.09090909090909091,
            "no_page_ticket_only": 0.18181818181818182
          },
          "predicted": "dmitri",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.2314951329999957,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2542
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "0": 0.19191919191919193,
            "1": 0.16161616161616163,
            "2": 0.31313131313131315,
            "3": 0.18181818181818182,
            "4": 0.15151515151515152
          },
          "ordinal_ev": 1.9393939393939394,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "hard-opus-b-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "credit_20_percent",
          "ok": true,
          "server_wall_seconds": 0.22841121000000442,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2607
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_10_percent": 0.47,
            "credit_20_percent": 0.19,
            "credit_25_percent": 0.09,
            "credit_50_percent": 0.09,
            "credit_full_month": 0.07,
            "no_credit": 0.09
          },
          "predicted": "credit_10_percent",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.19884828,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2284
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.41000000000000003,
            "yes": 0.59
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-01",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "merchant_wins",
          "ok": true,
          "server_wall_seconds": 0.17746347799999995,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 1885
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "customer_wins": 0.4059405940594059,
            "customer_withdraws": 0.1188118811881188,
            "merchant_wins": 0.4752475247524752
          },
          "predicted": "merchant_wins",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-02",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upstream_provider",
          "ok": true,
          "server_wall_seconds": 0.10571293399999604,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 467
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bad_push": 0.67,
            "database_hardware": 0.13,
            "upstream_provider": 0.2
          },
          "predicted": "bad_push",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.1546887570000024,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 1185
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.63,
            "yes": 0.37
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "attended",
          "ok": true,
          "server_wall_seconds": 0.11137235599999684,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 568
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "attended": 0.38,
            "late_cancellation": 0.23,
            "no_show": 0.39
          },
          "predicted": "no_show",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-06",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upheld",
          "ok": true,
          "server_wall_seconds": 0.13611011900000136,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 987
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "modified": 0.04,
            "overturned": 0.91,
            "upheld": 0.05
          },
          "predicted": "overturned",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-01",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "escalate_to_engineering",
          "ok": true,
          "server_wall_seconds": 0.12166271400000284,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 808
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "escalate_to_engineering": 0.7227722772277227,
            "escalate_to_security": 0.1188118811881188,
            "page_oncall": 0.0594059405940594,
            "process_refund": 0.04950495049504951,
            "send_kb_article": 0.04950495049504951
          },
          "predicted": "escalate_to_engineering",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-03",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "order_b",
          "ok": true,
          "server_wall_seconds": 0.11248486999999585,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 725
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "order_a": 0.12,
            "order_b": 0.21,
            "order_c": 0.44,
            "order_d": 0.23
          },
          "predicted": "order_c",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-06",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.11191193500000196,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 762
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.74,
            "yes": 0.26
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-07",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "p0_fix_24h",
          "ok": true,
          "server_wall_seconds": 0.11047221599999801,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 684
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "out_of_scope": 0.04,
            "p0_fix_24h": 0.11,
            "p1_fix_7d": 0.1,
            "p2_fix_30d": 0.65,
            "p3_backlog": 0.1
          },
          "predicted": "p2_fix_30d",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-08",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.11827568600000404,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 573
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.76,
            "yes": 0.24
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-12",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "refund_now",
          "ok": true,
          "server_wall_seconds": 0.10930390500000442,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 560
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_refund": 0.28,
            "escalate_to_fraud_team": 0.04,
            "offer_store_credit": 0.09,
            "refund_now": 0.25,
            "wait_for_inspection": 0.34
          },
          "predicted": "wait_for_inspection",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "erase_but_retain_transaction_records",
          "ok": true,
          "server_wall_seconds": 0.28279024200000435,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 3549
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "erase_all": 0.26,
            "erase_but_retain_transaction_records": 0.17,
            "erase_marketing_only": 0.13,
            "reject_unverified_request": 0.16,
            "restrict_all_pending_legal_hold": 0.28
          },
          "predicted": "restrict_all_pending_legal_hold",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-03",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "eur_400",
          "ok": true,
          "server_wall_seconds": 0.2563591850000009,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 3473
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_0": 0.25,
            "eur_150": 0.17,
            "eur_200": 0.17,
            "eur_300": 0.2,
            "eur_400": 0.09,
            "eur_600": 0.12
          },
          "predicted": "eur_0",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.2530256040000012,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 3284
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5800000000000001,
            "yes": 0.42
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.2675718850000024,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 3202
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.38,
            "1": 0.25,
            "2": 0.24,
            "3": 0.13
          },
          "ordinal_ev": 1.12,
          "predicted": "0",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.25726012799999864,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 3206
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.14,
            "1": 0.29,
            "2": 0.44,
            "3": 0.13
          },
          "ordinal_ev": 1.56,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-10",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "credit_minus_restocking_fee",
          "ok": true,
          "server_wall_seconds": 0.2639475110000049,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 3600
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_minus_restocking_fee": 0.28,
            "full_credit": 0.2,
            "reject_return": 0.28,
            "repair_or_replace_only": 0.24
          },
          "predicted": "credit_minus_restocking_fee",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.266060447000001,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 3371
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.35,
            "yes": 0.65
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-c-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "kuznets_technik",
          "ok": true,
          "server_wall_seconds": 0.11724805499999746,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 516
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "halvorsen_polymer": 0.2,
            "kuznets_technik": 0.34,
            "ortega_seal_works": 0.46
          },
          "predicted": "ortega_seal_works",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.13863723999999422,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 908
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.43000000000000005,
            "yes": 0.57
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-c-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_692_00",
          "ok": true,
          "server_wall_seconds": 0.13665781600000315,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 835
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_617_00": 0.28,
            "eur_688_00": 0.19,
            "eur_692_00": 0.19,
            "eur_693_90": 0.21,
            "eur_812_00": 0.13
          },
          "predicted": "eur_617_00",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "usd_2335_00",
          "ok": true,
          "server_wall_seconds": 0.10976730499999832,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 750
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "usd_2222_50": 0.23,
            "usd_2303_01": 0.21,
            "usd_2303_33": 0.16,
            "usd_2335_00": 0.22,
            "usd_2340_00": 0.18
          },
          "predicted": "usd_2222_50",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_18_book_set",
          "ok": true,
          "server_wall_seconds": 0.12531062200000065,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 952
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "none_declined": 0.25252525252525254,
            "sep_15_office_chair": 0.16161616161616163,
            "sep_16_taxi": 0.12121212121212122,
            "sep_17_parking": 0.12121212121212122,
            "sep_18_book_set": 0.14141414141414144,
            "sep_21_client_dinner": 0.20202020202020204
          },
          "predicted": "none_declined",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-08",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_498_81",
          "ok": true,
          "server_wall_seconds": 0.12432315600000265,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 954
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_497_40": 0.31,
            "eur_498_81": 0.28,
            "eur_506_06": 0.14,
            "eur_513_69": 0.16,
            "eur_532_01": 0.11
          },
          "predicted": "eur_497_40",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "expired_30_month_cap",
          "ok": true,
          "server_wall_seconds": 0.12310188300000391,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 790
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered": 0.29,
            "expired_24_month_term": 0.42,
            "expired_30_month_cap": 0.29
          },
          "predicted": "expired_24_month_term",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-01",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "deny_missing_approval",
          "ok": true,
          "server_wall_seconds": 0.09220569400000045,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 248
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_inactive_user": 0.15,
            "deny_missing_approval": 0.38,
            "grant_access": 0.1,
            "grant_owner_exception": 0.37
          },
          "predicted": "deny_missing_approval",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-06",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09272684800000519,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 212
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.81,
            "yes": 0.19
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-07",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "retain_until_2031",
          "ok": true,
          "server_wall_seconds": 0.0975137470000007,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 264
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_now": 0.65,
            "needs_signature_review": 0.1,
            "retain_indefinitely": 0.16,
            "retain_until_2031": 0.09
          },
          "predicted": "delete_now",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-08",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.0874848370000052,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 189
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.22999999999999998,
            "yes": 0.77
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-09",
          "tier": "hard",
          "family": "adversarial",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.08369879600000019,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 189
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.14,
            "1": 0.11,
            "2": 0.38,
            "3": 0.37
          },
          "ordinal_ev": 1.98,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-11",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "standard_research",
          "ok": true,
          "server_wall_seconds": 0.08561069599999627,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 270
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ethics_review": 0.04,
            "major_grants": 0.11,
            "manual_hold": 0.18,
            "standard_research": 0.67
          },
          "predicted": "standard_research",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-01",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "accept_s2_dusk",
          "ok": true,
          "server_wall_seconds": 0.19442051499999735,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2261
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "accept_s2_dusk": 0.26,
            "reject_cutoff": 0.31,
            "reject_tier": 0.21,
            "reroute_s3": 0.22
          },
          "predicted": "reject_cutoff",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-02",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "security_primary",
          "ok": true,
          "server_wall_seconds": 0.21569345500000026,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2265
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "executive_bridge": 0.07,
            "heron_q2": 0.22,
            "insufficient_information": 0.59,
            "security_primary": 0.12
          },
          "predicted": "insufficient_information",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_director",
          "ok": true,
          "server_wall_seconds": 0.2078803749999949,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2204
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "pay_manager_approved": 0.27,
            "reject_category": 0.4,
            "require_director": 0.21,
            "require_finance_board": 0.12
          },
          "predicted": "reject_category",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "admit",
          "ok": true,
          "server_wall_seconds": 0.2102701689999975,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2197
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "admit": 0.09,
            "deny_prerequisite": 0.28,
            "deny_score": 0.29,
            "manual_equivalency": 0.34
          },
          "predicted": "manual_equivalency",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "central_engineering",
          "ok": true,
          "server_wall_seconds": 0.1888839169999983,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2173
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "central_engineering": 0.26,
            "insufficient_log": 0.34,
            "local_maintenance": 0.16,
            "safety_shutdown": 0.24
          },
          "predicted": "insufficient_log",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_steward",
          "ok": true,
          "server_wall_seconds": 0.19241244300000204,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2172
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "confirm_l3": 0.1782178217821782,
            "reject_capacity": 0.24752475247524752,
            "require_steward": 0.22772277227722773,
            "switch_l4": 0.3465346534653465
          },
          "predicted": "switch_l4",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-10",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "approve_45_days",
          "ok": true,
          "server_wall_seconds": 0.2048067859999989,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2230
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_45_days": 0.18,
            "deny_sensitive": 0.38,
            "needs_field_review": 0.18,
            "reduce_to_30_days": 0.26
          },
          "predicted": "deny_sensitive",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "needs_authentication",
          "ok": true,
          "server_wall_seconds": 0.21529882699999803,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2179
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered_recurrence": 0.17,
            "expired": 0.24,
            "needs_authentication": 0.21,
            "wrong_issue_class": 0.38
          },
          "predicted": "wrong_issue_class",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-02",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.10586815999999999,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 263
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.74,
            "yes": 0.26
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-04",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09631472299999899,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 169
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.73,
            "yes": 0.27
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-05",
          "tier": "hard",
          "family": "trap",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.09046990699999924,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 299
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.71,
            "1": 0.08,
            "2": 0.13,
            "3": 0.08
          },
          "ordinal_ev": 0.5800000000000001,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-06",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "keep_open_unresolved",
          "ok": true,
          "server_wall_seconds": 0.09010370699999726,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 316
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "await_internal_metrics": 0.16831683168316833,
            "close_agent_verified": 0.1782178217821782,
            "close_confirmed": 0.37623762376237624,
            "keep_open_unresolved": 0.27722772277227725
          },
          "predicted": "close_confirmed",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-08",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "late",
          "ok": true,
          "server_wall_seconds": 0.08701652399999915,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 244
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "canceled": 0.05,
            "indeterminate": 0.22,
            "late": 0.23,
            "on_time": 0.5
          },
          "predicted": "on_time",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-10",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "blocked_second_review",
          "ok": true,
          "server_wall_seconds": 0.08062349200000085,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 170
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "blocked_failed_test": 0.15,
            "blocked_second_review": 0.62,
            "ready_one_review": 0.15,
            "ready_two_reviews": 0.08
          },
          "predicted": "blocked_second_review",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-13",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09325679100000173,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 177
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.91,
            "yes": 0.09
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-a-trap-15",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09906521799999979,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 279
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.69,
            "yes": 0.31
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-01",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.08895953700000092,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 255
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.13,
            "yes": 0.87
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-02",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09009608700000626,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 179
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.14,
            "yes": 0.86
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.09011387700000029,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 194
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.09999999999999998,
            "yes": 0.9
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09492119600000137,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 181
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.12,
            "yes": 0.88
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09058698100000129,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 200
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.29000000000000004,
            "yes": 0.71
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-14",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.08319523299999787,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 217
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.08999999999999997,
            "yes": 0.91
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.07997162699999905,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 173
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.26,
            "yes": 0.74
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-18",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.07783569899999776,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 140
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.15000000000000002,
            "yes": 0.85
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "reject_entire_claim",
          "ok": true,
          "server_wall_seconds": 0.25453284199999615,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2706
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_all": 0.51,
            "approve_lodging_only": 0.16,
            "approve_meals_only": 0.1,
            "reject_entire_claim": 0.23
          },
          "predicted": "approve_all",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "deny_current_export",
          "ok": true,
          "server_wall_seconds": 0.2417743399999992,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2557
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "analytics_owner_only": 0.31,
            "deny_current_export": 0.31,
            "privacy_and_security": 0.21,
            "privacy_only": 0.17
          },
          "predicted": "analytics_owner_only",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "delete_profile_omar_april_and_table",
          "ok": true,
          "server_wall_seconds": 0.21498960899999986,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2518
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_everything": 0.24,
            "delete_nothing": 0.12,
            "delete_only_profile": 0.14,
            "delete_profile_omar_april_and_table": 0.5
          },
          "predicted": "delete_profile_omar_april_and_table",
          "correct": true
        },
        {
          "id": "hard-sol-b-long_policy-06",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "east_ward",
          "ok": true,
          "server_wall_seconds": 0.2097587440000055,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2602
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "east_ward": 0.39603960396039606,
            "harbor_neighbors": 0.15841584158415842,
            "no_award": 0.1485148514851485,
            "river_arts": 0.297029702970297
          },
          "predicted": "east_ward",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-01",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "office_file_repair",
          "ok": true,
          "server_wall_seconds": 0.09189306400000419,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 274
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "data_pipeline": 0.0202020202020202,
            "desktop_support": 0.0101010101010101,
            "finance_review": 0.0101010101010101,
            "office_file_repair": 0.9393939393939393,
            "spreadsheet_formula": 0.0202020202020202
          },
          "predicted": "office_file_repair",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-02",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "ledger_reconciliation",
          "ok": true,
          "server_wall_seconds": 0.09639520499999321,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 255
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "customer_support": 0.0101010101010101,
            "fraud_review": 0.0,
            "ledger_reconciliation": 0.9393939393939393,
            "mobile_app": 0.0202020202020202,
            "payment_disputes": 0.0303030303030303
          },
          "predicted": "ledger_reconciliation",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-03",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "network_dns",
          "ok": true,
          "server_wall_seconds": 0.09797328700000207,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 255
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "api_platform": 0.16,
            "authoritative_dns": 0.62,
            "endpoint_devices": 0.04,
            "network_dns": 0.13,
            "tls_certificates": 0.05
          },
          "predicted": "authoritative_dns",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-07",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "release_notarization",
          "ok": true,
          "server_wall_seconds": 0.09450517399999825,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 238
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "ci_infrastructure": 0.13131313131313133,
            "code_signing": 0.24242424242424243,
            "endpoint_security": 0.030303030303030304,
            "macos_runtime": 0.13131313131313133,
            "release_notarization": 0.4646464646464647
          },
          "predicted": "release_notarization",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-09",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "source_access",
          "ok": true,
          "server_wall_seconds": 0.08933141499999664,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 244
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "git_client": 0.07,
            "identity_login": 0.06,
            "repository_admin": 0.07,
            "source_access": 0.67,
            "vpn_network": 0.13
          },
          "predicted": "source_access",
          "correct": true
        },
        {
          "id": "hard-sol-b-temporal_numeric-01",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "late_by_under_2h",
          "ok": true,
          "server_wall_seconds": 0.10541156700000442,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 251
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "before_open": 0.31313131313131315,
            "late_by_2h_or_more": 0.22222222222222224,
            "late_by_under_2h": 0.15151515151515152,
            "within_window": 0.31313131313131315
          },
          "predicted": "before_open",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "over_limit",
          "ok": true,
          "server_wall_seconds": 0.09520338299999764,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 265
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "at_least_100gb_below": 0.2,
            "exactly_equal": 0.16,
            "over_limit": 0.39,
            "under_100gb_below": 0.25
          },
          "predicted": "over_limit",
          "correct": true
        },
        {
          "id": "hard-sol-b-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "9_credits",
          "ok": true,
          "server_wall_seconds": 0.08604491000000536,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 199
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "12_credits": 0.48,
            "15_credits": 0.27,
            "6_credits": 0.13,
            "9_credits": 0.12
          },
          "predicted": "12_credits",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "pass",
          "ok": true,
          "server_wall_seconds": 0.08199892100000028,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 214
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "clear_fail": 0.5,
            "narrow_fail": 0.33,
            "pass": 0.17
          },
          "predicted": "clear_fail",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-03",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09097532199999847,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 360
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.13,
            "yes": 0.87
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.0887340120000033,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 322
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.12,
            "yes": 0.88
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-07",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09899549600000057,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 288
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.050000000000000044,
            "yes": 0.95
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.099838179999999,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 281
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.08999999999999997,
            "yes": 0.91
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-09",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.09495617800000389,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 311
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.06000000000000005,
            "yes": 0.94
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.10393292899999551,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 341
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.08999999999999997,
            "yes": 0.91
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-11",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.09836062999999484,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 309
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.17000000000000004,
            "yes": 0.83
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-13",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.100161417999999,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 301
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.08999999999999997,
            "yes": 0.91
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.08564921800000036,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 254
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.27,
            "yes": 0.73
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "programme_review",
          "ok": true,
          "server_wall_seconds": 0.2002428439999946,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2495
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "partial_release": 0.17,
            "programme_review": 0.36,
            "reject_draw": 0.22,
            "release_draw": 0.25
          },
          "predicted": "programme_review",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bus_substitution",
          "ok": true,
          "server_wall_seconds": 0.20834305099999995,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2514
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bus_substitution": 0.32,
            "call_fen_cross": 0.15,
            "pass_fen_cross": 0.24,
            "terminate_oakmere": 0.29
          },
          "predicted": "bus_substitution",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-11",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "coverage_excluded",
          "ok": true,
          "server_wall_seconds": 0.24520428100000657,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2592
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "adjuster_referral": 0.06,
            "coverage_confirmed": 0.17,
            "coverage_excluded": 0.54,
            "documentation_needed": 0.23
          },
          "predicted": "coverage_excluded",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "conservation_hold",
          "ok": true,
          "server_wall_seconds": 0.11820583200000101,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 513
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "conservation_hold": 0.22,
            "deny_request": 0.34,
            "digital_copy_only": 0.23,
            "supervised_delivery": 0.21
          },
          "predicted": "deny_request",
          "correct": false
        },
        {
          "id": "hard-sol-c-multi_hop-13",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "normal_release",
          "ok": true,
          "server_wall_seconds": 0.21391091899999992,
          "input_audit": {
            "state_truncated": false,
            "encoded_tokens": 2564
          },
          "valid": true,
          "strict_valid": false,
          "renormalized": true,
          "probs": {
            "engineering_review": 0.22772277227722773,
            "normal_release": 0.5742574257425742,
            "rework": 0.1188118811881188,
            "scrap_lot": 0.07920792079207921
          },
          "predicted": "normal_release",
          "correct": true
        }
      ],
      "raw_sha256": "bd2f2bf1c6c83c84d9629f202faeac70f68755cdeaf426cbfcf9d0af08d4de5d",
      "unrounded_diagnostic": {
        "note": "Secondary only; main score uses native rounded API output",
        "correct": 138,
        "prediction_changes": 2,
        "scored": [
          {
            "id": "original-policy-01-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.8386788964271545,
              "yes": 0.16132110357284546
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "original-policy-01-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.8655397891998291,
              "yes": 0.1344602108001709
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "original-policy-02-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.11539839953184128,
              "yes": 0.8846015930175781
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-policy-02-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.3147231340408325,
              "yes": 0.6852768063545227
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-policy-03-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.44442635774612427,
              "yes": 0.5555736422538757
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-policy-03-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.2605655789375305,
              "yes": 0.7394344210624695
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-policy-04-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.5838167667388916,
              "yes": 0.4161832332611084
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "original-policy-04-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.8673445582389832,
              "yes": 0.13265548646450043
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "original-policy-05-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.2780306041240692,
              "yes": 0.7219693660736084
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "original-policy-05-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.8012946844100952,
              "yes": 0.19870536029338837
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "original-policy-06-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.0833519771695137,
              "yes": 0.9166480302810669
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-policy-06-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.31776031851768494,
              "yes": 0.6822396516799927
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-intent-01-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.16200727224349976,
              "change_address": 0.004015171900391579,
              "other": 0.23090803623199463,
              "refund": 0.6000230312347412,
              "status": 0.0030464837327599525
            },
            "predicted": "refund",
            "correct": true
          },
          {
            "id": "original-intent-01-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.12896940112113953,
              "change_address": 0.008395141921937466,
              "other": 0.4711310565471649,
              "refund": 0.38542652130126953,
              "status": 0.006077858619391918
            },
            "predicted": "other",
            "correct": false
          },
          {
            "id": "original-intent-02-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.026980577036738396,
              "change_address": 0.19582444429397583,
              "other": 0.6798968315124512,
              "refund": 0.012799427844583988,
              "status": 0.08449865877628326
            },
            "predicted": "other",
            "correct": false
          },
          {
            "id": "original-intent-02-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.04598119109869003,
              "change_address": 0.07028789818286896,
              "other": 0.45001232624053955,
              "refund": 0.022948406636714935,
              "status": 0.4107701778411865
            },
            "predicted": "other",
            "correct": false
          },
          {
            "id": "original-intent-03-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.01677820272743702,
              "change_address": 0.6937251687049866,
              "other": 0.24519680440425873,
              "refund": 0.013280317187309265,
              "status": 0.031019464135169983
            },
            "predicted": "change_address",
            "correct": true
          },
          {
            "id": "original-intent-03-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.004665147978812456,
              "change_address": 0.8133889436721802,
              "other": 0.1725817620754242,
              "refund": 0.006555689033120871,
              "status": 0.002808394841849804
            },
            "predicted": "change_address",
            "correct": true
          },
          {
            "id": "original-intent-04-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.3155374526977539,
              "change_address": 0.006760961841791868,
              "other": 0.5707160234451294,
              "refund": 0.10226551443338394,
              "status": 0.004720055963844061
            },
            "predicted": "other",
            "correct": false
          },
          {
            "id": "original-intent-04-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.7299904227256775,
              "change_address": 0.0027535089757293463,
              "other": 0.2289748340845108,
              "refund": 0.03613849729299545,
              "status": 0.002142836106941104
            },
            "predicted": "cancel",
            "correct": true
          },
          {
            "id": "original-intent-05-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.03648090735077858,
              "change_address": 0.010190865956246853,
              "other": 0.7109861373901367,
              "refund": 0.23209720849990845,
              "status": 0.010244914330542088
            },
            "predicted": "other",
            "correct": true
          },
          {
            "id": "original-intent-05-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.03520656004548073,
              "change_address": 0.008726131170988083,
              "other": 0.44605711102485657,
              "refund": 0.5019126534461975,
              "status": 0.00809753593057394
            },
            "predicted": "refund",
            "correct": false
          },
          {
            "id": "original-intent-06-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.2294117659330368,
              "change_address": 0.048765357583761215,
              "other": 0.3498203754425049,
              "refund": 0.07457809895277023,
              "status": 0.29742440581321716
            },
            "predicted": "other",
            "correct": false
          },
          {
            "id": "original-intent-06-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cancel": 0.30450576543807983,
              "change_address": 0.026209641247987747,
              "other": 0.4425531327724457,
              "refund": 0.09620532393455505,
              "status": 0.13052614033222198
            },
            "predicted": "other",
            "correct": false
          },
          {
            "id": "original-ordinal-01-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.7840684056282043,
              "1": 0.10919222235679626,
              "2": 0.06933045387268066,
              "3": 0.03740894794464111
            },
            "ordinal_ev": 0.36007997393608093,
            "predicted": "0",
            "correct": true
          },
          {
            "id": "original-ordinal-01-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.9188807010650635,
              "1": 0.04504450410604477,
              "2": 0.02302812598645687,
              "3": 0.013046737760305405
            },
            "ordinal_ev": 0.13024096935987473,
            "predicted": "0",
            "correct": true
          },
          {
            "id": "original-ordinal-02-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.3161374032497406,
              "1": 0.46453797817230225,
              "2": 0.1618575006723404,
              "3": 0.05746708810329437
            },
            "ordinal_ev": 0.9606542438268661,
            "predicted": "1",
            "correct": true
          },
          {
            "id": "original-ordinal-02-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.27021893858909607,
              "1": 0.5866365432739258,
              "2": 0.08681365102529526,
              "3": 0.0563308410346508
            },
            "ordinal_ev": 0.9292563684284687,
            "predicted": "1",
            "correct": true
          },
          {
            "id": "original-ordinal-03-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.15541385114192963,
              "1": 0.1449718475341797,
              "2": 0.6152814030647278,
              "3": 0.0843329131603241
            },
            "ordinal_ev": 1.6285333931446075,
            "predicted": "2",
            "correct": true
          },
          {
            "id": "original-ordinal-03-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.2243546098470688,
              "1": 0.09127053618431091,
              "2": 0.6178698539733887,
              "3": 0.06650491803884506
            },
            "ordinal_ev": 1.5265249982476234,
            "predicted": "2",
            "correct": true
          },
          {
            "id": "original-ordinal-04-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.04961699992418289,
              "1": 0.013822833076119423,
              "2": 0.03531978651881218,
              "3": 0.901240348815918
            },
            "ordinal_ev": 2.7881834525614977,
            "predicted": "3",
            "correct": true
          },
          {
            "id": "original-ordinal-04-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.053166210651397705,
              "1": 0.02075055055320263,
              "2": 0.060374241322278976,
              "3": 0.8657090067863464
            },
            "ordinal_ev": 2.7386260535568,
            "predicted": "3",
            "correct": true
          },
          {
            "id": "original-ordinal-05-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.3345189690589905,
              "1": 0.07643093168735504,
              "2": 0.5056872963905334,
              "3": 0.08336280286312103
            },
            "ordinal_ev": 1.337893933057785,
            "predicted": "2",
            "correct": false
          },
          {
            "id": "original-ordinal-05-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.4546709358692169,
              "1": 0.09514999389648438,
              "2": 0.3975710868835449,
              "3": 0.052607983350753784
            },
            "ordinal_ev": 1.0481161177158356,
            "predicted": "0",
            "correct": true
          },
          {
            "id": "original-ordinal-06-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.22660908102989197,
              "1": 0.13620445132255554,
              "2": 0.602436363697052,
              "3": 0.034750089049339294
            },
            "ordinal_ev": 1.4453274458646774,
            "predicted": "2",
            "correct": true
          },
          {
            "id": "original-ordinal-06-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.25479137897491455,
              "1": 0.1654099076986313,
              "2": 0.5226662755012512,
              "3": 0.05713238939642906
            },
            "ordinal_ev": 1.382139626890421,
            "predicted": "2",
            "correct": true
          },
          {
            "id": "original-extraction-01-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.08198559284210205,
              "pickup": 0.3700275421142578,
              "post": 0.13041305541992188,
              "unknown": 0.41757383942604065
            },
            "predicted": "unknown",
            "correct": false
          },
          {
            "id": "original-extraction-01-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.046795204281806946,
              "pickup": 0.4719495475292206,
              "post": 0.06941290944814682,
              "unknown": 0.41184237599372864
            },
            "predicted": "pickup",
            "correct": true
          },
          {
            "id": "original-extraction-02-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.03479795530438423,
              "pickup": 0.04226212576031685,
              "post": 0.09703127294778824,
              "unknown": 0.8259086608886719
            },
            "predicted": "unknown",
            "correct": true
          },
          {
            "id": "original-extraction-02-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.02423354797065258,
              "pickup": 0.025219174101948738,
              "post": 0.1120418831706047,
              "unknown": 0.8385053873062134
            },
            "predicted": "unknown",
            "correct": true
          },
          {
            "id": "original-extraction-03-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.2940341830253601,
              "pickup": 0.11385902017354965,
              "post": 0.1568729430437088,
              "unknown": 0.43523386120796204
            },
            "predicted": "unknown",
            "correct": false
          },
          {
            "id": "original-extraction-03-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.7938085198402405,
              "pickup": 0.00736364396288991,
              "post": 0.10463446378707886,
              "unknown": 0.09419335424900055
            },
            "predicted": "courier",
            "correct": true
          },
          {
            "id": "original-extraction-04-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.030109843239188194,
              "pickup": 0.023950597271323204,
              "post": 0.7432087063789368,
              "unknown": 0.20273083448410034
            },
            "predicted": "post",
            "correct": true
          },
          {
            "id": "original-extraction-04-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.024639897048473358,
              "pickup": 0.01260473020374775,
              "post": 0.8174852132797241,
              "unknown": 0.14527012407779694
            },
            "predicted": "post",
            "correct": true
          },
          {
            "id": "original-extraction-05-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.022271210327744484,
              "pickup": 0.02701045572757721,
              "post": 0.019079728052020073,
              "unknown": 0.9316385984420776
            },
            "predicted": "unknown",
            "correct": true
          },
          {
            "id": "original-extraction-05-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.0866222158074379,
              "pickup": 0.03859835863113403,
              "post": 0.06279172748327255,
              "unknown": 0.8119876384735107
            },
            "predicted": "unknown",
            "correct": true
          },
          {
            "id": "original-extraction-06-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.7431442737579346,
              "pickup": 0.02791612036526203,
              "post": 0.06311632692813873,
              "unknown": 0.16582326591014862
            },
            "predicted": "courier",
            "correct": true
          },
          {
            "id": "original-extraction-06-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "courier": 0.7734956741333008,
              "pickup": 0.027482343837618828,
              "post": 0.020735500380396843,
              "unknown": 0.17828643321990967
            },
            "predicted": "courier",
            "correct": true
          },
          {
            "id": "original-adequacy-01-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.05173182860016823,
              "yes": 0.9482681751251221
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-adequacy-01-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.4025387465953827,
              "yes": 0.5974612236022949
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-adequacy-02-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.04744277521967888,
              "yes": 0.9525572657585144
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "original-adequacy-02-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.9393520355224609,
              "yes": 0.06064799800515175
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "original-adequacy-03-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.021280359476804733,
              "yes": 0.9787196516990662
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "original-adequacy-03-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.1001611277461052,
              "yes": 0.8998388051986694
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "original-adequacy-04-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.7721205949783325,
              "yes": 0.2278793752193451
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "original-adequacy-04-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.5336866974830627,
              "yes": 0.46631327271461487
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "original-adequacy-05-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.3727853298187256,
              "yes": 0.6272147297859192
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "original-adequacy-05-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.18424257636070251,
              "yes": 0.8157573938369751
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "original-adequacy-06-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.14164882898330688,
              "yes": 0.8583512306213379
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-adequacy-06-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.09352824091911316,
              "yes": 0.9064717888832092
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "original-routing-01-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.07309307903051376,
              "coding_agent": 0.009065038524568081,
              "document": 0.041633084416389465,
              "general": 0.04191281646490097,
              "math": 0.8026856780052185,
              "tools": 0.03161029517650604
            },
            "predicted": "math",
            "correct": true
          },
          {
            "id": "original-routing-01-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.04931257292628288,
              "coding_agent": 0.010179863311350346,
              "document": 0.05016465485095978,
              "general": 0.04788198694586754,
              "math": 0.8037091493606567,
              "tools": 0.03875180333852768
            },
            "predicted": "math",
            "correct": true
          },
          {
            "id": "original-routing-02-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.8789894580841064,
              "coding_agent": 0.013816106133162975,
              "document": 0.013437454588711262,
              "general": 0.0767718181014061,
              "math": 0.01022972259670496,
              "tools": 0.006755397189408541
            },
            "predicted": "coding",
            "correct": true
          },
          {
            "id": "original-routing-02-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.7991024255752563,
              "coding_agent": 0.05655611306428909,
              "document": 0.02263730764389038,
              "general": 0.10245455801486969,
              "math": 0.0076329815201461315,
              "tools": 0.011616529896855354
            },
            "predicted": "coding",
            "correct": true
          },
          {
            "id": "original-routing-03-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.07585307955741882,
              "coding_agent": 0.7937270998954773,
              "document": 0.01745128631591797,
              "general": 0.07603095471858978,
              "math": 0.003362160176038742,
              "tools": 0.033575400710105896
            },
            "predicted": "coding_agent",
            "correct": true
          },
          {
            "id": "original-routing-03-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.03983987122774124,
              "coding_agent": 0.8151764273643494,
              "document": 0.026542261242866516,
              "general": 0.07460379600524902,
              "math": 0.0029313426930457354,
              "tools": 0.04090630263090134
            },
            "predicted": "coding_agent",
            "correct": true
          },
          {
            "id": "original-routing-04-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.032438602298498154,
              "coding_agent": 0.030191857367753983,
              "document": 0.7420311570167542,
              "general": 0.13277338445186615,
              "math": 0.015928808599710464,
              "tools": 0.04663623124361038
            },
            "predicted": "document",
            "correct": true
          },
          {
            "id": "original-routing-04-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.05897926911711693,
              "coding_agent": 0.058390188962221146,
              "document": 0.665703535079956,
              "general": 0.0862223282456398,
              "math": 0.04653475433588028,
              "tools": 0.08416996151208878
            },
            "predicted": "document",
            "correct": true
          },
          {
            "id": "original-routing-05-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.023492395877838135,
              "coding_agent": 0.007391664199531078,
              "document": 0.04108307510614395,
              "general": 0.09880248457193375,
              "math": 0.003832418005913496,
              "tools": 0.8253979682922363
            },
            "predicted": "tools",
            "correct": true
          },
          {
            "id": "original-routing-05-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.04804099723696709,
              "coding_agent": 0.02849528379738331,
              "document": 0.15460330247879028,
              "general": 0.3730098009109497,
              "math": 0.013019077479839325,
              "tools": 0.38283154368400574
            },
            "predicted": "tools",
            "correct": true
          },
          {
            "id": "original-routing-06-0",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.0529228076338768,
              "coding_agent": 0.03143659234046936,
              "document": 0.11610125750303268,
              "general": 0.7231309413909912,
              "math": 0.012832600623369217,
              "tools": 0.06357578933238983
            },
            "predicted": "general",
            "correct": true
          },
          {
            "id": "original-routing-06-1",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "coding": 0.04794248566031456,
              "coding_agent": 0.026316598057746887,
              "document": 0.07950342446565628,
              "general": 0.7823469638824463,
              "math": 0.011729603633284569,
              "tools": 0.05216098949313164
            },
            "predicted": "general",
            "correct": true
          },
          {
            "id": "easy-intent-00",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "billing_question": 0.002641286002472043,
              "cancel_order": 0.011788918636739254,
              "change_address": 0.004078633617609739,
              "report_damage": 0.007148850709199905,
              "track_order": 0.9743423461914062
            },
            "predicted": "track_order",
            "correct": true
          },
          {
            "id": "easy-intent-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "billing_question": 0.0011598406126722693,
              "cancel_order": 0.9893296360969543,
              "change_address": 0.0009986282093450427,
              "report_damage": 0.007393687963485718,
              "track_order": 0.0011181439040228724
            },
            "predicted": "cancel_order",
            "correct": true
          },
          {
            "id": "easy-intent-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "billing_question": 0.0008558141998946667,
              "cancel_order": 0.007204018998891115,
              "change_address": 0.987266480922699,
              "report_damage": 0.0020023686811327934,
              "track_order": 0.0026712343096733093
            },
            "predicted": "change_address",
            "correct": true
          },
          {
            "id": "easy-intent-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "billing_question": 0.001163716078735888,
              "cancel_order": 0.014234941452741623,
              "change_address": 0.0010818245355039835,
              "report_damage": 0.9758220314979553,
              "track_order": 0.007697509601712227
            },
            "predicted": "report_damage",
            "correct": true
          },
          {
            "id": "easy-intent-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "billing_question": 0.8763135671615601,
              "cancel_order": 0.06544576585292816,
              "change_address": 0.011465511284768581,
              "report_damage": 0.031992409378290176,
              "track_order": 0.014782700687646866
            },
            "predicted": "billing_question",
            "correct": true
          },
          {
            "id": "easy-intent-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "billing_question": 0.030802372843027115,
              "cancel_order": 0.11834641546010971,
              "change_address": 0.028192240744829178,
              "report_damage": 0.05765002593398094,
              "track_order": 0.7650090456008911
            },
            "predicted": "track_order",
            "correct": true
          },
          {
            "id": "easy-intent-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "play_music": 0.0005431263707578182,
              "send_message": 0.15443047881126404,
              "set_alarm": 0.8421052694320679,
              "turn_off_lights": 0.0011603791499510407,
              "weather": 0.0017607645131647587
            },
            "predicted": "set_alarm",
            "correct": true
          },
          {
            "id": "easy-intent-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "play_music": 0.9911760091781616,
              "send_message": 0.0033059949055314064,
              "set_alarm": 0.002048364607617259,
              "turn_off_lights": 0.002503775293007493,
              "weather": 0.0009658037452027202
            },
            "predicted": "play_music",
            "correct": true
          },
          {
            "id": "easy-intent-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "play_music": 0.0007339987205341458,
              "send_message": 0.004242720082402229,
              "set_alarm": 0.0007615720969624817,
              "turn_off_lights": 0.0007223968859761953,
              "weather": 0.9935393929481506
            },
            "predicted": "weather",
            "correct": true
          },
          {
            "id": "easy-intent-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "play_music": 0.009079476818442345,
              "send_message": 0.9101366996765137,
              "set_alarm": 0.05612970516085625,
              "turn_off_lights": 0.00888892263174057,
              "weather": 0.015765182673931122
            },
            "predicted": "send_message",
            "correct": true
          },
          {
            "id": "easy-intent-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "play_music": 0.0009155551088042557,
              "send_message": 0.0027404159773141146,
              "set_alarm": 0.0019460574258118868,
              "turn_off_lights": 0.993006706237793,
              "weather": 0.0013912779977545142
            },
            "predicted": "turn_off_lights",
            "correct": true
          },
          {
            "id": "easy-intent-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "play_music": 0.0009037060663104057,
              "send_message": 0.002625074703246355,
              "set_alarm": 0.9931334853172302,
              "turn_off_lights": 0.002276445273309946,
              "weather": 0.001061334740370512
            },
            "predicted": "set_alarm",
            "correct": true
          },
          {
            "id": "easy-fact-00",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.034283772110939026,
              "yes": 0.9657162427902222
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "easy-fact-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.9762128591537476,
              "yes": 0.023787103593349457
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "easy-fact-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.01584840938448906,
              "yes": 0.9841516017913818
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "easy-fact-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.9896223545074463,
              "yes": 0.010377678088843822
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "easy-fact-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.049203962087631226,
              "yes": 0.9507960081100464
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "easy-fact-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.8709396719932556,
              "yes": 0.1290602684020996
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "easy-fact-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.01827847585082054,
              "yes": 0.9817215204238892
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "easy-fact-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.9709040522575378,
              "yes": 0.029095908626914024
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "easy-fact-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.011997099965810776,
              "yes": 0.9880028963088989
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "easy-fact-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.9884800910949707,
              "yes": 0.011519882827997208
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "easy-fact-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.010138199664652348,
              "yes": 0.9898618459701538
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "easy-fact-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.978111982345581,
              "yes": 0.02188800275325775
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "easy-extraction-00",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "bank_transfer": 0.005705555435270071,
              "cash": 0.0029326959047466516,
              "credit_card": 0.010485039092600346,
              "paypal": 0.9808767437934875
            },
            "predicted": "paypal",
            "correct": true
          },
          {
            "id": "easy-extraction-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "bank_transfer": 0.010208235122263432,
              "cash": 0.958379328250885,
              "credit_card": 0.025033866986632347,
              "paypal": 0.0063785226084291935
            },
            "predicted": "cash",
            "correct": true
          },
          {
            "id": "easy-extraction-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "bank_transfer": 0.966234564781189,
              "cash": 0.011082742363214493,
              "credit_card": 0.015811599791049957,
              "paypal": 0.00687115453183651
            },
            "predicted": "bank_transfer",
            "correct": true
          },
          {
            "id": "easy-extraction-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "high": 0.5854946970939636,
              "low": 0.019567472860217094,
              "medium": 0.029012948274612427,
              "urgent": 0.3659248650074005
            },
            "predicted": "high",
            "correct": true
          },
          {
            "id": "easy-extraction-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "high": 0.005562879145145416,
              "low": 0.9528421759605408,
              "medium": 0.03297973796725273,
              "urgent": 0.008615217171609402
            },
            "predicted": "low",
            "correct": true
          },
          {
            "id": "easy-extraction-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "high": 0.031708426773548126,
              "low": 0.009729194454848766,
              "medium": 0.012577124871313572,
              "urgent": 0.9459853172302246
            },
            "predicted": "urgent",
            "correct": true
          },
          {
            "id": "easy-extraction-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "L": 0.023736421018838882,
              "M": 0.895587682723999,
              "S": 0.06452646106481552,
              "XL": 0.016149429604411125
            },
            "predicted": "M",
            "correct": true
          },
          {
            "id": "easy-extraction-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "L": 0.019216492772102356,
              "M": 0.010408583097159863,
              "S": 0.006238934583961964,
              "XL": 0.964136004447937
            },
            "predicted": "XL",
            "correct": true
          },
          {
            "id": "easy-extraction-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "friday": 0.03580309450626373,
              "monday": 0.010451707988977432,
              "thursday": 0.9189052581787109,
              "tuesday": 0.017773400992155075,
              "wednesday": 0.01706649549305439
            },
            "predicted": "thursday",
            "correct": true
          },
          {
            "id": "easy-extraction-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "friday": 0.03251590207219124,
              "monday": 0.8080098628997803,
              "thursday": 0.0463910736143589,
              "tuesday": 0.049873556941747665,
              "wednesday": 0.06320958584547043
            },
            "predicted": "monday",
            "correct": true
          },
          {
            "id": "easy-extraction-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "friday": 0.9064440727233887,
              "monday": 0.015017958357930183,
              "thursday": 0.03083491139113903,
              "tuesday": 0.019117753952741623,
              "wednesday": 0.028585240244865417
            },
            "predicted": "friday",
            "correct": true
          },
          {
            "id": "easy-extraction-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "bank_transfer": 0.012043376453220844,
              "cash": 0.013001848012208939,
              "credit_card": 0.9613389372825623,
              "paypal": 0.013615811243653297
            },
            "predicted": "credit_card",
            "correct": true
          },
          {
            "id": "easy-tool_selection-00",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "convert_currency": 0.0006506135687232018,
              "create_calendar_event": 0.0011936001246795058,
              "get_weather": 0.99303138256073,
              "send_email": 0.003437215695157647,
              "translate_text": 0.0016871480038389564
            },
            "predicted": "get_weather",
            "correct": true
          },
          {
            "id": "easy-tool_selection-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "convert_currency": 0.0011770165292546153,
              "create_calendar_event": 0.0026351965498179197,
              "get_weather": 0.0021128994412720203,
              "send_email": 0.9898489713668823,
              "translate_text": 0.004225917626172304
            },
            "predicted": "send_email",
            "correct": true
          },
          {
            "id": "easy-tool_selection-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "convert_currency": 0.00046877042041160166,
              "create_calendar_event": 0.9927728772163391,
              "get_weather": 0.0009144147043116391,
              "send_email": 0.0049062324687838554,
              "translate_text": 0.0009376758825965226
            },
            "predicted": "create_calendar_event",
            "correct": true
          },
          {
            "id": "easy-tool_selection-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "convert_currency": 0.9880379438400269,
              "create_calendar_event": 0.0007025570957921445,
              "get_weather": 0.000935077725443989,
              "send_email": 0.0030461540445685387,
              "translate_text": 0.007278310135006905
            },
            "predicted": "convert_currency",
            "correct": true
          },
          {
            "id": "easy-tool_selection-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "convert_currency": 0.0014419001527130604,
              "create_calendar_event": 0.0014214224647730589,
              "get_weather": 0.0015844543231651187,
              "send_email": 0.005115137435495853,
              "translate_text": 0.9904370903968811
            },
            "predicted": "translate_text",
            "correct": true
          },
          {
            "id": "easy-tool_selection-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "convert_currency": 0.00024974337429739535,
              "create_calendar_event": 0.9923728108406067,
              "get_weather": 0.0007200388936325908,
              "send_email": 0.006151293404400349,
              "translate_text": 0.0005061687552370131
            },
            "predicted": "create_calendar_event",
            "correct": true
          },
          {
            "id": "easy-tool_selection-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "book_restaurant": 0.0018395129591226578,
              "call_taxi": 0.004136552568525076,
              "get_stock_price": 0.0009688755962997675,
              "search_flights": 0.9880024194717407,
              "set_reminder": 0.005052694585174322
            },
            "predicted": "search_flights",
            "correct": true
          },
          {
            "id": "easy-tool_selection-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "book_restaurant": 0.956574022769928,
              "call_taxi": 0.007457044906914234,
              "get_stock_price": 0.0012105456553399563,
              "search_flights": 0.003913055639714003,
              "set_reminder": 0.030845284461975098
            },
            "predicted": "book_restaurant",
            "correct": true
          },
          {
            "id": "easy-tool_selection-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "book_restaurant": 0.0013180827954784036,
              "call_taxi": 0.007674291264265776,
              "get_stock_price": 0.0011746537638828158,
              "search_flights": 0.0012332310434430838,
              "set_reminder": 0.9885998964309692
            },
            "predicted": "set_reminder",
            "correct": true
          },
          {
            "id": "easy-tool_selection-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "book_restaurant": 0.0008003750117495656,
              "call_taxi": 0.0010489763226360083,
              "get_stock_price": 0.9961070418357849,
              "search_flights": 0.0005824934341944754,
              "set_reminder": 0.0014611025108024478
            },
            "predicted": "get_stock_price",
            "correct": true
          },
          {
            "id": "easy-tool_selection-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "book_restaurant": 0.0007906971150077879,
              "call_taxi": 0.9949654936790466,
              "get_stock_price": 0.0005231574177742004,
              "search_flights": 0.0028736419044435024,
              "set_reminder": 0.0008470104658044875
            },
            "predicted": "call_taxi",
            "correct": true
          },
          {
            "id": "easy-tool_selection-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "book_restaurant": 0.0014909126330167055,
              "call_taxi": 0.004482998047024012,
              "get_stock_price": 0.001196530763991177,
              "search_flights": 0.0010867260862141848,
              "set_reminder": 0.9917427897453308
            },
            "predicted": "set_reminder",
            "correct": true
          },
          {
            "id": "hard-opus-a-long_policy-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "deny_repeated_seepage": 0.13780738413333893,
              "deny_vacancy_exclusion": 0.2565143406391144,
              "pay_full_estimate_less_deductible": 0.23389583826065063,
              "pay_subject_to_10000_sublimit": 0.182163268327713,
              "pay_subject_to_15000_sublimit": 0.18961913883686066
            },
            "predicted": "deny_vacancy_exclusion",
            "correct": false
          },
          {
            "id": "hard-opus-a-long_policy-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "budget_holder": 0.1356707364320755,
              "cfo": 0.13816748559474945,
              "department_head": 0.26679977774620056,
              "executive_committee": 0.11241935938596725,
              "vp_and_finance_director": 0.34694257378578186
            },
            "predicted": "vp_and_finance_director",
            "correct": false
          },
          {
            "id": "hard-opus-a-long_policy-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "full_repair_no_charge": 0.17757661640644073,
              "not_covered": 0.4099539816379547,
              "parts_covered_labour_charged": 0.27455922961235046,
              "pro_rata_replacement_credit": 0.1379101574420929
            },
            "predicted": "not_covered",
            "correct": false
          },
          {
            "id": "hard-opus-a-long_policy-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "license_exception_lvs": 0.3175589442253113,
              "license_required": 0.12486491352319717,
              "no_license_required": 0.3617250621318817,
              "shipment_prohibited": 0.19585104286670685
            },
            "predicted": "no_license_required",
            "correct": false
          },
          {
            "id": "hard-opus-a-long_policy-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.7077447175979614,
              "yes": 0.29225531220436096
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "hard-opus-a-long_policy-13",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.9201903343200684,
              "yes": 0.07980959117412567
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "hard-opus-a-long_policy-17",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "database_oncall": 0.1449994295835495,
              "no_page_ticket_only": 0.2676147520542145,
              "platform_oncall": 0.11769771575927734,
              "security_incident_response": 0.1392894834280014,
              "vendor_support": 0.3303985595703125
            },
            "predicted": "vendor_support",
            "correct": false
          },
          {
            "id": "hard-opus-a-long_policy-19",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.2054438292980194,
              "yes": 0.794556200504303
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-opus-a-probability-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.29460883140563965,
              "yes": 0.7053911685943604
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-opus-a-probability-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "early": 0.21797029674053192,
              "late": 0.549335241317749,
              "on_time": 0.23269446194171906
            },
            "predicted": "late",
            "correct": false
          },
          {
            "id": "hard-opus-a-probability-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "escalated": 0.3843245208263397,
              "reopened": 0.39280667901039124,
              "resolved_first_contact": 0.22286880016326904
            },
            "predicted": "reopened",
            "correct": false
          },
          {
            "id": "hard-opus-a-probability-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.4198921024799347,
              "yes": 0.5801079273223877
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-opus-a-temporal_numeric-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.8608112335205078,
              "yes": 0.1391887664794922
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "hard-opus-a-temporal_numeric-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.071785569190979,
              "yes": 0.928214430809021
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-opus-a-temporal_numeric-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "sep_24": 0.2932996451854706,
              "sep_25": 0.16746516525745392,
              "sep_26": 0.14449448883533478,
              "sep_27": 0.22180534899234772,
              "sep_28": 0.172935351729393
            },
            "predicted": "sep_24",
            "correct": false
          },
          {
            "id": "hard-opus-a-temporal_numeric-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.2681545317173004,
              "yes": 0.7318454384803772
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-opus-a-temporal_numeric-12",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.10704302787780762,
              "1": 0.12230099737644196,
              "2": 0.3145110607147217,
              "3": 0.45614489912986755
            },
            "ordinal_ev": 2.119757816195488,
            "predicted": "3",
            "correct": false
          },
          {
            "id": "hard-opus-b-ambiguous-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cannot_determine": 0.28440985083580017,
              "false_positive": 0.3729357421398163,
              "real_incident": 0.34265437722206116
            },
            "predicted": "false_positive",
            "correct": false
          },
          {
            "id": "hard-opus-b-ambiguous-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "insufficient_information": 0.6162480115890503,
              "time_barred": 0.320000559091568,
              "within_limitation": 0.06375143676996231
            },
            "predicted": "insufficient_information",
            "correct": false
          },
          {
            "id": "hard-opus-b-ambiguous-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "insufficient_information": 0.45400914549827576,
              "over_limit": 0.5192869305610657,
              "within_policy": 0.026703884825110435
            },
            "predicted": "over_limit",
            "correct": false
          },
          {
            "id": "hard-opus-b-ambiguous-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "compromised": 0.8866763114929199,
              "inconclusive": 0.09170122444629669,
              "legitimate": 0.021622391417622566
            },
            "predicted": "compromised",
            "correct": false
          },
          {
            "id": "hard-opus-b-ambiguous-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "ask_user_which_alex": 0.06587604433298111,
              "send_to_alex_moreno": 0.8560175895690918,
              "send_to_alexandra_lind": 0.07810629159212112
            },
            "predicted": "send_to_alex_moreno",
            "correct": false
          },
          {
            "id": "hard-opus-b-ambiguous-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cannot_determine": 0.30872076749801636,
              "grade_correct": 0.5117319226264954,
              "grade_should_be_higher": 0.1795472800731659
            },
            "predicted": "grade_correct",
            "correct": false
          },
          {
            "id": "hard-opus-b-ambiguous-13",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "cannot_determine": 0.48377543687820435,
              "charge_data_team": 0.20090962946414948,
              "charge_platform_team": 0.20210576057434082,
              "split_50_50": 0.11320915818214417
            },
            "predicted": "cannot_determine",
            "correct": true
          },
          {
            "id": "hard-opus-b-multi_hop-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "tier1_team_lead": 0.11677651852369308,
              "tier2_department_head": 0.5606659054756165,
              "tier3_cfo": 0.21938151121139526,
              "tier4_board": 0.10317610949277878
            },
            "predicted": "tier2_department_head",
            "correct": false
          },
          {
            "id": "hard-opus-b-multi_hop-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "ana": 0.11003507673740387,
              "bjorn": 0.11926865577697754,
              "chen": 0.11183587461709976,
              "dmitri": 0.38250038027763367,
              "esra": 0.09399048238992691,
              "no_page_ticket_only": 0.18236948549747467
            },
            "predicted": "dmitri",
            "correct": false
          },
          {
            "id": "hard-opus-b-multi_hop-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.1934874951839447,
              "1": 0.1559142917394638,
              "2": 0.31186726689338684,
              "3": 0.18477857112884521,
              "4": 0.15395234525203705
            },
            "ordinal_ev": 1.9497939199209213,
            "predicted": "2",
            "correct": true
          },
          {
            "id": "hard-opus-b-multi_hop-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "credit_10_percent": 0.47048524022102356,
              "credit_20_percent": 0.19281575083732605,
              "credit_25_percent": 0.08652066439390182,
              "credit_50_percent": 0.0856575295329094,
              "credit_full_month": 0.07254254072904587,
              "no_credit": 0.09197824448347092
            },
            "predicted": "credit_10_percent",
            "correct": false
          },
          {
            "id": "hard-opus-b-multi_hop-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.41199755668640137,
              "yes": 0.5880025029182434
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-opus-b-probability-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "customer_wins": 0.40680351853370667,
              "customer_withdraws": 0.11753184348344803,
              "merchant_wins": 0.4756646752357483
            },
            "predicted": "merchant_wins",
            "correct": true
          },
          {
            "id": "hard-opus-b-probability-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "bad_push": 0.6708141565322876,
              "database_hardware": 0.13199232518672943,
              "upstream_provider": 0.19719350337982178
            },
            "predicted": "bad_push",
            "correct": false
          },
          {
            "id": "hard-opus-b-probability-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.6256071925163269,
              "yes": 0.3743928074836731
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "hard-opus-b-probability-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "attended": 0.3794598877429962,
              "late_cancellation": 0.23318985104560852,
              "no_show": 0.3873502314090729
            },
            "predicted": "no_show",
            "correct": false
          },
          {
            "id": "hard-opus-b-probability-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "modified": 0.03856784850358963,
              "overturned": 0.9132384061813354,
              "upheld": 0.0481937900185585
            },
            "predicted": "overturned",
            "correct": false
          },
          {
            "id": "hard-opus-b-tradeoff-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "escalate_to_engineering": 0.7289229035377502,
              "escalate_to_security": 0.11565219610929489,
              "page_oncall": 0.05879027768969536,
              "process_refund": 0.05019955337047577,
              "send_kb_article": 0.04643508791923523
            },
            "predicted": "escalate_to_engineering",
            "correct": true
          },
          {
            "id": "hard-opus-b-tradeoff-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "order_a": 0.11761078238487244,
              "order_b": 0.21316738426685333,
              "order_c": 0.4426765739917755,
              "order_d": 0.2265452742576599
            },
            "predicted": "order_c",
            "correct": false
          },
          {
            "id": "hard-opus-b-tradeoff-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.7368577718734741,
              "yes": 0.2631422281265259
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "hard-opus-b-tradeoff-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "out_of_scope": 0.04095775634050369,
              "p0_fix_24h": 0.10578594356775284,
              "p1_fix_7d": 0.09867757558822632,
              "p2_fix_30d": 0.6519199013710022,
              "p3_backlog": 0.10265889018774033
            },
            "predicted": "p2_fix_30d",
            "correct": false
          },
          {
            "id": "hard-opus-b-tradeoff-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.7584527730941772,
              "yes": 0.24154725670814514
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "hard-opus-b-tradeoff-12",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "deny_refund": 0.2791862487792969,
              "escalate_to_fraud_team": 0.04250903055071831,
              "offer_store_credit": 0.08905017375946045,
              "refund_now": 0.24871502816677094,
              "wait_for_inspection": 0.3405395746231079
            },
            "predicted": "wait_for_inspection",
            "correct": false
          },
          {
            "id": "hard-opus-c-long_policy-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "erase_all": 0.2613794207572937,
              "erase_but_retain_transaction_records": 0.17493504285812378,
              "erase_marketing_only": 0.12633474171161652,
              "reject_unverified_request": 0.1557231992483139,
              "restrict_all_pending_legal_hold": 0.2816275656223297
            },
            "predicted": "restrict_all_pending_legal_hold",
            "correct": false
          },
          {
            "id": "hard-opus-c-long_policy-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "eur_0": 0.2496323585510254,
              "eur_150": 0.17080830037593842,
              "eur_200": 0.1654399037361145,
              "eur_300": 0.20441919565200806,
              "eur_400": 0.08728175610303879,
              "eur_600": 0.12241854518651962
            },
            "predicted": "eur_0",
            "correct": false
          },
          {
            "id": "hard-opus-c-long_policy-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.5794521570205688,
              "yes": 0.42054784297943115
            },
            "predicted": "no",
            "correct": false
          },
          {
            "id": "hard-opus-c-long_policy-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.38033759593963623,
              "1": 0.251056969165802,
              "2": 0.24091221392154694,
              "3": 0.12769322097301483
            },
            "ordinal_ev": 1.1159610599279404,
            "predicted": "0",
            "correct": false
          },
          {
            "id": "hard-opus-c-long_policy-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.1425655484199524,
              "1": 0.29393258690834045,
              "2": 0.4366069436073303,
              "3": 0.12689493596553802
            },
            "ordinal_ev": 1.5478312820196152,
            "predicted": "2",
            "correct": false
          },
          {
            "id": "hard-opus-c-long_policy-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "credit_minus_restocking_fee": 0.2836270034313202,
              "full_credit": 0.1966577172279358,
              "reject_return": 0.28424426913261414,
              "repair_or_replace_only": 0.2354709953069687
            },
            "predicted": "reject_return",
            "correct": false
          },
          {
            "id": "hard-opus-c-long_policy-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.350205659866333,
              "yes": 0.6497942805290222
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-opus-c-probability-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "halvorsen_polymer": 0.20398810505867004,
              "kuznets_technik": 0.34002307057380676,
              "ortega_seal_works": 0.4559888541698456
            },
            "predicted": "ortega_seal_works",
            "correct": false
          },
          {
            "id": "hard-opus-c-temporal_numeric-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.4340916574001312,
              "yes": 0.5659083724021912
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-opus-c-temporal_numeric-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "eur_617_00": 0.2758154571056366,
              "eur_688_00": 0.18626686930656433,
              "eur_692_00": 0.19176477193832397,
              "eur_693_90": 0.21155278384685516,
              "eur_812_00": 0.13460007309913635
            },
            "predicted": "eur_617_00",
            "correct": false
          },
          {
            "id": "hard-opus-c-temporal_numeric-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "usd_2222_50": 0.22965408861637115,
              "usd_2303_01": 0.2094644457101822,
              "usd_2303_33": 0.15947362780570984,
              "usd_2335_00": 0.2243468165397644,
              "usd_2340_00": 0.1770610809326172
            },
            "predicted": "usd_2222_50",
            "correct": false
          },
          {
            "id": "hard-opus-c-temporal_numeric-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "none_declined": 0.2533932328224182,
              "sep_15_office_chair": 0.16338367760181427,
              "sep_16_taxi": 0.12173942476511002,
              "sep_17_parking": 0.12418917566537857,
              "sep_18_book_set": 0.13621139526367188,
              "sep_21_client_dinner": 0.20108312368392944
            },
            "predicted": "none_declined",
            "correct": false
          },
          {
            "id": "hard-opus-c-temporal_numeric-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "eur_497_40": 0.31191301345825195,
              "eur_498_81": 0.2779386341571808,
              "eur_506_06": 0.13975103199481964,
              "eur_513_69": 0.1603330671787262,
              "eur_532_01": 0.11006425321102142
            },
            "predicted": "eur_497_40",
            "correct": false
          },
          {
            "id": "hard-opus-c-temporal_numeric-12",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "covered": 0.2880264222621918,
              "expired_24_month_term": 0.41821902990341187,
              "expired_30_month_cap": 0.29375457763671875
            },
            "predicted": "expired_24_month_term",
            "correct": false
          },
          {
            "id": "hard-sol-a-adversarial-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "deny_inactive_user": 0.1526007056236267,
              "deny_missing_approval": 0.3798134922981262,
              "grant_access": 0.09867750108242035,
              "grant_owner_exception": 0.3689083158969879
            },
            "predicted": "deny_missing_approval",
            "correct": true
          },
          {
            "id": "hard-sol-a-adversarial-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.810388445854187,
              "yes": 0.18961156904697418
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "hard-sol-a-adversarial-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "delete_now": 0.6533204913139343,
              "needs_signature_review": 0.09972269833087921,
              "retain_indefinitely": 0.15639328956604004,
              "retain_until_2031": 0.09056355804204941
            },
            "predicted": "delete_now",
            "correct": false
          },
          {
            "id": "hard-sol-a-adversarial-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.23271076381206512,
              "yes": 0.7672892808914185
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-sol-a-adversarial-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.14394040405750275,
              "1": 0.10754276812076569,
              "2": 0.3776400089263916,
              "3": 0.37087687849998474
            },
            "ordinal_ev": 1.9754534214735031,
            "predicted": "2",
            "correct": true
          },
          {
            "id": "hard-sol-a-adversarial-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "ethics_review": 0.03911445662379265,
              "major_grants": 0.11045869439840317,
              "manual_hold": 0.1758311539888382,
              "standard_research": 0.6745956540107727
            },
            "predicted": "standard_research",
            "correct": true
          },
          {
            "id": "hard-sol-a-multi_hop-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "accept_s2_dusk": 0.25737255811691284,
              "reject_cutoff": 0.31208741664886475,
              "reject_tier": 0.2069786936044693,
              "reroute_s3": 0.2235613465309143
            },
            "predicted": "reject_cutoff",
            "correct": false
          },
          {
            "id": "hard-sol-a-multi_hop-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "executive_bridge": 0.07075852155685425,
              "heron_q2": 0.22281071543693542,
              "insufficient_information": 0.590750515460968,
              "security_primary": 0.11568022519350052
            },
            "predicted": "insufficient_information",
            "correct": false
          },
          {
            "id": "hard-sol-a-multi_hop-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "pay_manager_approved": 0.27473148703575134,
              "reject_category": 0.39653950929641724,
              "require_director": 0.2096717208623886,
              "require_finance_board": 0.11905721575021744
            },
            "predicted": "reject_category",
            "correct": false
          },
          {
            "id": "hard-sol-a-multi_hop-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "admit": 0.08932296931743622,
              "deny_prerequisite": 0.2836725413799286,
              "deny_score": 0.2868422269821167,
              "manual_equivalency": 0.3401622474193573
            },
            "predicted": "manual_equivalency",
            "correct": false
          },
          {
            "id": "hard-sol-a-multi_hop-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "central_engineering": 0.25760459899902344,
              "insufficient_log": 0.3368498384952545,
              "local_maintenance": 0.16148585081100464,
              "safety_shutdown": 0.2440596967935562
            },
            "predicted": "insufficient_log",
            "correct": false
          },
          {
            "id": "hard-sol-a-multi_hop-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "confirm_l3": 0.17513254284858704,
              "reject_capacity": 0.24653971195220947,
              "require_steward": 0.23046796023845673,
              "switch_l4": 0.34785982966423035
            },
            "predicted": "switch_l4",
            "correct": false
          },
          {
            "id": "hard-sol-a-multi_hop-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "approve_45_days": 0.18158148229122162,
              "deny_sensitive": 0.3798380494117737,
              "needs_field_review": 0.18278971314430237,
              "reduce_to_30_days": 0.25579071044921875
            },
            "predicted": "deny_sensitive",
            "correct": false
          },
          {
            "id": "hard-sol-a-multi_hop-12",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "covered_recurrence": 0.1724662184715271,
              "expired": 0.23913638293743134,
              "needs_authentication": 0.20922034978866577,
              "wrong_issue_class": 0.379177063703537
            },
            "predicted": "wrong_issue_class",
            "correct": false
          },
          {
            "id": "hard-sol-a-trap-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.7434099316596985,
              "yes": 0.2565900385379791
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "hard-sol-a-trap-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.730274498462677,
              "yes": 0.2697254419326782
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "hard-sol-a-trap-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "0": 0.7118715643882751,
              "1": 0.08202363550662994,
              "2": 0.12529277801513672,
              "3": 0.08081195503473282
            },
            "ordinal_ev": 0.5750450566411018,
            "predicted": "0",
            "correct": true
          },
          {
            "id": "hard-sol-a-trap-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "await_internal_metrics": 0.16875363886356354,
              "close_agent_verified": 0.17773132026195526,
              "close_confirmed": 0.37548530101776123,
              "keep_open_unresolved": 0.27802973985671997
            },
            "predicted": "close_confirmed",
            "correct": false
          },
          {
            "id": "hard-sol-a-trap-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "canceled": 0.05341067537665367,
              "indeterminate": 0.22144946455955505,
              "late": 0.2257717251777649,
              "on_time": 0.49936816096305847
            },
            "predicted": "on_time",
            "correct": false
          },
          {
            "id": "hard-sol-a-trap-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "blocked_failed_test": 0.14685268700122833,
              "blocked_second_review": 0.6176565289497375,
              "ready_one_review": 0.15076839923858643,
              "ready_two_reviews": 0.08472247421741486
            },
            "predicted": "blocked_second_review",
            "correct": true
          },
          {
            "id": "hard-sol-a-trap-13",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.909955620765686,
              "yes": 0.09004437923431396
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "hard-sol-a-trap-15",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.6947142481803894,
              "yes": 0.3052857220172882
            },
            "predicted": "no",
            "correct": true
          },
          {
            "id": "hard-sol-b-judge_hard-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.1315310150384903,
              "yes": 0.8684689998626709
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-sol-b-judge_hard-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.14110992848873138,
              "yes": 0.8588901162147522
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-b-judge_hard-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.10145549476146698,
              "yes": 0.8985445499420166
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-sol-b-judge_hard-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.11919458955526352,
              "yes": 0.8808054327964783
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-b-judge_hard-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.29203060269355774,
              "yes": 0.7079693675041199
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-b-judge_hard-14",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.09308559447526932,
              "yes": 0.9069144129753113
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-b-judge_hard-15",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.26388096809387207,
              "yes": 0.7361190319061279
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-sol-b-judge_hard-18",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.15486544370651245,
              "yes": 0.8451345562934875
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-b-long_policy-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "approve_all": 0.5087080001831055,
              "approve_lodging_only": 0.1629502773284912,
              "approve_meals_only": 0.09951456636190414,
              "reject_entire_claim": 0.22882714867591858
            },
            "predicted": "approve_all",
            "correct": false
          },
          {
            "id": "hard-sol-b-long_policy-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "analytics_owner_only": 0.30910149216651917,
              "deny_current_export": 0.31250229477882385,
              "privacy_and_security": 0.209828719496727,
              "privacy_only": 0.1685674637556076
            },
            "predicted": "deny_current_export",
            "correct": true
          },
          {
            "id": "hard-sol-b-long_policy-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "delete_everything": 0.2409096360206604,
              "delete_nothing": 0.11932789534330368,
              "delete_only_profile": 0.14341115951538086,
              "delete_profile_omar_april_and_table": 0.49635130167007446
            },
            "predicted": "delete_profile_omar_april_and_table",
            "correct": true
          },
          {
            "id": "hard-sol-b-long_policy-06",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "east_ward": 0.3996564447879791,
              "harbor_neighbors": 0.15601423382759094,
              "no_award": 0.1463586837053299,
              "river_arts": 0.29797062277793884
            },
            "predicted": "east_ward",
            "correct": true
          },
          {
            "id": "hard-sol-b-routing_hard-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "data_pipeline": 0.019340617582201958,
              "desktop_support": 0.013968479819595814,
              "finance_review": 0.012922046706080437,
              "office_file_repair": 0.9340332746505737,
              "spreadsheet_formula": 0.019735587760806084
            },
            "predicted": "office_file_repair",
            "correct": true
          },
          {
            "id": "hard-sol-b-routing_hard-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "customer_support": 0.009882713668048382,
              "fraud_review": 0.004633682314306498,
              "ledger_reconciliation": 0.9325408935546875,
              "mobile_app": 0.020784074440598488,
              "payment_disputes": 0.03215861693024635
            },
            "predicted": "ledger_reconciliation",
            "correct": true
          },
          {
            "id": "hard-sol-b-routing_hard-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "api_platform": 0.16005395352840424,
              "authoritative_dns": 0.6193749904632568,
              "endpoint_devices": 0.042607229202985764,
              "network_dns": 0.1308065801858902,
              "tls_certificates": 0.04715719074010849
            },
            "predicted": "authoritative_dns",
            "correct": false
          },
          {
            "id": "hard-sol-b-routing_hard-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "ci_infrastructure": 0.13136199116706848,
              "code_signing": 0.23988661170005798,
              "endpoint_security": 0.032373808324337006,
              "macos_runtime": 0.1324843168258667,
              "release_notarization": 0.463893324136734
            },
            "predicted": "release_notarization",
            "correct": true
          },
          {
            "id": "hard-sol-b-routing_hard-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "git_client": 0.06719546765089035,
              "identity_login": 0.055769484490156174,
              "repository_admin": 0.0717688649892807,
              "source_access": 0.672076404094696,
              "vpn_network": 0.13318975269794464
            },
            "predicted": "source_access",
            "correct": true
          },
          {
            "id": "hard-sol-b-temporal_numeric-01",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "before_open": 0.3149806559085846,
              "late_by_2h_or_more": 0.22145475447177887,
              "late_by_under_2h": 0.15250636637210846,
              "within_window": 0.31105825304985046
            },
            "predicted": "before_open",
            "correct": false
          },
          {
            "id": "hard-sol-b-temporal_numeric-02",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "at_least_100gb_below": 0.19546040892601013,
              "exactly_equal": 0.15859277546405792,
              "over_limit": 0.39113572239875793,
              "under_100gb_below": 0.25481101870536804
            },
            "predicted": "over_limit",
            "correct": true
          },
          {
            "id": "hard-sol-b-temporal_numeric-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "12_credits": 0.4766136407852173,
              "15_credits": 0.2709546685218811,
              "6_credits": 0.12991055846214294,
              "9_credits": 0.12252113968133926
            },
            "predicted": "12_credits",
            "correct": false
          },
          {
            "id": "hard-sol-b-temporal_numeric-04",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "clear_fail": 0.4951353073120117,
              "narrow_fail": 0.3327992260456085,
              "pass": 0.1720653921365738
            },
            "predicted": "clear_fail",
            "correct": false
          },
          {
            "id": "hard-sol-c-judge_hard-03",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.1278839111328125,
              "yes": 0.8721160888671875
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-c-judge_hard-05",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.12228310108184814,
              "yes": 0.8777168989181519
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-sol-c-judge_hard-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.05045074224472046,
              "yes": 0.9495492577552795
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-c-judge_hard-08",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.09148705750703812,
              "yes": 0.9085129499435425
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-c-judge_hard-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.055882155895233154,
              "yes": 0.9441178441047668
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-sol-c-judge_hard-10",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.09264988452196121,
              "yes": 0.9073501229286194
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-sol-c-judge_hard-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.1708286702632904,
              "yes": 0.829171359539032
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-c-judge_hard-13",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.09432859718799591,
              "yes": 0.9056713581085205
            },
            "predicted": "yes",
            "correct": false
          },
          {
            "id": "hard-sol-c-judge_hard-15",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "no": 0.27382248640060425,
              "yes": 0.7261775135993958
            },
            "predicted": "yes",
            "correct": true
          },
          {
            "id": "hard-sol-c-multi_hop-07",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "partial_release": 0.16899947822093964,
              "programme_review": 0.35868072509765625,
              "reject_draw": 0.22346027195453644,
              "release_draw": 0.24885950982570648
            },
            "predicted": "programme_review",
            "correct": true
          },
          {
            "id": "hard-sol-c-multi_hop-09",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "bus_substitution": 0.32310542464256287,
              "call_fen_cross": 0.14587731659412384,
              "pass_fen_cross": 0.24387302994728088,
              "terminate_oakmere": 0.28714415431022644
            },
            "predicted": "bus_substitution",
            "correct": true
          },
          {
            "id": "hard-sol-c-multi_hop-11",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "adjuster_referral": 0.06468750536441803,
              "coverage_confirmed": 0.1712184101343155,
              "coverage_excluded": 0.5364008545875549,
              "documentation_needed": 0.22769322991371155
            },
            "predicted": "coverage_excluded",
            "correct": true
          },
          {
            "id": "hard-sol-c-multi_hop-12",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "conservation_hold": 0.21583281457424164,
              "deny_request": 0.33684253692626953,
              "digital_copy_only": 0.23492424190044403,
              "supervised_delivery": 0.21240034699440002
            },
            "predicted": "deny_request",
            "correct": false
          },
          {
            "id": "hard-sol-c-multi_hop-13",
            "valid": true,
            "strict_valid": true,
            "renormalized": false,
            "probs": {
              "engineering_review": 0.22581230103969574,
              "normal_release": 0.5755835771560669,
              "rework": 0.11615115404129028,
              "scrap_lot": 0.08245296776294708
            },
            "predicted": "normal_release",
            "correct": true
          }
        ]
      }
    },
    "laya-typed": {
      "model": "Laya typed-decisions",
      "runtime": {
        "model": "Laya typed-decisions",
        "checkpoint_revision": "f9ab0b228f0fc0f14d873dbc99038f135c2da1b2",
        "source_commit": "42626c348753fbb17572a813127df2278a1ec527",
        "gpu": "NVIDIA L40S",
        "python": "3.12.10",
        "software": {
          "torch": "2.8.0",
          "transformers": "5.0.0",
          "numpy": "2.5.3",
          "safetensors": "0.6.2"
        },
        "checkpoint_sha256": "4fa56de72383a9d3efa9cfa78955733c81b9fc8067a587ca4beb82c78107a24e",
        "weights_dtype": "torch.float32",
        "autocast_dtype": "torch.bfloat16",
        "max_length": 1024,
        "head_max_length": 256,
        "temperature": [
          1.0148024559020996,
          1.0374259948730469,
          1.0575125217437744
        ],
        "temperature_by_options": {
          "choice:3-5": 1.7601518630981445,
          "choice:6-10": 1.0000158548355103,
          "score:3-5": 1.2514300346374512,
          "noul:2": 1.983399510383606,
          "choice:11+": 0.10058280825614929,
          "choice:2": 1.9063563346862793
        },
        "probability_precision": "native API,4 decimal places",
        "prefix_cache": false
      },
      "phase": "public",
      "planned": 231,
      "received": 231,
      "correct": 124,
      "accuracy": 0.5367965367965368,
      "wilson95": [
        0.47241664546939055,
        0.59997261699461
      ],
      "operational_success": 231,
      "strict_valid": 231,
      "valid": 231,
      "renormalized": 0,
      "tiers": {
        "easy": {
          "n": 48,
          "correct": 47
        },
        "hard": {
          "n": 111,
          "correct": 30
        },
        "original": {
          "n": 72,
          "correct": 47
        }
      },
      "families": {
        "adequacy": {
          "n": 12,
          "correct": 7
        },
        "adversarial": {
          "n": 6,
          "correct": 2
        },
        "ambiguous": {
          "n": 7,
          "correct": 1
        },
        "extraction": {
          "n": 24,
          "correct": 19
        },
        "fact": {
          "n": 12,
          "correct": 11
        },
        "intent": {
          "n": 24,
          "correct": 18
        },
        "judge_hard": {
          "n": 17,
          "correct": 6
        },
        "long_policy": {
          "n": 19,
          "correct": 6
        },
        "multi_hop": {
          "n": 18,
          "correct": 3
        },
        "ordinal": {
          "n": 12,
          "correct": 9
        },
        "policy": {
          "n": 12,
          "correct": 11
        },
        "probability": {
          "n": 10,
          "correct": 4
        },
        "routing": {
          "n": 12,
          "correct": 7
        },
        "routing_hard": {
          "n": 5,
          "correct": 2
        },
        "temporal_numeric": {
          "n": 15,
          "correct": 3
        },
        "tool_selection": {
          "n": 12,
          "correct": 12
        },
        "tradeoff": {
          "n": 6,
          "correct": 3
        },
        "trap": {
          "n": 8,
          "correct": 0
        }
      },
      "types": {
        "choice": {
          "n": 139,
          "correct": 70
        },
        "noul": {
          "n": 74,
          "correct": 44
        },
        "score": {
          "n": 18,
          "correct": 10
        }
      },
      "brier_mean": 0.509093562900433,
      "ece": {
        "ece": 0.0772458874458874,
        "n": 231,
        "bins": [
          {
            "lo": 0.0,
            "hi": 0.1,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.1,
            "hi": 0.2,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "lo": 0.2,
            "hi": 0.3,
            "n": 19,
            "mean_confidence": 0.2621842105263158,
            "accuracy": 0.21052631578947367
          },
          {
            "lo": 0.3,
            "hi": 0.4,
            "n": 43,
            "mean_confidence": 0.34445348837209305,
            "accuracy": 0.13953488372093023
          },
          {
            "lo": 0.4,
            "hi": 0.5,
            "n": 31,
            "mean_confidence": 0.45067419354838717,
            "accuracy": 0.4838709677419355
          },
          {
            "lo": 0.5,
            "hi": 0.6,
            "n": 45,
            "mean_confidence": 0.5476866666666667,
            "accuracy": 0.5777777777777777
          },
          {
            "lo": 0.6,
            "hi": 0.7,
            "n": 42,
            "mean_confidence": 0.6442499999999998,
            "accuracy": 0.6428571428571429
          },
          {
            "lo": 0.7,
            "hi": 0.8,
            "n": 26,
            "mean_confidence": 0.7465999999999998,
            "accuracy": 0.8461538461538461
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "n": 25,
            "mean_confidence": 0.839172,
            "accuracy": 0.96
          },
          {
            "lo": 0.9,
            "hi": 1.0,
            "n": 0,
            "mean_confidence": null,
            "accuracy": null
          }
        ]
      },
      "ordinal_mae": 0.5729277777777777,
      "latency": {
        "n": 231,
        "p50_s": 0.037354086999997094,
        "p95_s": 0.04674963699999779
      },
      "peak_gpu_allocated_bytes": 2468433408,
      "truncation": {
        "state_truncated": 37,
        "instruction_truncated": 0,
        "options_truncated": 0
      },
      "scored": [
        {
          "id": "original-policy-01-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 1.139384819,
          "input_audit": {
            "state_tokens_before": 28,
            "state_tokens_kept": 28,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5177,
            "yes": 0.4823
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-01-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.04191797400000041,
          "input_audit": {
            "state_tokens_before": 30,
            "state_tokens_kept": 30,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8363,
            "yes": 0.1637
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-02-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03255794500000064,
          "input_audit": {
            "state_tokens_before": 20,
            "state_tokens_kept": 20,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.42479999999999996,
            "yes": 0.5752
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-02-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03502468200000308,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3105,
            "yes": 0.6895
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-03-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03665098400000133,
          "input_audit": {
            "state_tokens_before": 22,
            "state_tokens_kept": 22,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.44179999999999997,
            "yes": 0.5582
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-03-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03156346599999793,
          "input_audit": {
            "state_tokens_before": 22,
            "state_tokens_kept": 22,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3377,
            "yes": 0.6623
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-04-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03559447299999974,
          "input_audit": {
            "state_tokens_before": 30,
            "state_tokens_kept": 30,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7927,
            "yes": 0.2073
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-04-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.034400909999998674,
          "input_audit": {
            "state_tokens_before": 28,
            "state_tokens_kept": 28,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5992999999999999,
            "yes": 0.4007
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-05-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.034338799000000364,
          "input_audit": {
            "state_tokens_before": 33,
            "state_tokens_kept": 33,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.39880000000000004,
            "yes": 0.6012
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-policy-05-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03360231499999955,
          "input_audit": {
            "state_tokens_before": 26,
            "state_tokens_kept": 26,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6312,
            "yes": 0.3688
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-policy-06-0",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.0334814829999992,
          "input_audit": {
            "state_tokens_before": 31,
            "state_tokens_kept": 31,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.23050000000000004,
            "yes": 0.7695
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-policy-06-1",
          "tier": "original",
          "family": "policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.035093973000002165,
          "input_audit": {
            "state_tokens_before": 21,
            "state_tokens_kept": 21,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.31010000000000004,
            "yes": 0.6899
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-intent-01-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.04648164199999982,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0958,
            "change_address": 0.0477,
            "other": 0.1365,
            "refund": 0.5835,
            "status": 0.1364
          },
          "predicted": "refund",
          "correct": true
        },
        {
          "id": "original-intent-01-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "refund",
          "ok": true,
          "server_wall_seconds": 0.03376567800000174,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0993,
            "change_address": 0.0429,
            "other": 0.0817,
            "refund": 0.7287,
            "status": 0.0473
          },
          "predicted": "refund",
          "correct": true
        },
        {
          "id": "original-intent-02-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.0388711860000015,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0471,
            "change_address": 0.2799,
            "other": 0.3827,
            "refund": 0.0657,
            "status": 0.2246
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-02-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.03837463700000043,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0367,
            "change_address": 0.1923,
            "other": 0.3944,
            "refund": 0.0479,
            "status": 0.3287
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-03-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.03212077699999938,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0432,
            "change_address": 0.4663,
            "other": 0.1745,
            "refund": 0.0605,
            "status": 0.2555
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-03-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.03342366100000049,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.032,
            "change_address": 0.747,
            "other": 0.0708,
            "refund": 0.0392,
            "status": 0.111
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "original-intent-04-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.03441315999999972,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.2627,
            "change_address": 0.0583,
            "other": 0.428,
            "refund": 0.1181,
            "status": 0.1329
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-04-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "cancel",
          "ok": true,
          "server_wall_seconds": 0.03497783099999907,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.264,
            "change_address": 0.0625,
            "other": 0.3423,
            "refund": 0.1796,
            "status": 0.1516
          },
          "predicted": "other",
          "correct": false
        },
        {
          "id": "original-intent-05-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.030623347000002354,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0331,
            "change_address": 0.0516,
            "other": 0.2377,
            "refund": 0.517,
            "status": 0.1605
          },
          "predicted": "refund",
          "correct": false
        },
        {
          "id": "original-intent-05-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "other",
          "ok": true,
          "server_wall_seconds": 0.02986864300000036,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.0492,
            "change_address": 0.0519,
            "other": 0.181,
            "refund": 0.6019,
            "status": 0.116
          },
          "predicted": "refund",
          "correct": false
        },
        {
          "id": "original-intent-06-0",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.030833462000000367,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.1113,
            "change_address": 0.0581,
            "other": 0.1863,
            "refund": 0.0393,
            "status": 0.605
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-intent-06-1",
          "tier": "original",
          "family": "intent",
          "type": "choice",
          "expected": "status",
          "ok": true,
          "server_wall_seconds": 0.034234147000002935,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cancel": 0.2155,
            "change_address": 0.0685,
            "other": 0.1809,
            "refund": 0.0439,
            "status": 0.4912
          },
          "predicted": "status",
          "correct": true
        },
        {
          "id": "original-ordinal-01-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.036252054999998506,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.4033,
            "1": 0.3181,
            "2": 0.196,
            "3": 0.0826
          },
          "ordinal_ev": 0.9579,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-01-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.03155796499999752,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.3141,
            "1": 0.4955,
            "2": 0.1326,
            "3": 0.0579
          },
          "ordinal_ev": 0.9343999999999999,
          "predicted": "1",
          "correct": false
        },
        {
          "id": "original-ordinal-02-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.03334422999999731,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.1337,
            "1": 0.6717,
            "2": 0.1243,
            "3": 0.0703
          },
          "ordinal_ev": 1.1312,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-02-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.04212985799999913,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.1865,
            "1": 0.6802,
            "2": 0.0807,
            "3": 0.0526
          },
          "ordinal_ev": 0.9994000000000001,
          "predicted": "1",
          "correct": true
        },
        {
          "id": "original-ordinal-03-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.03716328300000171,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0544,
            "1": 0.1179,
            "2": 0.6213,
            "3": 0.2064
          },
          "ordinal_ev": 1.9797,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-03-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.03492980100000054,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0715,
            "1": 0.2266,
            "2": 0.4623,
            "3": 0.2396
          },
          "ordinal_ev": 1.87,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-ordinal-04-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.041579808000001606,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0145,
            "1": 0.0616,
            "2": 0.1662,
            "3": 0.7577
          },
          "ordinal_ev": 2.6671000000000005,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-04-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 3,
          "ok": true,
          "server_wall_seconds": 0.034302847999999386,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0167,
            "1": 0.0976,
            "2": 0.2214,
            "3": 0.6643
          },
          "ordinal_ev": 2.5333,
          "predicted": "3",
          "correct": true
        },
        {
          "id": "original-ordinal-05-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.03677983499999726,
          "input_audit": {
            "state_tokens_before": 16,
            "state_tokens_kept": 16,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.4746,
            "1": 0.2689,
            "2": 0.1889,
            "3": 0.0677
          },
          "ordinal_ev": 0.8498000000000001,
          "predicted": "0",
          "correct": true
        },
        {
          "id": "original-ordinal-05-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.0392983650000005,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.3097,
            "1": 0.3235,
            "2": 0.2272,
            "3": 0.1397
          },
          "ordinal_ev": 1.197,
          "predicted": "1",
          "correct": false
        },
        {
          "id": "original-ordinal-06-0",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.036563191000002604,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.2529,
            "1": 0.3842,
            "2": 0.2937,
            "3": 0.0692
          },
          "ordinal_ev": 1.1792,
          "predicted": "1",
          "correct": false
        },
        {
          "id": "original-ordinal-06-1",
          "tier": "original",
          "family": "ordinal",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.0738548370000025,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0971,
            "1": 0.2427,
            "2": 0.5255,
            "3": 0.1347
          },
          "ordinal_ev": 1.6977999999999998,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "original-extraction-01-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.037765483999997684,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.2772,
            "pickup": 0.4488,
            "post": 0.1231,
            "unknown": 0.1509
          },
          "predicted": "pickup",
          "correct": true
        },
        {
          "id": "original-extraction-01-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "pickup",
          "ok": true,
          "server_wall_seconds": 0.03293506099999988,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.2789,
            "pickup": 0.4231,
            "post": 0.0743,
            "unknown": 0.2238
          },
          "predicted": "pickup",
          "correct": true
        },
        {
          "id": "original-extraction-02-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.03189188200000004,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.1861,
            "pickup": 0.0898,
            "post": 0.3677,
            "unknown": 0.3564
          },
          "predicted": "post",
          "correct": false
        },
        {
          "id": "original-extraction-02-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.034764887000001465,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.242,
            "pickup": 0.0847,
            "post": 0.3723,
            "unknown": 0.3009
          },
          "predicted": "post",
          "correct": false
        },
        {
          "id": "original-extraction-03-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.033980402000000964,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.2028,
            "pickup": 0.5519,
            "post": 0.0848,
            "unknown": 0.1605
          },
          "predicted": "pickup",
          "correct": false
        },
        {
          "id": "original-extraction-03-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.03582556799999992,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.5346,
            "pickup": 0.1803,
            "post": 0.1146,
            "unknown": 0.1705
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-04-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.03287583100000191,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.1746,
            "pickup": 0.1628,
            "post": 0.4934,
            "unknown": 0.1692
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-04-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "post",
          "ok": true,
          "server_wall_seconds": 0.033340118999998225,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.1827,
            "pickup": 0.0996,
            "post": 0.5736,
            "unknown": 0.1441
          },
          "predicted": "post",
          "correct": true
        },
        {
          "id": "original-extraction-05-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.03668826399999858,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.5474,
            "pickup": 0.078,
            "post": 0.1375,
            "unknown": 0.2371
          },
          "predicted": "courier",
          "correct": false
        },
        {
          "id": "original-extraction-05-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "unknown",
          "ok": true,
          "server_wall_seconds": 0.034534693000001226,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.4128,
            "pickup": 0.0588,
            "post": 0.2086,
            "unknown": 0.3198
          },
          "predicted": "courier",
          "correct": false
        },
        {
          "id": "original-extraction-06-0",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.03518260599999934,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.6446,
            "pickup": 0.0664,
            "post": 0.1807,
            "unknown": 0.1082
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-extraction-06-1",
          "tier": "original",
          "family": "extraction",
          "type": "choice",
          "expected": "courier",
          "ok": true,
          "server_wall_seconds": 0.034688486000000296,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "courier": 0.5694,
            "pickup": 0.0899,
            "post": 0.192,
            "unknown": 0.1487
          },
          "predicted": "courier",
          "correct": true
        },
        {
          "id": "original-adequacy-01-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03009205799999748,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8235,
            "yes": 0.1765
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-adequacy-01-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03300003400000051,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5123,
            "yes": 0.4877
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-adequacy-02-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03824963499999967,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.29969999999999997,
            "yes": 0.7003
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-02-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.0346745360000007,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7237,
            "yes": 0.2763
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-03-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.0359386509999986,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.27549999999999997,
            "yes": 0.7245
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "original-adequacy-03-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03334266099999894,
          "input_audit": {
            "state_tokens_before": 21,
            "state_tokens_kept": 21,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5457000000000001,
            "yes": 0.4543
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-04-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.042101388999999045,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.1703,
            "yes": 0.8297
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-04-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.034513562000000775,
          "input_audit": {
            "state_tokens_before": 21,
            "state_tokens_kept": 21,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.18889999999999996,
            "yes": 0.8111
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-05-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03806461099999936,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7059,
            "yes": 0.2941
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-05-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03915014200000044,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.544,
            "yes": 0.456
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "original-adequacy-06-0",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03744466900000276,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3296,
            "yes": 0.6704
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "original-adequacy-06-1",
          "tier": "original",
          "family": "adequacy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03145030399999982,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6099,
            "yes": 0.3901
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "original-routing-01-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 0.03039881400000155,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1701,
            "coding_agent": 0.094,
            "document": 0.0413,
            "general": 0.1381,
            "math": 0.5069,
            "tools": 0.0496
          },
          "predicted": "math",
          "correct": true
        },
        {
          "id": "original-routing-01-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "math",
          "ok": true,
          "server_wall_seconds": 0.03335638000000074,
          "input_audit": {
            "state_tokens_before": 8,
            "state_tokens_kept": 8,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1699,
            "coding_agent": 0.1204,
            "document": 0.1619,
            "general": 0.25,
            "math": 0.2016,
            "tools": 0.0962
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-02-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 0.038978547999999336,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.623,
            "coding_agent": 0.1683,
            "document": 0.0166,
            "general": 0.1188,
            "math": 0.0156,
            "tools": 0.0577
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-02-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding",
          "ok": true,
          "server_wall_seconds": 0.035899409999998966,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.7385,
            "coding_agent": 0.0903,
            "document": 0.0257,
            "general": 0.0726,
            "math": 0.0201,
            "tools": 0.0529
          },
          "predicted": "coding",
          "correct": true
        },
        {
          "id": "original-routing-03-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 0.035122794999999485,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.3121,
            "coding_agent": 0.4367,
            "document": 0.0228,
            "general": 0.0998,
            "math": 0.0121,
            "tools": 0.1165
          },
          "predicted": "coding_agent",
          "correct": true
        },
        {
          "id": "original-routing-03-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "coding_agent",
          "ok": true,
          "server_wall_seconds": 0.03531876799999978,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.2641,
            "coding_agent": 0.5131,
            "document": 0.0292,
            "general": 0.1049,
            "math": 0.0144,
            "tools": 0.0742
          },
          "predicted": "coding_agent",
          "correct": true
        },
        {
          "id": "original-routing-04-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 0.037919728000002095,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.057,
            "coding_agent": 0.0648,
            "document": 0.2575,
            "general": 0.5757,
            "math": 0.0125,
            "tools": 0.0325
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-04-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "document",
          "ok": true,
          "server_wall_seconds": 0.032177088000000964,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1279,
            "coding_agent": 0.1221,
            "document": 0.2634,
            "general": 0.3773,
            "math": 0.029,
            "tools": 0.0803
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-05-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 0.03205224599999923,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1288,
            "coding_agent": 0.0624,
            "document": 0.1225,
            "general": 0.5969,
            "math": 0.0169,
            "tools": 0.0724
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-05-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "tools",
          "ok": true,
          "server_wall_seconds": 0.040030538999999976,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1175,
            "coding_agent": 0.0493,
            "document": 0.0885,
            "general": 0.6693,
            "math": 0.0165,
            "tools": 0.0589
          },
          "predicted": "general",
          "correct": false
        },
        {
          "id": "original-routing-06-0",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.037296536999999574,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1386,
            "coding_agent": 0.1277,
            "document": 0.0854,
            "general": 0.5768,
            "math": 0.0216,
            "tools": 0.0498
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "original-routing-06-1",
          "tier": "original",
          "family": "routing",
          "type": "choice",
          "expected": "general",
          "ok": true,
          "server_wall_seconds": 0.03916569099999734,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "coding": 0.1336,
            "coding_agent": 0.0911,
            "document": 0.0427,
            "general": 0.6302,
            "math": 0.0274,
            "tools": 0.075
          },
          "predicted": "general",
          "correct": true
        },
        {
          "id": "easy-intent-00",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.03607945299999926,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.037,
            "cancel_order": 0.0342,
            "change_address": 0.0354,
            "report_damage": 0.2362,
            "track_order": 0.6571
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-01",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "cancel_order",
          "ok": true,
          "server_wall_seconds": 0.03432325900000066,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0307,
            "cancel_order": 0.7438,
            "change_address": 0.039,
            "report_damage": 0.1406,
            "track_order": 0.0458
          },
          "predicted": "cancel_order",
          "correct": true
        },
        {
          "id": "easy-intent-02",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "change_address",
          "ok": true,
          "server_wall_seconds": 0.03650679000000068,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0379,
            "cancel_order": 0.0441,
            "change_address": 0.7893,
            "report_damage": 0.0612,
            "track_order": 0.0675
          },
          "predicted": "change_address",
          "correct": true
        },
        {
          "id": "easy-intent-03",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "report_damage",
          "ok": true,
          "server_wall_seconds": 0.03420743700000273,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0316,
            "cancel_order": 0.0374,
            "change_address": 0.0348,
            "report_damage": 0.8063,
            "track_order": 0.09
          },
          "predicted": "report_damage",
          "correct": true
        },
        {
          "id": "easy-intent-04",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "billing_question",
          "ok": true,
          "server_wall_seconds": 0.03916554199999922,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.5823,
            "cancel_order": 0.0532,
            "change_address": 0.0454,
            "report_damage": 0.1521,
            "track_order": 0.1671
          },
          "predicted": "billing_question",
          "correct": true
        },
        {
          "id": "easy-intent-05",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "track_order",
          "ok": true,
          "server_wall_seconds": 0.036834127000002326,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "billing_question": 0.0435,
            "cancel_order": 0.0427,
            "change_address": 0.0447,
            "report_damage": 0.1993,
            "track_order": 0.6698
          },
          "predicted": "track_order",
          "correct": true
        },
        {
          "id": "easy-intent-06",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.03210525700000133,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0369,
            "send_message": 0.0379,
            "set_alarm": 0.8478,
            "turn_off_lights": 0.0404,
            "weather": 0.0369
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-intent-07",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "play_music",
          "ok": true,
          "server_wall_seconds": 0.03177482999999981,
          "input_audit": {
            "state_tokens_before": 5,
            "state_tokens_kept": 5,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.8112,
            "send_message": 0.0376,
            "set_alarm": 0.0685,
            "turn_off_lights": 0.0383,
            "weather": 0.0445
          },
          "predicted": "play_music",
          "correct": true
        },
        {
          "id": "easy-intent-08",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "weather",
          "ok": true,
          "server_wall_seconds": 0.03162852699999874,
          "input_audit": {
            "state_tokens_before": 7,
            "state_tokens_kept": 7,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0367,
            "send_message": 0.039,
            "set_alarm": 0.0394,
            "turn_off_lights": 0.0434,
            "weather": 0.8416
          },
          "predicted": "weather",
          "correct": true
        },
        {
          "id": "easy-intent-09",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "send_message",
          "ok": true,
          "server_wall_seconds": 0.03155045599999795,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0348,
            "send_message": 0.8497,
            "set_alarm": 0.0412,
            "turn_off_lights": 0.0373,
            "weather": 0.037
          },
          "predicted": "send_message",
          "correct": true
        },
        {
          "id": "easy-intent-10",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "turn_off_lights",
          "ok": true,
          "server_wall_seconds": 0.03382514899999833,
          "input_audit": {
            "state_tokens_before": 8,
            "state_tokens_kept": 8,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0349,
            "send_message": 0.0365,
            "set_alarm": 0.1683,
            "turn_off_lights": 0.7143,
            "weather": 0.046
          },
          "predicted": "turn_off_lights",
          "correct": true
        },
        {
          "id": "easy-intent-11",
          "tier": "easy",
          "family": "intent",
          "type": "choice",
          "expected": "set_alarm",
          "ok": true,
          "server_wall_seconds": 0.03852978899999826,
          "input_audit": {
            "state_tokens_before": 7,
            "state_tokens_kept": 7,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "play_music": 0.0342,
            "send_message": 0.0351,
            "set_alarm": 0.8502,
            "turn_off_lights": 0.0454,
            "weather": 0.0351
          },
          "predicted": "set_alarm",
          "correct": true
        },
        {
          "id": "easy-fact-00",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03570550499999925,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.26070000000000004,
            "yes": 0.7393
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-01",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.032695468000000005,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6284000000000001,
            "yes": 0.3716
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-02",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03191074200000088,
          "input_audit": {
            "state_tokens_before": 20,
            "state_tokens_kept": 20,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.46230000000000004,
            "yes": 0.5377
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-03",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03658045199999904,
          "input_audit": {
            "state_tokens_before": 24,
            "state_tokens_kept": 24,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8251,
            "yes": 0.1749
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-04",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.035486361000000244,
          "input_audit": {
            "state_tokens_before": 17,
            "state_tokens_kept": 17,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.30879999999999996,
            "yes": 0.6912
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-05",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.036131083999997315,
          "input_audit": {
            "state_tokens_before": 18,
            "state_tokens_kept": 18,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7798,
            "yes": 0.2202
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-06",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.04715181599999951,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5845,
            "yes": 0.4155
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "easy-fact-07",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.032989243999999474,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8735999999999999,
            "yes": 0.1264
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-08",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03321340700000164,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3437,
            "yes": 0.6563
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-09",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03929962399999809,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7492,
            "yes": 0.2508
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-fact-10",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.039297724000000755,
          "input_audit": {
            "state_tokens_before": 14,
            "state_tokens_kept": 14,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.22840000000000005,
            "yes": 0.7716
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "easy-fact-11",
          "tier": "easy",
          "family": "fact",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.034487941999998384,
          "input_audit": {
            "state_tokens_before": 15,
            "state_tokens_kept": 15,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8286,
            "yes": 0.1714
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "easy-extraction-00",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "paypal",
          "ok": true,
          "server_wall_seconds": 0.0353087280000004,
          "input_audit": {
            "state_tokens_before": 7,
            "state_tokens_kept": 7,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.0947,
            "cash": 0.0875,
            "credit_card": 0.3474,
            "paypal": 0.4704
          },
          "predicted": "paypal",
          "correct": true
        },
        {
          "id": "easy-extraction-01",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "cash",
          "ok": true,
          "server_wall_seconds": 0.03550297099999966,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.1388,
            "cash": 0.7244,
            "credit_card": 0.0849,
            "paypal": 0.0519
          },
          "predicted": "cash",
          "correct": true
        },
        {
          "id": "easy-extraction-02",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "bank_transfer",
          "ok": true,
          "server_wall_seconds": 0.03868788199999784,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.7898,
            "cash": 0.0664,
            "credit_card": 0.0878,
            "paypal": 0.0561
          },
          "predicted": "bank_transfer",
          "correct": true
        },
        {
          "id": "easy-extraction-03",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "high",
          "ok": true,
          "server_wall_seconds": 0.03520540600000288,
          "input_audit": {
            "state_tokens_before": 19,
            "state_tokens_kept": 19,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.5566,
            "low": 0.0388,
            "medium": 0.092,
            "urgent": 0.3126
          },
          "predicted": "high",
          "correct": true
        },
        {
          "id": "easy-extraction-04",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "low",
          "ok": true,
          "server_wall_seconds": 0.03646847999999636,
          "input_audit": {
            "state_tokens_before": 16,
            "state_tokens_kept": 16,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.0559,
            "low": 0.6597,
            "medium": 0.1719,
            "urgent": 0.1125
          },
          "predicted": "low",
          "correct": true
        },
        {
          "id": "easy-extraction-05",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "urgent",
          "ok": true,
          "server_wall_seconds": 0.039615969999999834,
          "input_audit": {
            "state_tokens_before": 16,
            "state_tokens_kept": 16,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "high": 0.1598,
            "low": 0.0474,
            "medium": 0.0883,
            "urgent": 0.7045
          },
          "predicted": "urgent",
          "correct": true
        },
        {
          "id": "easy-extraction-06",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "M",
          "ok": true,
          "server_wall_seconds": 0.036254556000002935,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.1092,
            "M": 0.6735,
            "S": 0.147,
            "XL": 0.0703
          },
          "predicted": "M",
          "correct": true
        },
        {
          "id": "easy-extraction-07",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "XL",
          "ok": true,
          "server_wall_seconds": 0.04067344099999559,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "L": 0.2357,
            "M": 0.0631,
            "S": 0.0469,
            "XL": 0.6543
          },
          "predicted": "XL",
          "correct": true
        },
        {
          "id": "easy-extraction-08",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "thursday",
          "ok": true,
          "server_wall_seconds": 0.04520763800000083,
          "input_audit": {
            "state_tokens_before": 8,
            "state_tokens_kept": 8,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.031,
            "monday": 0.0397,
            "thursday": 0.8423,
            "tuesday": 0.0419,
            "wednesday": 0.045
          },
          "predicted": "thursday",
          "correct": true
        },
        {
          "id": "easy-extraction-09",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "monday",
          "ok": true,
          "server_wall_seconds": 0.037497379999997804,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.0656,
            "monday": 0.7803,
            "thursday": 0.0444,
            "tuesday": 0.0544,
            "wednesday": 0.0554
          },
          "predicted": "monday",
          "correct": true
        },
        {
          "id": "easy-extraction-10",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "friday",
          "ok": true,
          "server_wall_seconds": 0.03462720399999597,
          "input_audit": {
            "state_tokens_before": 8,
            "state_tokens_kept": 8,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "friday": 0.7058,
            "monday": 0.0842,
            "thursday": 0.0654,
            "tuesday": 0.0754,
            "wednesday": 0.0693
          },
          "predicted": "friday",
          "correct": true
        },
        {
          "id": "easy-extraction-11",
          "tier": "easy",
          "family": "extraction",
          "type": "choice",
          "expected": "credit_card",
          "ok": true,
          "server_wall_seconds": 0.03323793799999919,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bank_transfer": 0.1349,
            "cash": 0.1095,
            "credit_card": 0.6436,
            "paypal": 0.112
          },
          "predicted": "credit_card",
          "correct": true
        },
        {
          "id": "easy-tool_selection-00",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_weather",
          "ok": true,
          "server_wall_seconds": 0.035904399000003195,
          "input_audit": {
            "state_tokens_before": 10,
            "state_tokens_kept": 10,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0257,
            "create_calendar_event": 0.0332,
            "get_weather": 0.8876,
            "send_email": 0.0264,
            "translate_text": 0.0271
          },
          "predicted": "get_weather",
          "correct": true
        },
        {
          "id": "easy-tool_selection-01",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "send_email",
          "ok": true,
          "server_wall_seconds": 0.037451398999998275,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0388,
            "create_calendar_event": 0.1088,
            "get_weather": 0.0406,
            "send_email": 0.7533,
            "translate_text": 0.0584
          },
          "predicted": "send_email",
          "correct": true
        },
        {
          "id": "easy-tool_selection-02",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.034818577999999434,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0376,
            "create_calendar_event": 0.8405,
            "get_weather": 0.0393,
            "send_email": 0.0411,
            "translate_text": 0.0415
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-03",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "convert_currency",
          "ok": true,
          "server_wall_seconds": 0.033549843999999496,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.8606,
            "create_calendar_event": 0.0365,
            "get_weather": 0.0334,
            "send_email": 0.034,
            "translate_text": 0.0355
          },
          "predicted": "convert_currency",
          "correct": true
        },
        {
          "id": "easy-tool_selection-04",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "translate_text",
          "ok": true,
          "server_wall_seconds": 0.03374674799999866,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0453,
            "create_calendar_event": 0.1425,
            "get_weather": 0.047,
            "send_email": 0.0701,
            "translate_text": 0.6951
          },
          "predicted": "translate_text",
          "correct": true
        },
        {
          "id": "easy-tool_selection-05",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "create_calendar_event",
          "ok": true,
          "server_wall_seconds": 0.038270763999996404,
          "input_audit": {
            "state_tokens_before": 12,
            "state_tokens_kept": 12,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "convert_currency": 0.0326,
            "create_calendar_event": 0.8624,
            "get_weather": 0.0338,
            "send_email": 0.0359,
            "translate_text": 0.0353
          },
          "predicted": "create_calendar_event",
          "correct": true
        },
        {
          "id": "easy-tool_selection-06",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "search_flights",
          "ok": true,
          "server_wall_seconds": 0.033731987000003016,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0311,
            "call_taxi": 0.054,
            "get_stock_price": 0.0298,
            "search_flights": 0.8537,
            "set_reminder": 0.0314
          },
          "predicted": "search_flights",
          "correct": true
        },
        {
          "id": "easy-tool_selection-07",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "book_restaurant",
          "ok": true,
          "server_wall_seconds": 0.03804669100000524,
          "input_audit": {
            "state_tokens_before": 13,
            "state_tokens_kept": 13,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.8474,
            "call_taxi": 0.0396,
            "get_stock_price": 0.0353,
            "search_flights": 0.0369,
            "set_reminder": 0.0407
          },
          "predicted": "book_restaurant",
          "correct": true
        },
        {
          "id": "easy-tool_selection-08",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.0367523060000039,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0459,
            "call_taxi": 0.0524,
            "get_stock_price": 0.048,
            "search_flights": 0.0501,
            "set_reminder": 0.8036
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "easy-tool_selection-09",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "get_stock_price",
          "ok": true,
          "server_wall_seconds": 0.035086304000003565,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0302,
            "call_taxi": 0.0321,
            "get_stock_price": 0.8728,
            "search_flights": 0.0313,
            "set_reminder": 0.0336
          },
          "predicted": "get_stock_price",
          "correct": true
        },
        {
          "id": "easy-tool_selection-10",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "call_taxi",
          "ok": true,
          "server_wall_seconds": 0.036208735000002434,
          "input_audit": {
            "state_tokens_before": 9,
            "state_tokens_kept": 9,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0371,
            "call_taxi": 0.8438,
            "get_stock_price": 0.0358,
            "search_flights": 0.0402,
            "set_reminder": 0.0431
          },
          "predicted": "call_taxi",
          "correct": true
        },
        {
          "id": "easy-tool_selection-11",
          "tier": "easy",
          "family": "tool_selection",
          "type": "choice",
          "expected": "set_reminder",
          "ok": true,
          "server_wall_seconds": 0.03492333999999886,
          "input_audit": {
            "state_tokens_before": 11,
            "state_tokens_kept": 11,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "book_restaurant": 0.0408,
            "call_taxi": 0.0434,
            "get_stock_price": 0.0442,
            "search_flights": 0.0438,
            "set_reminder": 0.8277
          },
          "predicted": "set_reminder",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "pay_subject_to_15000_sublimit",
          "ok": true,
          "server_wall_seconds": 0.050664151999995966,
          "input_audit": {
            "state_tokens_before": 3514,
            "state_tokens_kept": 796,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_repeated_seepage": 0.1999,
            "deny_vacancy_exclusion": 0.1688,
            "pay_full_estimate_less_deductible": 0.242,
            "pay_subject_to_10000_sublimit": 0.1826,
            "pay_subject_to_15000_sublimit": 0.2067
          },
          "predicted": "pay_full_estimate_less_deductible",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "cfo",
          "ok": true,
          "server_wall_seconds": 0.04014757000000202,
          "input_audit": {
            "state_tokens_before": 2319,
            "state_tokens_kept": 873,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "budget_holder": 0.1763,
            "cfo": 0.1957,
            "department_head": 0.1918,
            "executive_committee": 0.252,
            "vp_and_finance_director": 0.1841
          },
          "predicted": "executive_committee",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "parts_covered_labour_charged",
          "ok": true,
          "server_wall_seconds": 0.041867623000001686,
          "input_audit": {
            "state_tokens_before": 2165,
            "state_tokens_kept": 885,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "full_repair_no_charge": 0.2602,
            "not_covered": 0.1285,
            "parts_covered_labour_charged": 0.3019,
            "pro_rata_replacement_credit": 0.3094
          },
          "predicted": "pro_rata_replacement_credit",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-09",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "license_required",
          "ok": true,
          "server_wall_seconds": 0.042812591000000566,
          "input_audit": {
            "state_tokens_before": 2280,
            "state_tokens_kept": 892,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "license_exception_lvs": 0.2848,
            "license_required": 0.3017,
            "no_license_required": 0.2183,
            "shipment_prohibited": 0.1953
          },
          "predicted": "license_required",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03715454299999976,
          "input_audit": {
            "state_tokens_before": 2052,
            "state_tokens_kept": 920,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5539000000000001,
            "yes": 0.4461
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-13",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03944527699999867,
          "input_audit": {
            "state_tokens_before": 2198,
            "state_tokens_kept": 932,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.46440000000000003,
            "yes": 0.5356
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-a-long_policy-17",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "database_oncall",
          "ok": true,
          "server_wall_seconds": 0.04057064799999921,
          "input_audit": {
            "state_tokens_before": 2173,
            "state_tokens_kept": 899,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "database_oncall": 0.202,
            "no_page_ticket_only": 0.0858,
            "platform_oncall": 0.2203,
            "security_incident_response": 0.3025,
            "vendor_support": 0.1894
          },
          "predicted": "security_incident_response",
          "correct": false
        },
        {
          "id": "hard-opus-a-long_policy-19",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.04128749199999504,
          "input_audit": {
            "state_tokens_before": 2038,
            "state_tokens_kept": 946,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3982,
            "yes": 0.6018
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.045980412999995224,
          "input_audit": {
            "state_tokens_before": 320,
            "state_tokens_kept": 320,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3972,
            "yes": 0.6028
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-a-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "on_time",
          "ok": true,
          "server_wall_seconds": 0.04852825099999336,
          "input_audit": {
            "state_tokens_before": 845,
            "state_tokens_kept": 845,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "early": 0.279,
            "late": 0.4963,
            "on_time": 0.2247
          },
          "predicted": "late",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-07",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "escalated",
          "ok": true,
          "server_wall_seconds": 0.04765737400000347,
          "input_audit": {
            "state_tokens_before": 897,
            "state_tokens_kept": 897,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "escalated": 0.336,
            "reopened": 0.1954,
            "resolved_first_contact": 0.4687
          },
          "predicted": "resolved_first_contact",
          "correct": false
        },
        {
          "id": "hard-opus-a-probability-08",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.040874164000001656,
          "input_audit": {
            "state_tokens_before": 272,
            "state_tokens_kept": 272,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5078,
            "yes": 0.4922
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-opus-a-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03821832199999875,
          "input_audit": {
            "state_tokens_before": 363,
            "state_tokens_kept": 363,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.48150000000000004,
            "yes": 0.5185
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-a-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.04067778000000288,
          "input_audit": {
            "state_tokens_before": 353,
            "state_tokens_kept": 353,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3546,
            "yes": 0.6454
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-07",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_26",
          "ok": true,
          "server_wall_seconds": 0.04497161199999766,
          "input_audit": {
            "state_tokens_before": 843,
            "state_tokens_kept": 843,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "sep_24": 0.3063,
            "sep_25": 0.1773,
            "sep_26": 0.1815,
            "sep_27": 0.1523,
            "sep_28": 0.1826
          },
          "predicted": "sep_24",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-09",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03529359699999901,
          "input_audit": {
            "state_tokens_before": 375,
            "state_tokens_kept": 375,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.42110000000000003,
            "yes": 0.5789
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-a-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.038964468000003194,
          "input_audit": {
            "state_tokens_before": 406,
            "state_tokens_kept": 406,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0701,
            "1": 0.1851,
            "2": 0.3782,
            "3": 0.3666
          },
          "ordinal_ev": 2.0412999999999997,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-02",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "real_incident",
          "ok": true,
          "server_wall_seconds": 0.04071994099999898,
          "input_audit": {
            "state_tokens_before": 410,
            "state_tokens_kept": 410,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.2557,
            "false_positive": 0.2885,
            "real_incident": 0.4558
          },
          "predicted": "real_incident",
          "correct": true
        },
        {
          "id": "hard-opus-b-ambiguous-03",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "time_barred",
          "ok": true,
          "server_wall_seconds": 0.03691141800000253,
          "input_audit": {
            "state_tokens_before": 353,
            "state_tokens_kept": 353,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.3737,
            "time_barred": 0.2087,
            "within_limitation": 0.4176
          },
          "predicted": "within_limitation",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-07",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "within_policy",
          "ok": true,
          "server_wall_seconds": 0.03602478100000184,
          "input_audit": {
            "state_tokens_before": 369,
            "state_tokens_kept": 369,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "insufficient_information": 0.3221,
            "over_limit": 0.4158,
            "within_policy": 0.2621
          },
          "predicted": "over_limit",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-09",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "legitimate",
          "ok": true,
          "server_wall_seconds": 0.04165895999999947,
          "input_audit": {
            "state_tokens_before": 518,
            "state_tokens_kept": 518,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "compromised": 0.4978,
            "inconclusive": 0.3406,
            "legitimate": 0.1616
          },
          "predicted": "compromised",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-10",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "ask_user_which_alex",
          "ok": true,
          "server_wall_seconds": 0.03717971300000045,
          "input_audit": {
            "state_tokens_before": 377,
            "state_tokens_kept": 377,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ask_user_which_alex": 0.2158,
            "send_to_alex_moreno": 0.4661,
            "send_to_alexandra_lind": 0.3182
          },
          "predicted": "send_to_alex_moreno",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-11",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.03689742799999607,
          "input_audit": {
            "state_tokens_before": 324,
            "state_tokens_kept": 324,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.3281,
            "grade_correct": 0.3565,
            "grade_should_be_higher": 0.3154
          },
          "predicted": "grade_correct",
          "correct": false
        },
        {
          "id": "hard-opus-b-ambiguous-13",
          "tier": "hard",
          "family": "ambiguous",
          "type": "choice",
          "expected": "cannot_determine",
          "ok": true,
          "server_wall_seconds": 0.04253291700000261,
          "input_audit": {
            "state_tokens_before": 423,
            "state_tokens_kept": 423,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "cannot_determine": 0.1204,
            "charge_data_team": 0.2074,
            "charge_platform_team": 0.207,
            "split_50_50": 0.4652
          },
          "predicted": "split_50_50",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-03",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "tier3_cfo",
          "ok": true,
          "server_wall_seconds": 0.04126809200000281,
          "input_audit": {
            "state_tokens_before": 2617,
            "state_tokens_kept": 934,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "tier1_team_lead": 0.2358,
            "tier2_department_head": 0.2342,
            "tier3_cfo": 0.2594,
            "tier4_board": 0.2706
          },
          "predicted": "tier4_board",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-04",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bjorn",
          "ok": true,
          "server_wall_seconds": 0.04058349000000305,
          "input_audit": {
            "state_tokens_before": 2124,
            "state_tokens_kept": 933,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ana": 0.1479,
            "bjorn": 0.0669,
            "chen": 0.0644,
            "dmitri": 0.0747,
            "esra": 0.3527,
            "no_page_ticket_only": 0.2935
          },
          "predicted": "esra",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.03800687900000099,
          "input_audit": {
            "state_tokens_before": 2224,
            "state_tokens_kept": 938,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0602,
            "1": 0.3166,
            "2": 0.2013,
            "3": 0.2218,
            "4": 0.2001
          },
          "ordinal_ev": 2.185,
          "predicted": "1",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "credit_20_percent",
          "ok": true,
          "server_wall_seconds": 0.03970862199999914,
          "input_audit": {
            "state_tokens_before": 2150,
            "state_tokens_kept": 901,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_10_percent": 0.1337,
            "credit_20_percent": 0.1296,
            "credit_25_percent": 0.1833,
            "credit_50_percent": 0.2345,
            "credit_full_month": 0.2486,
            "no_credit": 0.0703
          },
          "predicted": "credit_full_month",
          "correct": false
        },
        {
          "id": "hard-opus-b-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.044772048999995206,
          "input_audit": {
            "state_tokens_before": 1942,
            "state_tokens_kept": 935,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4488,
            "yes": 0.5512
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-01",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "merchant_wins",
          "ok": true,
          "server_wall_seconds": 0.03787391600000234,
          "input_audit": {
            "state_tokens_before": 1562,
            "state_tokens_kept": 938,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "customer_wins": 0.4083,
            "customer_withdraws": 0.2542,
            "merchant_wins": 0.3375
          },
          "predicted": "customer_wins",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-02",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upstream_provider",
          "ok": true,
          "server_wall_seconds": 0.04763333500000044,
          "input_audit": {
            "state_tokens_before": 367,
            "state_tokens_kept": 367,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bad_push": 0.5884,
            "database_hardware": 0.1719,
            "upstream_provider": 0.2398
          },
          "predicted": "bad_push",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.04655157300000212,
          "input_audit": {
            "state_tokens_before": 919,
            "state_tokens_kept": 919,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.47540000000000004,
            "yes": 0.5246
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-b-probability-04",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "attended",
          "ok": true,
          "server_wall_seconds": 0.04134222399999743,
          "input_audit": {
            "state_tokens_before": 391,
            "state_tokens_kept": 391,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "attended": 0.2856,
            "late_cancellation": 0.2269,
            "no_show": 0.4876
          },
          "predicted": "no_show",
          "correct": false
        },
        {
          "id": "hard-opus-b-probability-06",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "upheld",
          "ok": true,
          "server_wall_seconds": 0.041170429999993985,
          "input_audit": {
            "state_tokens_before": 820,
            "state_tokens_kept": 820,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "modified": 0.3018,
            "overturned": 0.346,
            "upheld": 0.3522
          },
          "predicted": "upheld",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-01",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "escalate_to_engineering",
          "ok": true,
          "server_wall_seconds": 0.03622370499999761,
          "input_audit": {
            "state_tokens_before": 667,
            "state_tokens_kept": 667,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "escalate_to_engineering": 0.3117,
            "escalate_to_security": 0.2453,
            "page_oncall": 0.1544,
            "process_refund": 0.1826,
            "send_kb_article": 0.106
          },
          "predicted": "escalate_to_engineering",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-03",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "order_b",
          "ok": true,
          "server_wall_seconds": 0.036599461999998084,
          "input_audit": {
            "state_tokens_before": 562,
            "state_tokens_kept": 562,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "order_a": 0.1148,
            "order_b": 0.1938,
            "order_c": 0.4543,
            "order_d": 0.2371
          },
          "predicted": "order_c",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-06",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.04340902399999891,
          "input_audit": {
            "state_tokens_before": 595,
            "state_tokens_kept": 595,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4887,
            "yes": 0.5113
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-07",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "p0_fix_24h",
          "ok": true,
          "server_wall_seconds": 0.039218222999998886,
          "input_audit": {
            "state_tokens_before": 549,
            "state_tokens_kept": 549,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "out_of_scope": 0.1948,
            "p0_fix_24h": 0.1892,
            "p1_fix_7d": 0.1826,
            "p2_fix_30d": 0.227,
            "p3_backlog": 0.2063
          },
          "predicted": "p2_fix_30d",
          "correct": false
        },
        {
          "id": "hard-opus-b-tradeoff-08",
          "tier": "hard",
          "family": "tradeoff",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03850692800000388,
          "input_audit": {
            "state_tokens_before": 440,
            "state_tokens_kept": 440,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.49829999999999997,
            "yes": 0.5017
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-b-tradeoff-12",
          "tier": "hard",
          "family": "tradeoff",
          "type": "choice",
          "expected": "refund_now",
          "ok": true,
          "server_wall_seconds": 0.03731098600000138,
          "input_audit": {
            "state_tokens_before": 431,
            "state_tokens_kept": 431,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_refund": 0.1112,
            "escalate_to_fraud_team": 0.1913,
            "offer_store_credit": 0.0745,
            "refund_now": 0.2581,
            "wait_for_inspection": 0.3649
          },
          "predicted": "wait_for_inspection",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "erase_but_retain_transaction_records",
          "ok": true,
          "server_wall_seconds": 0.040806832999997766,
          "input_audit": {
            "state_tokens_before": 3101,
            "state_tokens_kept": 845,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "erase_all": 0.1383,
            "erase_but_retain_transaction_records": 0.2589,
            "erase_marketing_only": 0.1838,
            "reject_unverified_request": 0.1386,
            "restrict_all_pending_legal_hold": 0.2804
          },
          "predicted": "restrict_all_pending_legal_hold",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-03",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "eur_400",
          "ok": true,
          "server_wall_seconds": 0.05067969199999567,
          "input_audit": {
            "state_tokens_before": 3129,
            "state_tokens_kept": 913,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_0": 0.0543,
            "eur_150": 0.1671,
            "eur_200": 0.1463,
            "eur_300": 0.2073,
            "eur_400": 0.2725,
            "eur_600": 0.1524
          },
          "predicted": "eur_400",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-04",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03975903300000283,
          "input_audit": {
            "state_tokens_before": 2881,
            "state_tokens_kept": 942,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4526,
            "yes": 0.5474
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.04153367700000388,
          "input_audit": {
            "state_tokens_before": 2958,
            "state_tokens_kept": 932,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.4219,
            "1": 0.1921,
            "2": 0.193,
            "3": 0.1929
          },
          "ordinal_ev": 1.1568,
          "predicted": "0",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-08",
          "tier": "hard",
          "family": "long_policy",
          "type": "score",
          "expected": 1,
          "ok": true,
          "server_wall_seconds": 0.04171414099999993,
          "input_audit": {
            "state_tokens_before": 2932,
            "state_tokens_kept": 922,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0806,
            "1": 0.3077,
            "2": 0.3096,
            "3": 0.302
          },
          "ordinal_ev": 1.8329,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-opus-c-long_policy-10",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "credit_minus_restocking_fee",
          "ok": true,
          "server_wall_seconds": 0.04694770099999346,
          "input_audit": {
            "state_tokens_before": 3145,
            "state_tokens_kept": 902,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "credit_minus_restocking_fee": 0.2956,
            "full_credit": 0.2588,
            "reject_return": 0.1799,
            "repair_or_replace_only": 0.2657
          },
          "predicted": "credit_minus_restocking_fee",
          "correct": true
        },
        {
          "id": "hard-opus-c-long_policy-11",
          "tier": "hard",
          "family": "long_policy",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.04015356100000389,
          "input_audit": {
            "state_tokens_before": 3127,
            "state_tokens_kept": 952,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.45009999999999994,
            "yes": 0.5499
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-opus-c-probability-03",
          "tier": "hard",
          "family": "probability",
          "type": "choice",
          "expected": "kuznets_technik",
          "ok": true,
          "server_wall_seconds": 0.04032566400000093,
          "input_audit": {
            "state_tokens_before": 421,
            "state_tokens_kept": 421,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "halvorsen_polymer": 0.3649,
            "kuznets_technik": 0.2968,
            "ortega_seal_works": 0.3383
          },
          "predicted": "halvorsen_polymer",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03217767699999996,
          "input_audit": {
            "state_tokens_before": 685,
            "state_tokens_kept": 685,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.39280000000000004,
            "yes": 0.6072
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-opus-c-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_692_00",
          "ok": true,
          "server_wall_seconds": 0.03943902700000024,
          "input_audit": {
            "state_tokens_before": 631,
            "state_tokens_kept": 631,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_617_00": 0.1666,
            "eur_688_00": 0.3002,
            "eur_692_00": 0.2159,
            "eur_693_90": 0.1349,
            "eur_812_00": 0.1824
          },
          "predicted": "eur_688_00",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "usd_2335_00",
          "ok": true,
          "server_wall_seconds": 0.04419992799999761,
          "input_audit": {
            "state_tokens_before": 497,
            "state_tokens_kept": 497,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "usd_2222_50": 0.1757,
            "usd_2303_01": 0.2044,
            "usd_2303_33": 0.2681,
            "usd_2335_00": 0.1964,
            "usd_2340_00": 0.1555
          },
          "predicted": "usd_2303_33",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-06",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "sep_18_book_set",
          "ok": true,
          "server_wall_seconds": 0.04496568300000092,
          "input_audit": {
            "state_tokens_before": 710,
            "state_tokens_kept": 710,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "none_declined": 0.0686,
            "sep_15_office_chair": 0.2553,
            "sep_16_taxi": 0.0788,
            "sep_17_parking": 0.1721,
            "sep_18_book_set": 0.1807,
            "sep_21_client_dinner": 0.2446
          },
          "predicted": "sep_15_office_chair",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-08",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "eur_498_81",
          "ok": true,
          "server_wall_seconds": 0.03990080599999857,
          "input_audit": {
            "state_tokens_before": 684,
            "state_tokens_kept": 684,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "eur_497_40": 0.2692,
            "eur_498_81": 0.2055,
            "eur_506_06": 0.1533,
            "eur_513_69": 0.152,
            "eur_532_01": 0.2199
          },
          "predicted": "eur_497_40",
          "correct": false
        },
        {
          "id": "hard-opus-c-temporal_numeric-12",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "expired_30_month_cap",
          "ok": true,
          "server_wall_seconds": 0.03676581499999543,
          "input_audit": {
            "state_tokens_before": 622,
            "state_tokens_kept": 622,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered": 0.3308,
            "expired_24_month_term": 0.3455,
            "expired_30_month_cap": 0.3236
          },
          "predicted": "expired_24_month_term",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-01",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "deny_missing_approval",
          "ok": true,
          "server_wall_seconds": 0.03724738500000058,
          "input_audit": {
            "state_tokens_before": 210,
            "state_tokens_kept": 210,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "deny_inactive_user": 0.1638,
            "deny_missing_approval": 0.2409,
            "grant_access": 0.2874,
            "grant_owner_exception": 0.3079
          },
          "predicted": "grant_owner_exception",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-06",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03980559400000061,
          "input_audit": {
            "state_tokens_before": 172,
            "state_tokens_kept": 172,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.44599999999999995,
            "yes": 0.554
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-07",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "retain_until_2031",
          "ok": true,
          "server_wall_seconds": 0.0342096870000006,
          "input_audit": {
            "state_tokens_before": 181,
            "state_tokens_kept": 181,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_now": 0.1824,
            "needs_signature_review": 0.2354,
            "retain_indefinitely": 0.2016,
            "retain_until_2031": 0.3807
          },
          "predicted": "retain_until_2031",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-08",
          "tier": "hard",
          "family": "adversarial",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.035950230000004524,
          "input_audit": {
            "state_tokens_before": 165,
            "state_tokens_kept": 165,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.523,
            "yes": 0.477
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-a-adversarial-09",
          "tier": "hard",
          "family": "adversarial",
          "type": "score",
          "expected": 2,
          "ok": true,
          "server_wall_seconds": 0.038570819999996786,
          "input_audit": {
            "state_tokens_before": 157,
            "state_tokens_kept": 157,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.0866,
            "1": 0.2035,
            "2": 0.4386,
            "3": 0.2712
          },
          "ordinal_ev": 1.8942999999999999,
          "predicted": "2",
          "correct": true
        },
        {
          "id": "hard-sol-a-adversarial-11",
          "tier": "hard",
          "family": "adversarial",
          "type": "choice",
          "expected": "standard_research",
          "ok": true,
          "server_wall_seconds": 0.036978380000000755,
          "input_audit": {
            "state_tokens_before": 199,
            "state_tokens_kept": 199,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ethics_review": 0.3109,
            "major_grants": 0.232,
            "manual_hold": 0.185,
            "standard_research": 0.2722
          },
          "predicted": "ethics_review",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-01",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "accept_s2_dusk",
          "ok": true,
          "server_wall_seconds": 0.04056859899999665,
          "input_audit": {
            "state_tokens_before": 2274,
            "state_tokens_kept": 946,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "accept_s2_dusk": 0.3957,
            "reject_cutoff": 0.2302,
            "reject_tier": 0.1819,
            "reroute_s3": 0.1922
          },
          "predicted": "accept_s2_dusk",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-02",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "security_primary",
          "ok": true,
          "server_wall_seconds": 0.04178646199999747,
          "input_audit": {
            "state_tokens_before": 2307,
            "state_tokens_kept": 954,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "executive_bridge": 0.1844,
            "heron_q2": 0.316,
            "insufficient_information": 0.2066,
            "security_primary": 0.293
          },
          "predicted": "heron_q2",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-05",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_director",
          "ok": true,
          "server_wall_seconds": 0.044354460000000984,
          "input_audit": {
            "state_tokens_before": 2248,
            "state_tokens_kept": 953,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "pay_manager_approved": 0.3506,
            "reject_category": 0.1688,
            "require_director": 0.2802,
            "require_finance_board": 0.2004
          },
          "predicted": "pay_manager_approved",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "admit",
          "ok": true,
          "server_wall_seconds": 0.03903860900000211,
          "input_audit": {
            "state_tokens_before": 2208,
            "state_tokens_kept": 955,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "admit": 0.3088,
            "deny_prerequisite": 0.2057,
            "deny_score": 0.1999,
            "manual_equivalency": 0.2857
          },
          "predicted": "admit",
          "correct": true
        },
        {
          "id": "hard-sol-a-multi_hop-08",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "central_engineering",
          "ok": true,
          "server_wall_seconds": 0.03846695800000077,
          "input_audit": {
            "state_tokens_before": 2199,
            "state_tokens_kept": 951,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "central_engineering": 0.276,
            "insufficient_log": 0.2057,
            "local_maintenance": 0.3267,
            "safety_shutdown": 0.1915
          },
          "predicted": "local_maintenance",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "require_steward",
          "ok": true,
          "server_wall_seconds": 0.038314055999997265,
          "input_audit": {
            "state_tokens_before": 2205,
            "state_tokens_kept": 947,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "confirm_l3": 0.3445,
            "reject_capacity": 0.1949,
            "require_steward": 0.2622,
            "switch_l4": 0.1984
          },
          "predicted": "confirm_l3",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-10",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "approve_45_days",
          "ok": true,
          "server_wall_seconds": 0.048089593000000264,
          "input_audit": {
            "state_tokens_before": 2238,
            "state_tokens_kept": 949,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_45_days": 0.2119,
            "deny_sensitive": 0.1968,
            "needs_field_review": 0.4436,
            "reduce_to_30_days": 0.1477
          },
          "predicted": "needs_field_review",
          "correct": false
        },
        {
          "id": "hard-sol-a-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "needs_authentication",
          "ok": true,
          "server_wall_seconds": 0.045391891000001294,
          "input_audit": {
            "state_tokens_before": 2201,
            "state_tokens_kept": 948,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "covered_recurrence": 0.3196,
            "expired": 0.1714,
            "needs_authentication": 0.2931,
            "wrong_issue_class": 0.216
          },
          "predicted": "covered_recurrence",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-02",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03958725000000385,
          "input_audit": {
            "state_tokens_before": 225,
            "state_tokens_kept": 225,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.36050000000000004,
            "yes": 0.6395
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-04",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03516057499999903,
          "input_audit": {
            "state_tokens_before": 133,
            "state_tokens_kept": 133,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.371,
            "yes": 0.629
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-05",
          "tier": "hard",
          "family": "trap",
          "type": "score",
          "expected": 0,
          "ok": true,
          "server_wall_seconds": 0.035672763999997414,
          "input_audit": {
            "state_tokens_before": 241,
            "state_tokens_kept": 241,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "0": 0.125,
            "1": 0.1967,
            "2": 0.3844,
            "3": 0.2939
          },
          "ordinal_ev": 1.8472,
          "predicted": "2",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-06",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "keep_open_unresolved",
          "ok": true,
          "server_wall_seconds": 0.034627305000000774,
          "input_audit": {
            "state_tokens_before": 231,
            "state_tokens_kept": 231,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "await_internal_metrics": 0.2299,
            "close_agent_verified": 0.2569,
            "close_confirmed": 0.2728,
            "keep_open_unresolved": 0.2404
          },
          "predicted": "close_confirmed",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-08",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "late",
          "ok": true,
          "server_wall_seconds": 0.03893711799999977,
          "input_audit": {
            "state_tokens_before": 168,
            "state_tokens_kept": 168,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "canceled": 0.0994,
            "indeterminate": 0.1922,
            "late": 0.2062,
            "on_time": 0.5022
          },
          "predicted": "on_time",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-10",
          "tier": "hard",
          "family": "trap",
          "type": "choice",
          "expected": "blocked_second_review",
          "ok": true,
          "server_wall_seconds": 0.037354086999997094,
          "input_audit": {
            "state_tokens_before": 103,
            "state_tokens_kept": 103,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "blocked_failed_test": 0.2687,
            "blocked_second_review": 0.2083,
            "ready_one_review": 0.1604,
            "ready_two_reviews": 0.3626
          },
          "predicted": "ready_two_reviews",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-13",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03970473200000413,
          "input_audit": {
            "state_tokens_before": 131,
            "state_tokens_kept": 131,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.35640000000000005,
            "yes": 0.6436
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-a-trap-15",
          "tier": "hard",
          "family": "trap",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03681432599999823,
          "input_audit": {
            "state_tokens_before": 217,
            "state_tokens_kept": 217,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3861,
            "yes": 0.6139
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-01",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03608698299999702,
          "input_audit": {
            "state_tokens_before": 158,
            "state_tokens_kept": 158,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5932999999999999,
            "yes": 0.4067
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-02",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03647909000000027,
          "input_audit": {
            "state_tokens_before": 113,
            "state_tokens_kept": 113,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.487,
            "yes": 0.513
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03810939199999552,
          "input_audit": {
            "state_tokens_before": 160,
            "state_tokens_kept": 160,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.46519999999999995,
            "yes": 0.5348
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.032920492000002355,
          "input_audit": {
            "state_tokens_before": 99,
            "state_tokens_kept": 99,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.7946,
            "yes": 0.2054
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.0384950580000023,
          "input_audit": {
            "state_tokens_before": 122,
            "state_tokens_kept": 122,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.34430000000000005,
            "yes": 0.6557
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-14",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03745766899999836,
          "input_audit": {
            "state_tokens_before": 154,
            "state_tokens_kept": 154,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5487,
            "yes": 0.4513
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-b-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.035884239000004925,
          "input_audit": {
            "state_tokens_before": 103,
            "state_tokens_kept": 103,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6223000000000001,
            "yes": 0.3777
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-b-judge_hard-18",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.03816078300000214,
          "input_audit": {
            "state_tokens_before": 89,
            "state_tokens_kept": 89,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.24919999999999998,
            "yes": 0.7508
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-01",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "reject_entire_claim",
          "ok": true,
          "server_wall_seconds": 0.04261458900000292,
          "input_audit": {
            "state_tokens_before": 2610,
            "state_tokens_kept": 925,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "approve_all": 0.3588,
            "approve_lodging_only": 0.2677,
            "approve_meals_only": 0.2038,
            "reject_entire_claim": 0.1697
          },
          "predicted": "approve_all",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-02",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "deny_current_export",
          "ok": true,
          "server_wall_seconds": 0.040259743000000014,
          "input_audit": {
            "state_tokens_before": 2498,
            "state_tokens_kept": 929,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "analytics_owner_only": 0.2388,
            "deny_current_export": 0.1551,
            "privacy_and_security": 0.3988,
            "privacy_only": 0.2072
          },
          "predicted": "privacy_and_security",
          "correct": false
        },
        {
          "id": "hard-sol-b-long_policy-05",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "delete_profile_omar_april_and_table",
          "ok": true,
          "server_wall_seconds": 0.04161625900000132,
          "input_audit": {
            "state_tokens_before": 2436,
            "state_tokens_kept": 919,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "delete_everything": 0.2691,
            "delete_nothing": 0.0928,
            "delete_only_profile": 0.2351,
            "delete_profile_omar_april_and_table": 0.4031
          },
          "predicted": "delete_profile_omar_april_and_table",
          "correct": true
        },
        {
          "id": "hard-sol-b-long_policy-06",
          "tier": "hard",
          "family": "long_policy",
          "type": "choice",
          "expected": "east_ward",
          "ok": true,
          "server_wall_seconds": 0.03882786600000543,
          "input_audit": {
            "state_tokens_before": 2468,
            "state_tokens_kept": 935,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "east_ward": 0.1909,
            "harbor_neighbors": 0.3534,
            "no_award": 0.1676,
            "river_arts": 0.2881
          },
          "predicted": "harbor_neighbors",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-01",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "office_file_repair",
          "ok": true,
          "server_wall_seconds": 0.03803274100000209,
          "input_audit": {
            "state_tokens_before": 181,
            "state_tokens_kept": 181,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "data_pipeline": 0.1449,
            "desktop_support": 0.1953,
            "finance_review": 0.1971,
            "office_file_repair": 0.2178,
            "spreadsheet_formula": 0.2449
          },
          "predicted": "spreadsheet_formula",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-02",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "ledger_reconciliation",
          "ok": true,
          "server_wall_seconds": 0.041576128000002655,
          "input_audit": {
            "state_tokens_before": 171,
            "state_tokens_kept": 171,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "customer_support": 0.1679,
            "fraud_review": 0.1341,
            "ledger_reconciliation": 0.2139,
            "mobile_app": 0.2749,
            "payment_disputes": 0.2092
          },
          "predicted": "mobile_app",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-03",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "network_dns",
          "ok": true,
          "server_wall_seconds": 0.0447052680000013,
          "input_audit": {
            "state_tokens_before": 182,
            "state_tokens_kept": 182,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "api_platform": 0.2705,
            "authoritative_dns": 0.2611,
            "endpoint_devices": 0.147,
            "network_dns": 0.1827,
            "tls_certificates": 0.1386
          },
          "predicted": "api_platform",
          "correct": false
        },
        {
          "id": "hard-sol-b-routing_hard-07",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "release_notarization",
          "ok": true,
          "server_wall_seconds": 0.03636838799999964,
          "input_audit": {
            "state_tokens_before": 156,
            "state_tokens_kept": 156,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "ci_infrastructure": 0.2005,
            "code_signing": 0.2027,
            "endpoint_security": 0.1242,
            "macos_runtime": 0.1991,
            "release_notarization": 0.2735
          },
          "predicted": "release_notarization",
          "correct": true
        },
        {
          "id": "hard-sol-b-routing_hard-09",
          "tier": "hard",
          "family": "routing_hard",
          "type": "choice",
          "expected": "source_access",
          "ok": true,
          "server_wall_seconds": 0.04730893800000047,
          "input_audit": {
            "state_tokens_before": 162,
            "state_tokens_kept": 162,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "git_client": 0.1908,
            "identity_login": 0.1704,
            "repository_admin": 0.2035,
            "source_access": 0.2199,
            "vpn_network": 0.2155
          },
          "predicted": "source_access",
          "correct": true
        },
        {
          "id": "hard-sol-b-temporal_numeric-01",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "late_by_under_2h",
          "ok": true,
          "server_wall_seconds": 0.04291978399999863,
          "input_audit": {
            "state_tokens_before": 133,
            "state_tokens_kept": 133,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "before_open": 0.1954,
            "late_by_2h_or_more": 0.2024,
            "late_by_under_2h": 0.1669,
            "within_window": 0.4353
          },
          "predicted": "within_window",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-02",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "over_limit",
          "ok": true,
          "server_wall_seconds": 0.042655738999997084,
          "input_audit": {
            "state_tokens_before": 136,
            "state_tokens_kept": 136,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "at_least_100gb_below": 0.2575,
            "exactly_equal": 0.3076,
            "over_limit": 0.2332,
            "under_100gb_below": 0.2018
          },
          "predicted": "exactly_equal",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-03",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "9_credits",
          "ok": true,
          "server_wall_seconds": 0.0374530400000026,
          "input_audit": {
            "state_tokens_before": 101,
            "state_tokens_kept": 101,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "12_credits": 0.3126,
            "15_credits": 0.249,
            "6_credits": 0.2097,
            "9_credits": 0.2286
          },
          "predicted": "12_credits",
          "correct": false
        },
        {
          "id": "hard-sol-b-temporal_numeric-04",
          "tier": "hard",
          "family": "temporal_numeric",
          "type": "choice",
          "expected": "pass",
          "ok": true,
          "server_wall_seconds": 0.04082713400000415,
          "input_audit": {
            "state_tokens_before": 133,
            "state_tokens_kept": 133,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "clear_fail": 0.2602,
            "narrow_fail": 0.3274,
            "pass": 0.4124
          },
          "predicted": "pass",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-03",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.040397116000001176,
          "input_audit": {
            "state_tokens_before": 331,
            "state_tokens_kept": 331,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3225,
            "yes": 0.6775
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-05",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.0381022119999983,
          "input_audit": {
            "state_tokens_before": 238,
            "state_tokens_kept": 238,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.5633,
            "yes": 0.4367
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-07",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.043088658000002056,
          "input_audit": {
            "state_tokens_before": 211,
            "state_tokens_kept": 211,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.629,
            "yes": 0.371
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-08",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.04279348100000391,
          "input_audit": {
            "state_tokens_before": 225,
            "state_tokens_kept": 225,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.4414,
            "yes": 0.5586
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-09",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03589851899999985,
          "input_audit": {
            "state_tokens_before": 219,
            "state_tokens_kept": 219,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6088,
            "yes": 0.3912
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-10",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03717967400000077,
          "input_audit": {
            "state_tokens_before": 236,
            "state_tokens_kept": 236,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.6036,
            "yes": 0.3964
          },
          "predicted": "no",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-11",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.04230706199999901,
          "input_audit": {
            "state_tokens_before": 255,
            "state_tokens_kept": 255,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.26770000000000005,
            "yes": 0.7323
          },
          "predicted": "yes",
          "correct": false
        },
        {
          "id": "hard-sol-c-judge_hard-13",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "no",
          "ok": true,
          "server_wall_seconds": 0.045023975000006544,
          "input_audit": {
            "state_tokens_before": 214,
            "state_tokens_kept": 214,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.8022,
            "yes": 0.1978
          },
          "predicted": "no",
          "correct": true
        },
        {
          "id": "hard-sol-c-judge_hard-15",
          "tier": "hard",
          "family": "judge_hard",
          "type": "noul",
          "expected": "yes",
          "ok": true,
          "server_wall_seconds": 0.03714652299999699,
          "input_audit": {
            "state_tokens_before": 199,
            "state_tokens_kept": 199,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "no": 0.3165,
            "yes": 0.6835
          },
          "predicted": "yes",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-07",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "programme_review",
          "ok": true,
          "server_wall_seconds": 0.05057480199999986,
          "input_audit": {
            "state_tokens_before": 2379,
            "state_tokens_kept": 942,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "partial_release": 0.1926,
            "programme_review": 0.4761,
            "reject_draw": 0.1332,
            "release_draw": 0.1981
          },
          "predicted": "programme_review",
          "correct": true
        },
        {
          "id": "hard-sol-c-multi_hop-09",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "bus_substitution",
          "ok": true,
          "server_wall_seconds": 0.04332071099999979,
          "input_audit": {
            "state_tokens_before": 2377,
            "state_tokens_kept": 935,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "bus_substitution": 0.2611,
            "call_fen_cross": 0.2188,
            "pass_fen_cross": 0.1445,
            "terminate_oakmere": 0.3756
          },
          "predicted": "terminate_oakmere",
          "correct": false
        },
        {
          "id": "hard-sol-c-multi_hop-11",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "coverage_excluded",
          "ok": true,
          "server_wall_seconds": 0.04491458200000409,
          "input_audit": {
            "state_tokens_before": 2366,
            "state_tokens_kept": 936,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "adjuster_referral": 0.2545,
            "coverage_confirmed": 0.2697,
            "coverage_excluded": 0.1821,
            "documentation_needed": 0.2937
          },
          "predicted": "documentation_needed",
          "correct": false
        },
        {
          "id": "hard-sol-c-multi_hop-12",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "conservation_hold",
          "ok": true,
          "server_wall_seconds": 0.04083738399999959,
          "input_audit": {
            "state_tokens_before": 402,
            "state_tokens_kept": 402,
            "state_truncated": false,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "conservation_hold": 0.2486,
            "deny_request": 0.0977,
            "digital_copy_only": 0.316,
            "supervised_delivery": 0.3377
          },
          "predicted": "supervised_delivery",
          "correct": false
        },
        {
          "id": "hard-sol-c-multi_hop-13",
          "tier": "hard",
          "family": "multi_hop",
          "type": "choice",
          "expected": "normal_release",
          "ok": true,
          "server_wall_seconds": 0.03778157600000043,
          "input_audit": {
            "state_tokens_before": 2445,
            "state_tokens_kept": 945,
            "state_truncated": true,
            "instruction_truncated": false,
            "options_truncated": false
          },
          "valid": true,
          "strict_valid": true,
          "renormalized": false,
          "probs": {
            "engineering_review": 0.3413,
            "normal_release": 0.3258,
            "rework": 0.2452,
            "scrap_lot": 0.0878
          },
          "predicted": "engineering_review",
          "correct": false
        }
      ],
      "raw_sha256": "50fb702758f60194886123ae5e13c464ffc67ab4e392c2132d1b1cf3acd6eb38",
      "protocol_sha256": "18587757df03d28e6c06d62375a21b11c6ed7dcadd2ef90a5f0015de305eb5e1"
    }
  },
  "baselines_protocol": {
    "status": "frozen_before_inference",
    "date": "2026-09-21",
    "tasks": 231,
    "dataset_sha256": "dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51",
    "jevbench_commit": "75e6224ed8103bbc3485ca74820a2eaf7ce8abe0",
    "measurement": "One serial stream of all 231 public tasks per model; no warmups, retries, answer selection or reordering. Native API probabilities scored, rounded as upstream returns them. State/question payloads and candidate order match our Open-Jev runs. Synchronized GPU-process timing excludes startup, network/RPC and the separate input audit; prediction failures retain elapsed time when inference started.",
    "scoring": "Unmodified upstream score_task: exact labels, sum tolerance 0.001, renormalize within 0.02, lexical argmax ties; failures and invalid probabilities count incorrect. Calibration uses valid distributions only. Ordinal accuracy is argmax; expected-value MAE separate.",
    "models": {
      "laya": {
        "model": "Laya English base",
        "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982"
      },
      "kev": {
        "model": "Kev-0.8B",
        "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8"
      }
    },
    "laya_settings": {
      "source_commit": "42626c348753fbb17572a813127df2278a1ec527",
      "weights": "FP32",
      "native_cuda_autocast": "BF16",
      "max_length": 512,
      "head_max_length": 192,
      "native_probability_decimals": 4,
      "temperature": "saved per question type and option count"
    },
    "kev_settings": {
      "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704",
      "dtype": "BF16",
      "adapter_merged": false,
      "attn": "sdpa",
      "max_state": 8192,
      "max_branch": 8192,
      "date_facts": false,
      "prefix_cache": false,
      "native_probability_decimals": 2,
      "unrounded_probabilities": "retained only as a secondary diagnostic",
      "temperature": 2.406050072164233
    },
    "budget": {
      "user_reported_account_cap_usd": 13,
      "start_only_below_metered_and_billed_usd": 7,
      "stop_at_metered_or_billed_usd": 8.5,
      "independent_client_deadline_seconds": 1500,
      "startup_seconds": 600,
      "call_seconds": 900,
      "inference_seconds": 720,
      "min_containers": 0,
      "max_containers_per_class": 1,
      "scaledown_seconds": 30,
      "gpu": "L40S",
      "sequential_models": true,
      "max_ordinary_attempt_compute_usd": 0.97
    },
    "code_sha256": {
      "modal_baselines.py": "e0203dfadf76204c53fe000741868bd19650106975c89eb3a5f678997999d388",
      "run_benchmark.py": "851ad20f968c9ee21d08b148104dfde53906a1096e43b56551371c3d4aa75f28",
      "guard_run.py": "b67a1c60e6b8e4be6d2c47e59a362041ae57ea939fc7340010339aac5fa7a8e3"
    }
  },
  "baselines_protocol_amendments": [
    {
      "date": "2026-09-21",
      "reason": "Kev startup-only checkpoint-verification filename error. The pinned base uses model.safetensors-00001-of-00001.safetensors, not model.safetensors. Corrected only the audit lookup, preserving the same SHA256 and model revision. No questions reached prediction; raw file is empty. Modal restarted the failing container three times before manual guardian interruption stopped the app.",
      "failed_attempt": "kev-startup-failure",
      "scored_responses_saved": 0,
      "inference_responses_produced": 0,
      "laya_run_unchanged": true,
      "effective_code_sha256": {
        "modal_baselines.py": "8eb675e0ce011a523e42f05a9234515b65dec09a2305f57d69ad0997c758e530",
        "run_benchmark.py": "f250c6993c98d9c4fc46ae15165b60e3936736bc6a4aeb6b9db8eec47b5edd03",
        "guard_run.py": "b67a1c60e6b8e4be6d2c47e59a362041ae57ea939fc7340010339aac5fa7a8e3"
      }
    }
  ],
  "typed_protocol": {
    "status": "frozen_before_inference",
    "date": "2026-09-21",
    "dataset": "JevBench public subset",
    "phase": "public",
    "questions": 231,
    "dataset_sha256": "dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51",
    "request_sha256": "f8e05ce96f39020d1cf9943c8252c5d1390509f7d150576118a348a30f869a81",
    "models": {
      "laya-typed": {
        "name": "Laya typed-decisions",
        "app": "dougdoes-laya-typed",
        "class": "LayaTyped",
        "method": "run",
        "revision": "f9ab0b228f0fc0f14d873dbc99038f135c2da1b2",
        "sha256": "4fa56de72383a9d3efa9cfa78955733c81b9fc8067a587ca4beb82c78107a24e"
      }
    },
    "family_counts": {
      "policy": 12,
      "intent": 24,
      "ordinal": 12,
      "extraction": 24,
      "adequacy": 12,
      "routing": 12,
      "fact": 12,
      "tool_selection": 12,
      "long_policy": 19,
      "probability": 10,
      "temporal_numeric": 15,
      "ambiguous": 7,
      "multi_hop": 18,
      "tradeoff": 6,
      "adversarial": 6,
      "trap": 8,
      "judge_hard": 17,
      "routing_hard": 5
    },
    "type_counts": {
      "noul": 74,
      "choice": 139,
      "score": 18
    },
    "source_code_sha256": {
      "modal_typed.py": "c905f7cbecb219957726b6e6e938242f637bc6c895e9d1a1c8e65a92f0a5d295",
      "run_suite.py": "307fe57c4979ea8370b25547f71b8eec03f4b41dea6f6f7a6ac5b9243fa6bade",
      "guard_suite.py": "042d59dc9875389352c1b3709f60bc6e98b39567c70b5312fe45f054c63dca24"
    },
    "method": "One serial response per question per model, native API probabilities, native saved calibration, no retries/selection/tuning. No dedicated warmup; containers may be reused between phases. Fixed denominator includes failures. Exact option order identical across models. JevBench score_task unmodified: strict sum0.001, permitted normalization0.02, lexical argmax ties. Score accuracy is argmax; expected-value MAE separate. Synchronized GPU-process latency excludes initialization/RPC/network and Laya/Kev separate input audit.",
    "budget": {
      "user_reported_account_cap_usd": 13,
      "start_below_metered_and_billed_usd": 7,
      "live_stop_usd": 8.5,
      "client_deadline_seconds": 1500,
      "sequential": true
    },
    "private_policy": "No training, checkpoint selection, calibration fitting, or task revisions based on these holdout outcomes. Private prompts/options/keys/rationales/IDs/responses stay outside public artifacts. Only allowlisted aggregate metrics and protocol commitment may be published."
  }
}