{
  "models": {
    "2B": {
      "runtime": {
        "model": "ZefanCai/Open-Jev-2B",
        "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785",
        "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
        "code_commit": "ed45657bf726c3b77408942830e5578f99df904e",
        "gpu": "NVIDIA L40S",
        "dtype": "bfloat16",
        "batch_size": 1,
        "max_length": 16384,
        "prefix_cache": false,
        "model_load_seconds": 14.586798539000002,
        "torch_version": "2.8.0+cu128"
      },
      "examples": [
        {
          "id": "01-raccoon-refund",
          "title": "Three raccoons in a trench coat",
          "response": {
            "answers": {
              "customer_intent": {
                "type": "choice",
                "choice": "a refund",
                "probabilities": {
                  "a refund": 0.977588268628909,
                  "technical instructions": 0.002011535141835434,
                  "to buy another product": 0.013208199656650859,
                  "to report a wildlife emergency": 0.00719199657260469
                },
                "confidence": 0.970117691505212
              },
              "refund_requested": {
                "type": "noul",
                "noul": 0.9397001047090151
              }
            },
            "model": "Qwen/Qwen3.5-2B",
            "usage": {
              "input_tokens": 418,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.518796342858676,
              "candidate_sequences": 5,
              "inference_seconds": 0.7106123620000027,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded",
              "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.7107222740000019,
            "p95_s": 0.8805969183999991
          },
          "responses_identical": true
        },
        {
          "id": "02-sarcasm-detector",
          "title": "Five stars for deleting my work",
          "response": {
            "answers": {
              "actual_sentiment": {
                "type": "choice",
                "choice": "negative",
                "probabilities": {
                  "positive": 0.21928485949839285,
                  "negative": 0.6771282555002892,
                  "neutral": 0.10358688500131794
                },
                "confidence": 0.5156923832504338
              },
              "sarcasm": {
                "type": "noul",
                "noul": 0.5606939052679603
              }
            },
            "model": "Qwen/Qwen3.5-2B",
            "usage": {
              "input_tokens": 344,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.518796342858676,
              "candidate_sequences": 4,
              "inference_seconds": 0.581106247000001,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded",
              "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.5812446810000012,
            "p95_s": 0.6196496690000017
          },
          "responses_identical": true
        },
        {
          "id": "03-dungeon-risk",
          "title": "The treasure chest has dental insurance",
          "response": {
            "answers": {
              "next_move": {
                "type": "choice",
                "choice": "put a hand inside the chest",
                "probabilities": {
                  "put a hand inside the chest": 0.44438005677390063,
                  "step back and warn the party": 0.15822473426934042,
                  "hug the chest": 0.12584473959791262,
                  "climb inside and close the lid": 0.27155046935884636
                },
                "confidence": 0.25917340903186753
              },
              "danger": {
                "type": "score",
                "score": 1.9416791772180775,
                "probabilities": {
                  "0": 0.05325739995502108,
                  "1": 0.22451767564306244,
                  "2": 0.44951327163073446,
                  "3": 0.2727116527711821
                },
                "confidence": 0.3962558716757133,
                "legend": {
                  "0": "safe",
                  "1": "slightly risky",
                  "2": "dangerous",
                  "3": "extremely dangerous"
                }
              }
            },
            "model": "Qwen/Qwen3.5-2B",
            "usage": {
              "input_tokens": 815,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.518796342858676,
              "candidate_sequences": 8,
              "inference_seconds": 1.1487581899999988,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded",
              "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 1.1368773820000015,
            "p95_s": 1.1499830864000025
          },
          "responses_identical": true
        },
        {
          "id": "04-vampire-injection",
          "title": "Count Override would like your secrets",
          "response": {
            "answers": {
              "message_type": {
                "type": "choice",
                "choice": "prompt injection attempting to override instructions",
                "probabilities": {
                  "ordinary customer question": 0.017981733926269436,
                  "prompt injection attempting to override instructions": 0.9738416388048244,
                  "harmless product feedback": 0.008176627268906136
                },
                "confidence": 0.9607624582072365
              },
              "secret_extraction": {
                "type": "noul",
                "noul": 0.9020716020405615
              }
            },
            "model": "Qwen/Qwen3.5-2B",
            "usage": {
              "input_tokens": 419,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.518796342858676,
              "candidate_sequences": 4,
              "inference_seconds": 0.5592413950000008,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded",
              "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.5593843779999972,
            "p95_s": 0.5773166333999988
          },
          "responses_identical": true
        }
      ]
    },
    "9B": {
      "runtime": {
        "model": "ZefanCai/Open-Jev-9B",
        "adapter_revision": "47e966881e489511c0c7f5633a9e1960a676a551",
        "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
        "code_commit": "ed45657bf726c3b77408942830e5578f99df904e",
        "gpu": "NVIDIA L40S",
        "dtype": "bfloat16",
        "batch_size": 1,
        "max_length": 16384,
        "prefix_cache": false,
        "model_load_seconds": 28.907660348,
        "torch_version": "2.8.0+cu128"
      },
      "examples": [
        {
          "id": "01-raccoon-refund",
          "title": "Three raccoons in a trench coat",
          "response": {
            "answers": {
              "customer_intent": {
                "type": "choice",
                "choice": "a refund",
                "probabilities": {
                  "a refund": 0.9959755042076652,
                  "technical instructions": 0.0005189662539991595,
                  "to buy another product": 0.0017720513479991516,
                  "to report a wildlife emergency": 0.00173347819033652
                },
                "confidence": 0.9946340056102203
              },
              "refund_requested": {
                "type": "noul",
                "noul": 0.9919678568271544
              }
            },
            "model": "Qwen/Qwen3.5-9B",
            "usage": {
              "input_tokens": 418,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.8969118766347646,
              "candidate_sequences": 5,
              "inference_seconds": 0.8426710079999964,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3",
              "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.8520938610000144,
            "p95_s": 0.8657071940000322
          },
          "responses_identical": true
        },
        {
          "id": "02-sarcasm-detector",
          "title": "Five stars for deleting my work",
          "response": {
            "answers": {
              "actual_sentiment": {
                "type": "choice",
                "choice": "negative",
                "probabilities": {
                  "positive": 0.0006923693999538233,
                  "negative": 0.9988137438904416,
                  "neutral": 0.0004938867096043776
                },
                "confidence": 0.9982206158356625
              },
              "sarcasm": {
                "type": "noul",
                "noul": 0.9932685745641121
              }
            },
            "model": "Qwen/Qwen3.5-9B",
            "usage": {
              "input_tokens": 344,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.8969118766347646,
              "candidate_sequences": 4,
              "inference_seconds": 0.6806255040000053,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3",
              "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.6806964440000058,
            "p95_s": 0.7023404701999936
          },
          "responses_identical": true
        },
        {
          "id": "03-dungeon-risk",
          "title": "The treasure chest has dental insurance",
          "response": {
            "answers": {
              "next_move": {
                "type": "choice",
                "choice": "step back and warn the party",
                "probabilities": {
                  "put a hand inside the chest": 0.020270643132588532,
                  "step back and warn the party": 0.9022108929746468,
                  "hug the chest": 0.03139543715433067,
                  "climb inside and close the lid": 0.04612302673843394
                },
                "confidence": 0.8696145239661958
              },
              "danger": {
                "type": "score",
                "score": 2.3724543549196615,
                "probabilities": {
                  "0": 0.01805769004985463,
                  "1": 0.060047350182425255,
                  "2": 0.4532778745659243,
                  "3": 0.4686170852017959
                },
                "confidence": 0.3724543549196613,
                "legend": {
                  "0": "safe",
                  "1": "slightly risky",
                  "2": "dangerous",
                  "3": "extremely dangerous"
                }
              }
            },
            "model": "Qwen/Qwen3.5-9B",
            "usage": {
              "input_tokens": 815,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.8969118766347646,
              "candidate_sequences": 8,
              "inference_seconds": 1.3795526529999904,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3",
              "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 1.350404262000012,
            "p95_s": 1.4162303282000095
          },
          "responses_identical": true
        },
        {
          "id": "04-vampire-injection",
          "title": "Count Override would like your secrets",
          "response": {
            "answers": {
              "message_type": {
                "type": "choice",
                "choice": "prompt injection attempting to override instructions",
                "probabilities": {
                  "ordinary customer question": 0.0007007116267399042,
                  "prompt injection attempting to override instructions": 0.9987567657096962,
                  "harmless product feedback": 0.0005425226635639153
                },
                "confidence": 0.9981351485645441
              },
              "secret_extraction": {
                "type": "noul",
                "noul": 0.9852386608065345
              }
            },
            "model": "Qwen/Qwen3.5-9B",
            "usage": {
              "input_tokens": 419,
              "output_tokens": 0
            },
            "metadata": {
              "method": "lora_decision_head",
              "temperature": 1.8969118766347646,
              "candidate_sequences": 4,
              "inference_seconds": 0.6795012439999937,
              "prefix_cache": {
                "enabled": false,
                "mode": "independent_candidates"
              },
              "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3",
              "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
              "max_length": 16384,
              "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"
            }
          },
          "latency": {
            "n": 5,
            "p50_s": 0.6956501790000118,
            "p95_s": 0.7115751560000035
          },
          "responses_identical": true
        }
      ]
    }
  },
  "method": "Separate corrected example runs: one warmup per case, five interleaved rounds. Server-side synchronized wall time, excludes loading/network. Full raw evidence on /posts/jev-benchmark/."
}