{
  "schema": 1,
  "date": "2026-09-23",
  "benchmark": "Creative Judge v1; separate JevK5 public latency reproduction",
  "source": "https://github.com/allebee/jevk5/tree/v0.2.0",
  "model_revision": "3c673298eb7f2dc7cb98019262c55b1fac01a0bc",
  "code_commit": "85238d7be5527370c43206fe54cd752eb3134c1b",
  "protocol_sha256": "592718e7d4c08048bab0e7ee92a0a96086ecbba5fb53a9f7d020aad40db74332",
  "temperature": 1.532,
  "rates": {
    "gpu_hour_usd": 0.8,
    "gpu_hour_usd_by_hardware": {
      "l4": 0.8,
      "h100": 3.95
    },
    "cpu_core_hour_usd": 0.0473,
    "memory_gib_hour_usd": 0.008
  },
  "method": "Native full probabilities call: tokenization, GPU work, CPU output synchronization and softmax. Batch 1, no prefix cache. First public pass keeps author order with no extra decision warmups; two repeats are separate. Eager control uses the same 120 short questions after three warmups. Model loading and all 13 CUDA graph captures are timed separately. Private 100 L4 run follows all public passes and the eager control, then three unrelated warmups; grading is unchanged.",
  "comparison_limit": "Author did not record exact runtime dependency versions. These are measured replications with pinned released weights/source and matching prompt token counts, not guarantees of identical hardware/software conditions.",
  "cost_method": "GPU-only estimates from measured seconds times the Modal GPU hourly rate, before credits; CPU/RAM, load/capture, build, idle and network excluded. Author ledger reservations and hosted token-price estimates are not Modal invoices.",
  "privacy": "Only aggregate counts, timings, costs, runtime metadata and commitments; no private questions, IDs, labels, probabilities or per-item outputs.",
  "reference": {
    "all": {
      "n": 231,
      "p50_s": 0.014792313799262047,
      "p95_s": 0.11755067156627774,
      "correct": 199
    },
    "easy": {
      "n": 48,
      "p50_s": 0.013536457903683186,
      "p95_s": 0.014758262084797025,
      "correct": 48
    },
    "original": {
      "n": 72,
      "p50_s": 0.013566636480391026,
      "p95_s": 0.014838527655228972,
      "correct": 69
    },
    "hard": {
      "n": 111,
      "p50_s": 0.02996736206114292,
      "p95_s": 0.16051249718293548,
      "correct": 82
    },
    "short": {
      "n": 120,
      "p50_s": 0.0135460807941854,
      "p95_s": 0.014829774154350162,
      "correct": 117
    }
  },
  "hardware_runs": {
    "l4": {
      "gpu": "L4",
      "gpu_name": "NVIDIA L4",
      "gpus": 1,
      "torch": "2.14.0+cu130",
      "transformers": "5.17.0",
      "flash_linear_attention": "0.5.2",
      "cuda": "13.0",
      "attention_implementation": "sdpa",
      "model_load_s": 17.952601204999997,
      "graph_capture_s": 65.333366048,
      "graph_lengths": [
        128,
        192,
        256,
        320,
        384,
        512,
        640,
        768,
        1024,
        1536,
        2048,
        3072,
        4096
      ],
      "peak_load_memory_bytes": 9101872640,
      "reserved_after_capture_bytes": 11628707840,
      "nvml_driver_memory_mib": "580.95.05, 11566, 23034",
      "packages": {
        "triton": "3.8.0",
        "einops": "0.8.2",
        "causal-conv1d": null,
        "kernels": "0.16.1"
      },
      "kernel_diagnostics": {
        "torch_chunk_gated_delta_rule": [
          {
            "implementation": "fla.ops.gated_delta_rule.chunk.chunk_gated_delta_rule",
            "native_extension": true
          }
        ],
        "causal_conv1d_fn": [
          {
            "implementation": "transformers.models.qwen3_5.modeling_qwen3_5.causal_conv1d_fn",
            "native_extension": false
          }
        ]
      },
      "lifecycle_seconds": 338.2134909629822,
      "lifecycle_resource_estimate_usd": 0.14671262107865016,
      "public_passes": [
        {
          "mode": "graph",
          "trial": 0,
          "all": {
            "n": 231,
            "valid": 231,
            "correct": 198,
            "p50_s": 0.06702240200000631,
            "p95_s": 0.765267348000009,
            "mean_s": 0.20762225187012973,
            "gpu_usd_per_1000": 0.046138278193362164,
            "tokens_min": 125,
            "tokens_max": 4033
          },
          "splits": {
            "easy": {
              "n": 48,
              "valid": 48,
              "correct": 48,
              "p50_s": 0.05695872149999559,
              "p95_s": 0.06618893260000362,
              "mean_s": 0.05753297810416728,
              "gpu_usd_per_1000": 0.012785106245370508,
              "tokens_min": 125,
              "tokens_max": 197
            },
            "original": {
              "n": 72,
              "valid": 72,
              "correct": 69,
              "p50_s": 0.05697522100000185,
              "p95_s": 0.06729516770000217,
              "mean_s": 0.059428510513888436,
              "gpu_usd_per_1000": 0.013206335669752985,
              "tokens_min": 131,
              "tokens_max": 214
            },
            "hard": {
              "n": 111,
              "valid": 111,
              "correct": 81,
              "p50_s": 0.16844614299999705,
              "p95_s": 1.0798774754999982,
              "mean_s": 0.3686513916756754,
              "gpu_usd_per_1000": 0.08192253148348343,
              "tokens_min": 217,
              "tokens_max": 4033
            },
            "short": {
              "n": 120,
              "valid": 120,
              "correct": 117,
              "p50_s": 0.05695872149999559,
              "p95_s": 0.06702595015000555,
              "mean_s": 0.05867029754999997,
              "gpu_usd_per_1000": 0.013037843899999993,
              "tokens_min": 125,
              "tokens_max": 214
            }
          },
          "reference_label_agreement": 0.987012987012987,
          "reference_probability_max_abs_diff": 0.056828320026397705,
          "reference_token_matches": 231
        },
        {
          "mode": "graph",
          "trial": 1,
          "all": {
            "n": 231,
            "valid": 231,
            "correct": 198,
            "p50_s": 0.06736356100000762,
            "p95_s": 0.7730194855000008,
            "mean_s": 0.20914507576623342,
            "gpu_usd_per_1000": 0.04647668350360743,
            "tokens_min": 125,
            "tokens_max": 4033
          },
          "splits": {
            "easy": {
              "n": 48,
              "valid": 48,
              "correct": 48,
              "p50_s": 0.05716456449999896,
              "p95_s": 0.06715994010000371,
              "mean_s": 0.05782910437499922,
              "gpu_usd_per_1000": 0.012850912083333159,
              "tokens_min": 125,
              "tokens_max": 197
            },
            "original": {
              "n": 72,
              "valid": 72,
              "correct": 69,
              "p50_s": 0.05693132899999398,
              "p95_s": 0.06720584899999978,
              "mean_s": 0.058508135166666385,
              "gpu_usd_per_1000": 0.013001807814814753,
              "tokens_min": 131,
              "tokens_max": 214
            },
            "hard": {
              "n": 111,
              "valid": 111,
              "correct": 81,
              "p50_s": 0.17014664600000629,
              "p95_s": 1.0892978760000176,
              "mean_s": 0.37228945729729707,
              "gpu_usd_per_1000": 0.08273099051051047,
              "tokens_min": 217,
              "tokens_max": 4033
            },
            "short": {
              "n": 120,
              "valid": 120,
              "correct": 117,
              "p50_s": 0.05698459100000264,
              "p95_s": 0.06722307690000093,
              "mean_s": 0.058236522849999514,
              "gpu_usd_per_1000": 0.012941449522222115,
              "tokens_min": 125,
              "tokens_max": 214
            }
          },
          "reference_label_agreement": 0.987012987012987,
          "reference_probability_max_abs_diff": 0.056828320026397705,
          "reference_token_matches": 231
        },
        {
          "mode": "graph",
          "trial": 2,
          "all": {
            "n": 231,
            "valid": 231,
            "correct": 198,
            "p50_s": 0.06821517500000596,
            "p95_s": 0.7789144424999961,
            "mean_s": 0.21148410984415514,
            "gpu_usd_per_1000": 0.0469964688542567,
            "tokens_min": 125,
            "tokens_max": 4033
          },
          "splits": {
            "easy": {
              "n": 48,
              "valid": 48,
              "correct": 48,
              "p50_s": 0.05777333400001794,
              "p95_s": 0.067826211000002,
              "mean_s": 0.058425942437499025,
              "gpu_usd_per_1000": 0.012983542763888673,
              "tokens_min": 125,
              "tokens_max": 197
            },
            "original": {
              "n": 72,
              "valid": 72,
              "correct": 69,
              "p50_s": 0.05765144899999086,
              "p95_s": 0.06836158259999366,
              "mean_s": 0.05928241318055427,
              "gpu_usd_per_1000": 0.013173869595678726,
              "tokens_min": 131,
              "tokens_max": 214
            },
            "hard": {
              "n": 111,
              "valid": 111,
              "correct": 81,
              "p50_s": 0.17179815299999746,
              "p95_s": 1.099581741999998,
              "mean_s": 0.37639685034234216,
              "gpu_usd_per_1000": 0.08364374452052047,
              "tokens_min": 217,
              "tokens_max": 4033
            },
            "short": {
              "n": 120,
              "valid": 120,
              "correct": 117,
              "p50_s": 0.057712866000002805,
              "p95_s": 0.06818578959999115,
              "mean_s": 0.05893982488333217,
              "gpu_usd_per_1000": 0.013097738862962706,
              "tokens_min": 125,
              "tokens_max": 214
            }
          },
          "reference_label_agreement": 0.987012987012987,
          "reference_probability_max_abs_diff": 0.056828320026397705,
          "reference_token_matches": 231
        },
        {
          "mode": "eager",
          "trial": 0,
          "all": {
            "n": 120,
            "valid": 120,
            "correct": 117,
            "p50_s": 0.06475880899999709,
            "p95_s": 0.06985882965000625,
            "mean_s": 0.06512907310000078,
            "gpu_usd_per_1000": 0.01447312735555573,
            "tokens_min": 125,
            "tokens_max": 214
          },
          "splits": {
            "easy": {
              "n": 48,
              "valid": 48,
              "correct": 48,
              "p50_s": 0.06408917150001514,
              "p95_s": 0.06914810669999127,
              "mean_s": 0.06426807254166711,
              "gpu_usd_per_1000": 0.014281793898148246,
              "tokens_min": 125,
              "tokens_max": 197
            },
            "original": {
              "n": 72,
              "valid": 72,
              "correct": 69,
              "p50_s": 0.06539855449997845,
              "p95_s": 0.07066862965001236,
              "mean_s": 0.06570307347222323,
              "gpu_usd_per_1000": 0.014600682993827387,
              "tokens_min": 131,
              "tokens_max": 214
            },
            "short": {
              "n": 120,
              "valid": 120,
              "correct": 117,
              "p50_s": 0.06475880899999709,
              "p95_s": 0.06985882965000625,
              "mean_s": 0.06512907310000078,
              "gpu_usd_per_1000": 0.01447312735555573,
              "tokens_min": 125,
              "tokens_max": 214
            }
          },
          "reference_label_agreement": 0.9833333333333333,
          "reference_probability_max_abs_diff": 0.04070621728897095,
          "reference_token_matches": 120,
          "first_graph_label_agreement": 1.0,
          "first_graph_max_probability_difference": 0.03762608766555786
        }
      ]
    },
    "h100": {
      "gpu": "H100",
      "gpu_name": "NVIDIA H100 80GB HBM3",
      "gpus": 1,
      "torch": "2.14.0+cu130",
      "transformers": "5.17.0",
      "flash_linear_attention": "0.5.2",
      "cuda": "13.0",
      "attention_implementation": "sdpa",
      "model_load_s": 7.7227527039999995,
      "graph_capture_s": 59.75462999300001,
      "graph_lengths": [
        128,
        192,
        256,
        320,
        384,
        512,
        640,
        768,
        1024,
        1536,
        2048,
        3072,
        4096
      ],
      "peak_load_memory_bytes": 9451703808,
      "reserved_after_capture_bytes": 11806965760,
      "nvml_driver_memory_mib": "580.95.05, 12109, 81559",
      "packages": {
        "triton": "3.8.0",
        "einops": "0.8.2",
        "causal-conv1d": null,
        "kernels": "0.16.1"
      },
      "kernel_diagnostics": {
        "torch_chunk_gated_delta_rule": [
          {
            "implementation": "fla.ops.gated_delta_rule.chunk.chunk_gated_delta_rule",
            "native_extension": true
          }
        ],
        "causal_conv1d_fn": [
          {
            "implementation": "transformers.models.qwen3_5.modeling_qwen3_5.causal_conv1d_fn",
            "native_extension": false
          }
        ]
      },
      "lifecycle_seconds": 130.4085829257965,
      "lifecycle_resource_estimate_usd": 0.20427142271502816,
      "public_passes": [
        {
          "mode": "graph",
          "trial": 0,
          "all": {
            "n": 231,
            "valid": 231,
            "correct": 198,
            "p50_s": 0.013168385000000171,
            "p95_s": 0.09439083299999851,
            "mean_s": 0.02952343870562777,
            "gpu_usd_per_1000": 0.03239377302423047,
            "tokens_min": 125,
            "tokens_max": 4033
          },
          "splits": {
            "easy": {
              "n": 48,
              "valid": 48,
              "correct": 48,
              "p50_s": 0.012036084000001779,
              "p95_s": 0.013057514949999672,
              "mean_s": 0.012104280229166578,
              "gpu_usd_per_1000": 0.013281085251446662,
              "tokens_min": 125,
              "tokens_max": 197
            },
            "original": {
              "n": 72,
              "valid": 72,
              "correct": 69,
              "p50_s": 0.012057189999993057,
              "p95_s": 0.013204621299997398,
              "mean_s": 0.012647764027777342,
              "gpu_usd_per_1000": 0.01387740775270014,
              "tokens_min": 131,
              "tokens_max": 214
            },
            "hard": {
              "n": 111,
              "valid": 111,
              "correct": 81,
              "p50_s": 0.02490882200000044,
              "p95_s": 0.12467757449999795,
              "mean_s": 0.04800243135135181,
              "gpu_usd_per_1000": 0.0526693343993999,
              "tokens_min": 217,
              "tokens_max": 4033
            },
            "short": {
              "n": 120,
              "valid": 120,
              "correct": 117,
              "p50_s": 0.01204834800000043,
              "p95_s": 0.013148009400001825,
              "mean_s": 0.012430370508333037,
              "gpu_usd_per_1000": 0.01363887875219875,
              "tokens_min": 125,
              "tokens_max": 214
            }
          },
          "reference_label_agreement": 0.987012987012987,
          "reference_probability_max_abs_diff": 0.04070621728897095,
          "reference_token_matches": 231
        },
        {
          "mode": "graph",
          "trial": 1,
          "all": {
            "n": 231,
            "valid": 231,
            "correct": 198,
            "p50_s": 0.013105269000007524,
            "p95_s": 0.09289328199999858,
            "mean_s": 0.029172915194804784,
            "gpu_usd_per_1000": 0.03200917083874414,
            "tokens_min": 125,
            "tokens_max": 4033
          },
          "splits": {
            "easy": {
              "n": 48,
              "valid": 48,
              "correct": 48,
              "p50_s": 0.01198703249999511,
              "p95_s": 0.0130781181499934,
              "mean_s": 0.012063680604165997,
              "gpu_usd_per_1000": 0.013236538440682136,
              "tokens_min": 125,
              "tokens_max": 197
            },
            "original": {
              "n": 72,
              "valid": 72,
              "correct": 69,
              "p50_s": 0.012000770500002034,
              "p95_s": 0.01308080350000509,
              "mean_s": 0.012196056736111337,
              "gpu_usd_per_1000": 0.013381784474344386,
              "tokens_min": 131,
              "tokens_max": 214
            },
            "hard": {
              "n": 111,
              "valid": 111,
              "correct": 81,
              "p50_s": 0.024746077000003197,
              "p95_s": 0.12328924850000789,
              "mean_s": 0.04758351942342271,
              "gpu_usd_per_1000": 0.05220969492292215,
              "tokens_min": 217,
              "tokens_max": 4033
            },
            "short": {
              "n": 120,
              "valid": 120,
              "correct": 117,
              "p50_s": 0.011997967500001039,
              "p95_s": 0.01308160455001186,
              "mean_s": 0.012143106283333201,
              "gpu_usd_per_1000": 0.013323686060879485,
              "tokens_min": 125,
              "tokens_max": 214
            }
          },
          "reference_label_agreement": 0.987012987012987,
          "reference_probability_max_abs_diff": 0.04070621728897095,
          "reference_token_matches": 231
        },
        {
          "mode": "graph",
          "trial": 2,
          "all": {
            "n": 231,
            "valid": 231,
            "correct": 198,
            "p50_s": 0.013088619000001245,
            "p95_s": 0.09321130800000788,
            "mean_s": 0.029158292645022035,
            "gpu_usd_per_1000": 0.031993126652176955,
            "tokens_min": 125,
            "tokens_max": 4033
          },
          "splits": {
            "easy": {
              "n": 48,
              "valid": 48,
              "correct": 48,
              "p50_s": 0.011975062000004755,
              "p95_s": 0.013013598749996903,
              "mean_s": 0.012039516395833338,
              "gpu_usd_per_1000": 0.013210024934317137,
              "tokens_min": 125,
              "tokens_max": 197
            },
            "original": {
              "n": 72,
              "valid": 72,
              "correct": 69,
              "p50_s": 0.012009490499998776,
              "p95_s": 0.01309631930000137,
              "mean_s": 0.01217277425000122,
              "gpu_usd_per_1000": 0.013356238413195784,
              "tokens_min": 131,
              "tokens_max": 214
            },
            "hard": {
              "n": 111,
              "valid": 111,
              "correct": 81,
              "p50_s": 0.024901591000002554,
              "p95_s": 0.12430924200000248,
              "mean_s": 0.047578640252252266,
              "gpu_usd_per_1000": 0.052204341387887904,
              "tokens_min": 217,
              "tokens_max": 4033
            },
            "short": {
              "n": 120,
              "valid": 120,
              "correct": 117,
              "p50_s": 0.012002386500000739,
              "p95_s": 0.013064494699996487,
              "mean_s": 0.012119471108334067,
              "gpu_usd_per_1000": 0.013297753021644324,
              "tokens_min": 125,
              "tokens_max": 214
            }
          },
          "reference_label_agreement": 0.987012987012987,
          "reference_probability_max_abs_diff": 0.04070621728897095,
          "reference_token_matches": 231
        },
        {
          "mode": "eager",
          "trial": 0,
          "all": {
            "n": 120,
            "valid": 120,
            "correct": 118,
            "p50_s": 0.05905652199999878,
            "p95_s": 0.11359110405000336,
            "mean_s": 0.07935790564166598,
            "gpu_usd_per_1000": 0.08707325757905017,
            "tokens_min": 125,
            "tokens_max": 214
          },
          "splits": {
            "easy": {
              "n": 48,
              "valid": 48,
              "correct": 48,
              "p50_s": 0.05905652199999878,
              "p95_s": 0.11182106745000268,
              "mean_s": 0.07626160422916684,
              "gpu_usd_per_1000": 0.08367592686255806,
              "tokens_min": 125,
              "tokens_max": 197
            },
            "original": {
              "n": 72,
              "valid": 72,
              "correct": 70,
              "p50_s": 0.05921404949999953,
              "p95_s": 0.11399334135000032,
              "mean_s": 0.08142210658333207,
              "gpu_usd_per_1000": 0.08933814472337824,
              "tokens_min": 131,
              "tokens_max": 214
            },
            "short": {
              "n": 120,
              "valid": 120,
              "correct": 118,
              "p50_s": 0.05905652199999878,
              "p95_s": 0.11359110405000336,
              "mean_s": 0.07935790564166598,
              "gpu_usd_per_1000": 0.08707325757905017,
              "tokens_min": 125,
              "tokens_max": 214
            }
          },
          "reference_label_agreement": 0.9916666666666667,
          "reference_probability_max_abs_diff": 0.04070621728897095,
          "reference_token_matches": 120,
          "first_graph_label_agreement": 0.9916666666666667,
          "first_graph_max_probability_difference": 0.020386874675750732
        }
      ]
    }
  },
  "models": {
    "jevk5": {
      "name": "JevK5 v0.2 · L4",
      "source": "https://github.com/allebee/jevk5/tree/v0.2.0",
      "model_repo": "alibiserikbay/JevK5",
      "model_revision": "3c673298eb7f2dc7cb98019262c55b1fac01a0bc",
      "code_commit": "85238d7be5527370c43206fe54cd752eb3134c1b",
      "context_tokens": 16384,
      "gpu": "L4",
      "dtype": "bfloat16",
      "probability_mode": "native calibrated option logits, CUDA graph replay, temperature1.532",
      "correct": 89,
      "n": 100,
      "valid": 100,
      "p50_s": 0.05808848149999335,
      "p95_s": 0.06875913165001464,
      "mean_s": 0.06022846326000149,
      "ece": 0.12801373451948164,
      "brier_mean": 0.17826996700853104,
      "ordinal_mae": 0.437806130386889,
      "serial_gpu_usd_per_1000": 0.013384102946666998,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 10
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 10
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 8
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 9
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 10
        },
        "instruction_following": {
          "n": 10,
          "correct": 10
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 6
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 7
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 9
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 10
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 39
        },
        "noul": {
          "n": 36,
          "correct": 35
        },
        "score": {
          "n": 20,
          "correct": 15
        }
      }
    }
  },
  "account_snapshot": {
    "metered_usd": 2.95,
    "budget_reservation_estimate_usd": 0.8509840437936783,
    "cumulative_conservative_bound_usd": 7.742779237288555,
    "active_containers": 0,
    "note": "Account metering can lag or revise; resource reservations are conservative estimates, not charges or proof of cap enforcement."
  }
}