{
  "schema": 1,
  "date": "2026-09-23",
  "benchmark": "Creative Judge v1",
  "questions": 100,
  "protocol_sha256": "cacdb1b3180d584b5a9f3e6dabbde5f21d9b1145724e322b8750f9f30c992e88",
  "prior_protocol_sha256": "b761470ed39c418f46b5357d208ad2281765f6c532fc2ea109231b5812e0030d",
  "intermediate_protocol_sha256": "0553a6ceedd470ce475ad7ba8ced7af5d233dd556e0833c2511a7dad044a27fc",
  "protocol_amendment": "Jevfire startup correction disables the optional FlashInfer sampler. A subsequent adapter correction serializes structured context as lossless JSON text for Jevfire and Diffusion. Jevfire had no successful private inference before these corrections: its first attempt failed startup and its next rejected all 100 request contexts. SemIf retains its original protocol commitment. Questions, grading, checkpoints and batch sizes are unchanged; failed attempt costs remain included.",
  "hardware": "1 × NVIDIA H100 per run; sequential deployments",
  "rates": {
    "gpu_hour_usd": 3.95,
    "cpu_core_hour_usd": 0.0473,
    "memory_gib_hour_usd": 0.008,
    "source": "Modal account billing rates API, September 23, 2026"
  },
  "quality_method": "One serial pass after three unrelated warmups. Frozen private questions and unchanged exact-label grading; fixed denominator includes failures. No checkpoint selection or calibration fitting on the suite.",
  "throughput_method": "Two repeated 100-question passes per batch size, original order, prefix caching disabled. Sweep stops at first OOM. Accuracy is from the serial pass; batch agreement is reported separately.",
  "cost_method": "GPU-only costs use measured duration × account GPU rate. Warm inference resource ceiling adds the configured maximum 8 CPU cores and 64 GiB RAM. Startup, build and idle time are separate. Lifecycle resource totals are conservative estimates, not verified maxima for CPU image builds. Metered account deltas can lag, include image builds, and are not per-model invoices.",
  "privacy": "Only model identities, coarse category counts, numerical aggregates and protocol commitments are public. No questions, expected labels, item IDs, responses or trial prediction lists.",
  "models": {
    "semif": {
      "name": "SemIf Qwen3.5-4B",
      "source": "https://github.com/TheoLeeCJ/SemIf",
      "model_repo": "Qwen/Qwen3.5-4B",
      "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
      "code_commit": "1f2dea3e25379f9dfc98cb83c324f00ab5deda37",
      "protocol_sha256": "b761470ed39c418f46b5357d208ad2281765f6c532fc2ea109231b5812e0030d",
      "context_tokens": 4096,
      "dtype": "bfloat16",
      "gpu": "NVIDIA H100 80GB HBM3",
      "probability_mode": "native conditional option softmax",
      "correct": 86,
      "n": 100,
      "valid": 100,
      "operational_success": 100,
      "p50_s": 0.06848917599999815,
      "p95_s": 0.08891518959999906,
      "mean_s": 0.06972967674000007,
      "serial_gpu_usd_per_1000": 0.07650895086750008,
      "ece": 0.06999327604725428,
      "brier_mean": 0.18379645503664693,
      "ordinal_mae": 0.45238568320233413,
      "load_seconds": 9.751196390999999,
      "lifecycle_seconds": 103.62086892127991,
      "lifecycle_resource_upper_usd": 0.17966069126327833,
      "account_metered_delta_usd": 0.0,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 9
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 10
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 7
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 6
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 10
        },
        "instruction_following": {
          "n": 10,
          "correct": 10
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 6
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 10
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 8
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 10
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 36
        },
        "noul": {
          "n": 36,
          "correct": 35
        },
        "score": {
          "n": 20,
          "correct": 15
        }
      },
      "input_tokens": {
        "min": 132,
        "max": 249,
        "mean": 174.01
      },
      "best_throughput_batch": 32,
      "batches": {
        "1": {
          "trials": 2,
          "decisions_per_second": 16.895303380420113,
          "gpu_usd_per_1000": 0.06494243977256946,
          "allocated_resource_upper_usd_per_1000": 0.07958161657598614,
          "agreement_with_primary": 1.0
        },
        "4": {
          "trials": 2,
          "decisions_per_second": 53.8675337439154,
          "gpu_usd_per_1000": 0.020368896549791626,
          "allocated_resource_upper_usd_per_1000": 0.024960406799901618,
          "agreement_with_primary": 0.98
        },
        "8": {
          "trials": 2,
          "decisions_per_second": 74.65512138154705,
          "gpu_usd_per_1000": 0.014697213023263923,
          "allocated_resource_upper_usd_per_1000": 0.01801022529564727,
          "agreement_with_primary": 0.99
        },
        "16": {
          "trials": 2,
          "decisions_per_second": 80.01493795676114,
          "gpu_usd_per_1000": 0.013712717278055562,
          "allocated_resource_upper_usd_per_1000": 0.0168038067627089,
          "agreement_with_primary": 1.0
        },
        "32": {
          "trials": 2,
          "decisions_per_second": 82.30437331736175,
          "gpu_usd_per_1000": 0.013331274852083323,
          "allocated_resource_upper_usd_per_1000": 0.016336380454183324,
          "agreement_with_primary": 0.99
        }
      },
      "trials": [
        {
          "batch_size": 1,
          "trial": 0,
          "n": 100,
          "seconds": 5.961270840000001,
          "decisions_per_second": 16.774946598467245,
          "batch_p50_s": 0.05904125449999853,
          "batch_p95_s": 0.06683061555000318,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 29,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 39.241379310344826,
            "gpu_util_p95_pct": 44.599999999999994,
            "peak_memory_mib": 8819.0,
            "mean_power_w": 207.40103448275863,
            "peak_temp_c": 49.0
          }
        },
        {
          "batch_size": 1,
          "trial": 1,
          "n": 100,
          "seconds": 5.876338435000001,
          "decisions_per_second": 17.017399713466297,
          "batch_p50_s": 0.057344464500001635,
          "batch_p95_s": 0.06505626450000025,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 29,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 40.37931034482759,
            "gpu_util_p95_pct": 45.0,
            "peak_memory_mib": 8819.0,
            "mean_power_w": 209.1155172413793,
            "peak_temp_c": 49.0
          }
        },
        {
          "batch_size": 4,
          "trial": 0,
          "n": 100,
          "seconds": 2.0198966999999968,
          "decisions_per_second": 49.50748223906706,
          "batch_p50_s": 0.07978670299999635,
          "batch_p95_s": 0.09077634019999863,
          "agreement_with_primary": 0.98,
          "telemetry": {
            "samples": 10,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 52.5,
            "gpu_util_p95_pct": 69.3,
            "peak_memory_mib": 9121.0,
            "mean_power_w": 290.00600000000003,
            "peak_temp_c": 53.0
          }
        },
        {
          "batch_size": 4,
          "trial": 1,
          "n": 100,
          "seconds": 1.6929148229999953,
          "decisions_per_second": 59.069717295517044,
          "batch_p50_s": 0.06694472400000251,
          "batch_p95_s": 0.07972505080000046,
          "agreement_with_primary": 0.98,
          "telemetry": {
            "samples": 9,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 62.111111111111114,
            "gpu_util_p95_pct": 83.0,
            "peak_memory_mib": 9121.0,
            "mean_power_w": 345.28333333333336,
            "peak_temp_c": 55.0
          }
        },
        {
          "batch_size": 8,
          "trial": 0,
          "n": 100,
          "seconds": 1.3905159150000017,
          "decisions_per_second": 71.91575365751918,
          "batch_p50_s": 0.11095407599999874,
          "batch_p95_s": 0.12477979479999987,
          "agreement_with_primary": 0.99,
          "telemetry": {
            "samples": 6,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 66.0,
            "gpu_util_p95_pct": 86.5,
            "peak_memory_mib": 9787.0,
            "mean_power_w": 385.22,
            "peak_temp_c": 57.0
          }
        },
        {
          "batch_size": 8,
          "trial": 1,
          "n": 100,
          "seconds": 1.2884697500000044,
          "decisions_per_second": 77.61144567033853,
          "batch_p50_s": 0.1014534299999994,
          "batch_p95_s": 0.11700675819999873,
          "agreement_with_primary": 0.99,
          "telemetry": {
            "samples": 7,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 71.85714285714286,
            "gpu_util_p95_pct": 87.7,
            "peak_memory_mib": 9787.0,
            "mean_power_w": 426.96142857142854,
            "peak_temp_c": 57.0
          }
        },
        {
          "batch_size": 16,
          "trial": 0,
          "n": 100,
          "seconds": 1.2823078350000046,
          "decisions_per_second": 77.98439444144833,
          "batch_p50_s": 0.1942493970000001,
          "batch_p95_s": 0.2427999425999992,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 6,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 74.66666666666667,
            "gpu_util_p95_pct": 97.0,
            "peak_memory_mib": 10167.0,
            "mean_power_w": 437.17833333333334,
            "peak_temp_c": 58.0
          }
        },
        {
          "batch_size": 16,
          "trial": 1,
          "n": 100,
          "seconds": 1.2172254409999965,
          "decisions_per_second": 82.15405021262637,
          "batch_p50_s": 0.19234473299999877,
          "batch_p95_s": 0.2117594688000011,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 6,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 76.66666666666667,
            "gpu_util_p95_pct": 99.0,
            "peak_memory_mib": 10167.0,
            "mean_power_w": 473.76166666666666,
            "peak_temp_c": 59.0
          }
        },
        {
          "batch_size": 32,
          "trial": 0,
          "n": 100,
          "seconds": 1.2135514260000022,
          "decisions_per_second": 82.40277079119004,
          "batch_p50_s": 0.36681938999999986,
          "batch_p95_s": 0.40932220584999573,
          "agreement_with_primary": 0.99,
          "telemetry": {
            "samples": 6,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 76.83333333333333,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 11571.0,
            "mean_power_w": 485.84333333333336,
            "peak_temp_c": 61.0
          }
        },
        {
          "batch_size": 32,
          "trial": 1,
          "n": 100,
          "seconds": 1.2164531039999957,
          "decisions_per_second": 82.20621055688503,
          "batch_p50_s": 0.3641740724999991,
          "batch_p95_s": 0.41581951445000126,
          "agreement_with_primary": 0.99,
          "telemetry": {
            "samples": 6,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 83.33333333333333,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 11571.0,
            "mean_power_w": 471.3233333333333,
            "peak_temp_c": 61.0
          }
        }
      ],
      "oom_batch_sizes": []
    },
    "jevfire": {
      "name": "Jevfire Qwen3.8-27B FP8",
      "source": "https://github.com/kikoncuo/jevfire",
      "model_repo": "Qwen/Qwen3.8-27B-FP8",
      "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
      "code_commit": "5df83b558bf635e006d31f8f2fc2798d0e3ff051",
      "protocol_sha256": "cacdb1b3180d584b5a9f3e6dabbde5f21d9b1145724e322b8750f9f30c992e88",
      "context_tokens": 16384,
      "dtype": "fp8",
      "gpu": "NVIDIA H100 80GB HBM3",
      "probability_mode": "native conditional option softmax",
      "correct": 94,
      "n": 100,
      "valid": 100,
      "operational_success": 100,
      "p50_s": 0.10787691150000711,
      "p95_s": 0.11089101779998883,
      "mean_s": 0.10931126653999924,
      "serial_gpu_usd_per_1000": 0.11993875078694363,
      "ece": 0.07410845813580216,
      "brier_mean": 0.09697763053869925,
      "ordinal_mae": 0.294316137708567,
      "load_seconds": 91.638258061,
      "lifecycle_seconds": 234.31250500679016,
      "lifecycle_resource_upper_usd": 0.35538284957422156,
      "account_metered_delta_usd": 0.3700000000000001,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 10
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 10
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 8
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 9
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 10
        },
        "instruction_following": {
          "n": 10,
          "correct": 10
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 8
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 9
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 10
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 10
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 41
        },
        "noul": {
          "n": 36,
          "correct": 36
        },
        "score": {
          "n": 20,
          "correct": 17
        }
      },
      "input_tokens": {
        "min": 154,
        "max": 286,
        "mean": 202.42
      },
      "best_throughput_batch": 16,
      "batches": {
        "1": {
          "trials": 2,
          "decisions_per_second": 9.30057877005035,
          "gpu_usd_per_1000": 0.1179735422225,
          "allocated_resource_upper_usd_per_1000": 0.14456686930982005,
          "agreement_with_primary": 1.0
        },
        "4": {
          "trials": 2,
          "decisions_per_second": 17.202315763775257,
          "gpu_usd_per_1000": 0.06378340203083353,
          "allocated_resource_upper_usd_per_1000": 0.07816131118735357,
          "agreement_with_primary": 0.99
        },
        "8": {
          "trials": 2,
          "decisions_per_second": 20.96800368972709,
          "gpu_usd_per_1000": 0.05232840657881929,
          "allocated_resource_upper_usd_per_1000": 0.06412415676053593,
          "agreement_with_primary": 0.97
        },
        "16": {
          "trials": 2,
          "decisions_per_second": 22.89639921245435,
          "gpu_usd_per_1000": 0.04792116926513909,
          "allocated_resource_upper_usd_per_1000": 0.058723450053412465,
          "agreement_with_primary": 0.97
        },
        "32": {
          "trials": 2,
          "decisions_per_second": 13.532566921530098,
          "gpu_usd_per_1000": 0.08108012534388868,
          "allocated_resource_upper_usd_per_1000": 0.099357022459382,
          "agreement_with_primary": 0.97
        }
      },
      "trials": [
        {
          "batch_size": 1,
          "trial": 0,
          "n": 100,
          "seconds": 10.667722179000009,
          "decisions_per_second": 9.374072395403719,
          "batch_p50_s": 0.10644178249999925,
          "batch_p95_s": 0.11011467525000569,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 52,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 45.05769230769231,
            "gpu_util_p95_pct": 57.0,
            "peak_memory_mib": 68787.0,
            "mean_power_w": 291.9973076923077,
            "peak_temp_c": 42.0
          }
        },
        {
          "batch_size": 1,
          "trial": 1,
          "n": 100,
          "seconds": 10.836315896999992,
          "decisions_per_second": 9.228228574222792,
          "batch_p50_s": 0.1086801269999853,
          "batch_p95_s": 0.11127929939999603,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 53,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 43.924528301886795,
            "gpu_util_p95_pct": 55.8,
            "peak_memory_mib": 68787.0,
            "mean_power_w": 291.19943396226415,
            "peak_temp_c": 42.0
          }
        },
        {
          "batch_size": 4,
          "trial": 0,
          "n": 100,
          "seconds": 5.819564543000013,
          "decisions_per_second": 17.183416260978444,
          "batch_p50_s": 0.22405242199999975,
          "batch_p95_s": 0.28768241839999853,
          "agreement_with_primary": 0.99,
          "telemetry": {
            "samples": 29,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 65.27586206896552,
            "gpu_util_p95_pct": 98.19999999999999,
            "peak_memory_mib": 68787.0,
            "mean_power_w": 409.98655172413794,
            "peak_temp_c": 49.0
          }
        },
        {
          "batch_size": 4,
          "trial": 1,
          "n": 100,
          "seconds": 5.806777093000022,
          "decisions_per_second": 17.221256886293848,
          "batch_p50_s": 0.22555150299999127,
          "batch_p95_s": 0.2853911145999972,
          "agreement_with_primary": 0.99,
          "telemetry": {
            "samples": 28,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 66.42857142857143,
            "gpu_util_p95_pct": 93.6,
            "peak_memory_mib": 68787.0,
            "mean_power_w": 418.5982142857143,
            "peak_temp_c": 50.0
          }
        },
        {
          "batch_size": 8,
          "trial": 0,
          "n": 100,
          "seconds": 4.770451407999985,
          "decisions_per_second": 20.962376816647016,
          "batch_p50_s": 0.3960815249999996,
          "batch_p95_s": 0.483085400799996,
          "agreement_with_primary": 0.97,
          "telemetry": {
            "samples": 26,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 72.34615384615384,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 69071.0,
            "mean_power_w": 522.0365384615385,
            "peak_temp_c": 54.0
          }
        },
        {
          "batch_size": 8,
          "trial": 1,
          "n": 100,
          "seconds": 4.767891056999986,
          "decisions_per_second": 20.973633584430345,
          "batch_p50_s": 0.4008234529999868,
          "batch_p95_s": 0.48059442600001034,
          "agreement_with_primary": 0.97,
          "telemetry": {
            "samples": 24,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 76.04166666666667,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 69071.0,
            "mean_power_w": 538.00625,
            "peak_temp_c": 56.0
          }
        },
        {
          "batch_size": 16,
          "trial": 0,
          "n": 100,
          "seconds": 4.365364913000008,
          "decisions_per_second": 22.907592376115254,
          "batch_p50_s": 0.808564414000017,
          "batch_p95_s": 0.8639061952999952,
          "agreement_with_primary": 0.97,
          "telemetry": {
            "samples": 24,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 78.5,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 69507.0,
            "mean_power_w": 580.99375,
            "peak_temp_c": 58.0
          }
        },
        {
          "batch_size": 16,
          "trial": 1,
          "n": 100,
          "seconds": 4.369633029000028,
          "decisions_per_second": 22.885216981913143,
          "batch_p50_s": 0.8063499390000004,
          "batch_p95_s": 0.8678293303000004,
          "agreement_with_primary": 0.97,
          "telemetry": {
            "samples": 24,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 78.875,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 69507.0,
            "mean_power_w": 589.3579166666667,
            "peak_temp_c": 59.0
          }
        },
        {
          "batch_size": 32,
          "trial": 0,
          "n": 100,
          "seconds": 7.413511374999985,
          "decisions_per_second": 13.48888467848344,
          "batch_p50_s": 2.312955328000001,
          "batch_p95_s": 2.5253956803999955,
          "agreement_with_primary": 0.97,
          "telemetry": {
            "samples": 37,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 93.5945945945946,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 70387.0,
            "mean_power_w": 633.437027027027,
            "peak_temp_c": 57.0
          }
        },
        {
          "batch_size": 32,
          "trial": 1,
          "n": 100,
          "seconds": 7.3656507129999795,
          "decisions_per_second": 13.576533003866903,
          "batch_p50_s": 2.307630520999993,
          "batch_p95_s": 2.4900228635999935,
          "agreement_with_primary": 0.97,
          "telemetry": {
            "samples": 39,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 89.2051282051282,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 70387.0,
            "mean_power_w": 632.0835897435898,
            "peak_temp_c": 58.0
          }
        }
      ],
      "oom_batch_sizes": []
    },
    "diffusion": {
      "name": "JoshuaSP DiffusionGemma 26B-A4B · 1 step",
      "source": "https://github.com/JoshuaSP/open-jev",
      "model_repo": "google/diffusiongemma-26B-A4B-it",
      "model_revision": "f7f5b7f5fa82ffc52addd066915886d497f5517b",
      "code_commit": "50d32c7e542dd714c89bb223e8fc8c0b2b3d0d9c",
      "protocol_sha256": "cacdb1b3180d584b5a9f3e6dabbde5f21d9b1145724e322b8750f9f30c992e88",
      "context_tokens": 16384,
      "dtype": "bfloat16",
      "gpu": "NVIDIA H100 80GB HBM3",
      "probability_mode": "label only; no fabricated confidence",
      "correct": 96,
      "n": 100,
      "valid": 100,
      "operational_success": 100,
      "p50_s": 0.2442433664999939,
      "p95_s": 0.3329541904000044,
      "mean_s": 0.29053775290999906,
      "serial_gpu_usd_per_1000": 0.3187844788873601,
      "ece": null,
      "brier_mean": null,
      "ordinal_mae": null,
      "load_seconds": 56.605311045,
      "lifecycle_seconds": 393.9470958709717,
      "lifecycle_resource_upper_usd": 0.5700204259000621,
      "account_metered_delta_usd": 1.3599999999999999,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 10
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 10
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 9
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 10
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 9
        },
        "instruction_following": {
          "n": 10,
          "correct": 10
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 10
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 9
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 9
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 10
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 42
        },
        "noul": {
          "n": 36,
          "correct": 36
        },
        "score": {
          "n": 20,
          "correct": 18
        }
      },
      "input_tokens": {
        "min": 114,
        "max": 235,
        "mean": 161.02
      },
      "best_throughput_batch": 8,
      "batches": {
        "1": {
          "trials": 2,
          "decisions_per_second": 6.464680075340953,
          "gpu_usd_per_1000": 0.16972568007000008,
          "allocated_resource_upper_usd_per_1000": 0.2079848561546401,
          "agreement_with_primary": 1.0
        },
        "2": {
          "trials": 2,
          "decisions_per_second": 9.688054836277983,
          "gpu_usd_per_1000": 0.11325516223479179,
          "allocated_resource_upper_usd_per_1000": 0.13878488285602184,
          "agreement_with_primary": 1.0
        },
        "4": {
          "trials": 2,
          "decisions_per_second": 16.490313342629054,
          "gpu_usd_per_1000": 0.06653737860673616,
          "allocated_resource_upper_usd_per_1000": 0.08153608288811284,
          "agreement_with_primary": 0.99
        },
        "8": {
          "trials": 2,
          "decisions_per_second": 22.68498255530686,
          "gpu_usd_per_1000": 0.04836777897215271,
          "allocated_resource_upper_usd_per_1000": 0.05927073350298937,
          "agreement_with_primary": 0.98
        }
      },
      "trials": [
        {
          "batch_size": 1,
          "trial": 0,
          "n": 100,
          "seconds": 15.358836076000003,
          "decisions_per_second": 6.510910039352645,
          "batch_p50_s": 0.15283191700000032,
          "batch_p95_s": 0.15848309335000152,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 76,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 53.30263157894737,
            "gpu_util_p95_pct": 57.0,
            "peak_memory_mib": 52143.0,
            "mean_power_w": 258.05092105263157,
            "peak_temp_c": 48.0
          }
        },
        {
          "batch_size": 1,
          "trial": 1,
          "n": 100,
          "seconds": 15.578503076000004,
          "decisions_per_second": 6.419101983813735,
          "batch_p50_s": 0.15393587949999699,
          "batch_p95_s": 0.15786447789999017,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 77,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 52.44155844155844,
            "gpu_util_p95_pct": 57.0,
            "peak_memory_mib": 52143.0,
            "mean_power_w": 257.9733766233766,
            "peak_temp_c": 48.0
          }
        },
        {
          "batch_size": 2,
          "trial": 0,
          "n": 100,
          "seconds": 12.420667967000014,
          "decisions_per_second": 8.051096790099058,
          "batch_p50_s": 0.26679021649998447,
          "batch_p95_s": 0.2840446504999917,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 61,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 41.950819672131146,
            "gpu_util_p95_pct": 65.0,
            "peak_memory_mib": 54299.0,
            "mean_power_w": 251.18688524590164,
            "peak_temp_c": 50.0
          }
        },
        {
          "batch_size": 2,
          "trial": 1,
          "n": 100,
          "seconds": 8.223310972000007,
          "decisions_per_second": 12.160551916435528,
          "batch_p50_s": 0.16343110000001104,
          "batch_p95_s": 0.17468635130000507,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 40,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 63.4,
            "gpu_util_p95_pct": 71.0,
            "peak_memory_mib": 54299.0,
            "mean_power_w": 317.26625,
            "peak_temp_c": 50.0
          }
        },
        {
          "batch_size": 4,
          "trial": 0,
          "n": 100,
          "seconds": 7.21145851899999,
          "decisions_per_second": 13.86682038543667,
          "batch_p50_s": 0.3028026519999969,
          "batch_p95_s": 0.3199036980000017,
          "agreement_with_primary": 0.99,
          "telemetry": {
            "samples": 35,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 50.02857142857143,
            "gpu_util_p95_pct": 89.0,
            "peak_memory_mib": 58563.0,
            "mean_power_w": 303.47828571428573,
            "peak_temp_c": 51.0
          }
        },
        {
          "batch_size": 4,
          "trial": 1,
          "n": 100,
          "seconds": 4.916873784000018,
          "decisions_per_second": 20.338126295901606,
          "batch_p50_s": 0.19515853400000083,
          "batch_p95_s": 0.20157419599998433,
          "agreement_with_primary": 0.99,
          "telemetry": {
            "samples": 24,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 70.25,
            "gpu_util_p95_pct": 82.24999999999999,
            "peak_memory_mib": 58563.0,
            "mean_power_w": 390.63916666666665,
            "peak_temp_c": 53.0
          }
        },
        {
          "batch_size": 8,
          "trial": 0,
          "n": 100,
          "seconds": 4.989236309000006,
          "decisions_per_second": 20.043147649593497,
          "batch_p50_s": 0.41963702900000044,
          "batch_p95_s": 0.4266038233999893,
          "agreement_with_primary": 0.98,
          "telemetry": {
            "samples": 25,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 49.88,
            "gpu_util_p95_pct": 97.99999999999997,
            "peak_memory_mib": 67177.0,
            "mean_power_w": 372.922,
            "peak_temp_c": 55.0
          }
        },
        {
          "batch_size": 8,
          "trial": 1,
          "n": 100,
          "seconds": 3.8271689719999813,
          "decisions_per_second": 26.12897437547487,
          "batch_p50_s": 0.30260834599999953,
          "batch_p95_s": 0.3158559308000008,
          "agreement_with_primary": 0.98,
          "telemetry": {
            "samples": 19,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 80.0,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 67177.0,
            "mean_power_w": 437.7026315789474,
            "peak_temp_c": 57.0
          }
        }
      ],
      "oom_batch_sizes": [
        16
      ]
    }
  },
  "experiment_cost": {
    "observed_metered_increase_usd": 2.62,
    "budget_reservation_estimate_usd": 4.377138787978384,
    "attempt_count": 6,
    "note": "Includes failed setup attempts. Metering may lag; account deltas are not per-model invoices. Reservation estimate includes deliberately conservative lifetime resources and build contingency, not a measured charge.",
    "active_containers_after_run": 0
  },
  "failed_setup_attempts": [
    {
      "model": "semif",
      "phase": "CPU image build",
      "elapsed_s": 103.21167993545532,
      "resource_lifetime_estimate_usd": 0.17911051105401254,
      "observed_account_metered_delta_usd": 0.0
    },
    {
      "model": "jevfire",
      "phase": "GPU initialization",
      "elapsed_s": 647.1089789867401,
      "resource_lifetime_estimate_usd": 0.9104106419777077,
      "observed_account_metered_delta_usd": 0.3500000000000001
    },
    {
      "model": "jevfire",
      "phase": "Request validation",
      "elapsed_s": 477.64259099960327,
      "resource_lifetime_estimate_usd": 0.6825536682091025,
      "observed_account_metered_delta_usd": 0.49
    }
  ]
}