{
  "schema": 1,
  "date": "2026-09-23",
  "benchmark": "Creative Judge v1",
  "questions": 100,
  "protocol_sha256": "99b22cbbb92189c0131c4d75c001c4a8701c83ec193cab0f3b9af5ad68093f09",
  "released_temperature": 2.207568021892729,
  "hardware": "1 × NVIDIA H100 per run; sequential deployments",
  "rates": {
    "gpu_hour_usd": 3.95,
    "cpu_core_hour_usd": 0.0473,
    "memory_gib_hour_usd": 0.008,
    "source": "Modal account billing rates API, September 23, 2026"
  },
  "quality_method": "One serial pass after three unrelated warmups. Frozen private questions and unchanged exact-label grading; fixed denominator includes failures. No checkpoint selection or calibration fitting on the suite.",
  "throughput_method": "Two repeated 100-question passes per batch size, original order, prefix caching disabled. Sweep stops at first OOM. Accuracy is from the serial pass; batch agreement is reported separately. Both passes remain in chart costs, including first-pass setup overhead; these are not steady-state-only estimates.",
  "cost_method": "GPU-only costs use measured duration × account GPU rate. Measured inference resource ceiling adds the configured maximum 8 CPU cores and 64 GiB RAM. Startup, build and idle time are separate. Lifecycle resource totals are conservative estimates, not verified maxima for CPU image builds. Metered account deltas can lag, include image builds, and are not per-model invoices.",
  "privacy": "Only model identities, coarse category counts, numerical aggregates and protocol commitments are public. No questions, expected labels, item IDs, responses or trial prediction lists.",
  "models": {
    "autojev": {
      "name": "AutoJev-27B",
      "source": "https://huggingface.co/denis-pplx/autojev-27b",
      "model_repo": "denis-pplx/autojev-27b",
      "model_revision": "6f5b557e037f5edb25c7dc92dbc6553e5a19c015",
      "code_commit": "ee63c1515980491a742f0bd0685c8dc5ca1f00c3",
      "protocol_sha256": "99b22cbbb92189c0131c4d75c001c4a8701c83ec193cab0f3b9af5ad68093f09",
      "context_tokens": 8192,
      "dtype": "bfloat16",
      "gpu": "NVIDIA H100 80GB HBM3",
      "probability_mode": "native decision readout softmax; released temperature 2.207568021892729",
      "correct": 99,
      "n": 100,
      "valid": 100,
      "operational_success": 100,
      "p50_s": 0.10129723800000079,
      "p95_s": 0.11264144520000983,
      "mean_s": 0.10365273339000068,
      "serial_gpu_usd_per_1000": 0.11373008246958409,
      "ece": 0.03708954945687659,
      "brier_mean": 0.02818015464936204,
      "ordinal_mae": 0.08697217581996117,
      "load_seconds": 56.65859411300001,
      "lifecycle_seconds": 901.0103812217712,
      "lifecycle_resource_upper_usd": 1.2517951934948763,
      "account_metered_delta_usd": -1.5199999999999996,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 10
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 10
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 10
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 10
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 10
        },
        "instruction_following": {
          "n": 10,
          "correct": 10
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 9
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 10
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 10
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 10
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 43
        },
        "noul": {
          "n": 36,
          "correct": 36
        },
        "score": {
          "n": 20,
          "correct": 20
        }
      },
      "input_tokens": {
        "min": 117,
        "max": 225,
        "mean": 153.43
      },
      "best_throughput_batch": 4,
      "batches": {
        "1": {
          "trials": 2,
          "decisions_per_second": 10.384455629990216,
          "gpu_usd_per_1000": 0.10566006166499997,
          "allocated_resource_upper_usd_per_1000": 0.12947771202107997,
          "agreement_with_primary": 1.0
        },
        "2": {
          "trials": 2,
          "decisions_per_second": 18.229673039035884,
          "gpu_usd_per_1000": 0.060188804257361006,
          "allocated_resource_upper_usd_per_1000": 0.07375642737400766,
          "agreement_with_primary": 1.0
        },
        "4": {
          "trials": 2,
          "decisions_per_second": 32.50712399676851,
          "gpu_usd_per_1000": 0.03375328504395823,
          "allocated_resource_upper_usd_per_1000": 0.041361873652348204,
          "agreement_with_primary": 1.0
        },
        "8": {
          "trials": 2,
          "decisions_per_second": 13.672851317826288,
          "gpu_usd_per_1000": 0.08024823767312485,
          "allocated_resource_upper_usd_per_1000": 0.09833761256531483,
          "agreement_with_primary": 1.0
        },
        "16": {
          "trials": 2,
          "decisions_per_second": 13.656089091384036,
          "gpu_usd_per_1000": 0.0803467387243751,
          "allocated_resource_upper_usd_per_1000": 0.09845831749910515,
          "agreement_with_primary": 1.0
        },
        "32": {
          "trials": 2,
          "decisions_per_second": 8.243140172617023,
          "gpu_usd_per_1000": 0.1331073109574306,
          "allocated_resource_upper_usd_per_1000": 0.16311205771097395,
          "agreement_with_primary": 1.0
        }
      },
      "trials": [
        {
          "batch_size": 1,
          "trial": 0,
          "n": 100,
          "seconds": 9.733065283000002,
          "decisions_per_second": 10.274255549755978,
          "batch_p50_s": 0.09696634449999664,
          "batch_p95_s": 0.10364004810000083,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 48,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 43.604166666666664,
            "gpu_util_p95_pct": 49.0,
            "peak_memory_mib": 50571.0,
            "mean_power_w": 282.714375,
            "peak_temp_c": 45.0
          }
        },
        {
          "batch_size": 1,
          "trial": 1,
          "n": 100,
          "seconds": 9.526490260999992,
          "decisions_per_second": 10.49704531892347,
          "batch_p50_s": 0.09475702999999669,
          "batch_p95_s": 0.10163390655000341,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 48,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 43.979166666666664,
            "gpu_util_p95_pct": 49.0,
            "peak_memory_mib": 50571.0,
            "mean_power_w": 289.9554166666667,
            "peak_temp_c": 46.0
          }
        },
        {
          "batch_size": 2,
          "trial": 0,
          "n": 100,
          "seconds": 6.002922937999983,
          "decisions_per_second": 16.658551347873438,
          "batch_p50_s": 0.10056388299999242,
          "batch_p95_s": 0.11695453984997925,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 30,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 49.833333333333336,
            "gpu_util_p95_pct": 61.0,
            "peak_memory_mib": 50811.0,
            "mean_power_w": 358.1003333333333,
            "peak_temp_c": 51.0
          }
        },
        {
          "batch_size": 2,
          "trial": 1,
          "n": 100,
          "seconds": 4.968200875999997,
          "decisions_per_second": 20.12801062112289,
          "batch_p50_s": 0.09891533949999598,
          "batch_p95_s": 0.10352830095000201,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 24,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 55.541666666666664,
            "gpu_util_p95_pct": 66.69999999999999,
            "peak_memory_mib": 50635.0,
            "mean_power_w": 393.78,
            "peak_temp_c": 52.0
          }
        },
        {
          "batch_size": 4,
          "trial": 0,
          "n": 100,
          "seconds": 3.4827455209999982,
          "decisions_per_second": 28.712979285172427,
          "batch_p50_s": 0.10557036599999492,
          "batch_p95_s": 0.13455548620000854,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 17,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 64.3529411764706,
            "gpu_util_p95_pct": 93.6,
            "peak_memory_mib": 50885.0,
            "mean_power_w": 473.32529411764705,
            "peak_temp_c": 60.0
          }
        },
        {
          "batch_size": 4,
          "trial": 1,
          "n": 100,
          "seconds": 2.6697520059999817,
          "decisions_per_second": 37.45666255714415,
          "batch_p50_s": 0.10177525699998569,
          "batch_p95_s": 0.13032829640000043,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 14,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 77.0,
            "gpu_util_p95_pct": 94.0,
            "peak_memory_mib": 50915.0,
            "mean_power_w": 567.9314285714286,
            "peak_temp_c": 62.0
          }
        },
        {
          "batch_size": 8,
          "trial": 0,
          "n": 100,
          "seconds": 12.252746575999993,
          "decisions_per_second": 8.161435428353224,
          "batch_p50_s": 0.18473365200000558,
          "batch_p95_s": 4.170576569,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 60,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 42.016666666666666,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 51211.0,
            "mean_power_w": 276.944,
            "peak_temp_c": 60.0
          }
        },
        {
          "batch_size": 8,
          "trial": 1,
          "n": 100,
          "seconds": 2.374780290999979,
          "decisions_per_second": 42.10915863626766,
          "batch_p50_s": 0.18408835700000736,
          "batch_p95_s": 0.2334698322000122,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 12,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 79.0,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 51227.0,
            "mean_power_w": 619.8675,
            "peak_temp_c": 62.0
          }
        },
        {
          "batch_size": 16,
          "trial": 0,
          "n": 100,
          "seconds": 12.291621437000003,
          "decisions_per_second": 8.1356231570053,
          "batch_p50_s": 0.35755821899999773,
          "batch_p95_s": 7.353173149999982,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 61,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 42.0655737704918,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 51541.0,
            "mean_power_w": 283.81360655737706,
            "peak_temp_c": 61.0
          }
        },
        {
          "batch_size": 16,
          "trial": 1,
          "n": 100,
          "seconds": 2.353860052000016,
          "decisions_per_second": 42.4834092897887,
          "batch_p50_s": 0.36215983299999266,
          "batch_p95_s": 0.42084805380001167,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 12,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 89.41666666666667,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 52067.0,
            "mean_power_w": 598.6949999999999,
            "peak_temp_c": 62.0
          }
        },
        {
          "batch_size": 32,
          "trial": 0,
          "n": 100,
          "seconds": 21.911353441000017,
          "decisions_per_second": 4.563844048669413,
          "batch_p50_s": 5.013515638999991,
          "batch_p95_s": 11.406179473449992,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 108,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 34.666666666666664,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 52359.0,
            "mean_power_w": 253.4,
            "peak_temp_c": 60.0
          }
        },
        {
          "batch_size": 32,
          "trial": 1,
          "n": 100,
          "seconds": 2.3512450119999926,
          "decisions_per_second": 42.530659071952265,
          "batch_p50_s": 0.6968561580000028,
          "batch_p95_s": 0.8206698786499871,
          "agreement_with_primary": 1.0,
          "telemetry": {
            "samples": 12,
            "sample_period_ms": 200,
            "gpu_util_mean_pct": 89.58333333333333,
            "gpu_util_p95_pct": 100.0,
            "peak_memory_mib": 52947.0,
            "mean_power_w": 618.3283333333334,
            "peak_temp_c": 62.0
          }
        }
      ],
      "fastest_repeat_pass": {
        "batch_size": 32,
        "n": 100,
        "decisions_per_second": 42.530659071952265,
        "gpu_usd_per_1000": 0.025798382770555475,
        "allocated_resource_upper_usd_per_1000": 0.03161379543356879,
        "agreement_with_primary": 1.0,
        "gpu_util_mean_pct": 89.58333333333333,
        "note": "Diagnostic second pass at the fastest repeat batch; 100 decisions. Chart costs retain both passes and their first-pass overhead."
      },
      "oom_batch_sizes": []
    }
  },
  "experiment_cost": {
    "account_metered_before_usd": 5.39,
    "account_metered_after_usd": 3.87,
    "maximum_observed_account_metered_usd": 5.59,
    "net_account_metered_change_usd": -1.5199999999999996,
    "budget_reservation_estimate_usd": 1.5017951934948763,
    "attempt_count": 1,
    "note": "Metering snapshots revised during the experiment and may lag. A negative account change is a reporting revision, not negative inference cost; account deltas are not per-model invoices. Reservation estimate includes deliberately conservative lifetime resources and build contingency, not a measured charge.",
    "active_containers_after_run": 0
  }
}