{
  "schema_version": 1,
  "name": "Doug Does Creative Judge v1",
  "status": "private_holdout",
  "questions": 100,
  "protocol_sha256": "8514f25341c8b60cc90ce62b799e238284050750e4b3557958415fae515206de",
  "method": "Original AI-authored and independently AI-reviewed synthetic judge questions; one native-API response each, no training or calibration on the holdout. Fixed100 denominator; native JevBench probability validation and lexical argmax scoring. Separate from the official public JevBench dataset.",
  "privacy": "Only aggregate results are public. Questions, answer keys, rationales and raw responses are retained privately.",
  "family_counts": {
    "grounding_and_citations": 10,
    "instruction_following": 10,
    "pairwise_answer_quality": 10,
    "agent_and_tool_judging": 10,
    "prompt_injection_resistance": 10,
    "code_and_query_judging": 10,
    "summary_and_extraction": 10,
    "appropriate_refusal_and_privacy": 10,
    "conversation_and_pragmatics": 10,
    "uncertainty_and_rubric_scoring": 10
  },
  "type_counts": {
    "noul": 36,
    "choice": 44,
    "score": 20
  },
  "models": {
    "laya": {
      "model": "Laya English base",
      "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
      "planned": 100,
      "received": 100,
      "correct": 50,
      "accuracy": 0.5,
      "operational_success": 100,
      "strict_valid": 100,
      "valid": 100,
      "renormalized": 0,
      "brier_mean": 0.6961804192,
      "ece": 0.2511749999999999,
      "ordinal_mae": 0.7541450000000001,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 4
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 5
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 4
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 6
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 3
        },
        "instruction_following": {
          "n": 10,
          "correct": 5
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 5
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 6
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 6
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 6
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 21
        },
        "noul": {
          "n": 36,
          "correct": 19
        },
        "score": {
          "n": 20,
          "correct": 10
        }
      },
      "truncation": {
        "state_truncated": 0,
        "instruction_truncated": 0,
        "options_truncated": 0
      },
      "truncation_audit_available": true
    },
    "laya-typed": {
      "model": "Laya typed-decisions",
      "checkpoint_revision": "f9ab0b228f0fc0f14d873dbc99038f135c2da1b2",
      "planned": 100,
      "received": 100,
      "correct": 51,
      "accuracy": 0.51,
      "operational_success": 100,
      "strict_valid": 100,
      "valid": 100,
      "renormalized": 0,
      "brier_mean": 0.6311231232,
      "ece": 0.12529600000000002,
      "ordinal_mae": 0.8123000000000001,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 2
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 6
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 5
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 8
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 3
        },
        "instruction_following": {
          "n": 10,
          "correct": 4
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 5
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 7
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 5
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 6
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 23
        },
        "noul": {
          "n": 36,
          "correct": 21
        },
        "score": {
          "n": 20,
          "correct": 7
        }
      },
      "truncation": {
        "state_truncated": 0,
        "instruction_truncated": 0,
        "options_truncated": 0
      },
      "truncation_audit_available": true
    },
    "kev": {
      "model": "Kev-0.8B",
      "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8",
      "planned": 100,
      "received": 100,
      "correct": 58,
      "accuracy": 0.58,
      "operational_success": 100,
      "strict_valid": 80,
      "valid": 100,
      "renormalized": 20,
      "brier_mean": 0.5630201031312052,
      "ece": 0.1407727472747274,
      "ordinal_mae": 0.6948020802080207,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 5
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 8
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 5
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 6
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 3
        },
        "instruction_following": {
          "n": 10,
          "correct": 5
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 5
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 7
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 7
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 7
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 24
        },
        "noul": {
          "n": 36,
          "correct": 22
        },
        "score": {
          "n": 20,
          "correct": 12
        }
      },
      "truncation": {
        "state_truncated": 0,
        "instruction_truncated": 0,
        "options_truncated": 0
      },
      "truncation_audit_available": true
    },
    "2B": {
      "model": "Open-Jev 2B",
      "checkpoint_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785",
      "planned": 100,
      "received": 100,
      "correct": 62,
      "accuracy": 0.62,
      "operational_success": 100,
      "strict_valid": 100,
      "valid": 100,
      "renormalized": 0,
      "brier_mean": 0.5339223541034063,
      "ece": 0.1429207121654829,
      "ordinal_mae": 0.70288844641531,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 6
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 7
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 5
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 8
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 7
        },
        "instruction_following": {
          "n": 10,
          "correct": 5
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 2
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 5
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 8
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 9
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 21
        },
        "noul": {
          "n": 36,
          "correct": 29
        },
        "score": {
          "n": 20,
          "correct": 12
        }
      },
      "truncation": null,
      "truncation_audit_available": false
    },
    "9B": {
      "model": "Open-Jev 9B",
      "checkpoint_revision": "47e966881e489511c0c7f5633a9e1960a676a551",
      "planned": 100,
      "received": 100,
      "correct": 78,
      "accuracy": 0.78,
      "operational_success": 100,
      "strict_valid": 100,
      "valid": 100,
      "renormalized": 0,
      "brier_mean": 0.29437034663246514,
      "ece": 0.09908060373341951,
      "ordinal_mae": 0.5593932022742534,
      "families": {
        "agent_and_tool_judging": {
          "n": 10,
          "correct": 9
        },
        "appropriate_refusal_and_privacy": {
          "n": 10,
          "correct": 8
        },
        "code_and_query_judging": {
          "n": 10,
          "correct": 7
        },
        "conversation_and_pragmatics": {
          "n": 10,
          "correct": 7
        },
        "grounding_and_citations": {
          "n": 10,
          "correct": 8
        },
        "instruction_following": {
          "n": 10,
          "correct": 6
        },
        "pairwise_answer_quality": {
          "n": 10,
          "correct": 7
        },
        "prompt_injection_resistance": {
          "n": 10,
          "correct": 7
        },
        "summary_and_extraction": {
          "n": 10,
          "correct": 9
        },
        "uncertainty_and_rubric_scoring": {
          "n": 10,
          "correct": 10
        }
      },
      "types": {
        "choice": {
          "n": 44,
          "correct": 35
        },
        "noul": {
          "n": 36,
          "correct": 31
        },
        "score": {
          "n": 20,
          "correct": 12
        }
      },
      "truncation": null,
      "truncation_audit_available": false
    }
  }
}