{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://manifoldmemory.ai/warrant-leaderboard/submission_schema.json",
  "title": "Warrant Reader Leaderboard — Submission Row v0",
  "description": "One reader configuration evaluated against the frozen retrieval artifact, judged by the canonical GPT-4o K=5 3-of-5 majority-vote protocol.",
  "type": "object",
  "required": [
    "schema_version",
    "submitted_at",
    "track",
    "reader",
    "model",
    "params",
    "pipeline",
    "evidence",
    "judge",
    "overall_acc",
    "overall_n",
    "ci_lo",
    "ci_hi",
    "refusal_rate",
    "qtype_acc"
  ],
  "properties": {
    "schema_version": {
      "type": "string",
      "const": "v0",
      "description": "Schema version. Bump on breaking changes."
    },
    "submitted_at": {
      "type": "string",
      "format": "date-time",
      "description": "ISO-8601 UTC timestamp of submission."
    },
    "track": {
      "type": "string",
      "enum": [
        "canonical",
        "experimental",
        "retrieval-only",
        "no-retrieval",
        "open-swap",
        "closed-reference"
      ],
      "description": "Where this row sits on the leaderboard. 'canonical' is reserved for the maintainers' published baseline."
    },
    "reader": {
      "type": "string",
      "description": "Display name (e.g. 'Llama-3.1-8B-Instruct')."
    },
    "model": {
      "type": "string",
      "description": "Canonical model identifier (HF repo id, API model id, or local weight hash)."
    },
    "params": {
      "type": "string",
      "description": "Active parameter count, human-readable (e.g. '8B', '4B (active)', 'closed')."
    },
    "pipeline": {
      "type": "string",
      "description": "Short human-readable description of the reader pipeline (router, fallbacks, prompt family)."
    },
    "evidence": {
      "type": "object",
      "required": ["artifact", "artifact_sha256", "top_k", "r_at_5"],
      "properties": {
        "artifact": {
          "type": "string",
          "const": "frozen_retrieval_topK_500q.v1.jsonl",
          "description": "Filename of the frozen retrieval artifact this submission consumed. Must match the published manifest."
        },
        "artifact_sha256": {
          "type": "string",
          "pattern": "^[0-9a-f]{64}$",
          "description": "SHA-256 of the consumed artifact, verifying no retrieval was substituted."
        },
        "top_k": {
          "type": "integer",
          "const": 10
        },
        "r_at_5": {
          "type": "number",
          "minimum": 0.0,
          "maximum": 1.0,
          "description": "Recall@5 of the frozen retrieval artifact. Must equal 0.962 (481/500) for v0."
        }
      },
      "additionalProperties": false
    },
    "judge": {
      "type": "object",
      "required": ["model", "n_seeds", "threshold", "protocol"],
      "properties": {
        "model": { "type": "string", "const": "gpt-4o" },
        "n_seeds": { "type": "integer", "const": 5 },
        "threshold": { "type": "integer", "const": 3 },
        "protocol": { "type": "string", "const": "K=5 3-of-5 majority vote" }
      },
      "additionalProperties": false
    },
    "overall_acc": {
      "type": "number",
      "minimum": 0.0,
      "maximum": 1.0,
      "description": "Overall accuracy: judged-correct / total."
    },
    "overall_n": {
      "type": "integer",
      "const": 500
    },
    "ci_lo": {
      "type": "number",
      "minimum": 0.0,
      "maximum": 1.0,
      "description": "95% Wilson lower bound."
    },
    "ci_hi": {
      "type": "number",
      "minimum": 0.0,
      "maximum": 1.0,
      "description": "95% Wilson upper bound."
    },
    "refusal_rate": {
      "type": "number",
      "minimum": 0.0,
      "maximum": 1.0,
      "description": "Fraction of questions where the reader refused (returned a refusal token instead of an answer)."
    },
    "qtype_acc": {
      "type": "object",
      "description": "Per-qtype accuracy and n. Required keys: SSA, MSA, TR, KU, MR.",
      "required": ["SSA", "MSA", "TR", "KU", "MR"],
      "additionalProperties": false,
      "patternProperties": {
        "^(SSA|MSA|TR|KU|MR)$": {
          "type": "object",
          "required": ["acc", "n"],
          "properties": {
            "acc": { "type": "number", "minimum": 0.0, "maximum": 1.0 },
            "n":   { "type": "integer", "minimum": 0 }
          },
          "additionalProperties": false
        }
      }
    },
    "notes": {
      "type": "string",
      "maxLength": 2000,
      "description": "Free-text disclosure of routing, fallbacks, prompt variants, oracle-leakage, etc. Required for canonical and experimental tracks."
    }
  },
  "additionalProperties": false
}