{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://evals.favur.dev/data/schema/canonical_dataset.schema.json",
  "title": "Favur Evals canonical dataset",
  "description": "The canonical, versioned Favur Evals benchmark dataset. Produced by ci_scripts.canonical_dataset from the scoring-engine site payload; the site and all public downloads derive from this artifact. Every score is pinned to the rubric_version that produced it.",
  "type": "object",
  "required": [
    "dataset_version",
    "generated_from",
    "rubric",
    "sows",
    "vendors",
    "models",
    "agent_roles",
    "runs"
  ],
  "properties": {
    "dataset_version": {
      "type": "string",
      "pattern": "^\\d+\\.\\d+\\.\\d+$",
      "description": "Semantic version of the dataset itself. Incremented when the rubric changes or the dataset shape changes; see data/dataset_changelog.json."
    },
    "generated_from": {
      "type": "object",
      "required": ["rubric_version", "schema_version", "manifest_version"],
      "properties": {
        "rubric_version": { "type": "integer", "minimum": 1 },
        "schema_version": { "type": "integer", "minimum": 1 },
        "manifest_version": { "type": "integer", "minimum": 1 }
      }
    },
    "rubric": {
      "type": "object",
      "description": "Everything needed to independently recompute a run's composite from this dataset alone: composite = blend(100 \u00d7 \u03a3(w_r \u00d7 score_r / cell_max) / \u03a3(w_r present), completion_score \u00d7 10).",
      "required": ["agent_role_weights", "composite_blend", "cell_max"],
      "properties": {
        "agent_role_weights": {
          "type": "object",
          "additionalProperties": { "type": "number" }
        },
        "composite_blend": {
          "type": "object",
          "required": ["role_weight", "completion_weight"],
          "properties": {
            "role_weight": { "type": "number" },
            "completion_weight": { "type": "number" }
          }
        },
        "cell_max": { "type": "number" }
      }
    },
    "sows": {
      "type": "array",
      "items": {
        "type": "object",
        "required": ["id", "run_count"],
        "properties": {
          "id": { "type": "string", "minLength": 1 },
          "run_count": { "type": "integer", "minimum": 0 },
          "first_run_at": { "type": ["string", "null"] },
          "last_run_at": { "type": ["string", "null"] }
        }
      }
    },
    "vendors": {
      "type": "array",
      "items": {
        "type": "object",
        "required": ["id"],
        "properties": {
          "id": { "type": "string", "minLength": 1 },
          "display": { "type": ["string", "null"] },
          "url": { "type": ["string", "null"] },
          "run_count": { "type": "integer", "minimum": 0 }
        }
      }
    },
    "models": {
      "type": "array",
      "items": {
        "type": "object",
        "required": ["model_key", "vendor"],
        "properties": {
          "model_key": { "type": "string", "minLength": 1 },
          "vendor": { "type": "string", "minLength": 1 },
          "family": { "type": ["string", "null"] },
          "version": { "type": ["string", "null"] },
          "run_count": {
            "type": "integer",
            "minimum": 0,
            "description": "Number of runs in which this model appears in the roster (any role)."
          }
        }
      }
    },
    "agent_roles": {
      "type": "array",
      "items": {
        "type": "object",
        "required": ["id"],
        "properties": {
          "id": { "type": "string", "minLength": 1 },
          "title": { "type": ["string", "null"] },
          "description": { "type": ["string", "null"] }
        }
      }
    },
    "runs": {
      "type": "array",
      "items": { "$ref": "#/$defs/run" }
    }
  },
  "$defs": {
    "run": {
      "type": "object",
      "required": [
        "run_id",
        "sow",
        "rubric_version",
        "started_at",
        "roster",
        "composite",
        "subjects",
        "agent_roles",
        "telemetry",
        "deliverable",
        "artifacts"
      ],
      "properties": {
        "run_id": { "type": "string", "minLength": 1 },
        "sow": { "type": "string", "minLength": 1 },
        "rubric_version": {
          "type": "integer",
          "minimum": 1,
          "description": "The rubric version that produced this run's scores."
        },
        "started_at": { "type": ["string", "null"] },
        "ended_at": { "type": ["string", "null"] },
        "run_status": { "type": ["string", "null"] },
        "is_mixed_run": { "type": ["boolean", "null"] },
        "roster": {
          "type": "object",
          "description": "role → model assignment for this run.",
          "additionalProperties": {
            "type": "object",
            "required": ["vendor"],
            "properties": {
              "vendor": { "type": "string" },
              "family": { "type": ["string", "null"] },
              "version": { "type": ["string", "null"] },
              "model_key": { "type": ["string", "null"] }
            }
          }
        },
        "composite": {
          "type": "object",
          "required": ["score", "coverage", "low_confidence"],
          "properties": {
            "score": { "type": "number", "minimum": 0, "maximum": 100 },
            "coverage": {
              "type": "number",
              "minimum": 0,
              "maximum": 1,
              "description": "Fraction of composite weight backed by real measurement."
            },
            "low_confidence": { "type": "boolean" },
            "completion_score": {
              "type": ["number", "null"],
              "minimum": 0,
              "maximum": 10,
              "description": "0-10 completion metric blended into the composite (25% when present)."
            },
            "pin": {
              "type": ["object", "null"],
              "description": "Pin-ledger entry: composite frozen when the run was first scored, and the rubric version it was pinned under.",
              "properties": {
                "composite_score": { "type": "number" },
                "pinned_at": { "type": "string" },
                "rubric_version": { "type": ["integer", "null"] }
              }
            },
            "pin_drift": { "type": ["number", "null"] }
          }
        },
        "subjects": {
          "type": "object",
          "additionalProperties": { "$ref": "#/$defs/subjectScore" }
        },
        "agent_roles": {
          "type": "object",
          "description": "role → per-role rubric cell for this run.",
          "additionalProperties": { "$ref": "#/$defs/agentCell" }
        },
        "telemetry": {
          "type": "object",
          "required": ["requests", "duration_min"],
          "properties": {
            "requests": { "type": ["number", "null"] },
            "failed_requests": { "type": ["number", "null"] },
            "duration_min": { "type": ["number", "null"] },
            "cost_usd": { "type": ["number", "null"] },
            "cache_hit_rate": { "type": ["number", "null"] },
            "input_tokens": { "type": ["number", "null"] },
            "output_tokens": { "type": ["number", "null"] },
            "reasoning_tokens": { "type": ["number", "null"] },
            "cached_input_tokens": { "type": ["number", "null"] },
            "concurrency_factor": { "type": ["number", "null"] },
            "tool_calls": { "type": ["number", "null"] },
            "tools_per_response": { "type": ["number", "null"] },
            "failure_breakdown": {
              "type": ["object", "null"],
              "additionalProperties": { "type": "number" }
            }
          }
        },
        "deliverable": {
          "type": "object",
          "description": "Static-analysis manifest of the produced deliverable.",
          "properties": {
            "src_loc": { "type": ["number", "null"] },
            "test_files": { "type": ["number", "null"] },
            "coverage_pct": { "type": ["number", "null"] },
            "lint_errors": { "type": ["number", "null"] },
            "avg_complexity": { "type": ["number", "null"] },
            "maintainability_index": { "type": ["number", "null"] },
            "files_changed": { "type": ["number", "null"] }
          }
        },
        "artifacts": {
          "type": "object",
          "required": ["run_folder"],
          "properties": {
            "run_folder": {
              "type": "string",
              "description": "Repo-relative path to the raw run artifacts (runs/<sow>/<folder>)."
            },
            "repo_url": { "type": ["string", "null"] },
            "page_url": { "type": ["string", "null"] }
          }
        }
      }
    },
    "subjectScore": {
      "type": "object",
      "required": ["raw", "max", "pct", "coverage", "low_confidence", "components"],
      "properties": {
        "raw": { "type": "number" },
        "max": { "type": "number" },
        "pct": { "type": "number" },
        "coverage": { "type": "number", "minimum": 0, "maximum": 1 },
        "low_confidence": { "type": "boolean" },
        "normalized_score": { "type": ["number", "null"] },
        "components": {
          "type": "object",
          "additionalProperties": { "$ref": "#/$defs/component" }
        }
      }
    },
    "agentCell": {
      "type": "object",
      "required": ["score", "coverage", "low_confidence"],
      "properties": {
        "score": { "type": ["number", "null"] },
        "coverage": { "type": ["number", "null"] },
        "low_confidence": { "type": ["boolean", "null"] },
        "components": {
          "type": "object",
          "additionalProperties": { "$ref": "#/$defs/component" }
        }
      }
    },
    "component": {
      "type": "object",
      "required": ["available", "weight"],
      "properties": {
        "available": { "type": "boolean" },
        "normalized": { "type": ["number", "null"] },
        "weight": { "type": "number" },
        "effective_weight": { "type": ["number", "null"] },
        "group": { "type": ["string", "null"] },
        "raw": {}
      }
    }
  }
}