#!/usr/bin/env python3
"""Rubric well-formedness gate: fail CI when an eval rubric is malformed BEFORE it grades anything.

This is the deterministic, repo-only half of P6 (Review Gate as CI). The Review Gate itself is an LLM
judgment and cannot run reproducibly in CI — but a *malformed rubric* (weights that don't sum to 1.0, a
missing pass threshold, a missing eval_type) is a deterministic defect we CAN catch. This guards the
scaffolding the evaluator skill scores against (D14 outcome-only, D15 weighted dimensions + threshold).

Scope: every rubric in planning/evals/ — a file is a rubric iff its frontmatter declares `type: Rubric`
(tolerates the `- type: Rubric` markdown-list frontmatter the P0/P1 rubrics use). Calibration logs and
method docs are not rubrics and are skipped. Three assertions per rubric:

  EVAL_TYPE   — eval_type is present and is `capability` or `regression` (D15 frontmatter field).
  THRESHOLD   — a pre-committed pass threshold in (0,1] exists (`pass_threshold:` or "pass threshold = ").
  WEIGHTS     — the first Weighted-dimensions table's Weight column sums to 1.0 (±0.011), excluding the
                explicit Total row. Partial-credit weighting is meaningless if the weights don't sum to 1.

Exit 0 = green. Exit 1 = a malformed rubric; prints which and why. No DB access, no secrets.
"""
import os, re, sys, glob

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
EVALS = os.path.join(ROOT, "planning", "evals")
fail = []
checked = 0


def read(p):
    return open(p, encoding="utf-8").read()


def num(s):
    """Parse a possibly-bold/aligned numeric cell like `**0.30**` or `0.35` -> float, else None."""
    s = s.replace("*", "").replace("`", "").strip()
    m = re.fullmatch(r"[0-9]*\.?[0-9]+", s)
    return float(m.group()) if m else None


def weight_sum(text):
    """Sum the Weight column of the FIRST markdown table that has a 'Weight' header, skipping the
    Total row. Returns (sum, n_rows) or (None, 0) if no weighted table is found."""
    lines = text.splitlines()
    for i, line in enumerate(lines):
        cells = [c.strip() for c in line.strip().strip("|").split("|")]
        wcol = next((j for j, c in enumerate(cells) if c.lower() == "weight"), None)
        if wcol is None:
            continue
        total, rows = 0.0, 0
        for row in lines[i + 1:]:
            if not row.lstrip().startswith("|"):
                break  # table ended
            rc = [c.strip() for c in row.strip().strip("|").split("|")]
            if set("".join(rc)) <= set("-: "):
                continue  # separator row |---|---|
            if len(rc) <= wcol:
                continue
            # the |Total|1.00| summary row is not a dimension — the "Total" label may sit in any
            # cell (an empty leading `#` column pushes it right), so scan every cell.
            if any(c.replace("*", "").strip().lower().startswith("total") for c in rc):
                continue
            w = num(rc[wcol])
            if w is not None:
                total += w
                rows += 1
        return round(total, 4), rows
    return None, 0


for path in sorted(glob.glob(os.path.join(EVALS, "*.md"))):
    rel = os.path.relpath(path, ROOT)
    text = read(path)
    # A rubric declares `type: Rubric` (YAML or `- ` markdown-list frontmatter). Skip everything else.
    if not re.search(r"^[-\s]*type:\s*Rubric\b", text, re.M | re.I):
        continue
    checked += 1

    m = re.search(r"^[-\s]*eval_type:\s*(\w+)", text, re.M | re.I)
    if not m:
        fail.append(f"{rel}: missing `eval_type` (must be capability|regression — D15)")
    elif m.group(1).lower() not in ("capability", "regression"):
        fail.append(f"{rel}: eval_type `{m.group(1)}` invalid (must be capability|regression)")

    t = re.search(r"pass_threshold:\s*([0-9]*\.?[0-9]+)", text, re.I) or \
        re.search(r"pass threshold\s*=?\s*([0-9]*\.?[0-9]+)", text, re.I)
    if not t:
        fail.append(f"{rel}: no pre-committed pass threshold (`pass_threshold:` or 'pass threshold = ')")
    elif not (0 < float(t.group(1)) <= 1):
        fail.append(f"{rel}: pass threshold {t.group(1)} out of range (0,1]")

    s, rows = weight_sum(text)
    if s is None:
        fail.append(f"{rel}: no weighted-dimensions table with a `Weight` column (D15)")
    elif rows < 2:
        fail.append(f"{rel}: weighted table has {rows} dimension(s) — needs >=2 for partial credit")
    elif abs(s - 1.0) > 0.011:
        fail.append(f"{rel}: dimension weights sum to {s}, not 1.0 (D15 partial-credit weighting)")

print(f"Rubric gate: {checked} rubric(s) checked in planning/evals/.")
if fail:
    print(f"\n✗ {len(fail)} malformed rubric issue(s):")
    for f in fail:
        print("  -", f)
    print("\nFix: weights must sum to 1.0, set a pass_threshold in (0,1], and an eval_type (D15).")
    sys.exit(1)
if checked == 0:
    print("✗ no rubrics found — expected at least one `type: Rubric` doc in planning/evals/")
    sys.exit(1)
print("✓ every rubric is well-formed (weights sum to 1.0, threshold + eval_type present).")