#!/usr/bin/env python3
"""raplib.py — the lane's shared helpers (the first shared Python module; 2026-09-02).

The architecture review counted the ffprobe duration reimplemented five times, the
consonant skeleton twice, text normalisation four times — class A1 in miniature: the same
concept spelled differently in two places, then disagreeing. What lives here:

  probe_seconds(path)          one container-duration probe
  sha256_file(path)            the hash every "did this input move?" check uses
  read_json(path)              None for a missing, zero-length or unparseable file — an
                               empty artifact is ABSENT, never done (song-words.json read
                               as done forever after a failed pass, ledger row 22)
  write_json_atomic(path, doc) tmp + rename, with a `_meta` header {generatedAt, inputs}
  audio_mode(plan)             the film's mode from the plan's constants (env fallback)

Sibling scripts import it as `import raplib` (they sit in the same directory).
"""
from __future__ import annotations

import hashlib
import json
import math
import os
import re
import subprocess
import time


def probe_seconds(path: str, ffprobe: str = "ffprobe") -> float:
    """Container duration in seconds; 0.0 when the file is missing or unreadable."""
    if not path or not os.path.isfile(path):
        return 0.0
    try:
        out = subprocess.run([ffprobe, "-v", "error", "-show_entries", "format=duration",
                              "-of", "csv=p=0", path], capture_output=True, text=True).stdout.strip()
        return float(out) if out else 0.0
    except (OSError, ValueError):
        return 0.0


def sha256_file(path: str) -> str:
    """Hex digest of the file's bytes; "" when it does not exist."""
    if not path or not os.path.isfile(path):
        return ""
    h = hashlib.sha256()
    with open(path, "rb") as fh:
        for chunk in iter(lambda: fh.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def read_json(path: str):
    """The document, or None when the file is missing, empty or not JSON. A zero-byte
    artifact is indistinguishable from a complete one to `-f`; here it is absent."""
    if not path or not os.path.isfile(path):
        return None
    try:
        if os.path.getsize(path) == 0:
            return None
        with open(path, encoding="utf-8") as fh:
            return json.load(fh)
    except (OSError, ValueError):
        return None


def write_json_atomic(path: str, doc: dict, inputs: dict | None = None, indent: int = 1) -> None:
    """Write via a sibling temp file and rename, so a crash mid-write leaves the old file
    (or none), never a truncated one. `inputs` ({name: path}) is recorded as hashes in
    `_meta`, so a consumer can refuse a document whose inputs moved under it."""
    meta = dict(doc.get("_meta") or {}) if isinstance(doc, dict) else {}
    meta["generatedAt"] = time.strftime("%Y-%m-%dT%H:%M:%S%z")
    if inputs:
        meta["inputs"] = {name: {"path": p, "sha256": sha256_file(p)} for name, p in inputs.items() if p}
    out = dict(doc)
    out["_meta"] = meta
    tmp = f"{path}.tmp.{os.getpid()}"
    os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
    with open(tmp, "w", encoding="utf-8") as fh:
        json.dump(out, fh, indent=indent, ensure_ascii=False)
    os.replace(tmp, path)


def moved_inputs(doc: dict) -> list[tuple[str, str]]:
    """(name, path) of every recorded input whose bytes no longer match its hash."""
    out = []
    for name, rec in ((doc or {}).get("_meta", {}).get("inputs") or {}).items():
        p = rec.get("path") or ""
        if rec.get("sha256") and sha256_file(p) != rec["sha256"]:
            out.append((name, p))
    return out


def audio_mode(plan: dict | None) -> str:
    """The film's audio mode: the plan's recorded constant, else the environment's
    AUDIO_MODE, else record. The plan is the source of truth once it exists."""
    c = (plan or {}).get("constants") or {}
    return str(c.get("audioMode") or os.environ.get("AUDIO_MODE") or "record").strip()


GAP_SECTION_MIN_S = 0.9   # select_cut / plan_coherence #3: a hole this long is a B-roll section


def gap_pieces(seconds: float, cap: float) -> int:
    """How many sections a hole of `seconds` becomes when a cutaway may be at most `cap`
    long. Each piece takes a fresh slice with a 0.2 s margin from a clip that is itself
    capped, so a hole longer than cap - 0.4 is split evenly — a 30 s hole on a 20 s
    route is two 15 s sections, not one that no clip can fill."""
    if cap is None or cap <= 0.4:
        return 1
    return max(1, math.ceil(float(seconds) / (float(cap) - 0.4) - 1e-9))


def plan_holes(plan: dict, min_s: float = GAP_SECTION_MIN_S, cap: float | None = None) -> list[dict]:
    """The instrumental sections the cut will open: between the plan's windows, and after
    the last one up to the plan's `hookSeconds`.

    select_cut's own tiling rule, in seconds (it snaps to frames): a window's picture runs
    startSec+trimStartSec .. startSec+seconds-tailSec, and a hole of min_s or more between
    two windows is a section OWNED BY THE WINDOW BEFORE IT — the cut goes from that world's
    performer into that world's own cutaway. Shorter holes were absorbed into a render by
    make_plan and are not sections. Shared by make_broll_plan (the pool is sized from
    these) and plan_coherence #6 (the pool must hold them), so the two never disagree
    with the cut about what needs filling (the duet fixture, 2026-09-02)."""
    jobs = sorted(plan.get("jobs") or [], key=lambda j: int(j.get("window", 0) or 0))
    out: list[dict] = []
    prev_end, prev_sid, prev_id = None, None, None
    for j in jobs:
        s0 = float(j.get("startSec") or 0) + float(j.get("trimStartSec") or 0)
        e0 = float(j.get("startSec") or 0) + float(j.get("seconds") or 0) - float(j.get("tailSec") or 0)
        if prev_end is not None and s0 - prev_end >= min_s:
            out.append({"start": round(prev_end, 3), "end": round(s0, 3), "seconds": round(s0 - prev_end, 3),
                        "styleId": prev_sid, "after": prev_id})
        prev_end, prev_sid, prev_id = e0, j.get("styleId"), j.get("id")
    hook = plan.get("hookSeconds")
    if hook is not None and prev_end is not None and float(hook) - prev_end >= min_s:
        out.append({"start": round(prev_end, 3), "end": round(float(hook), 3),
                    "seconds": round(float(hook) - prev_end, 3),
                    "styleId": prev_sid, "after": prev_id, "outro": True})
    if cap is None:
        return out
    # With a clip cap, a hole longer than one cutaway can hold is several sections
    # (select_cut opens the same pieces, frame-snapped).
    pieces: list[dict] = []
    for h in out:
        n = gap_pieces(float(h["seconds"]), cap)
        if n == 1:
            pieces.append(h)
            continue
        a, b = float(h["start"]), float(h["end"])
        step = (b - a) / n
        for k in range(n):
            s0 = a + k * step
            s1 = b if k == n - 1 else a + (k + 1) * step
            pieces.append({**h, "start": round(s0, 3), "end": round(s1, 3),
                           "seconds": round(s1 - s0, 3), "piece": f"{k + 1}/{n}"})
    return pieces


# ---- the voice, not just the words -----------------------------------------------------
# A take can sing every word of its slice in a different VOICE: w05 (Sita alone) carried
# a unison line rendered with a second, male voice, the consonant-skeleton judge passed it
# at 100 %, and vocal-swap carried that voice into the film (Baby Ram & Sita v7,
# 2026-09-02). Pitch is the cheap, honest tell: the take's vocal must sit in the record's
# own band for that span. YIN in plain numpy — no librosa on the lane.
VOICE_BAND = (0.50, 2.00)   # take f0 / record f0 within an octave: the same register
LOW_VOICE_HZ = 180.0        # below this a frame is an adult-male pitch; a child's voice rarely is
LOW_SHARE_MAX = 0.25        # a take with this much more low-pitched voice than the record is a stranger


def _pcm16k(path: str, ss: float | None = None, t: float | None = None):
    import numpy as np
    cmd = ["ffmpeg", "-v", "error"] + (["-ss", f"{ss:.3f}"] if ss is not None else []) \
        + (["-t", f"{t:.3f}"] if t else []) + ["-i", path, "-vn", "-ac", "1", "-ar", "16000", "-f", "f32le", "-"]
    r = subprocess.run(cmd, capture_output=True)
    return np.frombuffer(r.stdout, dtype=np.float32)


def f0_track(path: str, ss: float | None = None, t: float | None = None, *, sr: int = 16000,
             fmin: float = 70.0, fmax: float = 600.0, frame_s: float = 0.04, hop_s: float = 0.02,
             threshold: float = 0.15) -> list[float]:
    """The fundamental (Hz) of every voiced frame in path[ss, ss+t]. YIN: difference
    function -> cumulative-mean normalisation -> first dip under the threshold ->
    parabolic refinement. Frames below -40 dBFS are skipped."""
    import numpy as np
    x = _pcm16k(path, ss, t)
    n, hop = int(frame_s * sr), int(hop_s * sr)
    tau_min, tau_max = int(sr / fmax), int(sr / fmin)
    f0s: list[float] = []
    taus = np.arange(1, tau_max + 1)
    nfft = 1
    while nfft < 2 * (n + tau_max):
        nfft *= 2
    for i in range(0, len(x) - n - tau_max, hop):
        seg = x[i:i + n + tau_max].astype(np.float64)
        if np.sqrt(np.mean(seg[:n] ** 2)) < 0.01:
            continue
        # d(tau) = E(0) + E(tau) - 2 C(tau): energies by cumulative sum, the correlation
        # by FFT — the plain O(n · tau_max) form took the duet proof past ten minutes
        head = seg[:n]
        corr = np.fft.irfft(np.fft.rfft(seg, nfft) * np.conj(np.fft.rfft(head, nfft)), nfft)[:tau_max + 1]
        cs = np.concatenate(([0.0], np.cumsum(seg * seg)))
        e_tau = cs[taus + n] - cs[taus]
        d = cs[n] + e_tau - 2.0 * corr[1:]
        d = np.maximum(d, 0.0)
        cmnd = d * taus / np.maximum(np.cumsum(d), 1e-12)
        cand = np.where(cmnd[tau_min - 1:] < threshold)[0]
        if not len(cand):
            continue
        k = cand[0] + tau_min - 1
        while k + 1 < len(cmnd) and cmnd[k + 1] < cmnd[k]:      # walk down to the dip
            k += 1
        if 0 < k < len(cmnd) - 1:                                 # parabolic refinement
            a, b, c = cmnd[k - 1], cmnd[k], cmnd[k + 1]
            denom = a - 2 * b + c
            tau = (k + 1) + (0.5 * (a - c) / denom if denom else 0.0)
        else:
            tau = k + 1
        f0s.append(sr / tau)
    return f0s


def f0_median(path: str, ss: float | None = None, t: float | None = None, *, min_voiced: int = 12, **kw) -> float | None:
    """Median fundamental (Hz), or None when fewer than `min_voiced` frames carry a pitch
    (silence, noise, a stem with nothing on it)."""
    import numpy as np
    f0s = f0_track(path, ss, t, **kw)
    return float(np.median(f0s)) if len(f0s) >= min_voiced else None


def voice_profile(path: str, ss: float | None = None, t: float | None = None, *, min_voiced: int = 12) -> dict | None:
    """{median, lowShare} — the register, and how much of the voice sits under
    LOW_VOICE_HZ — or None when too little is voiced to say."""
    import numpy as np
    f0s = f0_track(path, ss, t)
    if len(f0s) < min_voiced:
        return None
    a = np.asarray(f0s)
    return {"median": float(np.median(a)), "lowShare": float((a < LOW_VOICE_HZ).mean()), "frames": int(len(a))}


def same_voice(take, record) -> bool | None:
    """Is the take's voice the record's? Both arguments are voice_profile() dicts (or a
    bare median for the register test alone). None when either is unknown.

    Two tells, learned on Baby Ram & Sita (2026-09-02): a take a whole register away
    (VOICE_BAND, an octave) is another voice; and a take carrying LOW_SHARE_MAX more
    low-pitched frames than the record carries an adult male the record does not —
    the operator's "the voice changes to male". A child's take that sings a few
    semitones higher than the record (w05: 410 vs 254 Hz, low share 5 % vs 6 %) is the
    same child, not a stranger; the first rule (±6 semitones) called it one."""
    if take is None or record is None:
        return None
    tm = take["median"] if isinstance(take, dict) else float(take)
    rm = record["median"] if isinstance(record, dict) else float(record)
    if not tm or not rm:
        return None
    if not (VOICE_BAND[0] <= tm / rm <= VOICE_BAND[1]):
        return False
    if isinstance(take, dict) and isinstance(record, dict):
        if take["lowShare"] - record["lowShare"] > LOW_SHARE_MAX:
            return False
    return True


def bursts(path: str, ss: float | None = None, t: float | None = None, *, min_gap_s: float = 0.12,
           rel: float = 0.35, dip: float = 0.5) -> int:
    """How many voice bursts (syllable-sized energy peaks) path[ss, ss+t] carries.

    A shouted, repeated word — "main, main, main" — is three bursts the transcriber
    never writes down (w09: two draws refused for a miss, a third passed at 100 % with
    six words and no "main"; Baby Ram & Sita, 2026-09-02). The envelope has the answer.
    Voice band (200–4000 Hz), 10 ms RMS, 3-frame smoothing; a burst is a local maximum
    above `rel` of the span's peak, at least `min_gap_s` from the last one, with the
    envelope dipping under `dip` of the smaller of the two peaks between them."""
    import numpy as np
    cmd = ["ffmpeg", "-v", "error"] + (["-ss", f"{ss:.3f}"] if ss is not None else []) \
        + (["-t", f"{t:.3f}"] if t else []) + ["-i", path, "-vn", "-ac", "1", "-ar", "16000",
                                              "-af", "highpass=f=200,lowpass=f=4000", "-f", "f32le", "-"]
    x = np.frombuffer(subprocess.run(cmd, capture_output=True).stdout, dtype=np.float32)
    w = 160
    if len(x) < 3 * w:
        return 0
    env = np.array([np.sqrt(np.mean(x[i:i + w] ** 2)) for i in range(0, len(x) - w, w)])
    env = np.convolve(env, np.ones(3) / 3.0, mode="same")
    peak = float(env.max())
    if peak <= 1e-4:
        return 0
    floor = rel * peak
    gap = max(1, int(min_gap_s * 100))
    count, last_i, last_v = 0, -10 ** 9, 0.0
    for i in range(1, len(env) - 1):
        v = env[i]
        if v < floor or v < env[i - 1] or v < env[i + 1]:
            continue
        if i - last_i < gap:
            continue
        if count and env[last_i:i].min() > dip * min(v, last_v):
            continue                       # no dip between the two: one burst, not two
        count, last_i, last_v = count + 1, i, v
    return count


# ---- what the operator types ----------------------------------------------------------
# He reads a defect off QuickTime's clock, which includes the title card; the tools wanted
# song seconds and, for a window, the selection's grid id (which counts gap sections).
# COVER_SPANS and VOCAL_SWAP_KEEP_RECORD now take any of: "1:48-1:53" (film clock, the
# bookends read from cards.json and subtracted), "108.0-113.0" (song seconds), and a
# window id that is the PLAN's (w04, what every page shows) or the grid's — the plan's
# wins when the selection knows it (film3 item 6, 2026-09-02).
_CLOCK = re.compile(r"^(\d+):(\d{1,2}(?:\.\d+)?)$")
_TIME = r"\d+(?:\.\d+)?(?::\d{1,2}(?:\.\d+)?)?"          # 108.5, or 1:48.5 on the film clock
_SPAN = re.compile(rf"^(?:(w\d+):)?({_TIME})-({_TIME})(?::([A-Za-z0-9_./-]+))?$")


def front_padding(cards_path: str) -> float:
    """Seconds of bookend before BODY, read from the cards manifest (line_subs' rule)."""
    manifest = json.load(open(cards_path, encoding="utf-8"))
    blocks = {b["id"]: b for b in manifest["blocks"]}
    total = 0.0
    for element in manifest["order"]:
        if element == "BODY":
            break
        total += float(blocks[element]["seconds"])
    return round(total, 3)


def parse_clock(tok: str) -> tuple[float, bool]:
    """"1:48.5" -> (108.5, True): the film clock. "108.5" -> (108.5, False): song seconds."""
    tok = tok.strip()
    m = _CLOCK.match(tok)
    if m:
        return int(m.group(1)) * 60 + float(m.group(2)), True
    return float(tok), False


def resolve_window(win: str, selection: dict | None) -> str:
    """A window id the operator typed -> the selection's grid id. The plan's id wins
    when the selection carries `planWindow`; a bare grid id passes through."""
    m = re.match(r"^w(\d+)$", win.strip())
    if not m:
        raise ValueError(f"window {win!r}: expected wNN")
    n = int(m.group(1))
    for w in ((selection or {}).get("windows") or []):
        if w.get("planWindow") == n and w.get("i") is not None:
            return f"w{int(w['i']):02d}"
    # The written selection carries no `windows`; its provenance names each grid
    # section's plan id (select_cut, 2026-09-06). A plan id found there wins.
    for grid, rec in ((selection or {}).get("provenance") or {}).items():
        if isinstance(rec, dict) and (rec.get("planId") == f"w{n:02d}" or rec.get("planWindow") == n):
            return grid
    return f"w{n:02d}"


def parse_span(spec: str, *, cards: str | None = None, selection: dict | None = None) -> dict:
    """"[wNN:]A-B[:BROLL]" -> {"start", "end", "window", "planId", "broll", "clock"} in
    SONG seconds. A film-clock time needs `cards` (bookends/cards.json) to subtract the
    front padding; without it the span is refused rather than guessed."""
    m = _SPAN.match(spec.strip())
    if not m:
        raise ValueError(f"span {spec!r}: expected [wNN:]A-B[:BROLL] — A and B as song seconds "
                         f"(108.0) or the film clock (1:48)")
    win, lo_s, hi_s, broll = m.group(1), m.group(2), m.group(3), m.group(4)
    lo, clo = parse_clock(lo_s)
    hi, chi = parse_clock(hi_s)
    clock = clo or chi
    if clock:
        if not cards or not os.path.isfile(cards):
            raise ValueError(f"span {spec!r} is on the film clock but there is no bookends/cards.json yet "
                             f"to subtract the title card — give song seconds, or deliver once first")
        pad = front_padding(cards)
        lo, hi = (lo - pad if clo else lo), (hi - pad if chi else hi)
    if hi <= lo:
        raise ValueError(f"span {spec!r}: the end must be after the start")
    return {"start": round(lo, 3), "end": round(hi, 3), "window": resolve_window(win, selection) if win else None,
            "planId": win, "broll": broll, "clock": clock}


def sheet_latin_share(path: str) -> tuple[int, int]:
    """(lines with Latin letters, lyric lines) of a lyric sheet — section tags and blank
    lines excluded. The lane's ear, aligner and judge read a ROMANISED sheet; a
    Devanagari sheet cost the first stage of film #2 and surfaced three stages later as
    "singer_diarize failed — need at least one array to stack" (film3 item 8)."""
    latin = total = 0
    for raw in open(path, encoding="utf-8"):
        line = raw.strip()
        if not line or line.startswith("["):
            continue
        total += 1
        if re.search(r"[A-Za-z]", line):
            latin += 1
    return latin, total


def sung_past_sheet(words: list, last_placed_end: float | None, sheet_text: str, *,
                    min_seconds: float = 4.0) -> dict | None:
    """Sung words AFTER the sheet's last placed line — the song sings past the sheet.

    Film #3 (2026-09-04): Suno repeated verses 37–40 and the closing Doha for 50 s
    after the sheet's last line. The brief placed nothing there, five windows were
    planned no-voice, and nothing said so until the plan was read by hand. The stem
    ear HEARD the words; the only check that would have caught it lived at the cut.

    `words` is either shape the lane uses — {t, w} (the word pass) or {t0, t1, raw,
    lat} (vocal_brief.load_words). `sheet_text` is the lyric sheet. Returns None when
    nothing sung follows the last placed line for at least `min_seconds`, or when the
    tail is a short block of words the sheet never uses (whisper's "thank you for
    watching" over the outro — drop_hallucinated_tail's case, not this one). Otherwise
    a record naming the span and whether it reads as a REPEAT (its words are the
    sheet's — add the repeat lines to the sheet) or as unplaced singing (cover it).
    Advisory: a repeat CAN be deliberate B-roll. Never a refusal.
    """
    if last_placed_end is None:
        return None
    rows = []
    for w in words or []:
        t = w.get("t", w.get("t0"))
        if t is None:
            continue
        text = w.get("lat") or w.get("w") or w.get("raw") or ""
        rows.append((float(t), float(w.get("t1", t)), str(text)))
    tail = [r for r in rows if r[0] > float(last_placed_end) + 0.5]
    if not tail:
        return None
    start, end = tail[0][0], max(tail[-1][1], tail[-1][0])
    seconds = round(end - start, 3)
    if seconds < min_seconds:
        return None
    sheet = set(re.findall(r"[a-z']+", (sheet_text or "").lower()))
    norm = [re.sub(r"[^a-z']", "", x[2].lower()) for x in tail]
    in_sheet = sum(1 for n in norm if n and n in sheet)
    ratio = round(in_sheet / len(tail), 3) if tail else 0.0
    if ratio < 0.5 and len(tail) <= 6:
        return None          # a short invented block over silence, not a sung passage
    likely_repeat = ratio >= 0.5
    return {
        "from": round(start, 3), "to": round(end, 3), "seconds": seconds,
        "words": len(tail), "inSheet": in_sheet, "ratio": ratio,
        "likelyRepeat": likely_repeat,
        "sample": " ".join(x[2] for x in tail[:8]),
        "advice": ("the words are the sheet's — a REPEAT the sheet does not carry; add the "
                   "repeat lines (verse numbers / the closing Doha) to references/lyrics.txt "
                   "and re-run from plan, or leave it as B-roll on purpose")
                  if likely_repeat else
                  ("sung words the sheet never uses — unplaced singing; cover those seconds "
                   "with B-roll or add the lines to the sheet"),
    }


def uncovered(a: float, b: float, covers: list[tuple[float, float]]) -> list[tuple[float, float]]:
    """The parts of [a, b] no (x, y) in `covers` reaches — in order, empty when covered."""
    out, cur = [], a
    for x, y in sorted((float(x), float(y)) for x, y in covers):
        if y <= cur:
            continue
        if x > cur:
            out.append((cur, min(x, b)))
        cur = max(cur, y)
        if cur >= b:
            break
    if cur < b:
        out.append((cur, b))
    return [(x, y) for x, y in out if y - x > 1e-6]


SINGING_UNVOICED_MAX_S = 1.5   # a placed line may run this long past a voiced window before it is a refusal


def singing_over_nothing(lines: list[dict], voiced: list[tuple[float, float]],
                         intended: list[tuple[float, float]] = (), min_s: float = SINGING_UNVOICED_MAX_S,
                         film_end: float | None = None) -> list[dict]:
    """Placed lines whose singing runs more than `min_s` continuous seconds outside every
    voiced span and every intended cover — the film3 item-10 invariant, at plan time
    (voiced = windows with a voice, or the other performer's by design) and at cut time
    (voiced = sections with a face, intended = the recorded cutaways). The review built
    this and found 7 s on film #2's corrected plan before any render.

    `film_end` bounds the judgement to the film: a line is clipped to [0, film_end] and
    a line that starts after it is not in the picture at all. Without it the preflight
    proof's synthetic copy — four windows, 18.75 s of a 4:33 song — was refused for every
    line the song sings AFTER the copy ends (film #3, 2026-09-03)."""
    out = []
    for ln in lines:
        a, b = float(ln.get("start") or 0), float(ln.get("end") or ln.get("start") or 0)
        if film_end is not None:
            b = min(b, float(film_end))
        if b - a <= 0:
            continue
        for x, y in uncovered(a, b, list(voiced) + list(intended)):
            if y - x > min_s:
                out.append({"text": ln.get("text", ""), "start": round(a, 2), "end": round(b, 2),
                            "from": round(x, 2), "to": round(y, 2), "seconds": round(y - x, 2)})
    return out

