#!/usr/bin/env python3
"""word_sync.py — decide, per window, what actually needs doing. From WORDS.

This is the gate that should run before anyone spends a render. `sync_profile.py`
measures energy and answers "how far off"; this answers "off in what WAY", which is the
question that decides whether a window costs nothing or costs ~100 minutes of a
single-slot queue.

Measured on one film, this disagreed with the energy gate on 4 of 14 windows and was
right every time:

  a window the energy gate called FIXED at 0ms was 730ms early    (shipped broken twice)
  a window it condemned as WANDERS was steady at -20ms            (a render nearly wasted)
  a window it scored -150ms was singing WORDS THAT ARE NOT IN THE SONG
  a window it called UNMEASURABLE read +70ms

The verdicts, cheapest first:

  OK          within the ship bar. Nothing to do.
  SLIDE       every word off by the same amount: a PLACEMENT error. sync_shift.py, free.
  RETIME      drift grows across the clip: the model performed at the wrong SPEED.
              Fixable by retiming, free. The energy gate can only call this "wanders"
              and demand a render that is not needed.
  WRONG-LYRIC the clip is not singing this window's words at all. No amount of sliding
              or retiming touches this, and it is INVISIBLE to correlation — a clip
              singing different words still has speech-shaped energy.
  RE-RENDER   ragged drift, or off by more than a cut can absorb.

TWO THINGS THAT MAKE THE NUMBERS REAL, both learned by getting them wrong:

  VOTING, NOT LONGEST-COMMON-SUBSEQUENCE. A lyric repeats, so "and"/"the"/"we'll" pair
  happily with an occurrence seconds away and enough of those outvote the truth — one
  window read +3520ms of drift inside an 8s window, which its own voice reference makes
  impossible. Here every same-word pair votes for the offset it implies; a repeated word
  scatters its votes, and only the true offset collects one from every word at once.

  A 300ms STEADY BAND, NOT 150ms. Whisper's own word timings on SUNG audio carry
  roughly +/-100ms each side, so a genuinely steady clip still shows ~200-250ms of
  spread. Calling that "ragged" condemns a slideable clip to a render.

Exit 5 if any window needs work, so a runner can gate on it.
"""
from __future__ import annotations

import argparse
import difflib
import json
import os
import hashlib
import statistics
import sys

MAX_OFFSET_S = 2.0        # beyond this a clip is on a different bar, not merely adrift
VOTE_BIN_S = 0.20
SHIP_MS = 80
STEADY_MS = 300           # whisper's noise floor on sung audio
RAGGED_MS = 500           # only ABOVE this is scatter bigger than the noise
MIN_MOUTHING_WORDS = 3    # fewer, and it is whisper hallucinating on ambience
SLIDEABLE_MS = 350        # a longer hold at the cut is more visible than the drift
MIN_WORDS = 3
MIN_COVERAGE = 0.70       # fraction of the window's lyric the clip must actually sing
FUZZY = 0.75              # whisper spells the same sung word differently take to take
SEGMENTS = 4              # check coverage per quarter: half a window can be invented


def vote_offset(clip_words, song_words, window_start):
    """Best offset by voting, plus the words that agree with it. ms, + = mouth early."""
    votes = {}
    for cw in clip_words:
        ct = window_start + cw["t"]
        for sw in song_words:
            if cw["w"] != sw["w"]:
                continue
            d = sw["t"] - ct
            if abs(d) <= MAX_OFFSET_S:
                votes.setdefault(round(d / VOTE_BIN_S), []).append((ct, sw["t"], cw["w"], d))
    if not votes:
        return None, []
    best = max(votes, key=lambda k: len(votes[k]))
    pool = votes.get(best - 1, []) + votes[best] + votes.get(best + 1, [])
    centre = statistics.median([d for *_, d in pool])
    seen, kept = set(), []
    for ct, st, w, d in sorted(pool):
        if abs(d - centre) <= VOTE_BIN_S * 1.5 and ct not in seen:
            seen.add(ct)
            kept.append((ct, st, w, d * 1000.0))
    return centre * 1000.0, kept


def classify(kept, window_start, seconds, silent_head):
    drifts = [(ct - window_start, d) for ct, _, _, d in kept]
    med = statistics.median([d for _, d in drifts])
    spread = max(d for _, d in drifts) - min(d for _, d in drifts)
    first = [d for t, d in drifts if t < seconds / 2]
    second = [d for t, d in drifts if t >= seconds / 2]
    slope = ((statistics.mean(second) - statistics.mean(first))
             if first and second else 0.0)
    # A window whose MEDIAN drift is inside the ship bar is landing on the beat. Spread
    # alone is not actionable: two whisper timings, each +/-100ms, produce up to ~400ms
    # of spread on a clip that is in fact steady. Condemning that to a ~100min render is
    # the same mistake as the energy gate's, wearing different clothes — it cost two
    # windows on this film (drift +20ms and +0ms, both called "ragged").
    if abs(med) <= SHIP_MS:
        note = (f"scatter {spread:.0f}ms is above the noise floor — worth a listen"
                if spread > RAGGED_MS else "")
        return ("RAGGED" if spread > RAGGED_MS else "OK"), med, spread, note
    if spread <= STEADY_MS:
        if abs(med) <= SLIDEABLE_MS:
            return "SLIDE", med, spread, f"slide {-med:+.0f}ms (free)"
        return "RE-RENDER", med, spread, f"{abs(med):.0f}ms is more than a cut can absorb"
    if abs(slope) > 150 and spread < 900:
        return "RETIME", med, spread, f"tempo drift {slope:+.0f}ms across the clip (free)"
    return "RE-RENDER", med, spread, f"ragged: {spread:.0f}ms scatter around {med:+.0f}ms"


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--store", required=True, help="clip-transcripts.json")
    ap.add_argument("--selection", required=True)
    ap.add_argument("--vocal-map", help="to report mouthing over silence")
    ap.add_argument("--json", help="write the decisions")
    ap.add_argument("--cutaways", help="spans deliberately replaced by B-roll")
    a = ap.parse_args()

    store = json.load(open(a.store, encoding="utf-8"))
    sel = json.load(open(a.selection, encoding="utf-8"))
    song = store["song"]["words"]

    # THE STORE MUST DESCRIBE THE CLIPS THAT ARE ON DISK NOW.
    #
    # A window keeps its filename when it is re-rendered, so nothing about a stale store
    # looks wrong: it is complete, it is well-formed, and every verdict in it is about a
    # take that no longer exists. Found on a real film — the store was written at 01:32
    # and three of its clips were re-rendered at 13:56, and this gate went on reporting
    # yesterday's words as the film's sync. Entries carry a `sourceId` (a content id of
    # the file they were made from) precisely so the drift is DETECTABLE.
    stale = []
    for cid, clip in store.get("clips", {}).items():
        recorded, path = clip.get("sourceId"), clip.get("path")
        if not recorded or not path or not os.path.isfile(path):
            continue                # an older store, or a clip that has moved: not proof
        with open(path, "rb") as fh:
            if hashlib.md5(fh.read(400_000)).hexdigest()[:12] != recorded:
                stale.append(cid)
    if stale:
        print(f"word_sync: REFUSED — the store describes {len(stale)} clip(s) that have "
              f"been re-rendered since it was written: {', '.join(sorted(stale))}",
              file=sys.stderr)
        print("  Every verdict about them would be about a take that no longer exists.\n"
              "  Re-run the transcribe stage (it now caches by content, so only the "
              "changed clips cost anything).", file=sys.stderr)
        return 6
    cutaways = []
    if a.cutaways:
        cutaways = json.load(open(a.cutaways, encoding="utf-8")).get("cutaways", [])
    silent = []
    if a.vocal_map:
        silent = [(float(g["start"]), float(g["end"]))
                  for g in json.load(open(a.vocal_map, encoding="utf-8")).get("silentStretches", [])]

    rows, todo = [], 0
    print(f"{'window':<7} {'film t':>8} {'drift':>9} {'spread':>8}  verdict")
    for s in sel["vocalMap"]:
        perf = s.get("performerClip")
        if not perf:
            continue
        start, end = float(s["start"]), float(s["end"])
        # An explicit exemption means this window was deliberately edited as a
        # picture montage rather than presented as a continuous lip-sync take.
        # build_mv_config and the assembler already honor this field; the word
        # gate must answer the same question or it re-flags an intentional edit.
        if s.get("lipSyncExempt"):
            print(f"{perf:<7} {start:8.1f} {'-':>9} {'-':>8}  EXEMPT  -> intentional visual montage")
            rows.append({"clip": perf, "startSec": start, "verdict": "EXEMPT",
                         "note": "intentional visual montage; no continuous lip-sync claim"})
            continue
        clip = store["clips"].get(perf)
        if clip is None:
            print(f"{perf:<7} {start:8.1f} {'-':>9} {'-':>8}  NO TRANSCRIPT — re-run transcribe_clips")
            rows.append({"clip": perf, "startSec": start, "verdict": "NO-TRANSCRIPT"})
            todo += 1
            continue
        near = [w for w in song if start - MAX_OFFSET_S <= w["t"] <= end + MAX_OFFSET_S]
        _, kept = vote_offset(clip["words"], near, start)

        head = 0.0
        for gs, ge in silent:
            if gs <= start < ge:
                head = min(ge, end) - start
        # Whisper hallucinates words onto music and ambience — the B-roll cutaways here
        # transcribed as "music sound music music" and "text on screen". Counting any
        # word in the silent head therefore re-flags a window that has ALREADY been
        # covered. Require several words that are actually in the song's vocabulary.
        lyric_vocab = {w["w"] for w in song}
        head_words = [w for w in clip["words"] if w["t"] < head and w["w"] in lyric_vocab]
        mouthing = head >= 1.0 and len(head_words) >= MIN_MOUTHING_WORDS

        if len(kept) < MIN_WORDS:
            # Distinguish "sings nothing here" from "sings the WRONG words" — the second
            # is a real defect and the first may be a legitimate instrumental window.
            verdict = "WRONG-LYRIC" if len(clip["words"]) >= 4 else "NO-VOCAL"
            note = (f'clip sings "{clip["text"][:44]}"' if verdict == "WRONG-LYRIC"
                    else "no readable vocal in the clip")
            print(f"{perf:<7} {start:8.1f} {'-':>9} {'-':>8}  {verdict}  -> {note}")
            rows.append({"clip": perf, "startSec": start, "verdict": verdict, "note": note})
            todo += 1
            continue

        # LYRIC COVERAGE — the check that "how well do the matched words align" cannot
        # make. A clip can invent the first half of a window and sing the second half
        # correctly; the matched words then agree beautifully and the median reports a
        # perfect +0ms. w06 did exactly that: 50% coverage, +0ms drift, 40ms spread —
        # the tightest number in the film — while the operator watched the man sing
        # "we love her god we show her to cut up" over "stumps stuck in the sand".
        # Alignment answers "are the words it DOES sing in the right place". Coverage
        # answers "is it singing the right words at all", and both must pass.
        # EXCLUDE CUTAWAY SPANS. A second covered by B-roll has no vocal in the clip by
        # design, so counting it as a missing word turns every deliberate edit into a
        # false WRONG-LYRIC — the gate flagged three intentionally-covered windows and
        # would have sent them all back for renders they did not need.
        def covered(t):
            return any(c["clip"] == perf and c["startSec"] <= t < c["endSec"] for c in cutaways)
        need = [w["w"] for w in song if start <= w["t"] <= end and not covered(w["t"])]
        have = [w["w"] for w in clip["words"]]
        # FUZZY, not exact: whisper renders the same sung word differently take to take
        # ("elena"/"elina", "carina"/"karina"). Exact matching scored two clean windows at
        # 69% and 73% purely on spelling.
        pool = list(have)
        hit = 0
        for w in need:
            m = max(pool, key=lambda x: difflib.SequenceMatcher(None, w, x).ratio(), default=None)
            if m and difflib.SequenceMatcher(None, w, m).ratio() >= FUZZY:
                pool.remove(m); hit += 1
        coverage = hit / len(need) if need else 1.0
        # Per-quarter, because a window can be half right — an overall figure passes a
        # clip whose opening bars are entirely invented.
        worst_q, worst_cov = None, 1.0
        if need:
            step = (end - start) / SEGMENTS
            for q in range(SEGMENTS):
                qa, qb = start + q * step, start + (q + 1) * step
                qneed = [w["w"] for w in song if qa <= w["t"] < qb and not covered(w["t"])]
                if len(qneed) < 3:
                    continue
                qpool, qhit = list(have), 0
                for w in qneed:
                    m = max(qpool, key=lambda x: difflib.SequenceMatcher(None, w, x).ratio(), default=None)
                    if m and difflib.SequenceMatcher(None, w, m).ratio() >= FUZZY:
                        qpool.remove(m); qhit += 1
                if qhit / len(qneed) < worst_cov:
                    worst_cov, worst_q = qhit / len(qneed), (qa, qb)

        verdict, med, spread, note = classify(kept, start, end - start, head)
        if coverage < MIN_COVERAGE or worst_cov < 0.5:
            where = (f"{worst_q[0]:.1f}-{worst_q[1]:.1f}s only {worst_cov*100:.0f}% covered"
                     if worst_q else "")
            verdict = "WRONG-LYRIC"
            note = (f"sings only {coverage*100:.0f}% of this window's words"
                    + (f"; {where}" if where else "")
                    + f' — clip: "{clip["text"][:40]}"')
        if mouthing:
            note = (note + "; " if note else "") + \
                   f"MOUTHING over silence for the first {head:.1f}s — cover_mouthing.py"
        if verdict != "OK" or mouthing:
            todo += 1
        print(f"{perf:<7} {start:8.1f} {med:+8.0f}ms {spread:7.0f}ms  {verdict}"
              + (f"  -> {note}" if note else ""))
        rows.append({"clip": perf, "startSec": start, "verdict": verdict,
                     "driftMs": round(med), "spreadMs": round(spread),
                     "words": len(kept), "mouthingHeadSec": head or None, "note": note})

    need = [r for r in rows if r["verdict"] in ("RE-RENDER", "WRONG-LYRIC", "NO-TRANSCRIPT")]
    print(f"\n{len(rows)} windows; {todo} need work")
    if need:
        print("MUST RE-RENDER (nothing free fixes these):")
        for r in need:
            print(f"  {r['clip']} at {r['startSec']:.0f}s — {r['verdict']}"
                  + (f": {r.get('note','')}" if r.get("note") else ""))
    else:
        print("Nothing needs a render — every remaining fix is free.")
    if a.json:
        json.dump({"shipBarMs": SHIP_MS, "windows": rows,
                   "mustRerender": [r["clip"] for r in need]}, open(a.json, "w"), indent=1)
    return 5 if todo else 0


if __name__ == "__main__":
    raise SystemExit(main())
