#!/usr/bin/env python3
"""rate_fit.py — a take that sings the right words at a UNIFORMLY wrong speed is retimed,
not redrawn.

THE FINDING (w11 A/B, 2026-08-29, nine renders). The free-lane Seedance sang a slow-jam
line 9–12% slower than the track on EVERY take — stem or mix slice, timestamp list or one
clock, locked shot, one line per window, slowness words scrubbed: fitted rates 1.089,
1.095, 1.112, 1.120. The word onsets lie on a straight line against the song's, they are
not scattered; a slide moves every word by the same amount and cannot fix a slope, and no
prompt change moved the slope. Running the video at the measured rate did: the operator
listened to the retimed take with the song under it — "yes this is better".

WHAT THIS DOES, per synced window of a selection: pair the clip transcript's words with
the song's words for that window (in order, skeleton-fuzzy, the same pairing the driver's
rhythm judge uses), fit take_t = a + rate * song_t by least squares, and when the fit is
CLEAN and the rate is MATERIAL —
    pairs >= MIN_PAIRS, |rate - 1| >= RATE_MIN, |rate - 1| <= RATE_MAX, residual <= RESIDUAL_MAX_MS
— write a retimed copy: video `setpts=PTS/rate` (tail padded by cloning the last frame back
to the window's length), audio `atempo=rate` (so the gates that correlate the clip's own
audio against the song still read true), and point the selection at it. The constant `a`
is left for sync_shift, which runs next and slides it.

Runs in the `slide` stage BEFORE sync_shift. Never touches a lipSyncExempt window, never
touches a window whose fit is noisy (that is a wander: cover it) or tiny (a slide's job).

TWO MORE GUARDS (film #3, 2026-09-07). Grid w12 fitted a clean-looking rate (1.136, 9
pairs) and got retimed — pushing a take that was already PASSING sync (corr 0.74 @
-375ms, before slide) below the master gate (0.22). Neither existing guard caught it:
9 pairs clears MIN_PAIRS, and the fit was genuinely straight. Two more checks now run:

  * ALREADY ON THE BAR. Pass `--sync-report build/sync-report-<style>.json` (check_sync's
    rows) and a window whose measured correlation already clears SYNC_MIN is left alone,
    whatever the fit says — a rate fit only has something to offer a window that is NOT
    yet on its bar.
  * A MIS-PAIRING IS NOT A SLOPE. `pair_words` matches in order and greedily consumes the
    clip's word list; w12's 9-of-29 pairing was words matched to the wrong neighbours, not
    a real tempo. `decide()` now needs the majority of a window's own song words paired
    (MIN_PAIR_FRACTION) before it will trust a fit at all.
"""
from __future__ import annotations

import argparse
import json
import os as _os, sys as _sys
_sys.path.insert(0, _os.path.dirname(_os.path.abspath(__file__)))   # a script loaded by path (tests) still finds its siblings
import raplib  # noqa: E402
import os
import re
import subprocess
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from sync_profile import clip_resolver  # noqa: E402
from check_sync import SYNC_MIN  # noqa: E402 — the same bar check_sync gates on (RAP_BAR=keep lowers it)

MIN_PAIRS = 5
# A MIS-PAIRING IS NOT A SLOPE. `pair_words` matches in order, greedily consuming the
# clip's word list, so a handful of accidental matches on a mostly-wrong take can still
# fit a straight line: w12 paired 9 of its 29 song words and fitted a clean-looking 1.136
# (film #3, 2026-09-07). Below this fraction of the window's own words, the fit is not
# trusted no matter how straight it looks.
MIN_PAIR_FRACTION = 0.6
RATE_MIN = 0.03           # below this a slide is enough
# Beyond RATE_MAX the picture is another take's problem (cover). The pack can raise it
# (RAP_RATE_MAX): the New Era song came back ~35% slow on EVERY window, hook and
# rapped verse alike (rates 1.34-1.36, straight lines), and a film where every window
# is covered has no singer in it — the operator judges the sped-up motion by eye.
RATE_MAX = float(os.environ.get("RAP_RATE_MAX") or 0.25)
RESIDUAL_MAX_MS = 200     # a straight line, not a wander


def _key(w: str) -> str:
    """Latin letters only — a Devanagari word is transliterated first. Stripping to
    [a-z] turned every Hindi word into "" and rate_fit paired 0 words on a whole film
    (film #3, 2026-09-06: eleven windows "too few paired words (0)"), so the two
    windows that sang 3-6% fast were covered instead of retimed. Same seam as
    line_subs._latin and vocal_brief."""
    if re.search(r"[\u0900-\u097f]", w):
        from vocal_brief import norm_latin, transliterate
        w = norm_latin(transliterate(w))
    return re.sub(r"[^a-z0-9']", "", w.lower())


def _skeleton(w: str) -> str:
    """consonant skeleton, doubled letters collapsed — whisper hears "citty" for "city"."""
    return re.sub(r"(.)\1+", r"\1", re.sub(r"[aeiou]", "", _key(w)))


def pair_words(song: list[dict], clip: list[dict]) -> list[tuple[float, float]]:
    """(song_t, clip_t) for each song word matched IN ORDER to the clip's first unpaired word."""
    pairs, gi = [], 0
    for w in song:
        k = _key(w["w"])
        if not k:
            continue
        for j in range(gi, len(clip)):
            g = _key(clip[j]["w"])
            if g and (g == k or (len(k) > 3 and _skeleton(g) == _skeleton(k))):
                pairs.append((float(w["t"]), float(clip[j]["t"])))
                gi = j + 1
                break
    return pairs


def fit(pairs: list[tuple[float, float]]) -> dict:
    """Least squares take_t = a + rate*song_t; residual = RMS in ms."""
    n = len(pairs)
    if n < 2:
        return {"pairs": n, "rate": None, "offsetMs": None, "residualMs": None}
    xs, ys = [p[0] for p in pairs], [p[1] for p in pairs]
    mx, my = sum(xs) / n, sum(ys) / n
    sxx = sum((x - mx) ** 2 for x in xs)
    if sxx <= 1e-9:
        return {"pairs": n, "rate": None, "offsetMs": None, "residualMs": None}
    rate = sum((x - mx) * (y - my) for x, y in pairs) / sxx
    a = my - rate * mx
    res = (sum((y - (a + rate * x)) ** 2 for x, y in pairs) / n) ** 0.5
    return {"pairs": n, "rate": round(rate, 4), "offsetMs": int(round(a * 1000)), "residualMs": int(round(res * 1000))}


def decide(f: dict, total_words: int | None = None) -> tuple[bool, str]:
    if f["rate"] is None or f["pairs"] < MIN_PAIRS:
        return False, f"too few paired words ({f['pairs']}) to fit a rate"
    # A MIS-PAIRING IS NOT A SLOPE (see MIN_PAIR_FRACTION above). `total_words` is the
    # song's own word count for this window; omit it (None) to skip the check — callers
    # that only have the fit, not the window, keep today's behaviour.
    if total_words and f["pairs"] < MIN_PAIR_FRACTION * total_words:
        return False, (f"only {f['pairs']} of {total_words} song words paired "
                       f"({f['pairs'] / total_words:.0%}) — a mis-pairing, not a slope")
    # THE RESIDUAL LIVES IN THE TAKE'S TIME BASE, WHICH IS STRETCHED BY `rate`. A fixed
    # millisecond bar is therefore stricter the slower the take sings: the New Era desk
    # verse fitted 1.348 with residual 218 ms and read as a WANDER, yet retimed x0.74 it
    # measured 1.005 / -220 ms and the operator watched it against the track and called
    # it perfect (2026-08-30). Scaling the bar by the rate keeps the same tightness per
    # unit of the take's own time; a real rubato (residual 373 ms at 1.363, the nineties
    # w02 line) still fails.
    bar = RESIDUAL_MAX_MS * max(1.0, f["rate"])
    if f["residualMs"] > bar:
        return False, (f"not a straight line (residual {f['residualMs']}ms past "
                       f"{bar:.0f}ms at {f['rate']:.2f}x) — a wander, cover it")
    dev = abs(f["rate"] - 1.0)
    if dev < RATE_MIN:
        return False, f"rate {f['rate']:.3f} is within {RATE_MIN:.0%} — a slide's job"
    if dev > RATE_MAX:
        return False, f"rate {f['rate']:.3f} is past {RATE_MAX:.0%} — retiming would look wrong, cover it"
    return True, f"rate {f['rate']:.3f} (residual {f['residualMs']}ms over {f['pairs']} words)"


def probe_seconds(path: str) -> float:
    r = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path],
                       capture_output=True, text=True)
    try:
        return float(r.stdout.strip())
    except ValueError:
        return 0.0


def retimable(want: list[dict], got: list[dict]) -> dict | None:
    """The fit, when this take is a straight line we can retime; else None.

    THE ONE HOME for that question. Three gates asked it separately and two of them
    got it wrong: a take singing every word at 1.1-1.35x cannot correlate with its
    slice — which is exactly what makes it retimable — and both the per-draw judge and
    `check_sync` refused those takes as "moving to different words" (New Era,
    2026-08-30: 4 draws parked, then the whole cut refused).
    """
    try:
        f = fit(pair_words(want, got))
        ok, _why = decide(f, total_words=len(want))
    except Exception:                     # noqa: BLE001 — a probe never kills a gate
        return None
    return f if ok else None


def load_sync_report(path: str) -> dict[int, float]:
    """segment index -> measured correlation, from check_sync's `--json-out` rows
    ({"segment", "clip", "corr", "offset_ms"}). select_cut writes the manifest check_sync
    measures and the selection's `vocalMap` from the SAME loop in the SAME order (see
    select_cut.py's `build()`), so a row's `segment` index lines up with `vocalMap[segment]`
    — no name matching needed. Missing/unreadable/empty path -> {} (opt-in; a window with
    no report entry falls through to the fit as before)."""
    if not path or not os.path.isfile(path):
        return {}
    try:
        rows = json.load(open(path, encoding="utf-8"))
    except Exception:                     # noqa: BLE001 — an unreadable report never kills the stage
        return {}
    out: dict[int, float] = {}
    for r in rows if isinstance(rows, list) else []:
        if not isinstance(r, dict) or r.get("segment") is None or "corr" not in r:
            continue
        try:
            out[int(r["segment"])] = float(r["corr"])
        except (TypeError, ValueError):
            continue
    return out


def retime(src: str, dst: str, rate: float, ffmpeg: str, fps: int, has_audio: bool) -> None:
    dur = probe_seconds(src)
    pad = max(0.0, dur - dur / rate) + 0.25
    vf = f"[0:v]setpts=PTS/{rate:.5f},tpad=stop_mode=clone:stop_duration={pad:.3f}[v]"
    cmd = [ffmpeg, "-y", "-v", "error", "-nostdin", "-i", src]
    if has_audio:
        cmd += ["-filter_complex", vf + f";[0:a]atempo={rate:.5f},apad[a]", "-map", "[v]", "-map", "[a]",
                "-c:a", "aac", "-b:a", "192k"]
    else:
        cmd += ["-filter_complex", vf, "-map", "[v]"]
    cmd += ["-t", f"{dur:.3f}", "-r", str(fps), "-c:v", "libx264", "-crf", "16", "-pix_fmt", "yuv420p", dst]
    p = subprocess.run(cmd, capture_output=True, text=True)
    if p.returncode:
        raise SystemExit(f"rate_fit: ffmpeg failed for {dst}:\n{p.stderr[-600:]}")




def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--selection", required=True)
    ap.add_argument("--store", required=True, help="artifacts/clip-transcripts.json (words per clip)")
    ap.add_argument("--song-words", required=True, help="build/song-words.json ({t,w} absolute)")
    ap.add_argument("--stage", default="", help="fallback dir for performerClip -> file")
    ap.add_argument("--out-dir", required=True, help="where retimed clips are written")
    ap.add_argument("--out-selection", default="", help="rewrite the selection onto the retimed clips")
    ap.add_argument("--json", default="", help="rate-report.json")
    ap.add_argument("--sync-report", default="",
                    help="check_sync's --json-out (build/sync-report-<style>.json). A window "
                         "whose measured correlation already clears the sync bar is left alone "
                         "— retiming a take that is already ON its bar only moves it off (grid "
                         "w12, film #3, 2026-09-07: a passing 0.74 fitted a clean-looking 1.136 "
                         "and got pushed below the master gate). Omit to skip this guard.")
    ap.add_argument("--fps", type=int, default=24)
    ap.add_argument("--ffmpeg", default=os.environ.get("FFMPEG_BIN", "ffmpeg"))
    a = ap.parse_args()

    sel = json.load(open(a.selection, encoding="utf-8"))
    store = json.load(open(a.store, encoding="utf-8"))
    sw = json.load(open(a.song_words, encoding="utf-8"))
    song_words = sw.get("words") if isinstance(sw, dict) else sw
    resolve = clip_resolver(sel, a.stage)
    sync_corr = load_sync_report(a.sync_report)
    os.makedirs(a.out_dir, exist_ok=True)
    rows, rewritten = [], {}
    rates: dict[str, float] = {}
    for idx, s in enumerate(sel.get("vocalMap", [])):
        perf = s.get("performerClip")
        if not perf or (s.get("lipSyncExempt") and s.get("lipSyncExemptBy") != "cover") \
                or s.get("type") not in (None, "rap", "sung", "synced"):
            continue
        src = resolve(perf)
        if not src:
            rows.append({"clip": perf, "action": "skipped", "why": "no file"})
            continue
        start, end = float(s["start"]), float(s["end"])
        stem = os.path.splitext(os.path.basename(src))[0]
        words = ((store.get("clips") or {}).get(stem) or {}).get("words") or []
        in_window = [{"t": float(w["t"]) - start, "w": w["w"]} for w in song_words if start <= float(w["t"]) < end]
        f = fit(pair_words(in_window, words))
        # ALREADY ON THE BAR. Checked before decide(), so a clean-looking fit never
        # overrides a window that is already passing — a rate fit only has something to
        # offer a window that is NOT yet synced (see the module docstring).
        corr = sync_corr.get(idx)
        if corr is not None and corr >= SYNC_MIN:
            why = f"already on the bar (corr {corr:.2f}) — a rate fit would move it"
            row = {"clip": perf, "startSec": start, **f, "action": "left", "why": why}
            rows.append(row)
            print(f"  {perf}: left — {why}")
            continue
        ok, why = decide(f, total_words=len(in_window))
        row = {"clip": perf, "startSec": start, **f, "why": why}
        if not ok:
            row["action"] = "left"
            rows.append(row)
            print(f"  {perf}: left — {why}")
            continue
        dst = os.path.join(a.out_dir, os.path.basename(src))
        has_audio = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "a", "-show_entries",
                                    "stream=codec_type", "-of", "csv=p=0", src], capture_output=True, text=True).stdout.strip() != ""
        retime(src, dst, f["rate"], a.ffmpeg, a.fps, has_audio)
        rewritten[perf] = os.path.abspath(dst)
        rates[perf] = float(f["rate"])
        row.update(action="retimed", path=os.path.abspath(dst))
        rows.append(row)
        print(f"  {perf}: RETIMED x{1 / f['rate']:.3f} — {why}; the residual {f['offsetMs']:+d}ms is left for slide")
    if rewritten and a.out_selection:
        by_id = {c["id"]: c for c in sel.get("clips", [])}
        for perf, path in rewritten.items():
            if perf in by_id:
                by_id[perf]["path"] = path
                _d = raplib.probe_seconds(path)        # the assembler clamps its in-point on durationSec
                if _d:
                    by_id[perf]["durationSec"] = round(_d, 3)
            else:
                sel.setdefault("clips", []).append({"id": perf, "path": path})
        # the retimed copy's lead-in is shorter by the rate; the assembler honours headSec
        for w in sel.get("vocalMap", []):
            perf = w.get("performerClip")
            if perf in rates and w.get("headSec"):
                w["headSec"] = round(float(w["headSec"]) / rates[perf], 3)
        json.dump(sel, open(a.out_selection, "w", encoding="utf-8"), indent=1)
        print(f"selection rewritten onto the retimed clips -> {a.out_selection}")
    if a.json:
        json.dump({"schemaVersion": 1, "windows": rows,
                   "thresholds": {"minPairs": MIN_PAIRS, "minPairFraction": MIN_PAIR_FRACTION,
                                  "rateMin": RATE_MIN, "rateMax": RATE_MAX, "residualMaxMs": RESIDUAL_MAX_MS,
                                  "syncMin": SYNC_MIN}},
                  open(a.json, "w", encoding="utf-8"), indent=1)
    n = sum(1 for r in rows if r.get("action") == "retimed")
    print(f"rate_fit: {n} window(s) retimed, {len(rows) - n} left")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
