#!/usr/bin/env python3
"""transcribe_clips.py — build the clip-transcript store the edit decisions run on.

WHY THIS EXISTS. On this route the model GENERATES the vocal, so a rendered clip's own
audio IS the performance. That makes the words it is singing recoverable — and once the
words are recoverable, every edit decision stops being a guess about energy and becomes
a lookup. Which bar is this clip on. Is he singing at all here. How far off is he, and
in what shape.

The cost of not having this, measured on one film: a shot was cut using envelope
correlation, measured "0ms", declared fixed, and delivered. Its words put it 730ms
early. The operator found it on screen. Word alignment fixed it to +50ms with no
re-render at all.

WHAT GOES IN THE STORE, and why both halves matter:

  the SONG, with WORD timestamps. Segment-level times are not enough. Interpolating
  words evenly across a sung line manufactures drift that is not in the audio — the
  first version of this analysis did exactly that and produced ragged nonsense out of
  clips that were steady. `--word_timestamps True` is not optional here.

  every CLIP, with word timestamps, transcribed from its own audio.

Transcribe the STAGE (the clips the master is actually built from), not only the raw
draws: a clip that has been slid or re-cut is a different performance in film time, so
measuring the original and reporting it as the film's sync measures the wrong object.

Caching is by CONTENT, not by filename. It used to be by stem, and that quietly broke
the layer: a re-rendered window keeps its name, so the cache went on returning the
PREVIOUS take's words. On a real film the store was written at 01:32 and three of its
clips were re-rendered at 13:56 — every verdict about those windows described takes that
no longer existed. Each clip's content id is recorded in the store so a consumer can tell
that the file on disk has moved on.

Usage:
  transcribe_clips.py --song <song.mp3> --clips <dir> [--clips <dir> ...]
                      --out <artifacts/clip-transcripts.json> [--model small]
                      [--language hi]   (default $SONG_LANGUAGE, else en)
"""
from __future__ import annotations

import argparse
import glob
import hashlib
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import unicodedata

WORD_RE = re.compile(r"[^a-z']")


def norm(word: str) -> str:
    """Lower-case, punctuation stripped. Latin keeps [a-z']; any non-ASCII LETTER is
    kept too — with the transcript in the song's own language (`SONG_LANGUAGE=hi`) the
    words arrive in Devanagari, and the old Latin-only filter erased every one of them
    into an empty word list that looked exactly like a silent song."""
    return "".join(c for c in word.lower()
                   if (c.isascii() and not WORD_RE.match(c))
                   or (not c.isascii() and unicodedata.category(c)[0] in ("L", "M")))


def content_id(path: str) -> str:
    """A short id for what this file CONTAINS, not what it is called."""
    with open(path, "rb") as fh:
        return hashlib.md5(fh.read(400_000)).hexdigest()[:12]


def song_language(language: str | None = None) -> str:
    """The language whisper is forced to — resolved at CALL time so the runner's env
    (`SONG_LANGUAGE=hi` from the pack) reaches every caller, including the ones that
    pass no argument. Forced to English, whisper TRANSLATES a Hindi lyric: the slice
    "Entry pe sab chup" came back as "I want to make you suffer", and no take could
    ever match it."""
    return language or os.environ.get("SONG_LANGUAGE") or "en"


def sheet_prompt(lyrics_path: str, cap: int = 700) -> str:
    """The lyric sheet as whisper's --initial_prompt: every non-section line, ". "-joined,
    cut at the runner's cap. ONE function for every caller — the map stage, the stem
    pass and the transcribe stage must prime with the SAME bytes, because the prompt is
    part of the cache key: a byte's difference is a second 75-minute pass."""
    if not lyrics_path or not os.path.isfile(lyrics_path):
        return ""
    lines = [l.strip() for l in open(lyrics_path, encoding="utf-8")
             if l.strip() and not l.strip().startswith("[")]
    return ". ".join(lines)[:cap]


def cache_path(media: str, cache_dir: str, language: str | None = None,
               prompt: str | None = None) -> str:
    """Where whisper_words keeps (and looks for) the transcript of THESE bytes in THIS
    language under THIS prompt. Exposed so a pass made elsewhere (the map stage's
    stereo-source pass) can be seeded into the cache instead of being made again."""
    stem = os.path.splitext(os.path.basename(media))[0]
    pkey = ("-p" + hashlib.sha1(prompt.encode("utf-8")).hexdigest()[:8]) if prompt else ""
    return os.path.join(cache_dir, f"{stem}-{content_id(media)}-{song_language(language)}{pkey}.json")


def seed_cache(media: str, cache_dir: str, whisper_json: str, language: str | None = None,
               prompt: str | None = None) -> str:
    """Put an existing openai-shaped whisper JSON where whisper_words will find it for
    `media`. The map stage transcribes the song once; the transcribe stage used to make
    the identical pass again (75 min on the slow engine, film #3). Returns the path."""
    os.makedirs(cache_dir, exist_ok=True)
    dest = cache_path(media, cache_dir, language, prompt)
    if not os.path.isfile(dest):
        shutil.copyfile(whisper_json, dest)
    return dest


WORDS_SCHEMA = 2


def words_from_doc(d: dict) -> dict:
    """openai-shaped whisper JSON -> {schema, text, words:[{t,e,w,eFrom}], segments}. The one reader.

    `e` IS THE END WHISPER WROTE for the word — and whisper TILES ends to the next word's
    onset (377 of 428 words on a real stem), so eFrom "whisper" means "until the next
    word", not "until the note stops"; consumers that need a real end use it only when
    eFrom is a measured source (see `measured_ends`, `word_ends.py`). Schema 1 kept the onset alone, and every
    consumer that needed a duration made one up: `vocal_brief.load_word_list` gave a
    fill word 0.4 s flat, `vocal_map.word_stretches` measured a gap start-to-start
    ("over-measures by one word's length"), `slice_check` called the last word's ONSET
    "last heard", and `make_plan.exact_onsets` sized the render from a line end built on
    those — so a held final note ("nimboodaaa…", 2.5 s) rendered as a 0.4 s word and the
    clip closed while the singer was still on it. `eFrom` names the source: "whisper"
    here; `word_ends.py` may replace it with "gemini" or "stem" (see WORD_ENDS)."""
    words = [{"t": round(float(w["start"]), 3),
              "e": round(max(float(w.get("end", w["start"])), float(w["start"])), 3),
              "w": norm(w["word"]), "eFrom": "whisper"}
             for s in d.get("segments", []) for w in s.get("words", []) if norm(w["word"])]
    return {"schema": WORDS_SCHEMA, "text": " ".join(w["w"] for w in words), "words": words,
            "segments": [{"start": float(s["start"]), "end": float(s["end"]),
                          "text": s["text"].strip()} for s in d.get("segments", [])]}


def measured_ends(words: list[dict], audio: str, *, hop_s: float = 0.02, floor_db: float = -40.0,
                  rel_db: float = -25.0, quiet_s: float = 0.16, peak_window_s: float = 1.0,
                  min_word_s: float = 0.08, hard_cap_s: float | None = None) -> list[dict]:
    """Measure where each word actually STOPS SOUNDING in the audio (the vocal stem) — the
    LOCAL source of word ends (`WORD_ENDS=stem`). Deterministic and free.

    WHY MEASURE RATHER THAN TRUST WHISPER'S END. Measured on a real stem (Hanuman Chalisa,
    2026-09-05): whisper's word end equalled the NEXT word's onset on 377 of 428 words —
    it tiles ends to the next start, so its `e` says "until the next word", not "until
    the note stops". That is wrong in both directions at once: "gansagar" tiled 4.1 s
    across 3.3 s of silence (a gap the onset-only map had found and the tiled end lost),
    while "roop" really was held for 3.9 s and the onset-only map called that silence.
    Only the audio can tell the two apart.

    Walk forward from the onset: the word ends at the first quiet_s of frames below
    max(floor_db, the word's own peak + rel_db), never before t + min_word_s, never past
    the NEXT word's onset (that is its word), never past hard_cap_s (the song end). `t` is
    never moved — whisper is the clock for onsets. A word whose end moved by more than a
    hop carries eFrom "stem" and its whisper end in whisperE."""
    import numpy as np
    sr = 16000
    r = subprocess.run(["ffmpeg", "-v", "error", "-i", audio, "-vn", "-ac", "1", "-ar", str(sr),
                        "-f", "f32le", "-"], capture_output=True)
    x = np.frombuffer(r.stdout, dtype=np.float32)
    hop = max(1, int(hop_s * sr))
    n = x.size // hop
    if n == 0:
        return [dict(w) for w in words]
    frames = x[: n * hop].reshape(n, hop)
    db = 20.0 * np.log10(np.sqrt(np.mean(frames * frames, axis=1)) + 1e-9)
    total_s = x.size / sr
    quiet_frames = max(1, int(round(quiet_s / hop_s)))
    out = []
    for i, w in enumerate(words):
        w = dict(w)
        t = float(w["t"])
        old_e = float(w.get("e", t))
        nxt = float(words[i + 1]["t"]) if i + 1 < len(words) else total_s
        cap = min(nxt, total_s)
        if hard_cap_s is not None:
            cap = min(cap, hard_cap_s)
        a = int(t / hop_s)
        b = min(n, int(cap / hop_s))
        if a >= n or b <= a:
            out.append(w)
            continue
        peak = float(np.max(db[a:min(b, a + max(1, int(peak_window_s / hop_s)))]))
        thresh = max(floor_db, peak + rel_db)
        k, quiet = a, 0
        new_e = cap
        while k < b:
            quiet = quiet + 1 if db[k] <= thresh else 0
            if quiet >= quiet_frames:
                new_e = (k - quiet_frames + 1) * hop_s
                break
            k += 1
        new_e = round(min(cap, max(t + min_word_s, new_e)), 3)
        if abs(new_e - old_e) > hop_s / 2:
            w["e"], w["eFrom"] = new_e, "stem"
            w.setdefault("whisperE", round(old_e, 3))
        out.append(w)
    return out


def whisper_words(media: str, model: str, cache_dir: str, language: str | None = None,
                  prompt: str | None = None) -> dict:
    """Transcribe one file to {text, words:[{t,w}]}, cached by CONTENT and LANGUAGE.

    It used to cache by stem, and that quietly broke the whole layer. A window that is
    re-rendered keeps its filename — `w06.mp4` is always `w06.mp4` — so the cache went
    on returning the previous take's words. Measured on a real film: the store was
    written at 01:32 and three of its clips were re-rendered at 13:56, so every verdict
    about those windows was about takes that no longer existed anywhere in the film.
    It cannot be spotted by reading the output; the store looks complete and current.

    This is the same failure as the adapter that skips its download when an mp4 is
    already in the output directory, and the answer is the same: key on content.
    """
    stem = os.path.splitext(os.path.basename(media))[0]
    lang = song_language(language)
    os.makedirs(cache_dir, exist_ok=True)   # first caller creates the cache dir
    # The language is part of the key: the same bytes transcribed in English and in
    # Hindi are two different transcripts, and the English one is a translation.
    # And so is the PROMPT: whisper primed with the card's lines hears a chant it
    # otherwise renders as "uh nuh nuh" (New Era, 2026-08-30), and that is a different
    # transcript of the same bytes — a stale unprimed entry must not answer for it.
    cached = cache_path(media, cache_dir, lang, prompt)
    if not os.path.isfile(cached):
        # A clip with NO audio stream is a legitimate render (intros go out with
        # generateAudio off) — its transcript is "no words", not a crash. ffmpeg's
        # "Output file does not contain any stream" killed the whole transcribe stage
        # on w00 the first time the back half ran with silent intros (2026-08-24).
        probe = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "a",
                                "-show_entries", "stream=codec_type", "-of", "csv=p=0", media],
                               capture_output=True, text=True)
        if probe.returncode == 0 and not probe.stdout.strip():
            doc = {"text": "", "words": [], "segments": [], "noAudioStream": True}
            with open(cached, "w", encoding="utf-8") as fh:
                json.dump(doc, fh)
            return doc
        wav = os.path.join(cache_dir, f"{stem}.wav")
        # NATIVE SAMPLE RATE, not 16k. Pre-resampling to 16k made whisper completely
        # deaf on a real take (w08 noir, 2026-08-24: 0 segments at 16k, full correct
        # lyric at the clip's own 48k — verified both ways on the same bytes). Whisper
        # resamples internally anyway; feeding it the native-rate mono keeps the one
        # resample ITS resample, and the marginal vocal above the VAD floor.
        subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", media, "-vn",
                        "-ac", "1", wav], check=True)
        r = subprocess.run(["whisper", wav, "--model", model, "--output_format", "json",
                            "--output_dir", cache_dir, "--language", lang,
                            "--word_timestamps", "True"]
                           + (["--initial_prompt", prompt[:700]] if prompt else []),
                           capture_output=True, text=True)
        # whisper names its output after the INPUT stem; move it to the content key so
        # the next render of the same window misses this entry instead of reusing it.
        produced = os.path.join(cache_dir, f"{stem}.json")
        if os.path.isfile(produced):
            os.replace(produced, cached)
        if not os.path.isfile(cached):
            raise SystemExit(f"whisper produced no JSON for {media}\n{r.stderr[-800:]}")
    return words_from_doc(json.load(open(cached, encoding="utf-8")))


def duration(path: str) -> float:
    r = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration",
                        "-of", "csv=p=0", path], capture_output=True, text=True)
    try:
        return round(float(r.stdout.strip()), 3)
    except ValueError:
        return 0.0


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--song", required=True)
    ap.add_argument("--clips", action="append", required=True,
                    help="directory of clips; repeatable (stage, b-roll, ...)")
    ap.add_argument("--out", required=True)
    ap.add_argument("--model", default="small")
    ap.add_argument("--cache", help="where whisper output lives (default: beside --out)")
    ap.add_argument("--language", default="",
                    help="whisper language code (default $SONG_LANGUAGE, else en)")
    ap.add_argument("--song-prompt", default="",
                    help="lyric sheet that primes the SONG pass (sheet_prompt); the same "
                         "prompt the map stage used, so its seeded transcript is the hit")
    a = ap.parse_args()
    lang = song_language(a.language or None)

    cache = a.cache or os.path.join(os.path.dirname(os.path.abspath(a.out)), "clip-tx")
    os.makedirs(cache, exist_ok=True)

    song_prompt = sheet_prompt(a.song_prompt) if a.song_prompt and os.path.isfile(a.song_prompt) else None
    song = whisper_words(a.song, a.model, cache, lang, prompt=song_prompt)
    if not song["words"]:
        raise SystemExit("the song transcribed with no word timestamps — every drift "
                         "measured against it would be interpolated, i.e. invented")
    print(f"song: {len(song['words'])} words, "
          f"{song['words'][0]['t']:.2f}s .. {song['words'][-1]['t']:.2f}s")

    clips = {}
    for d in a.clips:
        for f in sorted(glob.glob(os.path.join(d, "*.mp4"))):
            cid = os.path.splitext(os.path.basename(f))[0]
            if cid in clips:
                continue                      # first directory listed wins
            tx = whisper_words(f, a.model, cache, lang)
            clips[cid] = {"path": os.path.abspath(f), "durationSec": duration(f),
                          "sourceId": content_id(f), **tx}
            print(f"  {cid}: {len(tx['words'])} words  \"{tx['text'][:56]}\"")

    os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True)
    json.dump({"schemaVersion": 1, "model": a.model, "language": lang,
               "song": {"path": os.path.abspath(a.song), **song},
               "clips": clips}, open(a.out, "w"), indent=1)
    print(f"\n{len(clips)} clips -> {a.out}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
