#!/usr/bin/env python3
"""singer_diarize.py — WHO SINGS EACH LINE, measured from the vocal stem, then verified by ear.

WHY. A per-line pitch median called the hook's first two lines "female" (~250 Hz) and put
her face on HIS vocal for seven draws; the operator heard it in one play (2026-08-23). A
pitch number cannot tell an octave, a unison or a high male register from a voice. The
operator: "You need to hear the audio. If there are two different people singing, you
need to find out. Look online to figure out how you can get transcripts of two people."

HOW (what the singer-diarization literature does, and why each step):
  1. Separate the vocal stem with Demucs (htdemucs). Diarization on the mixed signal
     does not work for singing — the backing track dominates every embedding.
  2. Per LINE (the brief's word-aligned line table), compute a speaker embedding on
     the stem (resemblyzer's GE2E d-vector). A d-vector is what "the same voice" means
     to a diarizer: it is trained to put two recordings of one speaker close together
     whatever they are saying, and two speakers apart.
  3. Cluster the lines into two voices (agglomerative, cosine). Then LABEL the two
     clusters from an anchor the operator has confirmed by ear (`--anchor 21-31=male`)
     — never from a frequency. With no anchor the clusters are reported as A/B.
  4. A line whose embedding sits between the clusters (both voices, unison) is reported
     as `both` with its margin, not forced to one side.
  5. Write `build/singers.json` and `build/singers.html`: every line, its stem snippet
     (playable), the tool's label and confidence. THE OPERATOR LISTENS AND CORRECTS the
     page's table — `--corrections` applies his answers. The brief then takes `who`
     from this file, not from the sheet and not from a detector.

This runs in its own venv ($VCLAW_WORKSPACE/tools/singer-diarize/.venv: demucs, resemblyzer,
scikit-learn) because torch does not ship for the system Python.

Usage:
  singer_diarize.py --song build/hook.wav --brief build/vocal-brief.json --out build/singers.json
                    [--anchor 21-31=male] [--html build/singers.html]
                    [--stem build/stems/vocals.wav] [--corrections "25.52=male,63.04=both"]
"""
from __future__ import annotations

import argparse
import hashlib
import html
import json
import os
import subprocess
import sys
import os as _os, sys as _sys
_sys.path.insert(0, _os.path.dirname(_os.path.abspath(__file__)))
from lane_page import page as shell  # noqa: E402

VENV_PY = os.path.join(os.environ.get("VCLAW_WORKSPACE")
                       or os.path.expanduser(os.path.join("~", "videoclaw")),
                       "tools", "singer-diarize", ".venv", "bin", "python")


def say(msg: str) -> None:
    print(f"singer_diarize: {msg}", file=sys.stderr)


def ensure_venv() -> None:
    """Re-exec under the tool venv when the current interpreter lacks torch."""
    try:
        import resemblyzer  # noqa: F401
        import demucs  # noqa: F401
    except ImportError:
        if os.path.abspath(sys.executable) == os.path.abspath(VENV_PY):
            sys.exit("singer_diarize: the tool venv lacks demucs/resemblyzer — "
                     "pip install demucs resemblyzer scikit-learn soundfile in it")
        if not os.path.isfile(VENV_PY):
            sys.exit(f"singer_diarize: no tool venv at {VENV_PY} — "
                     "python3.11 -m venv it and pip install demucs resemblyzer scikit-learn soundfile")
        os.execv(VENV_PY, [VENV_PY, os.path.abspath(__file__), *sys.argv[1:]])


STEM_MODEL = "htdemucs"


def _sha256_file(path: str) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as fh:
        for chunk in iter(lambda: fh.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def stem_is_current(song: str, stem_path: str) -> bool:
    """The stem is only valid for the song it was cut from. Cached BY CONTENT, not by
    path: a re-picked hook (a new hook.wav) must not keep the previous song's vocals —
    the slices would then be sung to the wrong track and every gate downstream would
    agree with each other. Same rule as make_plan.build_voice."""
    if not (os.path.isfile(stem_path) and os.path.getsize(stem_path) > 0):
        return False
    try:
        side = json.load(open(stem_path + ".json", encoding="utf-8"))
    except (OSError, ValueError):
        return False
    return side.get("songSha") == _sha256_file(song) and side.get("model") == STEM_MODEL


def separate_vocals(song: str, stem_path: str) -> str:
    """Demucs htdemucs → vocals.wav, with a content sidecar (<stem>.json: songSha, model)."""
    if stem_is_current(song, stem_path):
        return stem_path
    out_dir = os.path.dirname(os.path.abspath(stem_path))
    os.makedirs(out_dir, exist_ok=True)
    if os.path.isfile(stem_path):
        say("rebuilding the vocal stem: the sidecar is missing or names a different song")
    say("separating the vocal stem with Demucs (htdemucs) — a few minutes on CPU/MPS...")
    tmp = os.path.join(out_dir, "_demucs")
    r = subprocess.run([sys.executable, "-m", "demucs", "--two-stems", "vocals", "-n", "htdemucs",
                        "-o", tmp, song], capture_output=True, text=True)
    if r.returncode:
        sys.exit(f"singer_diarize: demucs failed:\n{r.stderr[-1500:]}")
    base = os.path.splitext(os.path.basename(song))[0]
    produced = os.path.join(tmp, "htdemucs", base, "vocals.wav")
    if not os.path.isfile(produced):
        sys.exit(f"singer_diarize: demucs produced no vocals.wav under {tmp}")
    os.replace(produced, stem_path)
    with open(stem_path + ".json", "w", encoding="utf-8") as fh:
        json.dump({"songSha": _sha256_file(song), "model": STEM_MODEL}, fh)
    return stem_path


def line_table(brief_path: str) -> list[dict]:
    doc = json.load(open(brief_path, encoding="utf-8"))
    rows = []
    for ln in doc.get("lines", []):
        if ln.get("start") is None or ln.get("end") is None:
            continue
        if float(ln["end"]) - float(ln["start"]) < 0.8:
            continue
        rows.append({"start": float(ln["start"]), "end": float(ln["end"]),
                     "text": ln.get("text", ""), "sheet": ln.get("who", ""),
                     "adlib": bool(ln.get("adlib"))})
    return rows


def embed_lines(stem: str, rows: list[dict]):
    import numpy as np
    from resemblyzer import VoiceEncoder, preprocess_wav
    import soundfile as sf
    wav, sr = sf.read(stem, dtype="float32")
    if wav.ndim > 1:
        wav = wav.mean(axis=1)
    enc = VoiceEncoder(verbose=False)
    embs = []
    for r in rows:
        seg = wav[int(r["start"] * sr):int(r["end"] * sr)]
        pw = preprocess_wav(seg, source_sr=sr)
        if len(pw) < 1600:
            embs.append(None)
            continue
        embs.append(enc.embed_utterance(pw))
    return embs


def cluster(rows: list[dict], embs, anchors: dict) -> None:
    import numpy as np
    from sklearn.cluster import AgglomerativeClustering
    idx = [i for i, e in enumerate(embs) if e is not None]
    X = np.stack([embs[i] for i in idx])
    X = X / np.linalg.norm(X, axis=1, keepdims=True)
    if len(idx) < 2:
        for i in idx:
            rows[i].update({"voice": "A", "confidence": 0.0})
        return
    labels = AgglomerativeClustering(n_clusters=2, metric="cosine", linkage="average").fit_predict(X)
    cents = [X[labels == k].mean(axis=0) for k in (0, 1)]
    cents = [c / np.linalg.norm(c) for c in cents]
    sep = float(np.dot(cents[0], cents[1]))      # cosine between the two voices
    # name the clusters from the anchors (operator-confirmed spans), else A/B
    names = {0: "A", 1: "B"}
    for (a0, a1), who in anchors.items():
        votes = {0: 0, 1: 0}
        for j, i in enumerate(idx):
            if rows[i]["start"] >= a0 - 0.3 and rows[i]["end"] <= a1 + 0.3:
                votes[labels[j]] += 1
        if votes[0] != votes[1]:
            k = 0 if votes[0] > votes[1] else 1
            # a letter anchor (--anchor 42-50=a) names the performers of a same-gender duet
            other = {"male": "female", "female": "male", "a": "b", "b": "a"}.get(who, "B" if who == "A" else "A")
            names = {k: who, 1 - k: other}
    for j, i in enumerate(idx):
        d0, d1 = float(np.dot(X[j], cents[0])), float(np.dot(X[j], cents[1]))
        margin = abs(d0 - d1)
        k = labels[j]
        # between the two voices (unison / duet): small margin relative to the voices' own separation
        both = margin < 0.35 * (1.0 - sep) if sep < 0.95 else False
        rows[i].update({"voice": "both" if both else names[k],
                        "cluster": int(k), "simA": round(d0, 3), "simB": round(d1, 3),
                        "confidence": round(min(1.0, margin / max(1e-6, 1.0 - sep)), 2)})
    for i, e in enumerate(embs):
        if e is None:
            rows[i].update({"voice": "?", "confidence": 0.0})
    say(f"two voices, cosine between them {sep:.2f} (lower = more distinct); "
        f"clusters named {names[0]}/{names[1]}")


def parse_anchors(items: list[str]) -> dict:
    """`--anchor 21-31=male` (repeatable) or one comma-joined value, "8-22=a,32-43=b" — the
    pack's SINGER_ANCHOR arrives as ONE argument, and the comma form used to leave the
    second span glued to the first name ("a,32-43=b"), so no cluster was ever named."""
    out = {}
    for it in [x.strip() for one in items for x in str(one).split(",") if x.strip()]:
        span, _, who = it.partition("=")
        a, _, b = span.partition("-")
        out[(float(a), float(b))] = who.strip()
    return out


def write_html(path: str, rows: list[dict], stem: str, song: str) -> None:
    snips = os.path.join(os.path.dirname(os.path.abspath(path)), "singer-snips")
    os.makedirs(snips, exist_ok=True)
    cards = []
    for n, r in enumerate(rows):
        snip = os.path.join(snips, f"line{n:02d}.m4a")
        if not os.path.isfile(snip):
            subprocess.run(["ffmpeg", "-v", "error", "-y", "-ss", f"{r['start']:.3f}",
                            "-t", f"{r['end'] - r['start']:.3f}", "-i", song, "-c:a", "aac", "-b:a", "128k", snip],
                           check=False)
        stem_snip = os.path.join(snips, f"line{n:02d}-stem.m4a")
        if not os.path.isfile(stem_snip):
            subprocess.run(["ffmpeg", "-v", "error", "-y", "-ss", f"{r['start']:.3f}",
                            "-t", f"{r['end'] - r['start']:.3f}", "-i", stem, "-c:a", "aac", "-b:a", "128k", stem_snip],
                           check=False)
        cards.append(f"""<tr class="{html.escape(r.get('voice', '?'))}">
  <td>{r['start']:.1f}–{r['end']:.1f}s</td>
  <td>{html.escape(r['text'])}</td>
  <td>{html.escape(r.get('sheet', ''))}</td>
  <td><b>{html.escape(str(r.get('voice', '?')))}</b> <small>{r.get('confidence', 0):.2f}</small></td>
  <td><audio controls preload="none" src="file://{html.escape(snip)}"></audio></td>
  <td><audio controls preload="none" src="file://{html.escape(stem_snip)}"></audio></td>
  <td><input placeholder="your answer: male / female / both" data-start="{r['start']}" size="26"></td>
</tr>""")
    body = f"""<p>Column 4 is measured from the separated vocal stem (Demucs → speaker embedding → two voices, named from your anchor).
Play the song snippet and the stem snippet. Type your answer where the tool is wrong, then press the button to copy
the corrections line and paste it to the agent.</p>
<table><tr><th>when</th><th>line</th><th>sheet says</th><th>tool hears</th><th>song</th><th>vocal stem only</th><th>you</th></tr>
{''.join(cards)}</table>
<p><button onclick="copyC()">Copy corrections</button> <pre id="c"></pre></p>
<script>
function copyC(){{const xs=[...document.querySelectorAll('input')].filter(i=>i.value.trim()).map(i=>i.dataset.start+'='+i.value.trim());
const s='--corrections "'+xs.join(',')+'"';document.getElementById('c').textContent=s;navigator.clipboard&&navigator.clipboard.writeText(s);}}
</script>"""
    open(path, "w", encoding="utf-8").write(shell("Who sings each line — the tool's answer, for YOU to verify by ear",
        "Column 4 is measured from the separated vocal stem; play the song and the stem, type your answer where the tool is wrong, copy the corrections line.",
        body, extra_css="tr.male td:nth-child(4){color:#7db7ff}tr.female td:nth-child(4){color:#ff9ad5}tr.both td:nth-child(4){color:#c9f28f}"
                        "input{background:var(--panel);color:var(--ink);border:1px solid var(--line);padding:4px}td,th{vertical-align:middle}"))


def main() -> None:
    ensure_venv()
    # --separate-only: the `stem` stage wants the vocal stem and nothing else — it is
    # the clean clock the voice slices are cut from (VOICE_SOURCE=stem). No brief, no
    # clustering; exit 0 with the stem on disk (and its content sidecar) or non-zero.
    separate_only = "--separate-only" in sys.argv[1:]
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--song", required=True)
    ap.add_argument("--separate-only", action="store_true",
                    help="only separate the vocal stem (--song → --stem) and exit")
    ap.add_argument("--brief", required=not separate_only, help="vocal-brief.json (its word-aligned line table)")
    ap.add_argument("--out", required=not separate_only, help="singers.json")
    ap.add_argument("--html", default="")
    ap.add_argument("--stem", default="", help="vocals.wav (default <out dir>/stems/vocals.wav)")
    ap.add_argument("--anchor", action="append", default=[], metavar="A-B=male",
                    help="a span the OPERATOR confirmed by ear; names the clusters")
    ap.add_argument("--corrections", default="", help='"start=who,start=who" from the page')
    args = ap.parse_args()

    if args.separate_only:
        if not args.stem:
            sys.exit("singer_diarize --separate-only needs --stem <build/stems/vocals.wav>")
        stem = separate_vocals(args.song, args.stem)
        say(f"vocal stem: {stem}")
        return
    out_dir = os.path.dirname(os.path.abspath(args.out))
    stem = args.stem or os.path.join(out_dir, "stems", "vocals.wav")
    stem = separate_vocals(args.song, stem)
    rows = line_table(args.brief)
    say(f"{len(rows)} lines; embedding each on the vocal stem...")
    embs = embed_lines(stem, rows)
    cluster(rows, embs, parse_anchors(args.anchor))
    corrections = {}
    for it in [x for x in args.corrections.split(",") if x.strip()]:
        st, _, who = it.partition("=")
        corrections[round(float(st), 2)] = who.strip()
    for r in rows:
        c = corrections.get(round(r["start"], 2))
        if c:
            r["voice"], r["confidence"], r["by"] = c, 1.0, "operator"
    json.dump({"schemaVersion": 1, "song": os.path.abspath(args.song), "stem": os.path.abspath(stem),
               "anchors": args.anchor, "lines": rows}, open(args.out, "w"), indent=1, ensure_ascii=False)
    if args.html:
        write_html(args.html, rows, stem, args.song)
    for r in rows:
        flag = "" if r.get("voice") == r.get("sheet") or r.get("sheet") in ("both", "duet", "lead") else "  <-- differs from the sheet"
        say(f"{r['start']:7.2f}s {r.get('voice', '?'):7} {r.get('confidence', 0):.2f}  {r['text'][:42]}{flag}")
    say(f"wrote {args.out}" + (f" and {args.html}" if args.html else ""))


if __name__ == "__main__":
    main()
