#!/usr/bin/env python3
"""explore_queue.py — SUPERVISOR over the repo's overnight batch queue.

This is NOT a queue driver. `src/video/batch-queue.ts` + `src/cli/handlers/batch.ts`
already own submit/poll/download, `isExploreThrottled` + `nextBackoffMs` backoff,
resumable per-scene state (`batch-queue.json`), wedge detection and the
credit-guarded `--auto-resubmit`. Re-implementing any of that against the raw
useapi endpoints — which is how the WICC film was actually driven — means two
copies of the retry logic and only one of them getting the next fix.

So this script does the three things that layer genuinely lacks:

  1. **A heartbeat.** `batch-monitor --once` tells you the queue's state; nothing
     tells you whether the thing polling it is still alive. An overnight lane that
     died at 01:10 and a lane that is merely queued behind another machine look
     identical from the outside — for hours. `--status` distinguishes them: a
     `running` supervisor whose heartbeat is older than 3x its own interval is
     reported BROKEN (exit 1), not "still going".
  2. **A guard that cannot silently die, and cannot silently spin.** Every pass is
     wrapped in `except BaseException` — the WICC originals learned that the hard
     way, a submit pass raising takes the whole unattended background process with
     it. But `KeyboardInterrupt`/`SystemExit` are RE-RAISED (the originals swallowed
     Ctrl-C, so the only way to stop them was `kill`), and after
     MAX_ERROR_PASSES consecutive failing passes with nothing landing the run
     FAILS LOUD (state `failed`, exit 1). The originals looped a fixed count and
     then exited 0 whatever the outcome, which reads as success to any caller.
  3. **Two preconditions asserted BEFORE any submit** — both are silent failures
     that cost a whole night of queue time and report no error at all:
       * a transport that leaves the provider's audio OFF returns SILENT lip-sync
         clips, `check_sync.py` then measures ~0 correlation, and nothing anywhere
         says why. seedance-direct attaches the voice as a real audio reference and
         cites it in the prompt, so there is no env switch left to forget.
       * `DEFAULT_ASPECT_RATIO` is `16:9`. A 9:16 film whose manifest omits
         `defaults.aspectRatio` gets landscape clips — discovered after the queue
         drains, which on explore is hours later.

`plan.json` (skill-authored; this schema is the contract):

    {
      "schemaVersion": 1,
      "lane": "explore",                  // names build/queue/<lane>/
      "route": "seedance-direct",         // default AND the only free value; anything
                                          //   because only it resume-skips (see load_plan)
      "aspect": "9:16",                   // REQUIRED — no default, see (3) above
      "resolution": "720p",               // optional (720p | 1080p)
      "seconds": 10,                      // optional per-plan default duration
      "projectDir": "/abs/projects/<slug>",   // optional; used to derive queueDir
      "queueDir": "/abs/.../build/queue/explore",  // optional; --out wins
      "workspaceRoot": "/abs/videoclaw",  // optional; passed as --root
      "identityRef": "/abs/portrait.png", // optional plan-wide default
      "jobs": [
        {"id": "w05", "prompt": "...", "window": 5, "styleId": "neon-rain",
         "seconds": 9.4, "synced": true,
         "identityRef": "/abs/portrait.png",       // the FACE (image)
         "voiceRef": "/abs/voice/w05.mp4"}         // black-frame + track (video)
      ]
    }

Both refs go into the batch manifest's `characterRefs` and are split by EXTENSION
downstream: the still becomes the image reference and the `.mp4` the video/audio one
— which is the whole free-lip-sync trick.
They must never be `keyframe`, which would open the clip on the still.

Files written (all inside the queue dir):
  supervisor.json    the heartbeat + run state. THIS script's only state.
  provenance.json    {realpath(clip): {jobId, window, styleId, synced, voiceRef(bool),
                     voiceRefPath, clip, submittedAt}} — consumed by index_library.py /
                     select_cut.py, which is how `synced` stays CARRIED rather than
                     guessed from the picture. Keyed by clip path because that is all
                     the indexer has when it finds a file on disk.
  batch-manifest.json  the compiled manifest. On disk, not a tempfile, because
                     `handleVideoBatchSubmit` re-reads --manifest on EVERY resume
                     pass; a vanishing manifest breaks retry-after-refusal.
`batch-queue.json` / `batch-status.json` belong to the TS layer — never written here.

Between passes it RE-OFFERS anything still pending past the re-offer window by
re-running `batch-submit` (idempotent: the transport skips scenes it already sent).
Explore refuses most of a batch on the first offer, and a refused window that is
never re-offered is a hole in the film discovered in the morning.

Usage:
  explore_queue.py --plan <plan.json> [--out <dir>] [--interval 600] [--once]
                   [--max-minutes N] [--stall-minutes N] [--fail-wedged]
                   [--auto-resubmit] [--max-resubmits 2]
  explore_queue.py --status --out <dir> [--interval N]

Exit: 0 every job landed · 10 still outstanding (gate-compatible) · 1 broken
      (a failed job, a failed run, or a dead heartbeat) · 2 refused before any
      submit (bad plan / precondition).
"""
from __future__ import annotations

import argparse
import hashlib
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone

SUPERVISOR_FILE = "supervisor.json"
PROVENANCE_FILE = "provenance.json"
MANIFEST_FILE = "batch-manifest.json"
QUEUE_STATE_FILE = "batch-queue.json"
QUEUE_STATUS_FILE = "batch-status.json"

# The Runway account is suspended, so seedance-direct through the VENDORED
# Higgsfield engine is the free lane now. "Free" is CONDITIONAL there in a way it
# never was for Runway: resolveAdapterCommand only routes seedance-direct to the
# in-tree engine when it is bootstrapped, and otherwise falls through to the PAID
# native transport. Since this supervisor RE-OFFERS pending jobs, running against
# that fallback would re-buy the batch every pass — so the route is not trusted by
# name, it is proven free before anything is submitted (see free_engine_available).
DEFAULT_ROUTE = "seedance-direct"
# RUNWAY IS GONE (removed 2026-08-15). The account was suspended on 2026-08-10 and
# every path here refused it by name anyway; what remained were its ASSUMPTIONS, and
# those were still doing damage. The clip-length cap in make_plan.py was Runway's 10s
# explore limit applied to every route, so a film on Higgsfield seedance_2_5 — which
# the engine's own table gives (4, 20) — was refused above 10s. On a 112-second song
# that is fourteen renders instead of six on a single-slot lane.
#
# Removing it also makes this guard STRICTER rather than looser: seedance-direct must
# PROVE its free engine, and everything else is refused as paid. There is no longer a
# route that is free by name.
ROUTES = ("dreamina-useapi", "seedance-direct")
# Real routes that this supervisor does not drive — distinct from a typo, and from a
# route that is merely paid. Flow/Veo has its own submit path (native-veo / flow-r2v)
# and does not go through the explore queue at all.
ELSEWHERE_ROUTES = {
    "veo-useapi": "Flow/Veo renders through its own path (native-veo, flow-r2v), "
                  "not through the explore queue",
    "higgsfield-cli": "the PAID Higgsfield API pool is driven one window at a time by "
                      "paid_take.py (PAID_WINDOWS in the pack, inside render-all), never by "
                      "this re-offering supervisor — keep the plan on seedance-direct",
}
ASPECTS = ("16:9", "9:16", "1:1")
RESOLUTIONS = ("720p", "1080p")
# Mirrors the transport's own classifyReferences(). Kept in sync deliberately:
# the split is by extension there, so a ref whose extension we misread here is a
# ref that silently lands in the wrong provider slot.
IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
VIDEO_EXT = {".mp4", ".mov", ".webm", ".avi", ".mkv"}

MAX_ERROR_PASSES = 5
STALE_MULTIPLIER = 3          # heartbeat older than 3x the interval == dead


def now_iso() -> str:
    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()


def log(msg: str) -> None:
    print(f"[{time.strftime('%H:%M:%S')}] explore_queue: {msg}", file=sys.stderr, flush=True)


def die(msg: str, code: int = 2) -> None:
    print(f"explore_queue: {msg}", file=sys.stderr)
    raise SystemExit(code)


def repo_root() -> str:
    # skills/rap-avatar-mv/scripts/this.py -> repo root
    return os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))


def free_engine_available(env: dict | None = None) -> tuple[bool, str]:
    """Is seedance-direct going to resolve to the FREE in-tree engine?

    Mirrors hasFreeInTreeSeedanceEngine in src/video/execution-runtime.ts. Kept in
    step by hand, deliberately: getting this wrong means a re-offer pass buys the
    whole batch again, so it fails closed and says which piece is missing.
    """
    env = os.environ if env is None else env
    if env.get("VCLAW_SEEDANCE_DIRECT_NATIVE"):
        return False, ("VCLAW_SEEDANCE_DIRECT_NATIVE is set, which forces the PAID "
                       "native transport")
    engine = os.path.join(repo_root(), "engines", "seedance-direct")
    venv = os.path.join(engine, ".venv", "bin", "python")
    profile = env.get("HIGGS_VCLAW_PROFILE") or os.path.join(engine, ".cloak-profile")
    if not os.path.exists(venv):
        return False, f"the engine venv is missing ({venv}) — run its bootstrap.sh"
    if not os.path.exists(profile):
        return False, f"the browser profile is missing ({profile}) — run its bootstrap.sh"
    return True, ""


def is_url(ref: str) -> bool:
    return ref.startswith("http://") or ref.startswith("https://")


def ref_kind(ref: str) -> str:
    # Unknown extensions count as images, exactly as classifyReferences does, so the
    # budget checked here is the budget the transport will actually apply.
    ext = os.path.splitext(ref.split("?")[0])[1].lower()
    return "video" if ext in VIDEO_EXT else "image"


# --------------------------------------------------------------------------
# plan.json -> the repo's batch manifest
# --------------------------------------------------------------------------

def load_plan(path: str) -> dict:
    if not os.path.isfile(path):
        die(f"plan not found: {path}")
    try:
        plan = json.load(open(path, encoding="utf-8"))
    except Exception as exc:
        die(f"plan is not valid JSON ({path}): {exc}")
    if not isinstance(plan, dict):
        die(f"plan must be a JSON object: {path}")
    if plan.get("schemaVersion") != 1:
        die(f"plan schemaVersion must be 1, got: {json.dumps(plan.get('schemaVersion'))}")

    route = plan.get("route", DEFAULT_ROUTE)
    if route in ELSEWHERE_ROUTES:
        # A real route, driven by a different door. Worth its own message because
        # `veo-useapi` is the documented DEFAULT route for every other lane in this
        # repo, so an operator setting it here is following the house rule, not
        # making a typo — and "must be one of ..." reads like their route was
        # misspelt rather than deliberately out of scope.
        die(f"route {route!r} is real, but this supervisor cannot drive it: "
            f"{ELSEWHERE_ROUTES[route]}\n"
            "  The rap explore lane submits and RE-OFFERS batches on a free route. "
            "Set ROUTE=\"seedance-direct\" in the pack for this lane.")
    if route not in ROUTES:
        die(f"plan route must be one of {', '.join(ROUTES)}, got: {json.dumps(route)}")
    # FREE ROUTE ONLY, and this is a credit guard, not a preference. This supervisor
    # RE-OFFERS pending jobs (explore refuses most of a batch and a refused window
    # otherwise never retries) — but only a route that resumes into a stable job id
    # and skips scenes it has already sent. dreamina/seedance submit all-or-throw
    # with no resume, so the same re-offer would re-buy the WHOLE batch, every pass.
    # This supervisor RE-OFFERS anything still pending, so it may only ever drive a
    # route that costs nothing to re-submit. Two qualify, for different reasons.
    if route == "seedance-direct":
        ok, why = free_engine_available()
        if not ok:
            die(f"route 'seedance-direct' would NOT use the free in-tree engine: {why}.\n"
                "  It would fall through to the PAID native transport, and this supervisor "
                "re-offers pending jobs — so every pass would re-buy the whole batch. "
                "Bootstrap the engine, or drive a paid batch directly with "
                "`vclaw video batch-submit --confirm-spend`.")
    else:
        # No route is free by name any more. seedance-direct proves its engine above;
        # everything else is paid, and this supervisor re-offers pending jobs.
        die(f"route {route!r} is a PAID route and this supervisor only drives free lanes: "
            "its re-offer pass would re-submit (and re-buy) every task. Drive a paid batch "
            "directly with `vclaw video batch-submit --confirm-spend`.")

    # aspect is REQUIRED and has no default here. Defaulting it would re-create the
    # exact silent failure this script exists to prevent, one level up.
    aspect = plan.get("aspect")
    if aspect not in ASPECTS:
        die("plan must declare \"aspect\" (one of " + ", ".join(ASPECTS) + ") — it is the "
            "film's aspect and there is NO safe default: the batch layer falls back to "
            "16:9, so a 9:16 film that omits it gets landscape clips after hours of queueing")
    resolution = plan.get("resolution", "720p")
    if resolution not in RESOLUTIONS:
        die(f"plan resolution must be one of {', '.join(RESOLUTIONS)}, got: {json.dumps(resolution)}")

    jobs = plan.get("jobs")
    if not isinstance(jobs, list) or not jobs:
        die("plan requires a non-empty \"jobs\" array")

    plan_ref = plan.get("identityRef")
    seen: set[str] = set()
    for index, job in enumerate(jobs):
        if not isinstance(job, dict):
            die(f"plan job {index} must be an object")
        jid = job.get("id")
        if not isinstance(jid, str) or not jid.strip():
            die(f"plan job {index} requires a non-empty string \"id\"")
        # The id becomes clips/<id>.mp4 on disk, so refuse anything that would
        # escape the queue dir or produce a path the downstream index cannot match.
        if not all(c.isalnum() or c in "._-" for c in jid) or jid.startswith("."):
            die(f"plan job \"{jid}\": id must be [A-Za-z0-9._-] (it becomes clips/<id>.mp4)")
        if jid in seen:
            die(f"plan has a duplicate job id: \"{jid}\"")
        seen.add(jid)
        if not isinstance(job.get("prompt"), str) or not job["prompt"].strip():
            die(f"plan job \"{jid}\" requires a non-empty string \"prompt\"")
        if "keyframe" in job:
            die(f"plan job \"{jid}\" sets \"keyframe\": this lane cites the identity still as a "
                "REFERENCE, never as the first frame. buildBatchPayload prioritises characterRefs "
                "and would drop the keyframe silently — remove it, or use identityRef.")
        seconds = job.get("seconds", plan.get("seconds"))
        if seconds is not None and not (isinstance(seconds, (int, float)) and seconds > 0):
            die(f"plan job \"{jid}\" seconds must be a positive number when present")

        refs = []
        identity = job.get("identityRef", plan_ref)
        if identity is not None:
            if not isinstance(identity, str) or not identity.strip():
                die(f"plan job \"{jid}\" identityRef must be a non-empty string when present")
            refs.append(identity)
        # A `both` window carries a second STILL. It is an image, so the one-video
        # guard below is unaffected; but a second face with no first is a plan error.
        identity2 = job.get("identityRef2")
        if identity2 is not None:
            if not isinstance(identity2, str) or not identity2.strip():
                die(f"plan job \"{jid}\" identityRef2 must be a non-empty string when present")
            if identity is None:
                die(f"plan job \"{jid}\" has identityRef2 but no identityRef — the second "
                    "performer's still cannot be the only face")
            refs.append(identity2)
        identity3 = job.get("identityRef3")
        if identity3 is not None:
            if not isinstance(identity3, str) or not identity3.strip():
                die(f"plan job \"{jid}\" identityRef3 must be a non-empty string when present")
            if identity2 is None:
                die(f"plan job \"{jid}\" has identityRef3 but no identityRef2")
            refs.append(identity3)
        voice = job.get("voiceRef")
        if voice is not None:
            if not isinstance(voice, str) or not voice.strip():
                die(f"plan job \"{jid}\" voiceRef must be a non-empty string when present")
            refs.append(voice)
        for ref in refs:
            if not is_url(ref) and not os.path.isfile(ref):
                die(f"plan job \"{jid}\": reference not found on disk: {ref}")
        # THE MOTION DONOR is its own field (never a reference path): a second video in
        # referencePaths would silently drop the voice. It needs a voice to be a clock for.
        motion = job.get("motionRef")
        if motion is not None:
            if not isinstance(motion, str) or not motion.strip():
                die(f"plan job \"{jid}\" motionRef must be a non-empty string when present")
            if not os.path.isfile(motion):
                die(f"plan job \"{jid}\": motion reference not found on disk: {motion}")
            if voice is None:
                die(f"plan job \"{jid}\" has a motionRef but no voiceRef — a donor with no clock")
        videos = [r for r in refs if ref_kind(r) == "video"]
        images = [r for r in refs if ref_kind(r) == "image"]
        # The provider body carries ONE video reference. A second is not
        # rejected — it is silently dropped, which on a lip-sync job means the clip
        # renders against the wrong track and only check_sync.py ever notices.
        if len(videos) > 1:
            die(f"plan job \"{jid}\" has {len(videos)} video references; the provider body carries "
                "exactly one (videoAssetId) and drops the rest silently")
        if videos and not images:
            die(f"plan job \"{jid}\" has a voice video but no identity still — that renders a "
                "stranger's face mouthing your track. Give it an identityRef.")
        if job.get("synced") and not voice:
            die(f"plan job \"{jid}\" claims synced: true with no voiceRef. `synced` is CARRIED "
                "into provenance.json and believed downstream; a clip with no voice reference "
                "cannot be synced to its bar.")
    return {"schemaVersion": 1, "route": route, "aspect": aspect, "resolution": resolution,
            "jobs": jobs, "seconds": plan.get("seconds"), "identityRef": plan_ref,
            "verifyWords": plan.get("verifyWords", True),
            "pronouns": plan.get("pronouns"),
            "lane": plan.get("lane", "explore"), "queueDir": plan.get("queueDir"),
            "projectDir": plan.get("projectDir"), "workspaceRoot": plan.get("workspaceRoot")}


def build_manifest(plan: dict) -> dict:
    """plan.json -> the batch-queue manifest the TS layer already understands.

    The one translation that matters: identity still + voice video both go into
    `characterRefs`, never `keyframe`. characterRefs are delivered with
    `referenceRole: 'character'` and are never used as the first frame.
    """
    default_seconds = plan.get("seconds") if isinstance(plan.get("seconds"), (int, float)) else 10
    jobs = []
    for job in plan["jobs"]:
        refs = []
        identity = job.get("identityRef", plan.get("identityRef"))
        if identity:
            refs.append(identity)
        if job.get("identityRef2"):
            refs.append(job["identityRef2"])
        if job.get("identityRef3"):
            refs.append(job["identityRef3"])
        if job.get("voiceRef"):
            refs.append(job["voiceRef"])
        entry = {"id": job["id"], "prompt": job["prompt"]}
        if refs:
            entry["characterRefs"] = refs
        seconds = job.get("seconds")
        if isinstance(seconds, (int, float)):
            entry["seconds"] = seconds
        jobs.append(entry)
    return {
        "schemaVersion": 1,
        "route": plan["route"],
        "defaults": {"seconds": default_seconds,
                     "aspectRatio": plan["aspect"],
                     "resolution": plan["resolution"]},
        "jobs": jobs,
    }


# --------------------------------------------------------------------------
# preconditions
# --------------------------------------------------------------------------

def child_env(plan: dict) -> dict:
    """The environment the vclaw child runs in.

    This used to hold open `VCLAW_RUNWAY_AUDIO`, the only door through which
    a previous transport turned the provider's audio on — a queue submitted without it
    came back SILENT with no error anywhere. Runway is gone (2026-08-15), and
    seedance-direct attaches the voice as a real audio reference and cites it in the
    prompt, so there is no env door left to hold open and setting one would be cargo.
    """
    return dict(os.environ)


def assert_submit_preconditions(plan: dict, manifest: dict, args: argparse.Namespace) -> str:
    """Everything that must be true BEFORE a single job is submitted. Returns the
    resolved vclaw entrypoint."""
    # Re-assert the aspect on the COMPILED manifest, not just the plan: this is the
    # value the provider actually sees, and it is one edit of build_manifest away
    # from silently reverting to the 16:9 default.
    got = (manifest.get("defaults") or {}).get("aspectRatio")
    if got != plan["aspect"]:
        die(f"compiled manifest defaults.aspectRatio is {got!r} but the film's aspect is "
            f"{plan['aspect']!r} — refusing to queue clips in the wrong shape")

    # The route is already pinned to the free lane at plan-read time (see load_plan),
    # which is also the TS layer's --auto-resubmit guard — but that one only fires once
    # the monitor is already running, so an overnight lane would refuse at 03:00
    # instead of at launch.
    if args.auto_resubmit and not args.stall_minutes:
        # planResubmits() returns [] when stallMinutes <= 0, so the flag without a
        # stall window is a no-op that LOOKS armed.
        die("--auto-resubmit needs --stall-minutes <n>: wedge detection is disabled at 0, "
            "so the flag would silently do nothing all night")

    vclaw = args.vclaw or os.path.join(repo_root(), "dist", "cli", "vclaw.js")
    if not os.path.isfile(vclaw):
        die(f"vclaw entrypoint not found at {vclaw} — run `npm run build` (the queue is driven "
            "through the compiled CLI, not the TypeScript sources)")
    return vclaw


# --------------------------------------------------------------------------
# supervisor state (the heartbeat)
# --------------------------------------------------------------------------

def supervisor_path(out_dir: str) -> str:
    return os.path.join(out_dir, SUPERVISOR_FILE)


def write_supervisor(out_dir: str, doc: dict) -> None:
    doc["heartbeat"] = now_iso()
    path = supervisor_path(out_dir)
    tmp = f"{path}.tmp"
    # Atomic: --status may be read at any moment (a launchd tick, an operator, the
    # runner's gate) and a half-written heartbeat reads as a dead lane.
    with open(tmp, "w", encoding="utf-8") as handle:
        json.dump(doc, handle, indent=2)
        handle.write("\n")
    os.replace(tmp, path)


def read_json(path: str):
    try:
        return json.load(open(path, encoding="utf-8"))
    except Exception:
        return None


def read_rollup(out_dir: str):
    """The queue's rollup, preferring the TS layer's own batch-status.json."""
    status = read_json(os.path.join(out_dir, QUEUE_STATUS_FILE))
    if isinstance(status, dict) and isinstance(status.get("rollup"), dict):
        return status["rollup"]
    state = read_json(os.path.join(out_dir, QUEUE_STATE_FILE))
    if not isinstance(state, dict) or not isinstance(state.get("jobs"), list):
        return None
    jobs = state["jobs"]
    done = sum(1 for j in jobs if j.get("status") == "done")
    failed = sum(1 for j in jobs if j.get("status") == "failed")
    pending = len(jobs) - done - failed
    return {"total": len(jobs), "done": done, "pending": pending, "failed": failed,
            "terminal": pending == 0}


def age_seconds(iso: str | None) -> float | None:
    if not isinstance(iso, str) or not iso:
        return None
    try:
        stamp = datetime.fromisoformat(iso)
    except ValueError:
        return None
    if stamp.tzinfo is None:
        stamp = stamp.replace(tzinfo=timezone.utc)
    return (datetime.now(timezone.utc) - stamp).total_seconds()


# --------------------------------------------------------------------------
# provenance
# --------------------------------------------------------------------------

def write_provenance(out_dir: str, plan: dict) -> None:
    """{realpath(clip): {jobId, window, styleId, synced, voiceRef, clip, submittedAt}}.

    This is what lets `synced` be CARRIED to the cut instead of inferred from the
    picture — an open mouth in a contact strip proves nothing about WHICH words it
    is forming.

    KEYED BY REALPATH OF THE CLIP, not by job id: index_library.py finds a clip on
    disk and has only its path to look up with — it never sees the queue's job ids.
    realpath on both sides so a symlinked queue dir (/tmp -> /private/tmp on macOS)
    cannot make the two spellings of the same file miss each other. The clip does not
    exist yet at submit time, which is fine — realpath does not require existence, and
    the landed file arrives at exactly this path (clipPathForJob).

    `voiceRef` is a BOOL by consumer contract — "did this render carry a voice
    reference at all", which is the half select_cut.py tests. The path is kept
    alongside as `voiceRefPath` so the audit trail is not lost.

    submittedAt comes from the queue state (the real submit time), and an existing
    entry's submittedAt is never overwritten.
    """
    state = read_json(os.path.join(out_dir, QUEUE_STATE_FILE))
    submitted = {}
    if isinstance(state, dict):
        for job in state.get("jobs", []):
            if isinstance(job, dict) and isinstance(job.get("id"), str):
                submitted[job["id"]] = job.get("submittedAt")
    path = os.path.join(out_dir, PROVENANCE_FILE)
    existing = read_json(path)
    record = existing if isinstance(existing, dict) else {}
    for job in plan["jobs"]:
        jid = job["id"]
        clip = os.path.realpath(os.path.join(out_dir, "clips", f"{jid}.mp4"))
        prior = record.get(clip) if isinstance(record.get(clip), dict) else {}
        record[clip] = {
            "jobId": jid,
            "window": job.get("window"),
            "styleId": job.get("styleId"),
            "synced": bool(job.get("synced")),
            "voiceRef": bool(job.get("voiceRef")),
            "voiceRefPath": job.get("voiceRef"),
            "clip": clip,
            "submittedAt": prior.get("submittedAt") or submitted.get(jid),
            # the content key a landed clip is pinned to (lane_render.stale_prompt). STAMPED
            # ONLY WHEN THIS JOB WAS (RE)SUBMITTED: this writer runs for EVERY job in the
            # plan at the end of EVERY pass, so a re-plan that rewrote a prompt used to
            # re-stamp clips that never rendered under it — and the next pass parked three
            # good cutaways as "rendered from an older prompt" (Baby Nimbooda, 2026-09-05,
            # twice). A record keeps its sha until a NEW submission replaces the clip.
            "promptSha": (prior.get("promptSha")
                          if prior.get("promptSha") and prior.get("submittedAt") == submitted.get(jid)
                          else hashlib.sha256((job.get("prompt") or "").encode("utf-8")).hexdigest()[:12]),
        }
        if job.get("identityRef2"):
            record[clip]["identityRef2"] = job["identityRef2"]
        if job.get("identityRef3"):
            record[clip]["identityRef3"] = job["identityRef3"]
    with open(path, "w", encoding="utf-8") as handle:
        json.dump(record, handle, indent=2)
        handle.write("\n")


# --------------------------------------------------------------------------
# the vclaw children
# --------------------------------------------------------------------------

def run_vclaw(vclaw: str, argv: list[str], env: dict) -> subprocess.CompletedProcess:
    proc = subprocess.run([os.environ.get("NODE_BIN", "node"), vclaw, *argv],
                          stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
    if proc.returncode != 0:
        raise RuntimeError(f"vclaw {' '.join(argv[:3])} exited {proc.returncode}: "
                           f"{proc.stdout.decode('utf-8', 'replace').strip()[-400:]}")
    return proc


def submit_pass(vclaw: str, out_dir: str, plan: dict, env: dict) -> None:
    # No --confirm-spend anywhere in this script: the route is pinned to the free
    # explore lane, which needs no spend gate, and a supervisor that could pass that
    # flag on a loop is a supervisor that can re-buy a batch at 03:00.
    argv = ["video", "batch-submit", "--manifest", os.path.join(out_dir, MANIFEST_FILE),
            "--out", out_dir]
    if plan.get("workspaceRoot"):
        argv += ["--root", plan["workspaceRoot"]]
    run_vclaw(vclaw, argv, env)


def monitor_pass(vclaw: str, out_dir: str, args: argparse.Namespace, env: dict) -> None:
    # --once, always: the TS monitor's own loop cannot heartbeat, so a long internal
    # sleep is indistinguishable from a dead process. The waiting happens HERE.
    argv = ["video", "batch-monitor", "--out", out_dir, "--once"]
    if args.stall_minutes:
        argv += ["--stall-minutes", str(args.stall_minutes)]
    if args.fail_wedged:
        argv.append("--fail-wedged")
    if args.auto_resubmit:
        argv += ["--auto-resubmit", "--max-resubmits", str(args.max_resubmits)]
    run_vclaw(vclaw, argv, env)


def needs_reoffer(out_dir: str, older_than_sec: float) -> bool:
    """True when some job has been pending longer than the re-offer window.

    Explore admits two or three jobs at a time and REFUSES the rest, and a refused
    window otherwise sits there for the whole night — the single failure this
    supervisor exists to prevent. It cannot be detected by field: when the transport
    reports no `submittedSceneIndexes`, `handleVideoBatchSubmit` stamps `submittedAt`
    on EVERY job, and `providerTaskId` is documented as best-effort/absent. So the
    trigger is TIME, not a flag.

    Re-offering is cheap and safe only where the transport resumes from the job
    state and skips every scene already `submitted`/`completed`, so an accepted scene
    costs one no-op and only a genuinely refused one is re-sent. (This is exactly why
    the paid routes are refused up front — they have no such resume.)
    """
    state = read_json(os.path.join(out_dir, QUEUE_STATE_FILE))
    if not isinstance(state, dict):
        return False
    for job in state.get("jobs", []):
        if not isinstance(job, dict) or job.get("status") != "pending":
            continue
        stamp = job.get("submittedAt") or state.get("submittedAt")
        if not stamp:
            return True                      # never stamped at all — definitely re-offer
        age = age_seconds(stamp)
        if age is None or age >= older_than_sec:
            return True
    return False


# --------------------------------------------------------------------------
# --status
# --------------------------------------------------------------------------

def do_status(out_dir: str, interval_override: int | None) -> int:
    sup = read_json(supervisor_path(out_dir))
    rollup = read_rollup(out_dir)
    if sup is None and rollup is None:
        die(f"no queue at {out_dir}: neither {SUPERVISOR_FILE} nor {QUEUE_STATE_FILE} is readable", 1)

    state = (sup or {}).get("state", "unknown")
    interval = interval_override or (sup or {}).get("intervalSec") or 600
    heartbeat_age = age_seconds((sup or {}).get("heartbeat"))
    stale = False
    # Staleness is only meaningful while the supervisor claims to be RUNNING. A
    # finished lane's heartbeat is stale by definition and reporting that as broken
    # would fail every queue an hour after it landed.
    if state == "running" and (heartbeat_age is None or heartbeat_age > STALE_MULTIPLIER * interval):
        stale = True

    out = {"queueDir": os.path.abspath(out_dir), "state": state,
           "heartbeat": (sup or {}).get("heartbeat"),
           "heartbeatAgeSec": None if heartbeat_age is None else round(heartbeat_age),
           "intervalSec": interval, "stale": stale, "rollup": rollup}
    print(json.dumps(out, indent=2))

    if stale:
        print(f"explore_queue: BROKEN — supervisor says 'running' but its heartbeat is "
              f"{'missing' if heartbeat_age is None else str(round(heartbeat_age)) + 's old'} "
              f"(limit {STALE_MULTIPLIER * interval}s). The lane is DEAD, not slow.", file=sys.stderr)
        return 1
    if state == "failed":
        print(f"explore_queue: BROKEN — {(sup or {}).get('lastError') or 'the run failed'}",
              file=sys.stderr)
        return 1
    if rollup is None:
        print("explore_queue: no queue state yet — nothing has been submitted", file=sys.stderr)
        return 10
    if rollup.get("pending", 0) > 0:
        print(f"explore_queue: {rollup['pending']} of {rollup['total']} outstanding "
              f"({rollup['done']} landed, {rollup['failed']} failed)", file=sys.stderr)
        return 10
    if rollup.get("failed", 0) > 0:
        print(f"explore_queue: {rollup['failed']} of {rollup['total']} job(s) FAILED "
              f"({rollup['done']} landed)", file=sys.stderr)
        return 1
    print(f"explore_queue: all {rollup['total']} job(s) landed", file=sys.stderr)
    return 0


# --------------------------------------------------------------------------
# the run
# --------------------------------------------------------------------------

def do_run(args: argparse.Namespace) -> int:
    plan = load_plan(args.plan)
    out_dir = args.out or plan.get("queueDir") or (
        os.path.join(plan["projectDir"], "build", "queue", plan["lane"]) if plan.get("projectDir") else None)
    if not out_dir:
        die("no queue dir: pass --out, or give the plan a \"queueDir\" or \"projectDir\"")
    # Absolute from here on: provenance.json's `clip` paths are read by
    # index_library.py / select_cut.py from a different cwd, and a relative path
    # resolves to nothing there — silently, as a clip that "doesn't exist yet".
    out_dir = os.path.abspath(out_dir)
    os.makedirs(os.path.join(out_dir, "clips"), exist_ok=True)

    manifest = build_manifest(plan)
    env = child_env(plan)                                   # precondition 1
    vclaw = assert_submit_preconditions(plan, manifest, args)  # preconditions 2-4
    with open(os.path.join(out_dir, MANIFEST_FILE), "w", encoding="utf-8") as handle:
        json.dump(manifest, handle, indent=2)
        handle.write("\n")

    doc = {
        "schemaVersion": 1,
        "lane": plan["lane"],
        "route": plan["route"],
        "aspect": plan["aspect"],
        "queueDir": os.path.abspath(out_dir),
        "plan": os.path.abspath(args.plan),
        "manifest": os.path.join(os.path.abspath(out_dir), MANIFEST_FILE),
        "state": "running",
        "startedAt": now_iso(),
        "intervalSec": args.interval,
        "pass": 0,
        "consecutiveErrors": 0,
        "lastError": None,
        # A stale dist/ silently runs OLD batch logic against a NEW plan — the
        # mtime is the cheapest way to see that from the outside, months later.
        "vclaw": {"path": vclaw, "mtime": datetime.fromtimestamp(
            os.path.getmtime(vclaw), timezone.utc).replace(microsecond=0).isoformat()},
        "jobCount": len(plan["jobs"]),
        "rollup": None,
    }
    write_supervisor(out_dir, doc)
    log(f"lane {plan['lane']} · route {plan['route']} · {len(plan['jobs'])} job(s) · "
        f"{plan['aspect']} {plan['resolution']} · audio "
        f"{'ON' if doc['audioOn'] else 'off'} · vclaw {doc['vclaw']['mtime']}")

    deadline = time.time() + args.max_minutes * 60 if args.max_minutes else None
    # Starts at 0, not -1: a queue that has landed NOTHING must not read as progress
    # on the first pass, or the very first failure is forgiven and the fail-loud
    # threshold quietly becomes 6.
    best_done = 0
    errors = 0
    # How long a job may sit pending before it is re-offered. The wedge window when
    # one is set (same policy, one number); otherwise a couple of intervals, floored
    # at 15 minutes so a short --interval cannot turn this into a hammer.
    reoffer_after = args.stall_minutes * 60 if args.stall_minutes else max(2 * args.interval, 900)

    while True:
        doc["pass"] += 1
        try:
            if doc["pass"] == 1 or needs_reoffer(out_dir, reoffer_after):
                submit_pass(vclaw, out_dir, plan, env)
                write_provenance(out_dir, plan)
            monitor_pass(vclaw, out_dir, args, env)
            errors = 0
        except (KeyboardInterrupt, SystemExit):
            # RE-RAISED on purpose. The originals caught these too, so Ctrl-C did
            # nothing and the only way to stop an overnight run was to kill it.
            doc["state"] = "interrupted"
            write_supervisor(out_dir, doc)
            raise
        except BaseException as exc:                        # noqa: BLE001 — see below
            # Anything else — a network blip, a JSON parse error, a transport raise —
            # must NOT take the supervisor down. An unattended lane whose watcher
            # died leaves refused jobs unre-offered for the whole night.
            errors += 1
            doc["lastError"] = f"{type(exc).__name__}: {exc}"
            log(f"pass {doc['pass']} failed ({errors}/{MAX_ERROR_PASSES}): {doc['lastError']}")

        rollup = read_rollup(out_dir)
        doc["rollup"] = rollup
        doc["consecutiveErrors"] = errors
        done = (rollup or {}).get("done", 0)
        if done > best_done:
            # Progress clears the error streak: a lane that is landing clips despite
            # the odd bad pass is healthy, and must not be failed for the noise.
            best_done = done
            errors = 0
            doc["consecutiveErrors"] = 0
        write_supervisor(out_dir, doc)

        if errors >= MAX_ERROR_PASSES:
            # FAIL LOUD. The originals looped a fixed count and exited 0 whatever
            # happened, so a lane that never landed anything reported success.
            doc["state"] = "failed"
            doc["lastError"] = (f"{errors} consecutive failing passes with nothing landing "
                                f"— last: {doc['lastError']}")
            write_supervisor(out_dir, doc)
            log(f"FAILED: {doc['lastError']}")
            return 1

        if rollup and rollup.get("terminal"):
            doc["state"] = "finished"
            write_supervisor(out_dir, doc)
            if rollup.get("failed", 0) > 0:
                log(f"terminal with {rollup['failed']} failed job(s) ({rollup['done']} landed)")
                return 1
            log(f"all {rollup['total']} job(s) landed")
            return 0

        if args.once:
            doc["state"] = "idle"
            write_supervisor(out_dir, doc)
            log(f"one pass done — {(rollup or {}).get('pending', '?')} outstanding")
            return 10
        if deadline is not None and time.time() >= deadline:
            doc["state"] = "idle"
            write_supervisor(out_dir, doc)
            log(f"--max-minutes reached — {(rollup or {}).get('pending', '?')} outstanding")
            return 10

        log(f"pass {doc['pass']}: {(rollup or {}).get('done', 0)}/{(rollup or {}).get('total', '?')} "
            f"landed — sleeping {args.interval}s")
        time.sleep(args.interval)


def main() -> None:
    ap = argparse.ArgumentParser(description="supervisor over vclaw's overnight batch queue")
    ap.add_argument("--plan", help="skill-authored plan.json (see the module docstring)")
    ap.add_argument("--out", help="queue dir; defaults to the plan's queueDir/projectDir")
    ap.add_argument("--status", action="store_true",
                    help="report the queue + heartbeat and exit 0/10/1")
    ap.add_argument("--interval", type=int, default=None,
                    help="seconds between passes (default 600); with --status, overrides the "
                         "interval the supervisor recorded when judging heartbeat staleness")
    ap.add_argument("--once", action="store_true", help="one pass, then exit (launchd-friendly)")
    ap.add_argument("--max-minutes", type=float, default=0.0, help="stop after N minutes")
    ap.add_argument("--stall-minutes", type=int, default=0,
                    help="wedge window passed through to batch-monitor (0 = off)")
    ap.add_argument("--fail-wedged", action="store_true")
    ap.add_argument("--auto-resubmit", action="store_true",
                    help="free seedance-direct only, and needs --stall-minutes")
    ap.add_argument("--max-resubmits", type=int, default=2)
    ap.add_argument("--vclaw", help="path to dist/cli/vclaw.js (default: this repo's)")
    args = ap.parse_args()

    if args.status:
        if not args.out:
            die("--status needs --out <queue dir>")
        raise SystemExit(do_status(args.out, args.interval))
    if not args.plan:
        die("nothing to do: pass --plan <plan.json>, or --status --out <dir>")
    args.interval = args.interval or 600
    raise SystemExit(do_run(args))


if __name__ == "__main__":
    main()
