#!/usr/bin/env python3
"""
Subconscious Observer — Stop hook (async).

Reads recent conversation, greps two local corpora with fff (past-session
transcripts as memoirs, canon bundles as domain knowledge), and makes one
DeepSeek call — plus at most one more when the model escalates to a deep
archive read — synthesizing the hits into an insight cached for inject.sh.
No NotebookLM in this path — the mcp__context MCP tools remain the deep,
semantic pull surface.
"""

import os
import json
import subprocess
import hashlib
import time
import fcntl
import threading
import sys
from pathlib import Path

# fff + openai live in the extension's own venv (.venv). Version-globbed so a
# venv rebuilt on a newer python needs no code change; if the venv is absent we
# fall through to the ambient environment.
_VENV_SP = sorted((Path(__file__).resolve().parent / ".venv" / "lib").glob("python3.*/site-packages"))
if _VENV_SP:
    sys.path.insert(0, str(_VENV_SP[-1]))
try:
    from openai import OpenAI
    from fff import FileFinder
except ImportError as _e:
    _ext = Path(__file__).resolve().parent
    sys.stderr.write(
        f"subconscious: missing dependency ({_e}); create the venv:\n"
        f"  python3 -m venv {_ext}/.venv && {_ext}/.venv/bin/pip install fff-search openai\n"
    )
    raise

# ── paths ────────────────────────────────────────────────────────────────────
EXT_DIR     = Path(__file__).resolve().parent
CONFIG_PATH = EXT_DIR / "config.json"
CACHE_DIR   = EXT_DIR / "cache"
LOCK_FILE   = Path("/tmp/subconscious.lock")
LOG_FILE    = Path("/tmp/subconscious.log")
DIAG_FILE   = Path("/tmp/subconscious-diag.log")

# ── helpers ──────────────────────────────────────────────────────────────────

def log(msg: str):
    try:
        with open(LOG_FILE, "a") as f:
            f.write(f"[{time.strftime('%H:%M:%S')}] {msg}\n")
    except Exception:
        pass


def diag(label: str, payload):
    """Write structured diagnostic entry to /tmp/subconscious-diag.log"""
    try:
        entry = {
            "ts": time.strftime('%Y-%m-%d %H:%M:%S'),
            "source": "claude-code",
            "label": label,
            "payload": payload,
        }
        with open(DIAG_FILE, "a") as f:
            f.write(json.dumps(entry, default=str, ensure_ascii=False) + "\n")
    except Exception as e:
        log(f"diag write error: {e}")


def load_config() -> dict:
    override = os.environ.get("SUBCONSCIOUS_CONFIG")
    path = Path(override) if override else CONFIG_PATH
    if path.exists():
        return json.loads(path.read_text())
    return {}


def project_hash(cwd: str) -> str:
    return hashlib.sha256(cwd.encode()).hexdigest()[:16]


def cache_path(cwd: str) -> Path:
    return CACHE_DIR / f"{project_hash(cwd)}.json"


def read_cache(cwd: str) -> dict | None:
    p = cache_path(cwd)
    if p.exists():
        try:
            return json.loads(p.read_text())
        except Exception:
            pass
    return None


def write_cache(cwd: str, insight: str, session_id: str,
                terms: list[str] | None = None):
    CACHE_DIR.mkdir(parents=True, exist_ok=True)
    cache_path(cwd).write_text(json.dumps({
        "insight": insight,
        "timestamp": int(time.time()),
        "session_id": session_id,
        # gate.py compares these against the next prompt at inject time
        "terms": terms or [],
    }, indent=2))


def is_on_cooldown(cwd: str, cooldown: int) -> bool:
    cached = read_cache(cwd)
    if not cached:
        return False
    age = int(time.time()) - cached.get("timestamp", 0)
    return age < cooldown


PROJECTS_DIR = Path.home() / ".claude/projects"


def _resolve_cwd(newest_file: Path, session_id: str) -> str:
    """Extract cwd from the earliest JSONL entry that carries it (the first
    line can be a "last-prompt" marker without one), with index/env fallbacks."""
    cwd = ""
    try:
        with open(newest_file) as f:
            for _ in range(50):
                line = f.readline()
                if not line:
                    break
                try:
                    entry = json.loads(line)
                except json.JSONDecodeError:
                    continue
                if entry.get("cwd"):
                    cwd = entry["cwd"]
                    break
    except Exception as e:
        log(f"Could not read cwd from {newest_file.name}: {e}")

    # Fallback: try sessions-index.json in parent dir
    if not cwd:
        index_file = newest_file.parent / "sessions-index.json"
        if index_file.exists():
            try:
                index = json.loads(index_file.read_text())
                for entry in index.get("entries", []):
                    if entry.get("sessionId") == session_id:
                        cwd = entry.get("projectPath", "")
                        break
            except Exception as e:
                log(f"Could not read sessions-index.json: {e}")

    # Fallback #2: use CLAUDE_PROJECT_DIR env var (set by Claude Code for hooks)
    if not cwd:
        cwd = os.environ.get("CLAUDE_PROJECT_DIR", "")
        if cwd:
            log(f"Got cwd from CLAUDE_PROJECT_DIR: {cwd}")

    return cwd


def find_latest_transcript(max_idle_seconds: int = 1800) -> dict | None:
    """Find the transcript to observe. SUBCONSCIOUS_TRANSCRIPT pins a specific
    file (tests); otherwise the most recently modified .jsonl in
    ~/.claude/projects/. Returns dict with transcript_path, cwd, session_id,
    idle_seconds or None if nothing recent enough exists.
    """
    pinned = os.environ.get("SUBCONSCIOUS_TRANSCRIPT")
    if pinned:
        pinned_path = Path(pinned).expanduser()
        if not pinned_path.exists():
            log(f"SUBCONSCIOUS_TRANSCRIPT does not exist: {pinned}")
            return None
        newest_file = pinned_path
        idle = 0.0
    else:
        if not PROJECTS_DIR.exists():
            return None

        newest_mtime = 0
        newest_file = None

        for jsonl in PROJECTS_DIR.rglob("*.jsonl"):
            try:
                mt = jsonl.stat().st_mtime
                if mt > newest_mtime:
                    newest_mtime = mt
                    newest_file = jsonl
            except OSError:
                continue

        if not newest_file or newest_mtime == 0:
            return None

        idle = time.time() - newest_mtime
        if idle > max_idle_seconds:
            return None

    # Session ID is the filename stem (UUID)
    session_id = newest_file.stem

    cwd = _resolve_cwd(newest_file, session_id)
    if not cwd:
        log(f"Could not determine cwd for {newest_file.name}")
        diag("cwd_resolution_failed", {"file": newest_file.name, "parent_dir": newest_file.parent.name})
        return None

    return {
        "transcript_path": str(newest_file),
        "cwd": cwd,
        "session_id": session_id,
        "idle_seconds": int(idle),
    }


# ── transcript parsing ───────────────────────────────────────────────────────

CHARS_PER_TOKEN = 4  # conservative estimate for mixed code/English


def _estimate_tokens(text: str) -> int:
    return len(text) // CHARS_PER_TOKEN


def _total_tokens(messages: list[dict]) -> int:
    return sum(_estimate_tokens(m["text"]) for m in messages)


def _parse_raw_messages(path: str, tail_lines: int | None = 500) -> list[dict]:
    """Parse user/assistant messages from JSONL transcript tail (or the whole
    file when tail_lines is None). No truncation."""
    messages = []
    try:
        if tail_lines is None:
            with open(path, errors="replace") as f:
                stdout = f.read()
        else:
            result = subprocess.run(
                ["tail", "-n", str(tail_lines), path],
                capture_output=True, text=True
            )
            stdout = result.stdout
        for line in stdout.strip().split("\n"):
            if not line.strip():
                continue
            try:
                entry = json.loads(line)
            except json.JSONDecodeError:
                continue

            if entry.get("type") == "user":
                content = entry.get("message", {}).get("content", "")
                if isinstance(content, list):
                    text = " ".join(b.get("text", "") for b in content if b.get("type") == "text")
                else:
                    text = str(content)
                if text.strip():
                    messages.append({"role": "user", "text": text.strip()})

            elif entry.get("type") == "assistant":
                content = entry.get("message", {}).get("content", [])
                if isinstance(content, list):
                    text = " ".join(b.get("text", "") for b in content if b.get("type") == "text")
                elif isinstance(content, str):
                    text = content
                else:
                    text = ""
                if text.strip():
                    messages.append({"role": "assistant", "text": text.strip()})
    except Exception as e:
        log(f"Transcript parse error: {e}")
    return messages


def _group_into_pairs(messages: list[dict]) -> list[tuple[dict | None, dict | None]]:
    """Group messages into (user, assistant) pairs by consecutive turns."""
    pairs = []
    i = 0
    while i < len(messages):
        user_msg = None
        asst_msg = None

        if messages[i]["role"] == "user":
            user_msg = messages[i]
            i += 1
            if i < len(messages) and messages[i]["role"] == "assistant":
                asst_msg = messages[i]
                i += 1
        elif messages[i]["role"] == "assistant":
            asst_msg = messages[i]
            i += 1
        else:
            i += 1
            continue

        if user_msg or asst_msg:
            pairs.append((user_msg, asst_msg))
    return pairs


def _pairs_to_messages(pairs: list[tuple]) -> list[dict]:
    """Flatten pairs back into a message list."""
    msgs = []
    for user_msg, asst_msg in pairs:
        if user_msg:
            msgs.append(user_msg)
        if asst_msg:
            msgs.append(asst_msg)
    return msgs


def parse_transcript(path: str, token_budget: int = 60_000) -> list[dict]:
    """Read conversation from JSONL transcript with token-aware degradation.

    Strategy (tries in order):
    1. Greedily include as many pairs as fit within budget, working backwards
    2. Last 1 pair (if even 1 pair exceeds, but the pair itself fits)
    3. Last assistant response only
    4. Sliding window towards end of last assistant response
    """
    if not os.path.exists(path):
        return []

    raw = _parse_raw_messages(path)
    if not raw:
        return []

    pairs = _group_into_pairs(raw)

    if pairs:
        # Strategy 1: greedily pack pairs from the end
        selected_pairs = []
        running_tokens = 0
        for pair in reversed(pairs):
            pair_msgs = _pairs_to_messages([pair])
            pair_tokens = _total_tokens(pair_msgs)
            if running_tokens + pair_tokens <= token_budget:
                selected_pairs.insert(0, pair)
                running_tokens += pair_tokens
            else:
                break

        if selected_pairs:
            msgs = _pairs_to_messages(selected_pairs)
            strategy = f"last_{len(selected_pairs)}_pairs"
            log(f"Transcript strategy: {strategy} ({running_tokens} tokens, {len(msgs)} msgs)")
            diag("transcript_strategy", {"strategy": strategy, "tokens": running_tokens, "msg_count": len(msgs), "pair_count": len(selected_pairs)})
            return msgs

        # Strategy 2: single pair didn't fit — try last assistant only
        last_asst = pairs[-1][1]
        if last_asst and _estimate_tokens(last_asst["text"]) <= token_budget:
            log(f"Transcript strategy: last_assistant_only ({_estimate_tokens(last_asst['text'])} tokens)")
            diag("transcript_strategy", {"strategy": "last_assistant_only", "tokens": _estimate_tokens(last_asst["text"])})
            return [last_asst]

        # Strategy 3: sliding window of last assistant response (tail)
        if last_asst:
            char_budget = token_budget * CHARS_PER_TOKEN
            text = last_asst["text"][-char_budget:]
            log(f"Transcript strategy: assistant_sliding_window (truncated {len(last_asst['text'])} -> {len(text)} chars)")
            diag("transcript_strategy", {"strategy": "assistant_sliding_window", "original_chars": len(last_asst["text"]), "truncated_chars": len(text)})
            return [{"role": "assistant", "text": text}]

    # No pairs formed — fall back to last raw message
    last = raw[-1]
    tokens = _estimate_tokens(last["text"])
    if tokens <= token_budget:
        log(f"Transcript strategy: raw_last_message ({tokens} tokens)")
        diag("transcript_strategy", {"strategy": "raw_last_message", "tokens": tokens})
        return [last]

    char_budget = token_budget * CHARS_PER_TOKEN
    text = last["text"][-char_budget:]
    log(f"Transcript strategy: raw_sliding_window (truncated {len(last['text'])} -> {len(text)} chars)")
    diag("transcript_strategy", {"strategy": "raw_sliding_window", "original_chars": len(last["text"]), "truncated_chars": len(text)})
    return [{"role": last["role"], "text": text}]


# ── term extraction (mechanical, no LLM) ─────────────────────────────────────

STOPWORDS = frozenset("""
the a an and or but if then else of to in on for with as is are was were be
been being do does did how what why when where which who this that these those
it its i you we they my our your their can could should would will at by from
about into over after before than not no yes just like get got make made use
used using want need see say said know think thing things some any all more
most other only also very really actually there here now new one two let lets
hmm huh well okay yeah sure suppose guess ought indeed anyway basically maybe
next last out way somewhere along still again much many such nevermind etc
""".split())


def extract_terms(messages: list[dict], max_terms: int = 12) -> list[str]:
    """Search terms from the last 3 user messages + last assistant message.

    Tokenize on non-word chars, drop stopwords and short tokens, dedupe
    preserving order. NEWEST user message first so the current topic wins the
    term budget; older messages and the assistant reply fill what remains.
    """
    import re
    users = [m["text"] for m in messages if m["role"] == "user"][-3:]
    assts = [m["text"] for m in messages if m["role"] == "assistant"][-1:]

    terms: list[str] = []
    seen: set[str] = set()
    for text in users[::-1] + assts:
        for tok in re.split(r"[^a-zA-Z0-9_-]+", text.lower()):
            if len(tok) < 3 or tok in STOPWORDS or tok in seen:
                continue
            seen.add(tok)
            terms.append(tok)
            if len(terms) >= max_terms * 4:
                break
    return terms[:max_terms]


# ── fff retrieval ────────────────────────────────────────────────────────────

def _snippet(match, width: int) -> str:
    """Window line_content around the first match range. fff already caps
    line_content at ~512 chars; this trims further for the prompt."""
    text = match.line_content
    start = match.match_ranges[0].start if match.match_ranges else 0
    lo = max(0, start - width // 2)
    return text[lo:lo + width].replace("\n", " ").strip()


def fff_search_corpus(root: str, terms: list[str], cfg: dict,
                      exclude_substr: str | None = None,
                      max_hits: int = 25,
                      max_snippets: int = 2) -> list[dict]:
    """Grep one corpus root with fff. Returns hits ranked by
    (distinct-term coverage, mtime) descending — lexical relevance first,
    recency as tiebreak — at most 2 snippets per file."""
    root_path = Path(root).expanduser()
    if not root_path.is_dir():
        log(f"fff corpus missing: {root}")
        return []

    snippet_chars = cfg.get("snippet_chars", 240)
    budget_ms = cfg.get("grep_time_budget_ms", 3000)
    deadline = time.time() + (budget_ms / 1000.0) * 2

    by_file: dict[str, dict] = {}
    try:
        with FileFinder(str(root_path), watch=False) as finder:
            finder.wait_for_scan_blocking(timeout_ms=15000)

            for mode in ("plain", "fuzzy"):
                cursor = None
                while True:
                    res = finder.multi_grep(
                        terms, mode=mode, smart_case=True,
                        max_matches_per_file=3, page_limit=500,
                        cursor=cursor, time_budget_ms=budget_ms,
                    )
                    for m in res.items:
                        if m.is_binary:
                            continue
                        if exclude_substr and exclude_substr in m.relative_path:
                            continue
                        entry = by_file.setdefault(m.relative_path, {
                            "path": m.relative_path,
                            "abs": str(root_path / m.relative_path),
                            "modified": m.modified,
                            "terms": set(),
                            "snippets": [],
                        })
                        low = m.line_content.lower()
                        entry["terms"].update(t for t in terms if t in low)
                        if len(entry["snippets"]) < max_snippets:
                            entry["snippets"].append(_snippet(m, snippet_chars))
                    if not res.has_more or time.time() > deadline:
                        break
                    cursor = res.next_cursor()
                # plain-mode hits are enough; only fall through to fuzzy on drought
                if len(by_file) >= 5 or time.time() > deadline:
                    break
    except Exception as e:
        log(f"fff error on {root}: {e}")
        return []

    ranked = sorted(
        by_file.values(),
        key=lambda e: (len(e["terms"]), e["modified"]),
        reverse=True,
    )[:max_hits]
    # newest-first within the selection, so the prompt reads chronologically
    ranked.sort(key=lambda e: e["modified"], reverse=True)
    return ranked


def format_hits(hits: list[dict]) -> str:
    lines = []
    for h in hits:
        day = time.strftime("%Y-%m-%d", time.localtime(h["modified"]))
        for s in h["snippets"]:
            lines.append(f"- {h['path']} ({day}): {s}")
    return "\n".join(lines) if lines else "(no hits)"


# ── second turn: escalation + deep read ──────────────────────────────────────

def parse_escalation(reply: str) -> tuple[str | None, str | None]:
    """Decision protocol: the first non-empty line decides. Returns
    (mode, target): ('target', path) / ('broaden', None) / (None, None)."""
    for line in reply.strip().splitlines():
        line = line.strip()
        if not line:
            continue
        if line.upper().startswith("ESCALATE"):
            rest = line[len("ESCALATE"):].strip()
            if rest.upper().startswith("TARGET:"):
                return "target", rest[len("TARGET:"):].strip()
            return "broaden", None
        return None, None
    return None, None


def strip_answer_prefix(reply: str) -> str:
    """Drop a protocol 'ANSWER' first line, leaving the note itself."""
    lines = reply.strip().splitlines()
    if lines and lines[0].strip().upper() == "ANSWER":
        return "\n".join(lines[1:]).strip()
    return reply.strip()


def resolve_target(target: str, memoirs_hits: list[dict],
                   canon_hits: list[dict]) -> dict | None:
    """Map the model-chosen path back onto a real hit; a hallucinated path
    falls back to the top-ranked memoirs hit."""
    all_hits = memoirs_hits + canon_hits
    for h in all_hits:
        if target and (target in h["path"] or h["path"] in target
                       or target in h["abs"]):
            return h
    # models often rewrite directory prefixes; the basename usually survives
    base = Path(target).name if target else ""
    if base:
        for h in all_hits:
            if Path(h["path"]).name == base:
                log(f"Escalation target matched by basename: {h['path']}")
                return h
    log(f"Escalation target not in hit list, falling back to top hit: {target}")
    return all_hits[0] if all_hits else None


MSG_CHAR_CAP = 4000  # per-message cap inside deep-read windows


def _deep_read_jsonl(path: str, terms: list[str], char_budget: int) -> str:
    """Term-anchored windows over a whole session transcript: parse every
    user/assistant text message, window +/-2 messages around term hits, merge,
    spend the budget on the highest-scoring windows, emit chronologically."""
    msgs = _parse_raw_messages(path, tail_lines=None)
    if not msgs:
        return ""
    lowered = [m["text"].lower() for m in msgs]

    hit_idx = [i for i, low in enumerate(lowered)
               if any(t in low for t in terms)]
    if not hit_idx:  # no anchors — just take the tail of the session
        out, used = [], 0
        for m in reversed(msgs):
            text = m["text"][:MSG_CHAR_CAP]
            if used + len(text) > char_budget:
                break
            out.append(f"[{m['role']}]: {text}")
            used += len(text)
        return "\n".join(reversed(out))

    # contiguous windows of +/-2 messages around anchors
    include = sorted({j for i in hit_idx
                      for j in range(max(0, i - 2), min(len(msgs), i + 3))})
    windows: list[list[int]] = [[include[0]]]
    for j in include[1:]:
        if j == windows[-1][-1] + 1:
            windows[-1].append(j)
        else:
            windows.append([j])

    def window_score(w: list[int]) -> int:
        return sum(1 for t in terms if any(t in lowered[j] for j in w))

    selected: set[int] = set()
    used = 0
    for w in sorted(windows, key=window_score, reverse=True):
        w_chars = sum(min(len(msgs[j]["text"]), MSG_CHAR_CAP) for j in w)
        if used + w_chars > char_budget and selected:
            continue
        selected.update(w)
        used += w_chars
        if used >= char_budget:
            break

    parts = []
    prev = None
    for j in sorted(selected):
        if prev is not None and j != prev + 1:
            parts.append("[...]")
        parts.append(f"[{msgs[j]['role']}]: {msgs[j]['text'][:MSG_CHAR_CAP]}")
        prev = j
    return "\n".join(parts)


def _deep_read_text(path: str, terms: list[str], char_budget: int,
                    window: int = 4000) -> str:
    """Expanded windows around term matches in a plain-text/markdown file."""
    try:
        text = Path(path).read_text(errors="replace")
    except Exception as e:
        log(f"deep read failed for {path}: {e}")
        return ""
    low = text.lower()
    offsets = []
    for t in terms:
        start = 0
        while len(offsets) < 200:
            i = low.find(t, start)
            if i < 0:
                break
            offsets.append(i)
            start = i + len(t)
    if not offsets:
        return text[:char_budget]

    spans: list[list[int]] = []
    for o in sorted(offsets):
        lo, hi = max(0, o - window // 2), min(len(text), o + window // 2)
        if spans and lo <= spans[-1][1]:
            spans[-1][1] = max(spans[-1][1], hi)
        else:
            spans.append([lo, hi])
    parts, used = [], 0
    for lo, hi in spans:
        chunk = text[lo:hi]
        if used + len(chunk) > char_budget:
            chunk = chunk[:char_budget - used]
        parts.append(chunk)
        used += len(chunk)
        if used >= char_budget:
            break
    return "\n[...]\n".join(parts)


def deep_read(hits: list[dict], terms: list[str], char_budget: int,
              per_file_chars: int | None = None) -> tuple[str, dict]:
    """Deep content for the second turn. One hit with the full budget (TARGET)
    or many hits with per-file budgets (BROADEN). Returns (block, stats)."""
    t0 = time.time()
    blocks, used, files = [], 0, 0
    for h in hits:
        remaining = char_budget - used
        if remaining <= 0:
            break
        budget = min(per_file_chars, remaining) if per_file_chars else remaining
        path = h["abs"]
        body = (_deep_read_jsonl(path, terms, budget)
                if path.endswith(".jsonl")
                else _deep_read_text(path, terms, budget))
        if not body:
            continue
        day = time.strftime("%Y-%m-%d", time.localtime(h["modified"]))
        blocks.append(f"### {h['path']} ({day})\n{body}")
        used += len(body)
        files += 1
    stats = {"files": files, "chars": used,
             "seconds": round(time.time() - t0, 2)}
    return "\n\n".join(blocks), stats


def _mock_reply(stage: int) -> str:
    if stage == 1:
        return os.environ.get("SUBCONSCIOUS_MOCK_T1", "ESCALATE BROADEN")
    return os.environ.get("SUBCONSCIOUS_MOCK_T2", "Mock second-turn insight.")


def _llm_is_mock() -> bool:
    return os.environ.get("SUBCONSCIOUS_LLM") == "mock"


# ── the DeepSeek calls (one, plus at most one escalation) ────────────────────

BASE_SYSTEM_PROMPT = """You are the Subconscious — a background memory layer for a coding assistant.

You receive the developer's recent conversation plus raw lexical search hits from two local corpora:
- PAST SESSIONS: snippets from earlier assistant-conversation transcripts and the developer's curated memory notes (newest first). Transcript snippets are raw JSON fragments — read through the escaping to the content.
- CANON BUNDLES: snippets from the developer's ingested domain-knowledge bundles (Jira, Confluence, Slack, code docs).

Synthesize a concise note that will be injected before the developer's next prompt.

Rules:
- Max 300 words
- Be direct, technical, no filler
- Highlight: relevant past solutions, architectural constraints, gotchas, patterns
- The hits are noisy keyword matches — use only what is genuinely relevant, ignore the rest
- Never treat a snippet as authoritative just because it matched; prefer snippets that clearly relate to the conversation topic
- Use bullet points for multiple items
- If nothing in the hits is useful for the current conversation, output exactly: NO_UPDATE
- Don't repeat what the developer already knows from the current conversation
- Frame as background context, not instructions"""

ESCALATION_ADDENDUM = """

THIS TURN USES A DECISION PROTOCOL. Your reply MUST start with exactly one of these three first lines:
ESCALATE TARGET:<one path copied verbatim from the hits>
ESCALATE BROADEN
ANSWER

ESCALATE is the DEFAULT whenever any hit relates to the conversation. Pick TARGET:<path> when one hit file is clearly the place where the relevant past work happened; otherwise BROADEN.

ANSWER is permitted ONLY when both are true:
- The conversation contains NO reference to specific past work (no experiment, incident, past decision, nothing like "when we did X" or a recollection of what happened). If the developer recounts past events, you MUST escalate — the snippets are 240-char fragments; confirming or correcting a recollection from fragments is forbidden, the developer's memory is sometimes wrong, and only the full archive text can tell.
- Either the hits are clearly unrelated (then ANSWER + NO_UPDATE), or the note you would write needs no facts beyond what the snippets literally contain.

An ESCALATE first line must be the ONLY line in the reply. You will then receive the expanded material and answer again. With ANSWER, the note (or NO_UPDATE) follows from the second line."""


USER_MSG_CHARS = 1500       # per included user message
ASST_MSG_CHARS = 8000       # final assistant message (head+tail split when over)
TRUNC_MARK = "[... truncated for length, the message continues ...]"


def _clip_head(text: str, limit: int) -> str:
    if len(text) <= limit:
        return text
    return text[:limit] + f"\n{TRUNC_MARK}"


def _clip_head_tail(text: str, limit: int) -> str:
    """Keep the start and the end; conclusions live at the end of a reply.
    The marker prevents the model reading a cut as 'the assistant was cut off'."""
    if len(text) <= limit:
        return text
    head = limit * 3 // 10
    tail = limit - head
    return f"{text[:head]}\n{TRUNC_MARK}\n{text[-tail:]}"


def _conversation_text(messages: list[dict]) -> str:
    parts = [f"[{m['role']}]: {_clip_head(m['text'], USER_MSG_CHARS)}"
             for m in [m for m in messages if m["role"] == "user"][-3:]]
    last_asst = [m for m in messages if m["role"] == "assistant"][-1:]
    if last_asst:
        parts.append(f"[assistant]: {_clip_head_tail(last_asst[0]['text'], ASST_MSG_CHARS)}")
    return "\n".join(parts)


def flash_synthesize(client: OpenAI | None, model: str, messages: list[dict],
                     memoirs_block: str, canon_block: str,
                     allow_escalation: bool = False) -> str:
    """Turn 1: conversation + recency-ordered local search hits in; injection
    text, NO_UPDATE, or (when allowed) an ESCALATE line out."""
    if _llm_is_mock():
        return _mock_reply(1)

    system = BASE_SYSTEM_PROMPT + (ESCALATION_ADDENDUM if allow_escalation else "")
    response = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": system},
            {"role": "user", "content": f"Current conversation:\n{_conversation_text(messages)}\n\n--- PAST SESSIONS (newest first) ---\n{memoirs_block}\n\n--- CANON BUNDLES ---\n{canon_block}"}
        ],
        # low temperature: the first line is a protocol decision, not prose
        temperature=0.1 if allow_escalation else 0.4,
        max_tokens=600,
        extra_body={"thinking": {"type": "disabled"}},
    )
    return response.choices[0].message.content.strip()


DEEP_ADDENDUM = """

You escalated and now have ARCHIVE EXCERPTS: large verbatim excerpts from the archived record. Two extra rules bind this turn:
- The ARCHIVE EXCERPTS are the record; the live conversation is recollection. When they conflict, the archive wins — state the corrected version explicitly and name the source file.
- Never restate the developer's claim as fact unless the archive supports it.
No further escalation is available; answer with the note or NO_UPDATE."""


def flash_deep_synthesize(client: OpenAI | None, model: str,
                          messages: list[dict], memoirs_block: str,
                          canon_block: str, deep_block: str) -> str:
    """Turn 2: same inputs plus the deep-read block; insight or NO_UPDATE out."""
    if _llm_is_mock():
        return _mock_reply(2)

    response = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": BASE_SYSTEM_PROMPT + DEEP_ADDENDUM},
            {"role": "user", "content": f"Current conversation:\n{_conversation_text(messages)}\n\n--- PAST SESSIONS (newest first) ---\n{memoirs_block}\n\n--- CANON BUNDLES ---\n{canon_block}\n\n--- ARCHIVE EXCERPTS (deep read) ---\n{deep_block}"}
        ],
        temperature=0.4,
        max_tokens=800,
        extra_body={"thinking": {"type": "disabled"}},
    )
    return response.choices[0].message.content.strip()


# ── main ─────────────────────────────────────────────────────────────────────

def main():
    # Drain stdin in background so hook doesn't hang if Claude Code pipes data
    threading.Thread(target=lambda: sys.stdin.read(), daemon=True).start()

    log("=" * 50)
    log("observe.py started")
    diag("startup", {"pid": os.getpid(), "env_keys": sorted([k for k in os.environ if "DEEP" in k or "API" in k or "CLAUDE" in k])})

    config = load_config()
    diag("config", config)
    model = config.get("model", "deepseek-v4-flash")
    cooldown = config.get("cooldown_seconds", 600)
    min_msgs = config.get("min_messages", 2)
    fff_cfg = config.get("fff", {})
    transcripts_root = fff_cfg.get("transcripts_root", "~/.claude/projects")
    canon_roots = fff_cfg.get("canon_roots", [])
    max_terms = fff_cfg.get("max_terms", 12)
    memoirs_max = fff_cfg.get("memoirs_hits", 25)
    canon_max = fff_cfg.get("canon_hits", 15)
    st_cfg = config.get("second_turn", {})

    # A missing corpus root silently shrinks the push surface (memoirs-only
    # canon drought looks like "no relevant hits") — warn on every run.
    missing_roots = [r for r in [transcripts_root, *canon_roots]
                     if not Path(r).expanduser().is_dir()]
    if missing_roots:
        for r in missing_roots:
            log(f"WARNING: corpus root missing: {r} — its corpus is absent from this run")
        sys.stderr.write("subconscious: corpus roots missing: " + ", ".join(missing_roots) + "\n")
        diag("missing_corpus_roots", {"missing": missing_roots})

    # Find the most recently modified transcript in ~/.claude/projects/
    latest = find_latest_transcript(max_idle_seconds=480)
    if not latest:
        log("No recent transcript found (all >30min old or no projects). Skipping.")
        return

    transcript_path = latest["transcript_path"]
    cwd = latest["cwd"]
    session_id = latest["session_id"]
    idle = latest["idle_seconds"]

    log(f"Project: {cwd}")
    log(f"Session: {session_id}")
    log(f"Transcript idle: {idle}s")
    diag("transcript_found", {
        "cwd": cwd,
        "session_id": session_id,
        "idle_seconds": idle,
        "transcript_path": transcript_path,
        "claude_project_dir": os.environ.get("CLAUDE_PROJECT_DIR", ""),
    })

    # Cooldown check
    if os.environ.get("SUBCONSCIOUS_FORCE") == "1":
        log("SUBCONSCIOUS_FORCE=1, skipping cooldown check")
    elif is_on_cooldown(cwd, cooldown):
        cached = read_cache(cwd)
        age = int(time.time()) - cached.get("timestamp", 0)
        log(f"On cooldown ({age}s / {cooldown}s). Skipping.")
        return

    # Lock: one observer at a time (concurrent Stop hooks from parallel sessions)
    try:
        lock_fd = open(LOCK_FILE, "w")
        fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except (IOError, OSError):
        log("Lock held by another process. Skipping.")
        return

    try:
        t_start = time.time()

        # Parse transcript (token-aware: fills up to 60k tokens greedily)
        messages = parse_transcript(transcript_path)
        if len(messages) < min_msgs:
            log(f"Only {len(messages)} messages, need {min_msgs}. Skipping.")
            diag("skipped_insufficient_messages", {"count": len(messages), "min": min_msgs})
            return
        log(f"Parsed {len(messages)} messages")
        diag("parsed_messages", messages)

        # ── Step 1: mechanical term extraction ──
        terms = extract_terms(messages, max_terms)
        if not terms:
            log("No search terms extracted. Skipping.")
            return
        log(f"Terms: {terms}")
        diag("terms", {"terms": terms})

        # ── Step 2: fff over both corpora ──
        # Exclude the current session's own transcript/artifacts, else the top
        # recency hits are this conversation echoing itself.
        t_fff = time.time()
        memoirs_hits = fff_search_corpus(
            transcripts_root, terms, fff_cfg,
            exclude_substr=session_id, max_hits=memoirs_max,
        )
        canon_hits: list[dict] = []
        per_root = max(1, canon_max // max(1, len(canon_roots))) if canon_roots else 0
        for root in canon_roots:
            canon_hits.extend(fff_search_corpus(root, terms, fff_cfg, max_hits=per_root))
        canon_hits.sort(key=lambda e: e["modified"], reverse=True)
        fff_secs = round(time.time() - t_fff, 2)
        log(f"fff: {len(memoirs_hits)} memoirs files, {len(canon_hits)} canon files in {fff_secs}s")
        diag("fff_hits", {
            "memoirs_files": [h["path"] for h in memoirs_hits],
            "canon_files": [h["path"] for h in canon_hits],
            "seconds": fff_secs,
        })

        if not memoirs_hits and not canon_hits:
            log("No hits in either corpus. Skipping.")
            return

        # ── Step 3: DeepSeek turn 1 (escalation offered only on cache miss) ──
        client = None
        if not _llm_is_mock():
            api_key = os.environ.get("DEEPSEEK_API_KEY")
            if not api_key:
                log("DEEPSEEK_API_KEY not set")
                return
            client = OpenAI(api_key=api_key, base_url="https://api.deepseek.com")

        # escalation is offered on every run (~0.7 cents worst case); the old
        # cache-miss gate let a fresh-but-wrong insight block its own correction
        eligible = bool(st_cfg.get("enabled"))
        log(f"Second turn eligible: {eligible}")

        memoirs_block = format_hits(memoirs_hits)
        canon_block = format_hits(canon_hits)

        log("DeepSeek: synthesizing...")
        insight = flash_synthesize(
            client, model, messages, memoirs_block, canon_block,
            allow_escalation=eligible,
        )
        diag("synthesized_insight", {"insight": insight})

        # ── Step 4: at most ONE escalation ──
        mode, target = parse_escalation(insight)
        if mode and not eligible:
            log(f"Model escalated ({mode}) while ineligible. Dropping.")
            diag("escalation_dropped_ineligible", {"mode": mode, "target": target})
            return
        if mode:
            log(f"Escalation: {mode}" + (f" -> {target}" if target else ""))
            deep_budget = st_cfg.get("deep_read_chars", 160000)
            per_file = st_cfg.get("per_file_chars", 20000)
            threshold = st_cfg.get("broaden_file_threshold", 12)

            if mode == "target":
                hit = resolve_target(target, memoirs_hits, canon_hits)
                if not hit:
                    log("No hit to deep-read. Stopping.")
                    return
                deep_block, stats = deep_read([hit], terms, deep_budget)
                chosen = [hit["path"]]
            else:  # broaden
                pool = memoirs_hits + canon_hits
                if len(pool) >= threshold:
                    # plenty of files: widen the result set, then read the top ones
                    wide_memoirs = fff_search_corpus(
                        transcripts_root, terms, fff_cfg,
                        exclude_substr=session_id,
                        max_hits=memoirs_max * 2, max_snippets=5,
                    )
                    wide_canon: list[dict] = []
                    for root in canon_roots:
                        wide_canon.extend(fff_search_corpus(
                            root, terms, fff_cfg,
                            max_hits=max(1, canon_max * 2 // max(1, len(canon_roots))),
                            max_snippets=5,
                        ))
                    pool = wide_memoirs + wide_canon
                deep_block, stats = deep_read(pool, terms, deep_budget,
                                              per_file_chars=per_file)
                chosen = [h["path"] for h in pool[:stats["files"]]]

            diag("escalation_decision", {"mode": mode, "target": target,
                                         "chosen": chosen})
            diag("deep_read_stats", stats)
            log(f"Deep read: {stats['files']} files, {stats['chars']} chars "
                f"in {stats['seconds']}s")
            if not deep_block:
                log("Deep read produced nothing. Stopping.")
                return

            log("DeepSeek: deep synthesizing...")
            insight = flash_deep_synthesize(
                client, model, messages, memoirs_block, canon_block, deep_block,
            )
            diag("deep_synthesized_insight", {"insight": insight})

        insight = strip_answer_prefix(insight)
        if insight == "NO_UPDATE" or not insight or insight.upper().startswith("ESCALATE"):
            log("No useful insight. Not updating cache.")
            return

        # Write cache
        write_cache(cwd, insight, session_id, terms)
        total_secs = round(time.time() - t_start, 2)
        log(f"Cache written ({len(insight)} chars) in {total_secs}s total")
        log("Done.")
        diag("cache_written", {"cwd": cwd, "session_id": session_id,
                               "insight_len": len(insight), "total_seconds": total_secs})

    except Exception as e:
        log(f"Fatal error: {e}")
        import traceback
        log(traceback.format_exc())
    finally:
        # Release lock
        try:
            fcntl.flock(lock_fd, fcntl.LOCK_UN)
            lock_fd.close()
        except Exception:
            pass


if __name__ == "__main__":
    main()