#!/usr/bin/env python3
"""Survey a repo and produce a grounded plan for a deep LLM-wiki backfill (REQ-061).

When an established repo updates into the wiki-capable plugin version, its
existing knowledge should be captured in one deliberate pass, not accreted slowly.
The installer flags that with a `.prd_plugin/local/wiki-backfill-needed` marker;
this read-only surveyor inventories what the repo already knows — code modules,
docs, README, `.prd_plugin` state, git-history themes, existing memory — and
groups it into proposed wiki topics.

It writes nothing to the repo. The agent runs `--plan`, compiles the wiki via the
`project-llm-wiki` skill's Backfill mode, then removes the marker.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import subprocess
import sys
from pathlib import Path
from urllib.parse import quote

MARKER_REL = Path(".prd_plugin") / "local" / "wiki-backfill-needed"

# Directory *names* (matched at any depth) that are never repo *knowledge* —
# plugin plumbing, vendor code, VCS, and generated report/output trees. The last
# row are generated artifacts (e.g. request reports and their calibration report
# sub-trees), not durable domain/codebase knowledge, so they must not be surveyed.
SKIP_DIRS = {
    ".git", ".hg", ".svn", "node_modules", ".venv", "venv", "__pycache__",
    ".prd_plugin", ".claude", ".agents", ".opencode", ".codex", ".codex-plugin",
    ".claude-plugin", "dist", "build", "out", "target", ".next", ".cache",
    "wiki", "raw", "coverage", ".pytest_cache", ".mypy_cache",
    "request-report",
}
# Multi-segment path prefixes (relative to the repo root) that are generated
# outputs rather than authored docs — e.g. captured evidence records. A bare
# "evidence" name would over-match legitimate source dirs, so anchor it to docs/.
SKIP_PATH_PREFIXES = {
    ("docs", "evidence"),
}
CODE_EXTS = {".py", ".js", ".ts", ".tsx", ".jsx", ".rs", ".go", ".java", ".rb",
             ".c", ".cc", ".cpp", ".h", ".hpp", ".cs", ".swift", ".kt", ".scala",
             ".php", ".sql", ".sh", ".mjs", ".cjs"}
DOC_EXTS = {".md", ".rst", ".adoc", ".txt"}


def _read_json(path):
    try:
        return json.loads(Path(path).read_text(encoding="utf-8-sig"))
    except Exception:
        return None


def wiki_exists(root):
    root = Path(root)
    return (root / "wiki" / "index.md").is_file()


def backfill_pending(root):
    """True when the marker is present and no wiki has been built yet."""
    root = Path(root)
    if wiki_exists(root):
        return False
    return (root / MARKER_REL).is_file()


def _skip_path(rel):
    """True when a repo-relative path is plugin/vendor plumbing or a generated
    report/output tree (matched by directory name or by anchored path prefix)."""
    parts = rel.parts
    if set(parts) & SKIP_DIRS:
        return True
    for prefix in SKIP_PATH_PREFIXES:
        if parts[: len(prefix)] == prefix:
            return True
    return False


def _iter_files(root):
    root = Path(root)
    for path in root.rglob("*"):
        if not path.is_file():
            continue
        if _skip_path(path.relative_to(root)):
            continue
        yield path


def _top_module(root, path):
    rel = path.relative_to(root)
    return rel.parts[0] if len(rel.parts) > 1 else rel.name


def _survey_code(root):
    root = Path(root)
    modules = {}
    for path in _iter_files(root):
        if path.suffix.lower() not in CODE_EXTS:
            continue
        key = _top_module(root, path)
        modules.setdefault(key, 0)
        modules[key] += 1
    return dict(sorted(modules.items(), key=lambda kv: (-kv[1], kv[0])))


def _survey_docs(root):
    root = Path(root)
    docs = []
    for path in _iter_files(root):
        if path.suffix.lower() in DOC_EXTS:
            docs.append(str(path.relative_to(root)).replace("\\", "/"))
    return sorted(docs)


def _survey_state(root):
    root = Path(root)
    state = root / ".prd_plugin" / "state"
    out = {}
    for name, key in (("requests.json", "requests"), ("decisions.json", "decisions"),
                      ("changelog.json", "changes"), ("evidence.json", "records"),
                      ("tracking.json", "records")):
        data = _read_json(state / name)
        label = name[:-5]
        if isinstance(data, dict) and isinstance(data.get(key), list):
            out[label] = len(data[key])
        else:
            out[label] = 0
    return out


def _survey_git_themes(root, limit=40):
    try:
        r = subprocess.run(
            ["git", "-C", str(root), "log", "--no-merges", f"-n{limit}", "--pretty=%s"],
            capture_output=True, text=True, timeout=15,
        )
        if r.returncode != 0:
            return []
        return [line.strip() for line in r.stdout.splitlines() if line.strip()]
    except Exception:
        return []


def _head_commit(root):
    """Short SHA of HEAD, or 'unknown' when this is not a git repo. Articles stamp
    this so a reader can see how current an article is relative to HEAD."""
    try:
        r = subprocess.run(
            ["git", "-C", str(root), "rev-parse", "--short", "HEAD"],
            capture_output=True, text=True, timeout=15,
        )
        if r.returncode == 0 and r.stdout.strip():
            return r.stdout.strip()
    except Exception:
        pass
    return "unknown"


def _commits_behind(root, commit):
    """How many commits HEAD is ahead of `commit`, or None if not determinable."""
    if not commit or commit == "unknown":
        return None
    try:
        r = subprocess.run(
            ["git", "-C", str(root), "rev-list", "--count", f"{commit}..HEAD"],
            capture_output=True, text=True, timeout=15,
        )
        if r.returncode == 0 and r.stdout.strip().isdigit():
            return int(r.stdout.strip())
    except Exception:
        pass
    return None


_COMMIT_RE = re.compile(r"^>\s*Commit:\s*(\S+)", re.MULTILINE)

_MARKDOWN_REFERENCE_RE = re.compile(
    r"(?<![A-Za-z0-9_])(?P<tick>`?)"
    r"(?P<reference>(?:\.{1,2}[\\/])?(?:[A-Za-z0-9_.-]+[\\/])*[A-Za-z0-9_.-]+\.md)"
    r"(?P=tick)(?![A-Za-z0-9_])"
)
_INLINE_LINK_RE = re.compile(r"!?\[[^\]]*\]\([^)]*\)")
_URL_RE = re.compile(r"(?:https?|file)://\S+")
_FENCE_RE = re.compile(r"^\s*(```|~~~)")
_REFERENCE_SEARCH_SKIP_PARTS = {
    ".git", "node_modules", ".venv", "venv", "__pycache__",
    ".agents", ".claude", ".opencode", ".codex",
}


def _inside_any(start, end, spans):
    return any(start >= left and end <= right for left, right in spans)


def _authored_markdown_matches(root, name):
    matches = []
    for path in root.rglob(name):
        if not path.is_file():
            continue
        rel = path.relative_to(root)
        if set(rel.parts) & _REFERENCE_SEARCH_SKIP_PARTS:
            continue
        if rel.parts[:2] == ("templates", "repo-skeleton"):
            continue
        matches.append(path.resolve())
    return sorted(set(matches), key=lambda p: p.as_posix())


def _resolve_markdown_reference(root, article, reference):
    """Resolve without guessing. Exact relative/root paths win; a bare filename
    is fixable only when it has one authored match after host/template mirrors are
    excluded."""
    normalized = reference.replace("\\", "/")
    exact = []
    for candidate in (article.parent / normalized, root / normalized):
        resolved = candidate.resolve()
        if resolved.is_file() and resolved.suffix.lower() == ".md":
            exact.append(resolved)
    exact = sorted(set(exact), key=lambda p: p.as_posix())
    if len(exact) == 1:
        return exact[0], "resolved"
    if len(exact) > 1:
        return None, "ambiguous"
    if "/" in normalized:
        return None, "missing"
    matches = _authored_markdown_matches(root, normalized)
    if len(matches) == 1:
        return matches[0], "resolved"
    return (None, "ambiguous") if matches else (None, "missing")


def _line_reference_matches(root, article, line, line_number):
    protected = [(m.start(), m.end()) for m in _INLINE_LINK_RE.finditer(line)]
    protected += [(m.start(), m.end()) for m in _URL_RE.finditer(line)]
    findings = []
    for match in _MARKDOWN_REFERENCE_RE.finditer(line):
        if _inside_any(match.start(), match.end(), protected):
            continue
        reference = match.group("reference")
        target, resolution = _resolve_markdown_reference(root, article, reference)
        if target == article.resolve():
            continue
        rel_article = article.relative_to(root).as_posix()
        target_rel = target.relative_to(root).as_posix() if target else None
        findings.append({
            "category": "wiki_inline_markdown_link",
            "severity": "error",
            "path": rel_article,
            "line": line_number,
            "reference": reference,
            "target": target_rel,
            "resolution": resolution,
            "fixable": target is not None,
            "start": match.start(),
            "end": match.end(),
            "tick": bool(match.group("tick")),
            "summary": (
                f"{rel_article}:{line_number} references {reference} without an inline Markdown link"
                if target else
                f"{rel_article}:{line_number} has an unlinked {resolution} Markdown reference: {reference}"
            ),
        })
    return findings


def _scan_wiki_links(root, wiki_dir):
    findings = []
    wiki = root / wiki_dir
    if not wiki.is_dir():
        return findings
    for article in sorted(wiki.rglob("*.md")):
        in_fence = False
        for number, line in enumerate(article.read_text(encoding="utf-8-sig").splitlines(keepends=True), 1):
            if _FENCE_RE.match(line):
                in_fence = not in_fence
                continue
            if not in_fence:
                findings.extend(_line_reference_matches(root, article, line, number))
    return findings


def _relative_link(article, target):
    rel = Path(os.path.relpath(target, article.parent)).as_posix()
    return quote(rel, safe="/._-~")


def _apply_wiki_link_fixes(root, findings):
    by_path = {}
    for finding in findings:
        if finding["fixable"]:
            by_path.setdefault(finding["path"], []).append(finding)
    fixed = 0
    for rel, file_findings in by_path.items():
        article = root / rel
        lines = article.read_text(encoding="utf-8-sig").splitlines(keepends=True)
        by_line = {}
        for finding in file_findings:
            by_line.setdefault(finding["line"] - 1, []).append(finding)
        for index, line_findings in by_line.items():
            for finding in sorted(line_findings, key=lambda f: f["start"], reverse=True):
                label = f"`{finding['reference']}`" if finding["tick"] else finding["reference"]
                target = root / finding["target"]
                replacement = f"[{label}]({_relative_link(article, target)})"
                lines[index] = lines[index][:finding["start"]] + replacement + lines[index][finding["end"]:]
                fixed += 1
        article.write_text("".join(lines), encoding="utf-8")
    return fixed


def wiki_link_report(root, fix=False, wiki_dir=None):
    """Audit wiki prose for bare local Markdown references and optionally repair
    only references with one deterministic target."""
    root = Path(root).resolve()
    if wiki_dir is None:
        config = _read_json(root / ".prd_plugin" / "config.json") or {}
        wiki_dir = (((config.get("knowledge") or {}).get("llm_wiki") or {}).get("wiki_dir") or "wiki")
    findings = _scan_wiki_links(root, Path(wiki_dir))
    fixed = _apply_wiki_link_fixes(root, findings) if fix else 0
    remaining = _scan_wiki_links(root, Path(wiki_dir)) if fix else findings
    public = [{k: v for k, v in finding.items() if k not in {"start", "end", "tick"}}
              for finding in remaining]
    return {
        "status": "fail" if public else "ok",
        "repo_root": str(root),
        "wiki_dir": Path(wiki_dir).as_posix(),
        "summary": {"unlinked": len(public), "fixed": fixed},
        "findings": public,
    }


def wiki_drift(root, threshold=25):
    """Detect wiki staleness for the drift monitor: articles whose Commit stamp is
    far behind HEAD, articles that carry no Commit stamp, and index inconsistency.
    Read-only, fail-open. Returns {status, findings, summary} — the validator
    contract. When there is no wiki, there is nothing to drift (status ok)."""
    root = Path(root)
    findings = []
    try:
        config = _read_json(root / ".prd_plugin" / "config.json") or {}
        wiki_dir = (((config.get("knowledge") or {}).get("llm_wiki") or {}).get("wiki_dir") or "wiki")
        wiki = root / wiki_dir
        index = wiki / "index.md"
        if not index.is_file():
            return {"status": "ok", "findings": [],
                    "summary": "no wiki (nothing to drift)"}
        index_text = index.read_text(encoding="utf-8-sig")
        articles = [p for p in wiki.rglob("*.md") if p.name not in ("index.md", "log.md")]
        stale = unstamped = missing_from_index = 0
        for art in articles:
            rel = str(art.relative_to(wiki)).replace("\\", "/")
            text = art.read_text(encoding="utf-8-sig")
            if rel not in index_text:
                missing_from_index += 1
                findings.append({"category": "wiki_drift", "severity": "warning",
                                 "summary": f"wiki article not in index: {rel}"})
            m = _COMMIT_RE.search(text)
            commit = m.group(1) if m else None
            if not commit or commit == "unknown":
                unstamped += 1
                findings.append({"category": "wiki_drift", "severity": "info",
                                 "summary": f"wiki article has no Commit stamp: {rel}"})
                continue
            behind = _commits_behind(root, commit)
            if behind is not None and behind >= threshold:
                stale += 1
                findings.append({"category": "wiki_drift", "severity": "warning",
                                 "summary": f"wiki article {behind} commits behind HEAD "
                                            f"(re-ingest candidate): {rel}"})
        link_report = wiki_link_report(root, wiki_dir=wiki_dir)
        findings.extend(link_report["findings"])
        unlinked = link_report["summary"]["unlinked"]
        summary = (f"{len(articles)} articles: {stale} stale (>= {threshold} behind), "
                   f"{unstamped} unstamped, {missing_from_index} missing from index, "
                   f"{unlinked} unlinked Markdown references")
        return {"status": "attention" if findings else "ok",
                "findings": findings, "summary": summary}
    except Exception as exc:  # pragma: no cover - fail-open
        return {"status": "ok", "findings": [], "summary": f"wiki drift skipped: {exc}"}


def _memory_notes(root):
    data = _read_json(Path(root) / ".prd_plugin" / "state" / "memory.json")
    if isinstance(data, dict):
        for key in ("records", "memories", "notes"):
            if isinstance(data.get(key), list):
                return len(data[key])
    return 0


def _propose_topics(code, docs, state):
    """Suggest one-level wiki topics from the inventory. Advisory — the agent
    refines these while compiling."""
    topics = []
    for module, count in code.items():
        topics.append({"topic": module, "kind": "code",
                       "hint": f"{count} source file(s) under {module}/"})
    if docs:
        topics.append({"topic": "docs", "kind": "docs",
                       "hint": f"{len(docs)} existing doc(s) to distill: " + ", ".join(docs[:8])})
    if any(state.values()):
        topics.append({"topic": "decisions", "kind": "state",
                       "hint": "settled decisions + accepted requests from .prd_plugin state"})
        topics.append({"topic": "workflow", "kind": "state",
                       "hint": "how work is tracked and shipped in this repo"})
    return topics


def build_plan(root):
    root = Path(root)
    code = _survey_code(root)
    docs = _survey_docs(root)
    state = _survey_state(root)
    return {
        "repo": str(root.resolve().name),
        "head_commit": _head_commit(root),
        "wiki_exists": wiki_exists(root),
        "backfill_pending": backfill_pending(root),
        "code_modules": code,
        "docs": docs,
        "state": state,
        "memory_notes": _memory_notes(root),
        "git_themes": _survey_git_themes(root),
        "proposed_topics": _propose_topics(code, docs, state),
        "instructions": (
            "Compile the wiki via the project-llm-wiki skill's Backfill mode: for "
            "each proposed topic, record raw sources (or reference existing docs) and "
            "write a synthesized article grounded ONLY in what the code/docs/state "
            "show. Cite the REQ-*/DEC-*/EV- that produced knowledge, and stamp each "
            "article's Commit field with head_commit. Build the index, "
            "log a backfill entry, run Lint, then remove "
            ".prd_plugin/local/wiki-backfill-needed."
        ),
    }


def _format_plan(plan):
    lines = [f"# Wiki backfill plan — {plan['repo']}", ""]
    lines.append(f"Backfill pending: {plan['backfill_pending']}  |  "
                 f"wiki exists: {plan['wiki_exists']}  |  "
                 f"memory notes: {plan['memory_notes']}")
    lines.append("")
    lines.append("## Proposed topics")
    for t in plan["proposed_topics"]:
        lines.append(f"- **{t['topic']}** ({t['kind']}) — {t['hint']}")
    lines.append("")
    lines.append("## Code modules")
    for mod, n in plan["code_modules"].items():
        lines.append(f"- {mod}: {n} file(s)")
    lines.append("")
    st = plan["state"]
    lines.append("## PRD state to distill")
    lines.append("  " + ", ".join(f"{k}: {v}" for k, v in st.items()))
    if plan["docs"]:
        lines.append("")
        lines.append("## Existing docs")
        for d in plan["docs"][:40]:
            lines.append(f"- {d}")
    if plan["git_themes"]:
        lines.append("")
        lines.append("## Recent git themes")
        for s in plan["git_themes"][:20]:
            lines.append(f"- {s}")
    lines.append("")
    lines.append("## Next")
    lines.append(plan["instructions"])
    return "\n".join(lines) + "\n"


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--repo-root", default=".")
    parser.add_argument("--plan", action="store_true", help="Emit the backfill plan.")
    parser.add_argument("--status", action="store_true",
                        help="Report whether a backfill is pending.")
    parser.add_argument("--drift", action="store_true",
                        help="Report wiki staleness for the drift monitor.")
    parser.add_argument("--lint-links", action="store_true",
                        help="Require local .md references in wiki prose to be inline links.")
    parser.add_argument("--fix", action="store_true",
                        help="With --lint-links, repair references that resolve unambiguously.")
    parser.add_argument("--format", choices=("json", "markdown"), default=None)
    parser.add_argument("--output", help="Write the result here instead of stdout.")
    parser.add_argument("--json", action="store_true", help="Emit JSON.")
    args = parser.parse_args(argv)
    root = Path(args.repo_root)
    as_json = args.json or args.format == "json"

    if args.fix and not args.lint_links:
        parser.error("--fix requires --lint-links")

    if args.lint_links:
        result = wiki_link_report(root, fix=args.fix)
        text = json.dumps(result, indent=2) if as_json or args.format is None else (
            f"wiki inline Markdown links: {result['summary']['unlinked']} unlinked, "
            f"{result['summary']['fixed']} fixed"
        )
        if args.output:
            Path(args.output).write_text(text + "\n", encoding="utf-8")
        else:
            print(text)
        return 1 if result["findings"] else 0

    if args.drift:
        result = wiki_drift(root)
        text = json.dumps(result, indent=2) if as_json or args.format is None else result["summary"]
        if args.output:
            Path(args.output).write_text(text + "\n", encoding="utf-8")
        else:
            print(text)
        return 0

    if args.status or not args.plan:
        pending = backfill_pending(root)
        if args.json:
            print(json.dumps({"backfill_pending": pending}))
        else:
            print("LLM wiki backfill: pending" if pending
                  else "LLM wiki backfill: not pending")
        if not args.plan:
            return 0

    plan = build_plan(root)
    print(json.dumps(plan, indent=2) if args.json else _format_plan(plan))
    return 0


if __name__ == "__main__":
    sys.exit(main())