#!/usr/bin/env python3
"""List the facts a codebase tour cites, each with the file and line it came from.

Reads a repository and prints, to stdout:
  - build and tool manifests found (package.json, pyproject.toml, Makefile, ...)
  - commands the repo itself defines: package.json scripts, Makefile and justfile
    targets, CI `run:` / `script:` steps, tox and nox sessions, pyproject entry points
  - commands the README or CONTRIBUTING names that no manifest or CI file defines, and
    paths they name that do not exist
  - entry points (main functions, `__main__` guards, Dockerfile CMD/ENTRYPOINT, bin)
  - test files grouped by directory, and test-runner config files
  - files that say they are generated, vendored directories, and lockfiles
  - environment variables the code reads, with the line that reads each
  - the directories holding the most files

It is read-only: it opens files under the root it is given and writes nothing.
It cannot tell which command works; it tells you where each one is written down.

Usage:
    python3 repo_facts.py <repo-root>
    python3 repo_facts.py <repo-root> --json
    python3 repo_facts.py <repo-root> --max 25     # cap items per section (default 40)

Exit codes: 0 facts printed, 2 bad input.
"""

import argparse
import json
import os
import re
import sys
from collections import Counter, defaultdict

SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", ".tox",
             ".nox", "target", ".mypy_cache", ".pytest_cache", ".ruff_cache", ".next",
             "coverage", ".gradle", ".idea", ".vscode", ".terraform"}
MANIFESTS = {"package.json", "pyproject.toml", "setup.cfg", "setup.py", "tox.ini", "noxfile.py",
             "Makefile", "makefile", "GNUmakefile", "justfile", "Taskfile.yml", "go.mod",
             "Cargo.toml", "pom.xml", "build.gradle", "build.gradle.kts", "Gemfile",
             "composer.json", "Dockerfile", "docker-compose.yml", "docker-compose.yaml",
             "compose.yml", "compose.yaml", "requirements.txt", "requirements-dev.txt",
             ".tool-versions", ".nvmrc", ".python-version", "CONTRIBUTING.md", "Procfile"}
LOCKFILES = {"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "poetry.lock", "uv.lock",
             "Pipfile.lock", "Cargo.lock", "go.sum", "Gemfile.lock", "composer.lock"}
VENDOR_DIRS = {"vendor", "third_party", "third-party", "vendored", "external"}
SOURCE_EXT = {".py", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".go", ".rb", ".java",
              ".kt", ".rs", ".php", ".cs", ".sh", ".scala", ".swift"}
TEST_FILE = re.compile(r"(^test_.*\.py$|_test\.py$|_test\.go$|\.(test|spec)\.[cm]?[jt]sx?$|"
                       r"Tests?\.(java|kt|cs)$|_spec\.rb$|_test\.rb$)")
TEST_CONFIG = re.compile(r"^(pytest\.ini|conftest\.py|jest\.config\.\w+|vitest\.config\.\w+|"
                         r"karma\.conf\.\w+|\.mocharc\.\w+|phpunit\.xml(\.dist)?)$")
GENERATED = re.compile(r"(?i)(do not edit|@generated|code generated|auto-?generated|"
                       r"this file is generated|generated by|don't change)")
# Environment reads. `[.]` keeps each pattern literal without a backslash-dot sequence.
ENV_READS = [
    re.compile(r"os[.]environ(?:[.]get)?\s*[\[(]\s*[\"']([A-Za-z_][A-Za-z0-9_]*)"),
    re.compile(r"os[.]getenv\s*\(\s*[\"']([A-Za-z_][A-Za-z0-9_]*)"),
    re.compile(r"process[.]env[.]([A-Za-z_][A-Za-z0-9_]*)"),
    re.compile(r"process[.]env\[\s*[\"'`]([A-Za-z_][A-Za-z0-9_]*)"),
    re.compile(r"os[.](?:Getenv|LookupEnv)\(\s*\"([A-Za-z_][A-Za-z0-9_]*)\""),
    re.compile(r"ENV(?:[.]fetch\(|\[)\s*[\"']([A-Za-z_][A-Za-z0-9_]*)"),
    re.compile(r"System[.]getenv\(\s*\"([A-Za-z_][A-Za-z0-9_]*)\""),
    re.compile(r"env::var\(\s*\"([A-Za-z_][A-Za-z0-9_]*)\""),
]
RUNNERS = {"npm", "yarn", "pnpm", "make", "just", "nox", "tox"}
MAX_BYTES = 512 * 1024


def walk(root):
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS)
        for name in sorted(filenames):
            full = os.path.join(dirpath, name)
            yield os.path.relpath(full, root).replace(os.sep, "/"), full


def read_lines(full):
    try:
        if os.path.getsize(full) > MAX_BYTES:
            return []
        with open(full, encoding="utf-8", errors="replace") as fh:
            return fh.read().splitlines()
    except OSError:
        return []


def line_of(lines, pattern, start=0):
    for i in range(start, len(lines)):
        if re.search(pattern, lines[i]):
            return i + 1
    return 0


def package_scripts(rel, lines, out, targets):
    try:
        doc = json.loads("\n".join(lines))
    except ValueError:
        return
    for name, cmd in (doc.get("scripts") or {}).items():
        n = line_of(lines, r'^\s*"%s"\s*:' % re.escape(name))
        out.append({"command": "npm run %s" % name, "runs": cmd, "where": "%s:%d" % (rel, n)})
        targets["npm"].add(name)
    for key in ("main", "bin"):
        if key in doc:
            yield key, doc[key], "%s:%d" % (rel, line_of(lines, r'^\s*"%s"\s*:' % key))


def make_targets(rel, lines, out, targets, runner):
    for i, line in enumerate(lines):
        m = re.match(r"^([A-Za-z0-9][\w.-]*)\s*(?:[^:=\n]*)?:(?!=)", line)
        if not m or m.group(1).startswith(".") or "=" in line.split(":")[0]:
            continue
        recipe = ""
        for nxt in lines[i + 1:i + 4]:
            if nxt.startswith(("\t", "    ")) and nxt.strip():
                recipe = nxt.strip()
                break
        out.append({"command": "%s %s" % (runner, m.group(1)), "runs": recipe,
                    "where": "%s:%d" % (rel, i + 1)})
        targets[runner].add(m.group(1))


def ci_steps(rel, lines, out):
    i = 0
    while i < len(lines):
        m = re.match(r"^(\s*)(?:-\s*)?run:\s*(.*)$", lines[i])
        if m:
            indent, value = len(m.group(1)), m.group(2).strip()
            if value in ("|", ">", "|-", ">-", ""):
                j = i + 1
                while j < len(lines) and (not lines[j].strip() or
                                          len(lines[j]) - len(lines[j].lstrip()) > indent):
                    if lines[j].strip() and not lines[j].strip().startswith("#"):
                        out.append({"command": lines[j].strip(), "where": "%s:%d" % (rel, j + 1)})
                    j += 1
                i = j
                continue
            out.append({"command": value, "where": "%s:%d" % (rel, i + 1)})
        i += 1


def gitlab_steps(rel, lines, out):
    inside = None
    for i, line in enumerate(lines):
        m = re.match(r"^(\s*)(before_script|script|after_script):\s*$", line)
        if m:
            inside = len(m.group(1))
            continue
        if inside is not None:
            item = re.match(r"^(\s*)-\s+(.*)$", line)
            if item and len(item.group(1)) >= inside:
                out.append({"command": item.group(2).strip().strip("'\""), "where": "%s:%d" % (rel, i + 1)})
            elif line.strip():
                inside = None


def tox_nox(rel, name, lines, out, targets):
    if name == "tox.ini":
        env = None
        for i, line in enumerate(lines):
            sec = re.match(r"^\[(testenv(?::(\S+))?)\]", line)
            if sec:
                env = sec.group(2) or "(default)"
                if sec.group(2):
                    targets["tox"].add(sec.group(2))
            elif env and re.match(r"^\s*commands\s*=", line):
                cmds = [line.split("=", 1)[1].strip()]
                for nxt in lines[i + 1:]:
                    if not nxt.strip() or not nxt[0].isspace():
                        break
                    cmds.append(nxt.strip())
                cmd = "; ".join(c for c in cmds if c and not c.startswith("#"))
                out.append({"command": "tox -e %s" % env, "runs": cmd, "where": "%s:%d" % (rel, i + 1)})
    else:
        for i, line in enumerate(lines):
            if re.match(r"^\s*@nox\.session", line):
                fn = next((re.match(r"^\s*def\s+(\w+)", lines[j]) for j in range(i + 1, min(i + 4, len(lines)))
                           if re.match(r"^\s*def\s+\w+", lines[j])), None)
                if fn:
                    targets["nox"].add(fn.group(1))
                    body = []
                    for j in range(i + 1, len(lines)):
                        if j > i + 1 and re.match(r"^(@|def |class )", lines[j]):
                            break
                        body.append(lines[j])
                    runs = [b.strip() for b in body if "session.run(" in b]
                    out.append({"command": "nox -s %s" % fn.group(1), "runs": "; ".join(runs),
                                "where": "%s:%d" % (rel, i + 1)})


def pyproject(rel, lines, out, entries):
    table = ""
    for i, line in enumerate(lines):
        sec = re.match(r"^\[([^\]]+)\]", line)
        if sec:
            table = sec.group(1).strip()
            continue
        kv = re.match(r"^\s*([\w.-]+)\s*=\s*(.+)$", line)
        if not kv:
            continue
        if table in ("project.scripts", "tool.poetry.scripts"):
            entries.append({"what": "console script `%s`" % kv.group(1), "target": kv.group(2).strip('"\''),
                            "where": "%s:%d" % (rel, i + 1)})
        elif table.startswith("tool.hatch.envs.") and table.endswith(".scripts"):
            out.append({"command": "hatch run %s" % kv.group(1), "runs": kv.group(2),
                        "where": "%s:%d" % (rel, i + 1)})
        elif table == "tool.pytest.ini_options" and kv.group(1) in ("addopts", "testpaths", "markers"):
            out.append({"command": "(pytest config) %s" % kv.group(1), "runs": kv.group(2),
                        "where": "%s:%d" % (rel, i + 1)})


def doc_commands(root, rel, lines, targets, missing):
    fence = False
    for i, line in enumerate(lines):
        if line.strip().startswith("```"):
            fence = not fence
            continue
        indented = line.startswith(("    ", "\t")) and not line.lstrip().startswith(("-", "*"))
        if fence or indented:
            candidates = [line.strip().lstrip("$ ").strip()]
        else:
            candidates = re.findall(r"`([^`]+)`", line)
        for cand in candidates:
            words = cand.split()
            if len(words) > 1 and words[0] in ("python", "python3") and words[1].endswith(".py"):
                if not os.path.exists(os.path.join(root, words[1])):
                    missing.append({"command": cand, "where": "%s:%d" % (rel, i + 1),
                                    "why": "`%s` does not exist" % words[1]})
                continue
            if len(words) < 2 and not (words and words[0] in ("make", "just")):
                continue
            if not words or words[0] not in RUNNERS:
                continue
            runner, target = words[0], None
            if runner in ("npm", "pnpm") and len(words) > 1:
                target = words[2] if words[1] == "run" and len(words) > 2 else (
                    words[1] if words[1] in ("test", "start") else None)
                runner = "npm"
            elif runner == "yarn" and len(words) > 1 and words[1] not in ("install", "add"):
                target = words[2] if words[1] == "run" and len(words) > 2 else words[1]
                runner = "npm"
            elif runner in ("make", "just"):
                target = words[1] if len(words) > 1 else None
            elif runner == "nox" and "-s" in words:
                target = words[words.index("-s") + 1] if words.index("-s") + 1 < len(words) else None
            elif runner == "tox" and "-e" in words:
                target = words[words.index("-e") + 1] if words.index("-e") + 1 < len(words) else None
            if runner in ("make", "just") and target is None:
                if not targets[runner]:
                    missing.append({"command": cand, "where": "%s:%d" % (rel, i + 1),
                                    "why": "no %s file defines any target" % runner})
                continue
            if target and target not in targets[runner]:
                source = {"npm": "package.json scripts", "make": "a Makefile", "just": "a justfile",
                          "nox": "noxfile.py", "tox": "tox.ini"}[runner]
                missing.append({"command": cand, "where": "%s:%d" % (rel, i + 1),
                                "why": "`%s` is not defined in %s" % (target, source)})


def doc_paths(root, rel, lines, missing):
    for i, line in enumerate(lines):
        for cand in re.findall(r"`([^`\s]+)`", line):
            looks_like_path = "/" in cand or re.search(r"\.(json|ya?ml|toml|ini|cfg|py|[jt]sx?|go|rb|sql|md)$", cand)
            if not looks_like_path or cand.startswith(("http", "-", "@", "$")) or "*" in cand or "<" in cand:
                continue
            if not os.path.exists(os.path.join(root, cand.strip("/"))):
                missing.append({"path": cand, "where": "%s:%d" % (rel, i + 1)})


def collect(root):
    facts = {"manifests": [], "commands": [], "doc_commands_not_found": [], "doc_paths_not_found": [],
             "entry_points": [],
             "tests": {}, "test_config": [], "generated": [], "vendored": [], "lockfiles": [],
             "env_vars": [], "largest_dirs": []}
    targets = defaultdict(set)
    docs, dir_count, tests = [], Counter(), defaultdict(int)
    for rel, full in walk(root):
        name = os.path.basename(rel)
        parts = rel.split("/")
        dir_count["/".join(parts[:2]) if len(parts) > 2 else (parts[0] if len(parts) > 1 else ".")] += 1
        vend = next((p for p in parts[:-1] if p in VENDOR_DIRS), None)
        if vend:
            vdir = "/".join(parts[:parts.index(vend) + 1])
            if vdir not in facts["vendored"]:
                facts["vendored"].append(vdir)
            continue
        if name in LOCKFILES:
            facts["lockfiles"].append(rel)
            continue
        ext = os.path.splitext(name)[1].lower()
        is_ci = rel.startswith(".github/workflows/") and ext in (".yml", ".yaml")
        is_gitlab = name == ".gitlab-ci.yml"
        if name in MANIFESTS or is_ci or is_gitlab or name.lower().startswith("readme"):
            facts["manifests"].append(rel)
        if TEST_FILE.search(name):
            tests["/".join(parts[:-1]) or "."] += 1
        if TEST_CONFIG.match(name):
            facts["test_config"].append(rel)
        wanted = (name in MANIFESTS or is_ci or is_gitlab or ext in SOURCE_EXT or name == ".gitattributes"
                  or name.lower().startswith("readme") or ext in (".md", ".yml", ".yaml", ".json", ".sql",
                                                                 ".toml", ".cfg", ".ini", ".proto"))
        if not wanted:
            continue
        lines = read_lines(full)
        if not lines:
            continue
        if name == "package.json":
            for key, val, where in package_scripts(rel, lines, facts["commands"], targets):
                facts["entry_points"].append({"what": "package.json %s" % key, "target": val, "where": where})
        elif name in ("Makefile", "makefile", "GNUmakefile"):
            make_targets(rel, lines, facts["commands"], targets, "make")
        elif name == "justfile":
            make_targets(rel, lines, facts["commands"], targets, "just")
        elif is_ci:
            ci_steps(rel, lines, facts["commands"])
        elif is_gitlab:
            gitlab_steps(rel, lines, facts["commands"])
        elif name in ("tox.ini", "noxfile.py"):
            tox_nox(rel, name, lines, facts["commands"], targets)
        elif name == "pyproject.toml":
            pyproject(rel, lines, facts["commands"], facts["entry_points"])
        elif name == "Dockerfile":
            for i, line in enumerate(lines):
                if re.match(r"^\s*(CMD|ENTRYPOINT)\b", line):
                    facts["entry_points"].append({"what": "Dockerfile", "target": line.strip(),
                                                  "where": "%s:%d" % (rel, i + 1)})
        elif name == ".gitattributes":
            for i, line in enumerate(lines):
                if "linguist-generated" in line or "linguist-vendored" in line:
                    facts["generated"].append({"path": line.split()[0], "marker": line.strip(),
                                               "where": "%s:%d" % (rel, i + 1)})
        if name.lower().startswith(("readme", "contributing")) and ext in (".md", ".rst", ""):
            docs.append((rel, lines))
        for i, line in enumerate(lines[:10]):
            if ext in SOURCE_EXT | {".json", ".yml", ".yaml", ".sql", ".toml"} and GENERATED.search(line):
                facts["generated"].append({"path": rel, "marker": line.strip()[:100], "where": "%s:%d" % (rel, i + 1)})
                break
        if ext in SOURCE_EXT:
            for i, line in enumerate(lines):
                if (re.match(r"^\s*func main\(\)", line) or re.match(r"^if __name__ == [\"']__main__[\"']", line)):
                    facts["entry_points"].append({"what": "main", "target": rel, "where": "%s:%d" % (rel, i + 1)})
                for pat in ENV_READS:
                    for var in pat.findall(line):
                        facts["env_vars"].append({"name": var, "where": "%s:%d" % (rel, i + 1)})
    for rel, lines in docs:
        doc_commands(root, rel, lines, targets, facts["doc_commands_not_found"])
        doc_paths(root, rel, lines, facts["doc_paths_not_found"])
    facts["tests"] = dict(sorted(tests.items()))
    facts["largest_dirs"] = [{"dir": d, "files": n} for d, n in dir_count.most_common(10)]
    return facts


def render(facts, cap):
    out = []

    def section(title, items, fmt, empty):
        out.append("## %s" % title)
        if not items:
            out.append("- %s" % empty)
        for item in items[:cap]:
            out.append("- " + fmt(item))
        if len(items) > cap:
            out.append("- ... %d more (raise --max to see them)" % (len(items) - cap))
        out.append("")

    out.append("# Repository facts\n")
    out.append("Where each fact is written down. Nothing here was run.\n")
    section("Manifests, CI and docs", facts["manifests"], lambda m: "`%s`" % m, "none found")
    section("Commands the repo defines", facts["commands"],
            lambda c: "`%s`%s — %s" % (c["command"], (" → `%s`" % c["runs"]) if c.get("runs") else "", c["where"]),
            "none found: no package scripts, Makefile, CI steps, tox or nox sessions")
    section("Commands the docs name that nothing defines", facts["doc_commands_not_found"],
            lambda c: "`%s` — %s — %s" % (c["command"], c["where"], c["why"]), "none")
    section("Paths the docs name that do not exist", facts["doc_paths_not_found"],
            lambda p: "`%s` — %s" % (p["path"], p["where"]), "none")
    section("Entry points", facts["entry_points"],
            lambda e: "%s: `%s` — %s" % (e["what"], e["target"], e["where"]), "none found")
    tests = [{"dir": d, "n": n} for d, n in facts["tests"].items()]
    section("Test files by directory", tests, lambda t: "`%s/` — %d file(s)" % (t["dir"], t["n"]),
            "no test files found")
    section("Test runner config", facts["test_config"], lambda t: "`%s`" % t, "none found")
    section("Generated files", facts["generated"],
            lambda g: "`%s` — %s — \"%s\"" % (g["path"], g["where"], g["marker"]), "no generated-file markers found")
    section("Vendored directories", facts["vendored"], lambda v: "`%s/`" % v, "none")
    section("Lockfiles", facts["lockfiles"], lambda f: "`%s`" % f, "none")
    section("Environment variables the code reads", facts["env_vars"],
            lambda e: "`%s` — %s" % (e["name"], e["where"]), "none found")
    section("Largest directories (file count)", facts["largest_dirs"],
            lambda d: "`%s` — %d" % (d["dir"], d["files"]), "empty")
    return "\n".join(out)


def main(argv=None):
    ap = argparse.ArgumentParser(description="List a repo's commands, tests, generated files and env reads with file:line.")
    ap.add_argument("root", help="repository root to read")
    ap.add_argument("--json", action="store_true", help="print JSON instead of Markdown")
    ap.add_argument("--max", type=int, default=40, help="items per section in Markdown (default 40)")
    args = ap.parse_args(argv)
    if not os.path.isdir(args.root):
        print("error: %s is not a directory" % args.root, file=sys.stderr)
        return 2
    facts = collect(args.root)
    print(json.dumps(facts, indent=2) if args.json else render(facts, max(1, args.max)))
    return 0


if __name__ == "__main__":
    sys.exit(main())