/** * Repository digest: the machine-generated project context that gets pasted into * the ChatGPT thread so it can actually analyse the code instead of guessing. * * Design constraints, in order: * 1. Deterministic. Same commit + same working tree produces the same digest, * so ChatGPT can tell when something changed and caching/bypassing works. * 2. Bounded. It is re-sent on every turn, so it must fit a token budget. Full * file contents are the expensive part and are allocated last. * 3. Honest. If something was omitted or truncated, the digest says so, rather * than letting ChatGPT reason about a project it cannot fully see. * * Zero pi dependencies: uses `git` through node:child_process. */ import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { existsSync, readFileSync, statSync } from "node:fs"; import { join, relative } from "node:path"; import { estimateTokens } from "../core/protocol.ts"; export interface DigestOptions { cwd: string; /** Hard ceiling for the whole digest. */ budgetTokens?: number; /** How many recent commits to list. */ commitCount?: number; /** Force these paths to be included in full (relative to repo root). */ focusPaths?: string[]; /** Glob-ish suffixes treated as source worth outlining. */ sourceSuffixes?: string[]; /** Max lines of any single file included in full. */ maxFileLines?: number; } export interface DigestResult { markdown: string; hash: string; tokens: number; truncated: boolean; stats: { trackedFiles: number; dirtyFiles: number; filesIncludedInFull: number; filesOutlined: number; branch: string; head: string; }; } const DEFAULT_SOURCE_SUFFIXES = [".ts", ".tsx", ".js", ".mjs", ".cjs", ".py", ".go", ".rs", ".java", ".c", ".cpp", ".h"]; const SKIP_DIRS = /(^|\/)(node_modules|\.git|dist|build|out|vendor|\.next|coverage|__pycache__)(\/|$)/; function git(cwd: string, args: string[]): string { try { return execFileSync("git", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 32 * 1024 * 1024 }).trim(); } catch { return ""; } } /** * A cheap symbol outline: enough for ChatGPT to know what a file offers without * paying for the whole file. */ function outline(text: string, maxEntries = 40): string[] { const lines = text.split("\n"); const found: string[] = []; const patterns: RegExp[] = [ /^\s*export\s+(?:async\s+)?function\s+([A-Za-z0-9_$]+)/, /^\s*export\s+class\s+([A-Za-z0-9_$]+)/, /^\s*export\s+(?:const|let|var)\s+([A-Za-z0-9_$]+)/, /^\s*export\s+interface\s+([A-Za-z0-9_$]+)/, /^\s*export\s+type\s+([A-Za-z0-9_$]+)/, /^\s*export\s+default\s+(?:function|class)\s*([A-Za-z0-9_$]*)/, /^\s*(?:export\s+)?(?:async\s+)?function\s+([A-Za-z0-9_$]+)/, /^\s*(?:export\s+)?class\s+([A-Za-z0-9_$]+)/, /^\s*(?:export\s+)?(?:const|let)\s+([A-Za-z0-9_$]+)\s*=\s*(?:async\s*)?\(/, /^\s*(?:pub\s+)?(?:fn|struct|enum|impl)\s+([A-Za-z0-9_$]+)/, /^\s*(?:def|class)\s+([A-Za-z0-9_$]+)/, ]; for (let i = 0; i < lines.length; i += 1) { const line = lines[i]; for (const pattern of patterns) { const match = pattern.exec(line); if (match) { const name = match[1] || "(default)"; found.push(` - ${name} @L${i + 1}`); break; } } if (found.length >= maxEntries) { found.push(` - …(更多符号已省略)`); break; } } return found; } function readTextFile(path: string): string | null { try { const stat = statSync(path); if (!stat.isFile() || stat.size > 2 * 1024 * 1024) return null; const text = readFileSync(path, "utf8"); // crude binary guard if (text.includes("\u0000")) return null; return text; } catch { return null; } } export function buildRepoDigest(options: DigestOptions): DigestResult { const cwd = options.cwd; const budget = options.budgetTokens ?? 6000; const commitCount = options.commitCount ?? 8; const sourceSuffixes = options.sourceSuffixes ?? DEFAULT_SOURCE_SUFFIXES; const maxFileLines = options.maxFileLines ?? 400; if (!existsSync(join(cwd, ".git"))) { const markdown = [ "## 代码库", `(当前目录不是 git 仓库:${cwd})`, "ChatGPT 看不到任何源码。请让 pi 先初始化仓库,或在配置里指定 repoRoot。", ].join("\n"); return { markdown, hash: createHash("sha256").update(markdown).digest("hex").slice(0, 12), tokens: estimateTokens(markdown), truncated: false, stats: { trackedFiles: 0, dirtyFiles: 0, filesIncludedInFull: 0, filesOutlined: 0, branch: "", head: "" }, }; } const branch = git(cwd, ["rev-parse", "--abbrev-ref", "HEAD"]) || "(unknown)"; const head = git(cwd, ["rev-parse", "--short", "HEAD"]) || "(none)"; const remote = git(cwd, ["remote", "get-url", "origin"]) || "(no remote)"; const upstream = git(cwd, ["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]); const ahead = upstream ? git(cwd, ["rev-list", "--left-right", "--count", `${upstream}...HEAD`]) : ""; const status = git(cwd, ["status", "--porcelain=v1", "--untracked-files=normal"]); const dirtyLines = status ? status.split("\n").filter(Boolean) : []; const log = git(cwd, ["log", `-${commitCount}`, "--pretty=format:%h %ad %s", "--date=short"]); const tracked = (git(cwd, ["ls-files"]) || "").split("\n").filter(Boolean); // Line counts per tracked file (cheap: wc-style, streamed from git). const lineCounts = new Map(); for (const file of tracked) { const text = readTextFile(join(cwd, file)); if (text === null) continue; lineCounts.set(file, text.split("\n").length); } const sections: string[] = []; const stats = { trackedFiles: tracked.length, dirtyFiles: dirtyLines.length, filesIncludedInFull: 0, filesOutlined: 0, branch, head, }; sections.push( [ "### 仓库状态", `- 远端: ${remote}`, `- 分支: ${branch} @ ${head}${upstream ? ` (upstream ${upstream}, ahead/behind ${ahead || "?"})` : " (无 upstream)"}`, `- 跟踪文件: ${tracked.length} 个`, `- 工作区改动: ${dirtyLines.length} 个${dirtyLines.length ? "" : "(干净)"}`, dirtyLines.length ? "- 改动清单:\n" + dirtyLines.slice(0, 40).map((l) => ` ${l}`).join("\n") : "", dirtyLines.length > 40 ? ` …(还有 ${dirtyLines.length - 40} 个)` : "", "", "### 最近提交", log ? log.split("\n").map((l) => `- ${l}`).join("\n") : "- (无提交)", ] .filter(Boolean) .join("\n"), ); // File inventory (always complete: it is cheap and it tells ChatGPT what exists). const inventory = tracked .filter((file) => !SKIP_DIRS.test(file)) .map((file) => { const count = lineCounts.get(file); return count === undefined ? `- ${file}` : `- ${file} (${count} 行)`; }); sections.push(`### 文件清单(共 ${inventory.length} 个)\n${inventory.join("\n")}`); // Budget allocation. // // `used` must account for every block already accumulated -- including the // outlines sitting in their own array. Tracking only `sections` let the // outlines grow unchecked and blew the budget by ~3x in the first test run. let used = estimateTokens(sections.join("\n\n")); const remaining = () => budget - used; // 1. Files explicitly focused, then files touched by the working tree. const focus = new Set(options.focusPaths ?? []); for (const line of dirtyLines) { const path = line.slice(3).trim(); if (path) focus.add(path); } const bodies: string[] = []; const outlined: string[] = []; let truncated = false; const sourceFiles = tracked.filter( (file) => !SKIP_DIRS.test(file) && sourceSuffixes.some((suffix) => file.endsWith(suffix)), ); // 2. Outlines first (cheap, broad coverage), newest-first for relevance. for (const file of sourceFiles) { if (focus.has(file)) continue; if (remaining() < 400) { truncated = true; break; } const text = readTextFile(join(cwd, file)); if (text === null) continue; const lines = outline(text); if (lines.length === 0) continue; const block = `**${file}**\n${lines.join("\n")}`; const cost = estimateTokens(block) + 2; if (cost > remaining()) { truncated = true; break; } outlined.push(block); used += cost; stats.filesOutlined += 1; } // 3. Full bodies for the focused files, while the budget allows. for (const file of [...focus].sort()) { const text = readTextFile(join(cwd, file)); if (text === null) continue; const rawLines = text.split("\n"); const clipped = rawLines.length > maxFileLines; const body = (clipped ? rawLines.slice(0, maxFileLines).join("\n") : text).trimEnd(); const block = `**${file}**${clipped ? ` (仅前 ${maxFileLines} / ${rawLines.length} 行)` : ""}\n\`\`\`\n${body}\n\`\`\``; const cost = estimateTokens(block) + 2; if (cost > remaining()) { truncated = true; const note = `**${file}** (因 token 预算未包含正文;用 COMMAND 让 pi 打印它)`; bodies.push(note); used += estimateTokens(note); continue; } bodies.push(block); used += cost; stats.filesIncludedInFull += 1; if (clipped) truncated = true; } if (bodies.length) sections.push(`### 重点文件全文\n${bodies.join("\n\n")}`); if (outlined.length) sections.push(`### 其余源码符号大纲\n${outlined.join("\n\n")}`); if (truncated) { sections.push( "### 注意\n上面有内容因 token 预算被省略。需要看某个文件的完整内容时,在 COMMANDS 里写一条命令让 pi 输出它。", ); } const joined = sections.join("\n\n"); const clamped = estimateTokens(joined) > budget ? clampDigest(joined, budget) : { text: joined, truncated: false }; const markdown = clamped.text; if (clamped.truncated) truncated = true; const tokens = estimateTokens(markdown); const hash = createHash("sha256") .update(markdown) .digest("hex") .slice(0, 12); return { markdown, hash, tokens, truncated, stats }; } /** * Last-resort trim, applied only if the allocation above still overshot. * Keeps the head (repo state, inventory) and drops the tail (bodies, outlines). */ function clampDigest(text: string, budget: number): { text: string; truncated: boolean } { const keepChars = Math.max(200, budget * 3 - 160); const tail = "\n\n### 注意\n" + "摘要因 token 预算被硬截断(保留了仓库状态与文件清单,省略了部分正文与大纲)。" + "\n需要看具体文件时,在 COMMANDS 里让 pi 打印它。"; return { text: text.slice(0, keepChars) + tail, truncated: true }; } /** Compact one-line description for status output. */ export function describeDigest(result: DigestResult): string { const s = result.stats; return ( `digest ${result.hash} · ~${result.tokens} tokens · ${s.branch}@${s.head} · ` + `${s.trackedFiles} 文件 / ${s.dirtyFiles} 改动 · 全文 ${s.filesIncludedInFull} + 大纲 ${s.filesOutlined}` + `${result.truncated ? " · 已截断" : ""}` ); } /** Shortest useful hello-world for docs and tests. */ export function relativeFrom(cwd: string, path: string): string { return relative(cwd, path); }