import { stat, writeFile } from "node:fs/promises"; import { resolve as resolvePath } from "node:path"; import type { Session } from "neo4j-driver"; /** * Pure llms.txt + llms-full.txt generator for an account's published * KnowledgeDocument set. llmstxt.org draft format. Plugin-agnostic: * the caller injects its own neo4j session factory so the lib can * be consumed from any plugin that already has a configured driver * (aeo, admin's publish-site tool, future hook callers). */ export interface WriteLlmsTxtInput { accountId: string; siteName: string; siteDescription?: string; siteDir?: string; } export interface WriteLlmsTxtDeps { getSession: () => Session; } export interface WriteLlmsTxtResult { llmsTxt: string; llmsFullTxt: string; pageCount: number; skippedNoUrl: number; sizeBytes: { index: number; full: number }; writtenPaths?: { index: string; full: string }; } interface PageRow { title: string; url: string; summary: string; body: string; } const HEADER_RX = /^#\s/m; function escapeMarkdown(text: string): string { return text.replace(/[\[\]\(\)\n\r]/g, " ").trim(); } function summariseBody(body: string): string { const flat = body.replace(/\s+/g, " ").trim(); if (flat.length <= 200) return flat; return flat.slice(0, 197).trimEnd() + "…"; } export async function writeLlmsTxt( input: WriteLlmsTxtInput, deps: WriteLlmsTxtDeps, ): Promise { const session = deps.getSession(); let rows: PageRow[] = []; let skippedNoUrl = 0; try { const r = await session.run( `MATCH (d:KnowledgeDocument {accountId: $accountId}) RETURN d.title AS title, d.url AS url, d.summary AS summary, coalesce(d.body, d.abstract, '') AS body ORDER BY d.title ASC`, { accountId: input.accountId }, ); for (const rec of r.records) { const title = (rec.get("title") as string | null) ?? ""; const url = (rec.get("url") as string | null) ?? ""; const summary = (rec.get("summary") as string | null) ?? ""; const body = (rec.get("body") as string | null) ?? ""; if (!url || !title) { skippedNoUrl += 1; continue; } rows.push({ title: escapeMarkdown(title), url: url.trim(), summary: escapeMarkdown(summary) || summariseBody(body) || "(no summary)", body: body.trim(), }); } } finally { await session.close(); } const indexLines: string[] = []; indexLines.push(`# ${escapeMarkdown(input.siteName)}`); indexLines.push(""); if (input.siteDescription) { indexLines.push(`> ${escapeMarkdown(input.siteDescription)}`); indexLines.push(""); } indexLines.push("## Pages"); indexLines.push(""); for (const row of rows) { indexLines.push(`- [${row.title}](${row.url}): ${row.summary}`); } indexLines.push(""); indexLines.push( "", ); const llmsTxt = indexLines.join("\n"); const fullLines: string[] = []; fullLines.push(`# ${escapeMarkdown(input.siteName)}`); fullLines.push(""); if (input.siteDescription) { fullLines.push(`> ${escapeMarkdown(input.siteDescription)}`); fullLines.push(""); } for (const row of rows) { fullLines.push(`# ${row.title}`); fullLines.push(""); fullLines.push(`Source: ${row.url}`); fullLines.push(""); if (row.body) { const safeBody = row.body.replace(HEADER_RX, "## "); fullLines.push(safeBody); } else { fullLines.push(row.summary); } fullLines.push(""); fullLines.push("---"); fullLines.push(""); } const llmsFullTxt = fullLines.join("\n"); const sizeBytes = { index: Buffer.byteLength(llmsTxt, "utf-8"), full: Buffer.byteLength(llmsFullTxt, "utf-8"), }; process.stderr.write( `[aeo-llms-txt] site=${input.siteName} pages=${rows.length} skippedNoUrl=${skippedNoUrl} indexBytes=${sizeBytes.index} fullBytes=${sizeBytes.full}\n`, ); let writtenPaths: { index: string; full: string } | undefined; if (input.siteDir) { const dir = resolvePath(input.siteDir); let dirStat; try { dirStat = await stat(dir); } catch (err) { const code = (err as NodeJS.ErrnoException)?.code; throw new Error( `siteDir does not exist (${code ?? "unknown"}): ${dir}`, ); } if (!dirStat.isDirectory()) { throw new Error(`siteDir is not a directory: ${dir}`); } const indexPath = resolvePath(dir, "llms.txt"); const fullPath = resolvePath(dir, "llms-full.txt"); await writeFile(indexPath, llmsTxt, "utf-8"); await writeFile(fullPath, llmsFullTxt, "utf-8"); writtenPaths = { index: indexPath, full: fullPath }; process.stderr.write( `[aeo-llms-txt] wrote site=${dir} indexBytes=${sizeBytes.index} fullBytes=${sizeBytes.full}\n`, ); } return { llmsTxt, llmsFullTxt, pageCount: rows.length, skippedNoUrl, sizeBytes, writtenPaths, }; }