import { posix } from 'node:path' import { toString as mdastToString } from 'mdast-util-to-string' import remarkFrontmatter from 'remark-frontmatter' import remarkGfm from 'remark-gfm' import remarkParse from 'remark-parse' import { unified } from 'unified' import ts from 'typescript' import { visit } from 'unist-util-visit' import { parse as parseYaml, parseDocument as parseYamlDocument, isMap, isScalar } from 'yaml' import type { Root, RootContent } from 'mdast' import { sha256NormalizedV1 } from '../index-builder/content-hash.js' import { createFuzzyCandidateIndex, resolveFuzzyReference, type FuzzyCandidateIndex } from '../lib/fuzzy-match.js' import { toPosix } from '../lib/paths.js' import type { Evidence, KnowledgeRelation } from '../schemas/knowledge.js' import { relationId } from './identity.js' import { surfaceFactEntityId } from '../storage/facts.js' /** * The Markdown analyzer. * * Documentation was previously read with regular expressions: frontmatter by one, the `docbridge` * block by a hand-written YAML subset, and the prose not at all. Headings, links and inline code * were discarded — so a repository where 23 documents linked to other documents and 14 cited * source paths produced no edges from any of it, and one document in ninety-two counted as * documented. A real parser turns that prose into evidence-backed relations with line numbers, * which is the difference between a graph that knows what its documentation says and one that * only knows the documentation exists. * * Every edge here is `observed` with file and line evidence from the node's own position. Nothing * is inferred from a near-match unless the near-match is unambiguous. */ export const MARKDOWN_ANALYZER_VERSION = '1.7.0' /** Headings deeper than this are structure, not subject matter. */ const MAX_HEADING_DEPTH = 3 const MAX_HEADINGS = 64 const MAX_SUMMARY_LENGTH = 400 const MAX_TITLE_LENGTH = 256 /** A document with more relations than this is an index page; the tail adds noise, not knowledge. */ export const MARKDOWN_RELATION_CAP = 64 export const MARKDOWN_FACT_RELATION_CAP = 256 /** * A region a generator owns. The analyzer skips it when collecting mentions, so Doc Bridge does * not read its own output back in as evidence about the repository. */ const GENERATED_OPEN = // const GENERATED_CLOSE = // const EXTERNAL_LINK = /^(?:[a-z][a-z0-9+.-]*:|\/\/|#|mailto:)/i export type MarkdownHeading = { readonly depth: number readonly text: string readonly line: number } export type MarkdownReference = { /** The link target, or the inline code token. */ readonly value: string /** Link text, when the reference came from a link. */ readonly text?: string readonly line: number readonly configExample?: boolean readonly configPath?: string readonly configRoots?: readonly string[] } export type MarkdownGeneratedRegion = { readonly lineStart: number readonly lineEnd: number readonly hash?: string } /** The frontmatter fields the knowledge model understands. Everything else is left alone. */ export type MarkdownFrontmatter = { readonly type?: string readonly audience?: string readonly owner?: string readonly lifecycle?: string readonly tier?: string } export type MarkdownDocumentV1 = { readonly path: string readonly title?: string readonly headings: readonly MarkdownHeading[] readonly summary?: string readonly wordCount: number readonly frontmatter: MarkdownFrontmatter readonly generatedRegions: readonly MarkdownGeneratedRegion[] readonly contentHash: string readonly citationRegions?: readonly { lineStart: number; lineEnd: number; hash: string }[] /** Links outside generated regions. */ readonly links: readonly MarkdownReference[] /** Inline code tokens outside generated regions. */ readonly codeTokens: readonly MarkdownReference[] /** Bounded syntactic references in fenced blocks; exact resolution only. */ readonly fenceTokens?: readonly (MarkdownReference & { readonly configOnly?: boolean })[] readonly fenceTokensTruncated?: boolean /** Lexical CLI citations; shell tokens still require a known bin during resolution. */ readonly cliTokens: readonly MarkdownCliReference[] /** Raw frontmatter text and the line it starts on, for the declaration parser. */ readonly frontmatterBlock?: { readonly value: string; readonly line: number } } /** Preserve object paths so unrelated artifacts cannot borrow a config leaf's name. */ function objectConfigReferences(source: ts.SourceFile): (MarkdownReference & { configOnly?: boolean })[] { const references: (MarkdownReference & { configOnly?: boolean })[] = [] const name = (node: ts.PropertyName) => ts.isIdentifier(node) || ts.isStringLiteral(node) ? node.text : undefined const collect = (object: ts.ObjectLiteralExpression, prefix: string, roots: string[], depth = 0): void => { if (depth > 32) return for (const property of object.properties) { if (!ts.isPropertyAssignment(property)) continue const key = name(property.name) if (!key) continue const path = prefix ? `${prefix}.${key}` : key references.push({value:key,line:source.getLineAndCharacterOfPosition(property.name.getStart(source)).line,configOnly:true,configExample:true,configPath:path,configRoots:roots}) if (ts.isObjectLiteralExpression(property.initializer)) collect(property.initializer,path,roots,depth+1) } } const walk = (node: ts.Node): void => { if (ts.isArrayLiteralExpression(node)) return if (ts.isObjectLiteralExpression(node)) { // Unknown spreads/computed/shorthand keys cannot establish a config example. const roots = node.properties.map(property => ts.isPropertyAssignment(property) ? name(property.name) : undefined) if (roots.every((key): key is string => key !== undefined)) collect(node,'',roots) return } ts.forEachChild(node,walk) } walk(source) return references } /** Parse code examples without treating keys, strings or member names as API references. */ function fenceReferences(value: string, language: string, configSection?: string): (MarkdownReference & { configOnly?: boolean })[] { if (['json', 'jsonc', 'yaml', 'yml', 'toml'].includes(language)) { const references: (MarkdownReference & { configOnly?: boolean })[] = value.split(/\r?\n/).flatMap((text, line) => [...text.matchAll(/(? ({ value: match[0], line, configOnly: true }))) if (language === 'json' || language === 'jsonc') { const source = ts.parseJsonText('config.json', value) if (!(source as ts.SourceFile & { parseDiagnostics?: readonly ts.Diagnostic[] }).parseDiagnostics?.length) references.push(...objectConfigReferences(source)) } else if (language === 'yaml' || language === 'yml') { const parsed = parseYamlDocument(value) if (!parsed.errors.length && isMap(parsed.contents)) { const roots = parsed.contents.items.map(item => String(isScalar(item.key) ? item.key.value : '')) const walk = (node: unknown, prefix = '', depth = 0): void => { if (!isMap(node) || depth > 32) return for (const item of node.items) { if (!isScalar(item.key) || typeof item.key.value !== 'string' || !item.key.range) continue const key = item.key.value, path = prefix ? `${prefix}.${key}` : key references.push({value:key, line:value.slice(0,item.key.range[0]).split(/\r?\n/).length-1, configOnly:true, configExample:true, configPath:path, configRoots:roots}) walk(item.value, path, depth+1) } } walk(parsed.contents) } } else { // ponytail: flat TOML assignments only; table/array syntax needs a TOML parser. const keys = value.split(/\r?\n/).flatMap((text,line) => { const match = /^\s*(?:"([A-Za-z_$][\w$]*)"|'([A-Za-z_$][\w$]*)'|([A-Za-z_$][\w$]*))\s*=/.exec(text) return match ? [{value:(match[1]??match[2]??match[3])!,line}] : [] }) if (!/^\s*\[/m.test(value)) references.push(...keys.map(key => ({...key,configOnly:true,configExample:true,configPath:key.value,configRoots:keys.map(key=>key.value)}))) } return references } if (!['ts', 'tsx', 'js', 'jsx', 'mjs', 'cjs'].includes(language)) return [] const source = ts.createSourceFile('fence.' + language, value, ts.ScriptTarget.Latest, true, language === 'tsx' || language === 'jsx' ? ts.ScriptKind.TSX : ts.ScriptKind.TS) const references: (MarkdownReference & { configOnly?: boolean })[] = [] const add = (node: ts.Node, name: string, configOnly = false) => { references.push({ value: name, line: source.getLineAndCharacterOfPosition(node.getStart(source)).line, ...(configOnly ? { configOnly } : {}) }) } const walk = (node: ts.Node): void => { // Dotted configuration references remain separate from export/member citations. if (ts.isPropertyAccessExpression(node)) { const name = node.getText(source) if (/^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+$/.test(name)) add(node, name, true) } if (ts.isIdentifier(node)) { const parent = node.parent const reference = (ts.isImportSpecifier(parent) && (parent.propertyName ?? parent.name) === node) || (ts.isExportSpecifier(parent) && (parent.propertyName ?? parent.name) === node) || (ts.isImportClause(parent) && parent.name === node) || (ts.isNamespaceImport(parent) && parent.name === node) || (ts.isCallExpression(parent) && (parent.expression === node || parent.arguments.includes(node))) || (ts.isNewExpression(parent) && (parent.expression === node || !!parent.arguments?.includes(node))) || ((ts.isJsxOpeningElement(parent) || ts.isJsxSelfClosingElement(parent)) && parent.tagName === node) || (ts.isTypeReferenceNode(parent) && parent.typeName === node) || (ts.isExpressionWithTypeArguments(parent) && parent.expression === node) || (ts.isVariableDeclaration(parent) && parent.initializer === node) || (ts.isBinaryExpression(parent) && parent.operatorToken.kind === ts.SyntaxKind.EqualsToken && parent.right === node) || (ts.isPropertyAssignment(parent) && parent.initializer === node) || (ts.isBindingElement(parent) && (parent.propertyName ?? parent.name) === node && ts.isObjectBindingPattern(parent.parent) && ts.isVariableDeclaration(parent.parent.parent) && !!parent.parent.parent.initializer && ts.isCallExpression(parent.parent.parent.initializer) && ['require', 'import'].includes(parent.parent.parent.initializer.expression.getText(source)) && parent.parent.parent.initializer.arguments.some(ts.isStringLiteral)) if (reference) add(node, node.text) } ts.forEachChild(node, walk) } walk(source) references.push(...objectConfigReferences(source)) if (configSection) { // Illustrative JSDoc may contain glob text with */ inside it; closing lines still bound the comment. const signatureSource = ts.createSourceFile('section.ts', value.replace(/^[ \t]*\/\*\*[ \t]*\r?\n[\s\S]*?^[ \t]*\*\//gm, comment => comment.replace(/[^\r\n]/g, ' ')), ts.ScriptTarget.Latest, true) const declaration = signatureSource.statements.find(node => ts.isInterfaceDeclaration(node) || ts.isTypeAliasDeclaration(node) && ts.isTypeLiteralNode(node.type)) const members = declaration && (ts.isInterfaceDeclaration(declaration) ? declaration.members : ts.isTypeAliasDeclaration(declaration) && ts.isTypeLiteralNode(declaration.type) ? declaration.type.members : undefined) const collect = (members: ts.NodeArray, prefix: string, depth = 0): void => { if (depth > 32) return for (const member of members) { if (!ts.isPropertySignature(member) || !member.type || !member.name || !(ts.isIdentifier(member.name) || ts.isStringLiteral(member.name))) continue const path = `${prefix}.${member.name.text}` references.push({ value: path, line: signatureSource.getLineAndCharacterOfPosition(member.name.getStart(signatureSource)).line, configOnly: true }) if (ts.isTypeLiteralNode(member.type)) collect(member.type.members, path, depth + 1) } } if (members) collect(members, configSection) } return references } const processor = unified() .use(remarkParse) .use(remarkFrontmatter, ['yaml']) .use(remarkGfm) export type MarkdownCliReference = MarkdownReference & { readonly kind: 'cli-command' | 'cli-flag'; readonly bin?: string; readonly runner?: boolean; readonly packageRunner?: boolean; readonly command?: string; readonly ownerIds?: readonly string[] } const cliWords = (value: string) => { const words = value.trim().split(/\s+/) const runner = ['npx', 'pnpm', 'yarn', 'bunx'].includes(words[0] ?? '') const executable = runner ? words.shift() : undefined const packageRunner = executable === 'npx' || executable === 'bunx' || executable === 'pnpm' && ['dlx', 'exec'].includes(words[0] ?? '') || executable === 'yarn' && words[0] === 'dlx' if (executable === 'npx' && ['-y', '--yes'].includes(words[0] ?? '') || executable === 'pnpm' && ['dlx', 'exec'].includes(words[0] ?? '') || executable === 'yarn' && words[0] === 'dlx') words.shift() return { words, runner, packageRunner } } /** Qualified command prefixes and dash-prefixed flags, without evaluating shell syntax. */ export const cliCommandTokens = (value: string, line: number, shell = false): MarkdownCliReference[] => { const { words, runner, packageRunner } = cliWords(value) if (!/^(?:@[\w.-]+\/)?[A-Za-z][\w.-]*(?:@[^\s@]+)?$/.test(words[0] ?? '')) return [] const result: MarkdownCliReference[] = [] const prefix: string[] = [] for (const word of words) { if (prefix.length && !/^[A-Za-z][\w.-]*$/.test(word)) break prefix.push(word) // A bare word is only a command when it is the first word of a shell line. if (prefix.length >= (shell || runner || words[1]?.startsWith('-') ? 1 : 2)) result.push({ value: prefix.join(' '), line, kind: 'cli-command', bin: words[0]!, ...(runner ? { runner, packageRunner } : {}) }) } return result } export const cliFlagTokens = (value: string, line: number, bin?: string): MarkdownCliReference[] => { const { words, runner, packageRunner } = cliWords(value) const command = cliCommandTokens(value, line, true).at(-1)?.value const context = command ? { bin: words[0]!, command, ...(runner ? { runner, packageRunner } : {}) } : bin ? { bin } : {} return words.flatMap(word => { const name = word.split('=')[0]! return /^--?[A-Za-z][\w.-]*$/.test(name) ? [{ value: name, line, kind: 'cli-flag' as const, ...context }] : [] }) } /** Resolve invocation context once for discovery and historical citation verification. */ export const cliCitationTokens = (tokens: readonly MarkdownCliReference[], resolution: MarkdownResolution): MarkdownCliReference[] => { const commands = [...(resolution.facts?.values() ?? [])].flat().filter(fact => fact.kind === 'cli-command') const bins = new Set(commands.map(fact => fact.name.split(' ')[0]!)) return tokens.flatMap(token => { if (!token.bin) return [token] if (token.packageRunner) { const name = token.bin.replace(/@[^@/]+$/, '') const packages = resolution.cliPackages?.filter(pkg => pkg.name === name) ?? [] if (packages.length) { if (packages.length !== 1) return [] const declaration = packages[0]!.bin const entries = typeof declaration === 'string' ? [[name.split('/').pop()!, declaration]] : declaration && typeof declaration === 'object' && !Array.isArray(declaration) ? Object.entries(declaration) : [] if (!entries.length || entries.some(([alias, entry]) => !/^[\w][\w.-]*$/.test(alias!) || typeof entry !== 'string')) return [] const sameExecutable = new Set(entries.map(([, entry]) => posix.normalize(entry as string))).size === 1 const aliases = sameExecutable ? entries.map(([alias]) => alias!) : entries.filter(([alias]) => alias === name.split('/').pop()).map(([alias]) => alias!) const canonical = aliases.find(alias => alias === name.split('/').pop()) ?? aliases.sort()[0] return (canonical ? [canonical] : []).flatMap(bin => cliCitationTokens([{ ...token, packageRunner: false, bin, value: token.kind === 'cli-command' ? token.value.replace(token.bin!, bin) : token.value, ...(token.command ? { command: token.command.replace(token.bin!, bin) } : {}) }], resolution)) } } if (!token.bin || !bins.has(token.bin)) return [] if (token.kind === 'cli-command') return [token] const words = token.command?.split(' ') ?? [token.bin] const owners = new Set(words.map((_, index) => words.slice(0, index + 1).join(' ')).flatMap(name => (resolution.facts?.get(name) ?? []).filter(fact => fact.kind === 'cli-command').map(fact => surfaceFactEntityId('cli-command', fact.ownerId, fact.name)))) const ownerIds = [...new Set((resolution.facts?.get(token.value) ?? []).filter(fact => fact.kind === 'cli-flag' && owners.has(fact.ownerId)).map(fact => fact.ownerId))] return ownerIds.length ? [{ ...token, ownerIds }] : [] }) } type Positioned = { readonly position?: { readonly start: { readonly line: number } } | undefined } const lineOf = (node: Positioned): number => node.position?.start.line ?? 1 const withinGenerated = (line: number, regions: readonly MarkdownGeneratedRegion[]): boolean => regions.some((region) => line >= region.lineStart && line <= region.lineEnd) /** Cap a summary at a sentence boundary when there is one, and never mid-word. */ const boundedSummary = (value: string): string | undefined => { const text = value.replace(/\s+/g, ' ').trim() if (!text) return undefined if (text.length <= MAX_SUMMARY_LENGTH) return text const sliced = text.slice(0, MAX_SUMMARY_LENGTH) const sentenceEnd = Math.max(sliced.lastIndexOf('. '), sliced.lastIndexOf('! '), sliced.lastIndexOf('? ')) if (sentenceEnd > MAX_SUMMARY_LENGTH * 0.4) return sliced.slice(0, sentenceEnd + 1).trim() const wordEnd = sliced.lastIndexOf(' ') return (wordEnd > 0 ? sliced.slice(0, wordEnd) : sliced).trim() } const frontmatterField = (data: unknown, key: keyof MarkdownFrontmatter): string | undefined => { if (typeof data !== 'object' || data === null || Array.isArray(data)) return undefined const value = (data as Record)[key] if (typeof value === 'string' && value.trim()) return value.trim().slice(0, 128) if (typeof value === 'number' || typeof value === 'boolean') return String(value) return undefined } /** * The frontmatter fields the knowledge model reads, parsed as real YAML. * * Unparseable frontmatter yields no fields rather than a guess: the declaration layer is what * reports the syntax error, with a line number, and one report is better than two. */ const frontmatterSubset = (value: string | undefined): MarkdownFrontmatter => { if (!value) return {} let data: unknown try { data = parseYaml(value) } catch { return {} } const subset: Record = {} for (const key of ['type', 'audience', 'owner', 'lifecycle', 'tier'] as const) { const field = frontmatterField(data, key) if (field !== undefined) subset[key] = field } return subset } /** * The audience a document declares about itself. * * `audience` is authoritative by definition — it is the author naming their reader. `type` counts * only when it happens to name an audience, because in practice it names a document kind * (`package`, `module`, `index`), and putting that in the audience field would corrupt the * coverage counts that read it. */ const AUDIENCES = new Set(['agent', 'human', 'archive', 'project', 'internal', 'external', 'unclassified']) export const declaredAudience = (frontmatter: MarkdownFrontmatter): string | undefined => { if (frontmatter.audience) return frontmatter.audience.toLowerCase() const type = frontmatter.type?.toLowerCase() return type && AUDIENCES.has(type) ? type : undefined } const generatedRegions = (tree: Root, totalLines: number): MarkdownGeneratedRegion[] => { const regions: MarkdownGeneratedRegion[] = [] let open: { line: number; hash?: string } | undefined visit(tree, 'html', (node) => { const line = lineOf(node) const closing = GENERATED_CLOSE.exec(node.value) if (closing) { if (open) regions.push({ lineStart: open.line, lineEnd: node.position?.end.line ?? line, ...(open.hash ? { hash: open.hash } : {}) }) open = undefined return } const opening = GENERATED_OPEN.exec(node.value) if (!opening) return // A second opening marker closes the previous region: generators emit regions, not nests. if (open) regions.push({ lineStart: open.line, lineEnd: line - 1, ...(open.hash ? { hash: open.hash } : {}) }) open = { line, ...(opening[1] ? { hash: opening[1] } : {}) } }) // An unclosed marker owns the rest of the file, which is what a trailing generated block is. if (open) regions.push({ lineStart: open.line, lineEnd: totalLines, ...(open.hash ? { hash: open.hash } : {}) }) return regions } /** * The hash of a document's content, as the entity records it. * * A byte-order mark is not content — a file that only gained one parses to the same tree — so it * is stripped before hashing. Exported because deciding whether a document needs parsing at all * means computing the same hash without parsing it. */ export const markdownContentHash = (content: string): string => sha256NormalizedV1(content.replace(/^\uFEFF/, '')) export const parseMarkdownDocument = (path: string, content: string): MarkdownDocumentV1 => { const normalized = content.replace(/^\uFEFF/, '') const tree = processor.parse(normalized) as Root const totalLines = normalized.split(/\r?\n/).length const citationRegions: { lineStart: number; lineEnd: number; hash: string }[] = [] visit(tree, node => { if (!['paragraph', 'code', 'heading', 'tableRow'].includes(node.type) || !node.position || citationRegions.length >= 4096) return const { start, end } = node.position citationRegions.push({ lineStart: start.line, lineEnd: end.line, hash: sha256NormalizedV1(normalized.slice(start.offset, end.offset)) }) }) const frontmatterNode = tree.children.find((child): child is RootContent & { type: 'yaml'; value: string } => child.type === 'yaml') const regions = generatedRegions(tree, totalLines) const headings: MarkdownHeading[] = [] const links: MarkdownReference[] = [] const codeTokens: MarkdownReference[] = [] const fenceTokens: (MarkdownReference & { configOnly?: boolean })[] = [] let fenceTokensTruncated = false const cliTokens: MarkdownCliReference[] = [] let title: string | undefined let summary: string | undefined let configSection: string | undefined visit(tree, (node, _index, parent) => { if (node.type === 'yaml') return const line = lineOf(node as { position?: { start: { line: number } } }) if (node.type === 'heading') { const section = node.children.filter(child => child.type === 'inlineCode') configSection = section.length === 1 && section[0]?.type === 'inlineCode' && /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(section[0].value) ? section[0].value : undefined const text = mdastToString(node).trim() if (!text) return if (!title && node.depth === 1) title = text.slice(0, MAX_TITLE_LENGTH) if (node.depth <= MAX_HEADING_DEPTH && headings.length < MAX_HEADINGS) { headings.push({ depth: node.depth, text: text.slice(0, MAX_TITLE_LENGTH), line }) } return } if (node.type === 'paragraph' && !summary && !withinGenerated(line, regions)) { // A paragraph that is only a link is navigation, not a description of the document. const onlyLink = node.children.length === 1 && node.children[0]?.type === 'link' if (!onlyLink) summary = boundedSummary(mdastToString(node)) return } if (node.type === 'link' && !withinGenerated(line, regions)) { const text = mdastToString(node).trim() links.push({ value: node.url, ...(text ? { text } : {}), line }) return } if (node.type === 'code') { for (const token of fenceReferences(node.value, node.lang ?? '', configSection)) { const citationLine = line + token.line + 1 if (withinGenerated(citationLine, regions)) continue if (fenceTokens.length >= 4096) { fenceTokensTruncated = true; break } fenceTokens.push({ ...token, line: citationLine }) } } if (node.type === 'code' && ['sh', 'bash', 'shell', 'zsh', 'console'].includes(node.lang ?? '')) { for (const [offset, text] of node.value.split(/\r?\n/).entries()) { const shellLine = line + offset + 1 if (withinGenerated(shellLine, regions)) continue const bin = text.trim().split(/\s+/)[0]! cliTokens.push(...cliCommandTokens(text, shellLine, true), ...cliFlagTokens(text, shellLine, bin)) } } if (node.type === 'inlineCode' && !withinGenerated(line, regions)) { const value = node.value.trim() if (value) { codeTokens.push({ value, line, ...(/\bconfig(?:uration)?\s+example\s*:?\s*$/i.test(parent ? parent.children.slice(0, _index ?? 0).map(child => mdastToString(child)).join('') : '') ? { configExample: true } : {}) }) cliTokens.push(...cliCommandTokens(value, line), ...cliFlagTokens(value, line)) } } }) const frontmatter = frontmatterSubset(frontmatterNode?.value) const body = tree.children.filter((child) => child.type !== 'yaml') const prose = body.map((child) => mdastToString(child)).join(' ') const wordCount = prose.split(/\s+/).filter(Boolean).length return { path, ...(title ? { title } : {}), headings, ...(summary ? { summary } : {}), wordCount, frontmatter, generatedRegions: regions, contentHash: markdownContentHash(normalized), citationRegions, links, codeTokens, cliTokens, fenceTokens, fenceTokensTruncated, ...(frontmatterNode ? { frontmatterBlock: { value: frontmatterNode.value, line: lineOf(frontmatterNode) } } : {}), } } export type MarkdownResolution = { /** Repository-relative paths of every scanned document. */ readonly documents: ReadonlyMap /** Repository-relative module path to entity id. */ readonly modules: ReadonlyMap /** Repository-relative directory path to area entity id. */ readonly areas?: ReadonlyMap /** Package name, and short name, to entity id. */ readonly packages: ReadonlyMap readonly cliPackages?: readonly { readonly name: string; readonly bin: unknown }[] /** Exported symbol to the entity ids of every module exporting it. */ readonly symbols: ReadonlyMap /** Module entity ids of entrypoint scripts (see `isEntrypointScript`); bare words do not resolve to them unless anchored. */ readonly entrypoints?: ReadonlySet readonly facts?: ReadonlyMap readonly packagePaths?: readonly { readonly id: string; readonly path: string }[] readonly relationCap?: number /** * Path candidates for near-miss resolution, indexed by length. * * Built once per snapshot by the caller. Deriving it here meant rebuilding it for every document * — with four thousand documents and nine thousand modules, fifty-five million string copies * before any analysis, and a similarity scan over the whole universe per unresolved reference. * `markdownPathCandidateIndex` builds it from the same three maps, so a caller that omits it * still gets identical results, only slowly. */ readonly pathIndex?: FuzzyCandidateIndex } /** The candidate index the analyzer wants, built once from a resolution universe. */ export const markdownPathCandidateIndex = ( resolution: Pick, ): FuzzyCandidateIndex => createFuzzyCandidateIndex([ ...resolution.documents.keys(), ...resolution.modules.keys(), ...(resolution.areas ?? new Map()).keys(), ]) export type MarkdownNote = { readonly scope: string readonly reason: string readonly evidence: readonly Evidence[] } export type AmbiguousSymbolReference = { readonly symbol: string readonly candidateModuleIds: readonly string[] readonly candidateCount: number readonly lines: readonly number[] } export type MarkdownFact = { readonly kind: string; readonly name: string; readonly ownerId: string; readonly evidence?: readonly Evidence[] } export const configCitationKey = (token: MarkdownReference): string => token.value.includes('.') ? token.value : JSON.stringify([token.value, token.line, token.configPath, token.configRoots]) /** Bare keys require example syntax and an independently cited schema owner. */ export const configKeyCitationIndex = (facts: ReadonlyMap | undefined, documentPath: string, packages: MarkdownResolution['packagePaths'], document?: MarkdownDocumentV1, symbols?: MarkdownResolution['symbols']): ReadonlyMap => { const index = new Map() const fixture = (path: string) => { const match = /(?:^|\/)(?:tests?|__tests__)\/fixtures\/[^/]+/.exec(path); return match ? path.slice(0, match.index + match[0].length) : '' } const orderedPackages = [...(packages ?? [])].sort((a, b) => b.path.length - a.path.length) const packageId = (path: string) => orderedPackages.find(pkg => pkg.path === '.' || path.startsWith(`${pkg.path}/`))?.id const documentFixture = fixture(documentPath), documentPackage = packageId(documentPath) for (const fact of facts?.values() ?? []) for (const item of fact) { if (item.kind !== 'config-key') continue const paths = item.evidence?.map(proof => proof.path) ?? [item.ownerId.replace(/^module:/, '')] if (!paths.some(path => fixture(path) === documentFixture && packageId(path) === documentPackage)) continue const matches = index.get(item.name) ?? [] if (!matches.some(match => match.ownerId === item.ownerId)) matches.push(item) index.set(item.name, matches) } if (!document) return index const tokens = [...document.codeTokens, ...(document.fenceTokens ?? [])] const anchors = new Set() for (const token of tokens.filter(token => token.value.includes('.'))) for (const fact of index.get(token.value) ?? []) anchors.add(fact.ownerId) for (const matches of index.values()) for (const fact of matches) { const paths = [fact.ownerId.replace(/^module:/, ''), ...(fact.evidence?.map(item => item.path) ?? [])] if (document.codeTokens.some(token => paths.includes(token.value) || symbols?.get(token.value)?.includes(fact.ownerId)) || document.links.some(link => paths.includes(resolveRelative(document.path, link.value) ?? ''))) anchors.add(fact.ownerId) } const result = new Map([...index].filter(([name]) => name.includes('.'))) for (const token of tokens.filter(token => token.configExample && !token.value.includes('.'))) { const ownerFacts = [...index.values()].flat() const matches = ownerFacts.filter(fact => anchors.has(fact.ownerId) && fact.name.split('.').at(-1) === token.value && !ownerFacts.some(child => child.ownerId === fact.ownerId && child.name.startsWith(`${fact.name}.`)) && (!token.configRoots || token.configRoots.every(key => index.get(key)?.some(root => root.ownerId === fact.ownerId)) || token.configRoots.length === 1 && token.configPath === token.value) && (!token.configPath?.includes('.') || token.configPath === fact.name)) const unique = matches.filter(fact => ownerFacts.filter(other => other.ownerId === fact.ownerId && other.name.split('.').at(-1) === token.value && (!token.configPath?.includes('.') || other.name === token.configPath)).length === 1) if (unique.length) result.set(configCitationKey(token), unique) } return result } export type AmbiguousFactReference = { readonly factKind: string readonly factName: string readonly candidateOwnerIds: readonly string[] readonly candidateCount: number readonly lines: readonly number[] } export type MarkdownAnalysis = { readonly ambiguousFactReferences: readonly AmbiguousFactReference[] readonly ambiguousFactReferencesTruncated: boolean readonly ambiguousSymbolReferences: readonly AmbiguousSymbolReference[] readonly ambiguousSymbolReferencesTruncated: boolean readonly relations: readonly KnowledgeRelation[] readonly notes: readonly MarkdownNote[] readonly truncated: boolean readonly factReferencesTruncated: boolean } const documentEvidence = (path: string, line: number): Evidence => ({ source: 'documentation', path, lineStart: line, lineEnd: line, }) /** Resolve a relative link against the document's own directory, POSIX-style. */ const resolveRelative = (from: string, target: string): string | undefined => { if (!target || EXTERNAL_LINK.test(target)) return undefined const clean = target.split('#')[0]?.split('?')[0] ?? '' if (!clean) return undefined const base = clean.startsWith('/') ? [] : from.split('/').slice(0, -1) const segments = [...base] for (const segment of clean.replace(/^\//, '').split('/')) { if (!segment || segment === '.') continue if (segment === '..') { if (!segments.length) return undefined segments.pop() continue } segments.push(segment) } return segments.length ? toPosix(segments.join('/')) : undefined } /** A token that could be a repository path, as opposed to prose or a symbol. */ const pathShaped = (value: string): boolean => /[/.]/.test(value) && !/\s/.test(value) /** * A bare lowercase word such as `main`: the whole citation span, with no separator or capital. * * Such a word is usually prose (a branch name, a verb) but it is also a legitimate name for a * library function. Only entrypoint scripts are filtered out of its candidates (see `citableOwners`). */ const BARE_WORD = /^[a-z]+$/ export const analyzeMarkdownDocument = ( document: MarkdownDocumentV1, documentId: string, resolution: MarkdownResolution, ): MarkdownAnalysis => { const cap = resolution.relationCap ?? MARKDOWN_RELATION_CAP const configKeys = configKeyCitationIndex(resolution.facts, document.path, resolution.packagePaths, document, resolution.symbols) const relations = new Map() const notes: MarkdownNote[] = [] const ambiguous = new Map() /** Owners that survived citable filtering at each ambiguous line, unioned per symbol. */ const ambiguousOwners = new Map>() const ambiguousFacts = new Map() let truncated = false let factsTruncated = false let legacyCount = 0 const reserveRelation = (fact: MarkdownFact | undefined): boolean => { if (fact) { return true } else { if (legacyCount >= cap) { truncated = true; return false } legacyCount++ } return true } const add = (kind: string, to: string, line: number, confidence?: 'fuzzy', symbol?: string, fact?: MarkdownFact, codeFence = false): void => { if (to === documentId) return const id = relationId(documentId, kind, to, fact ? `${fact.kind}:${fact.name}` : symbol) const region = document.citationRegions?.find(item => item.lineStart <= line && item.lineEnd >= line) const citationRegion = region ? { line, hash: region.hash } : undefined const existing = relations.get(id) if (existing) { // One relation, every place the document says it — evidence accumulates, the edge does not. if (existing.evidence.length < 8 && (!codeFence || !existing.evidence.some(item => item.lineStart === line))) { relations.set(id, { ...existing, evidence: [...existing.evidence, documentEvidence(document.path, line)], ...(citationRegion ? { metadata: { ...existing.metadata, citationRegions: [...(Array.isArray(existing.metadata?.citationRegions) ? existing.metadata.citationRegions : []), citationRegion] } } : {}) }) } return } if (!reserveRelation(fact)) return relations.set(id, { id, kind, from: documentId, to, provenance: 'observed', evidence: [documentEvidence(document.path, line)], ...((fact || symbol || confidence || codeFence || citationRegion) ? { metadata: { ...(citationRegion ? { citationRegions: [citationRegion] } : {}), ...(fact ? { factKind: fact.kind, factName: fact.name } : symbol ? { symbol } : confidence ? { confidence } : {}), ...(codeFence ? { citationContext: 'code-fence' } : {}) } } : {}), }) } const areas = resolution.areas ?? new Map() const pathCandidates = resolution.pathIndex ?? markdownPathCandidateIndex(resolution) /* * An entrypoint script owns names such as `main` that prose uses for other things. A bare word * may resolve to one only when the same citation region also cites that script's path or its * package name, which is the same anchor rule configuration keys use. Elsewhere the script is * filtered out before resolution, so one owner resolves and several owners are reported only * among owners that could plausibly be meant. */ const entrypoints = resolution.entrypoints ?? new Set() const modulePathById = new Map([...resolution.modules].map(([path, id]) => [id, path])) const packagePaths = [...(resolution.packagePaths ?? [])].sort((a, b) => b.path.length - a.path.length) const citedAround = new Map>() const citedNear = (line: number): Set => { const region = (document.citationRegions ?? []).filter(item => item.lineStart <= line && item.lineEnd >= line).sort((a, b) => (a.lineEnd - a.lineStart) - (b.lineEnd - b.lineStart))[0] const key = region ? `${region.lineStart}:${region.lineEnd}` : `line:${line}` const memo = citedAround.get(key) if (memo) return memo const inScope = (cited: number) => region ? cited >= region.lineStart && cited <= region.lineEnd : cited === line const cited = new Set() for (const token of document.codeTokens) if (inScope(token.line)) cited.add(token.value.replace(/^\.\//, '')) for (const link of document.links) if (inScope(link.line)) { cited.add(link.value.replace(/^\.\//, '')) const resolved = resolveRelative(document.path, link.value) if (resolved) cited.add(resolved) } citedAround.set(key, cited) return cited } const anchoredOwner = (ownerId: string, line: number): boolean => { const cited = citedNear(line) const path = modulePathById.get(ownerId) if (path === undefined) return false if (cited.has(path)) return true const pkg = packagePaths.find(item => item.path === '.' || path.startsWith(`${item.path}/`)) return !!pkg && [...resolution.packages.entries()].some(([name, id]) => id === pkg.id && cited.has(name)) } /** Owners a bare-word citation may resolve to; every other span is returned unchanged. */ const citableOwners = (owners: readonly string[], value: string, line: number, exempt: boolean): readonly string[] => { if (exempt || !BARE_WORD.test(value) || !owners.some(id => entrypoints.has(id))) return owners return owners.filter(id => !entrypoints.has(id) || anchoredOwner(id, line)) } /** A path-shaped reference: a document link, a module mention, or an unambiguous near-miss. */ const resolvePath = (candidate: string, line: number, linkKind: 'links-to' | 'mentions'): boolean => { const documentEntity = resolution.documents.get(candidate) if (documentEntity) { add(linkKind === 'links-to' ? 'links-to' : 'mentions', documentEntity, line) return true } const moduleEntity = resolution.modules.get(candidate) if (moduleEntity) { add('mentions', moduleEntity, line) return true } // A directory is a unit of architecture now: naming one is a mention of the area. const areaEntity = areas.get(candidate) if (areaEntity) { add('mentions', areaEntity, line) return true } const fuzzy = resolveFuzzyReference(candidate, pathCandidates) if (!fuzzy) return false const target = resolution.documents.get(fuzzy.candidate) ?? resolution.modules.get(fuzzy.candidate) ?? areas.get(fuzzy.candidate) if (!target) return false add(resolution.documents.has(fuzzy.candidate) ? linkKind : 'mentions', target, line, 'fuzzy') return true } for (const link of document.links) { const resolved = resolveRelative(document.path, link.value) if (resolved) resolvePath(resolved, link.line, 'links-to') // Link text can name a path or package even when the href points elsewhere. if (link.text) resolveToken(link.text, link.line) } for (const token of document.codeTokens) resolveToken(token.value, token.line, undefined, false, undefined, token) for (const token of cliCitationTokens(document.cliTokens, resolution)) { resolveToken(token.value, token.line, token.kind, false, token.ownerIds) } for (const token of document.fenceTokens ?? []) { const candidates = resolution.facts?.get(token.value) if (token.configOnly) { if (configKeys.has(configCitationKey(token))) resolveToken(token.value, token.line, 'config-key', true, undefined, token) } else if (resolution.symbols.has(token.value) || candidates?.some(fact => fact.kind !== 'cli-command' && fact.kind !== 'cli-flag')) resolveToken(token.value, token.line, undefined, true) } // Signature facts reuse a uniquely resolved symbol citation, preserving its legacy edge. function resolveSignatureSymbol(value: string, line: number, codeFence: boolean): boolean { const owners = [...new Set(citableOwners(resolution.symbols.get(value) ?? [], value, line, codeFence))] if (owners.length !== 1) return false const signature = resolution.facts?.get(value)?.find(fact => fact.kind === 'signature' && fact.ownerId === owners[0]) if (!signature) return false if (!resolution.facts?.get(value)?.some(fact => fact.kind === 'symbol')) add('mentions-symbol', owners[0]!, line, undefined, value, undefined, codeFence) add('mentions-symbol', signature.ownerId, line, undefined, undefined, signature, codeFence) return true } /** * Resolve exact paths, codec facts, legacy packages/exports, then fuzzy paths. * Codec facts take precedence over legacy exports; distinct kinds resolve independently. * A legacy symbol resolves only when exactly one module exports it, because sending an * agent to one of two possible definitions is worse than sending it nowhere. */ function resolveToken(raw: string, line: number, cliKind?: 'cli-command' | 'cli-flag' | 'config-key', exactOnly = false, ownerIds?: readonly string[], configReference?: MarkdownReference): void { const value = raw.trim() if (!value || value.length > 256) return if (!exactOnly && !cliKind && (pathShaped(value) || areas.has(value))) { const direct = value.replace(/^\.\//, '') if (resolution.documents.has(direct) || resolution.modules.has(direct) || areas.has(direct)) { resolvePath(direct, line, 'mentions') return } } const signatureSymbol = !cliKind && resolveSignatureSymbol(value, line, exactOnly) const candidates = resolution.facts?.get(value)?.filter(fact => !ownerIds || ownerIds.includes(fact.ownerId)) const citable = new Set(citableOwners(candidates?.map(fact => fact.ownerId) ?? [], value, line, exactOnly || !!cliKind)) const facts = [...(candidates ?? []).filter(fact => citable.has(fact.ownerId) && fact.kind !== 'config-key'), ...(cliKind && cliKind !== 'config-key' ? [] : (value.includes('.') || configReference?.configExample ? configKeys.get(configCitationKey(configReference ?? {value,line})) : undefined) ?? [])].filter(fact => (cliKind ? fact.kind === cliKind : fact.kind !== 'cli-command' && fact.kind !== 'cli-flag') && (fact.kind !== 'signature' || (!resolution.symbols.has(value) && !resolution.symbols.has(value.split('.')[0]!)))) if (facts?.length) { for (const kind of [...new Set(facts.map(fact => fact.kind))].sort()) { const candidates = facts.filter(fact => fact.kind === kind) const owners = [...new Set(candidates.map(fact => fact.ownerId))].sort() if (owners.length === 1) { add('mentions-symbol', owners[0]!, line, undefined, kind === 'symbol' ? value : undefined, kind === 'symbol' ? undefined : candidates[0], exactOnly) } else { const key = `${kind}:${value}` const prior = ambiguousFacts.get(key) ambiguousFacts.set(key, { factKind: kind, factName: value, candidateOwnerIds: owners.slice(0, 32), candidateCount: owners.length, lines: [...new Set([...(prior?.lines ?? []), line])].sort((a, b) => a - b).slice(0, 8) }) } } return } if (cliKind || candidates?.some(fact => fact.kind === 'cli-command' ? value.includes(' ') : fact.kind === 'cli-flag' && /^--?[A-Za-z][\w.-]*$/.test(value))) return if (signatureSymbol) return const packageEntity = resolution.packages.get(value) if (!exactOnly && packageEntity) { add('mentions', packageEntity, line) return } const declared = resolution.symbols.get(value) const modules = declared && citableOwners(declared, value, line, exactOnly) if (declared?.length && !modules?.length) return if (modules?.length === 1 && modules[0]) { add('mentions-symbol', modules[0], line, undefined, value, undefined, exactOnly) return } if (modules && modules.length > 1) { const evidence = ambiguous.get(value) ?? [] if (evidence.length < 8) evidence.push(documentEvidence(document.path, line)) ambiguous.set(value, evidence) ambiguousOwners.set(value, new Set([...(ambiguousOwners.get(value) ?? []), ...modules])) return } if (!exactOnly && pathShaped(value)) resolvePath(value.replace(/^\.\//, ''), line, 'mentions') } const factRelations = [...relations.values()].filter(relation => typeof relation.metadata?.factKind === 'string') .sort((a, b) => String(b.metadata?.factName).split('.').length - String(a.metadata?.factName).split('.').length || a.id.localeCompare(b.id)) if (factRelations.length > MARKDOWN_FACT_RELATION_CAP) { factsTruncated = true for (const relation of factRelations.slice(MARKDOWN_FACT_RELATION_CAP)) relations.delete(relation.id) } for (const [token, evidence] of [...ambiguous.entries()].sort(([a], [b]) => a.localeCompare(b))) { notes.push({ scope: `mentions-symbol:${document.path}:${token}`, reason: `"${token}" is exported by ${ambiguousOwners.get(token)?.size ?? 0} modules; the reference is ambiguous and produced no relation.`, evidence, }) } for (const [key, ambiguity] of [...ambiguousFacts.entries()].sort(([a], [b]) => a.localeCompare(b))) { notes.push({ scope: `mentions-fact:${document.path}:${key}`, reason: `"${ambiguity.factName}" has ${ambiguity.candidateCount} ${ambiguity.factKind} owners; the reference is ambiguous and produced no relation.`, evidence: ambiguity.lines.map(line => documentEvidence(document.path, line)) }) } if (document.fenceTokensTruncated) notes.push({ scope: `fence-tokens:${document.path}`, reason: 'Document exceeds 4096 fenced identifier tokens; remaining citations were not analyzed.', evidence: [documentEvidence(document.path, 1)] }) if (truncated) { notes.push({ scope: `relations:${document.path}`, reason: `Document references more than ${cap} entities; the remainder was not recorded.`, evidence: [documentEvidence(document.path, 1)], }) } if (factsTruncated) notes.push({ scope: `fact-relations:${document.path}`, reason: `Document references more than ${MARKDOWN_FACT_RELATION_CAP} facts; the remainder was not recorded.`, evidence: [documentEvidence(document.path, 1)] }) const ambiguousSymbolReferences = [...ambiguous.entries()] .sort(([a], [b]) => a.localeCompare(b)) .slice(0, 64) .map(([symbol, evidence]) => { const candidates = [...(ambiguousOwners.get(symbol) ?? [])].sort() return { symbol, candidateModuleIds: candidates.slice(0, 32), candidateCount: candidates.length, lines: [...new Set(evidence.map((item) => item.lineStart as number))].sort((a, b) => a - b) } }) return { ambiguousFactReferences: [...ambiguousFacts.entries()].sort(([a], [b]) => a.localeCompare(b)).slice(0, 64).map(([, value]) => value), ambiguousFactReferencesTruncated: ambiguousFacts.size > 64, ambiguousSymbolReferences, ambiguousSymbolReferencesTruncated: ambiguous.size > 64, relations: [...relations.values()].sort((a, b) => a.id.localeCompare(b.id)), notes, truncated, factReferencesTruncated: factsTruncated, } }