/** * Batched wiki/markdown target resolution for seed-scoped graph expansion. * * @module src/store/sqlite/graph-link-resolver */ import type { Database } from "bun:sqlite"; import type { WorkspaceResolutionReason } from "./workspace-link-resolver"; import { stripWikiMdExt } from "../../core/links"; import { GRAPH_LINK_BULK_MAX_DOCUMENTS, resolveGraphLinkTargetsBulk, } from "./graph-link-bulk-resolver"; import { loadLinkWorkspaceMemberships, resolveWorkspaceTargets, workspaceKeyForDocument, workspaceMemberCollections, type WorkspaceTargetInput, } from "./workspace-link-resolver"; export interface GraphLinkSourceIdentity { collection: string; relPath: string; /** The link carried an explicit `collection:` prefix. */ explicit: boolean; } export interface GraphLinkTarget { targetRefNorm: string; /** Explicit prefix collection, else the source collection. */ targetCollection: string; linkType: "wiki" | "markdown"; /** * Source document identity. Plain wiki links from a document inside a link * workspace resolve across the workspace; without it (or for explicit, * markdown and non-workspace links) resolution stays collection-scoped. */ source?: GraphLinkSourceIdentity; } /** Where a resolved link landed relative to its source collection. */ export type LinkResolutionScope = | "same-collection" | "cross-collection" | "explicit-collection"; export interface ResolvedGraphLinkTarget { targetId: number; targetDocid: string; matchRank: number; matchCount: number; /** Collection of the resolved target document. */ targetCollection?: string; scope?: LinkResolutionScope; /** Workspace resolution reason; absent for collection-scoped resolution. */ reason?: WorkspaceResolutionReason; /** False for a tied workspace link: audit evidence only, never an edge. */ traversable?: boolean; /** Tied candidates of a non-traversable link, canonical path order. */ candidates?: Array<{ id: number; docid: string; collection: string }>; } /** Resolved and allowed to become a graph edge. */ export const isTraversableResolution = ( resolution: ResolvedGraphLinkTarget | null | undefined ): resolution is ResolvedGraphLinkTarget => resolution !== null && resolution !== undefined && resolution.traversable !== false; /** Source identity for a raw doc_links row. */ export const linkSourceIdentity = (row: { source_collection: string; source_rel_path: string; target_collection: string | null; }): GraphLinkSourceIdentity => ({ collection: row.source_collection, relPath: row.source_rel_path, explicit: row.target_collection !== null && row.target_collection !== "", }); const MAX_SQL_PARAMS = 900; const BULK_RESOLUTION_THRESHOLD = 128; export const AUDIT_LINK_SNAPSHOT_MAX_DOCUMENTS = 50_000; export const AUDIT_LINK_SNAPSHOT_MAX_LINKS = 50_000; export interface AuditLinkSnapshotDocument { id: number; docid: string; uri: string; collection: string; relPath: string; recordSourcePath: string | null; title: string | null; mirrorHash: string | null; } export interface AuditLinkSnapshotLink { sourceId: number; sourceDocid: string; sourceUri: string; sourceCollection: string; sourceRelPath: string; targetRef: string; targetRefNorm: string; targetAnchor: string | null; targetCollection: string; /** The link carried an explicit `collection:` prefix. */ explicitCollection?: boolean; linkType: "wiki" | "markdown"; startLine: number; startCol: number; endLine: number; endCol: number; resolved: ResolvedGraphLinkTarget | null; /** * Unresolved workspace wiki link whose target exists as a file in the * workspace but is not an indexed document. Existence only: no edge. */ outsideIndex?: boolean; } export interface AuditLinkSnapshot { documents: AuditLinkSnapshotDocument[]; /** Optional finding scope when documents also include graph-wide evidence. */ auditedDocumentIds?: number[]; /** * Requested collection scope of a scoped audit. Connectivity still uses * the whole graph, but candidate identities outside it are withheld. */ scopeCollections?: string[]; links: AuditLinkSnapshotLink[]; /** Why some workspace file listings were incomplete, when they were. */ outsideIndexDiagnostic?: string; totals: { documents: number; links: number }; truncated: { documents: boolean; links: boolean }; metrics: { documentRowsExamined: number; linkRowsExamined: number; uniqueTargetsResolved: number; batchedResolution: true; }; } export interface AuditLinkSnapshotOptions { collections?: readonly string[]; pathPrefixes?: readonly string[]; maxDocuments?: number; maxLinks?: number; } const chunkArray = (items: T[], chunkSize: number): T[][] => { const chunks: T[][] = []; for (let offset = 0; offset < items.length; offset += chunkSize) { chunks.push(items.slice(offset, offset + chunkSize)); } return chunks; }; const suffixMatch = (targetExpr: string, valueExpr: string): string => `(substr(${targetExpr}, -length(${valueExpr})) = ${valueExpr} AND (length(${targetExpr}) = length(${valueExpr}) OR substr(${targetExpr}, -length(${valueExpr}) - 1, 1) = '/'))`; export const resolveGraphLinkTargetsSql = ( db: Database, targets: GraphLinkTarget[] ): Array => { const results: Array = Array.from( { length: targets.length }, () => null ); const wikiTargets: Array<{ idx: number; collection: string; baseRef: string; baseRefMd: string; }> = []; const markdownTargets: Array<{ idx: number; collection: string; relPath: string; }> = []; for (const [idx, target] of targets.entries()) { if (target.linkType === "wiki") { const baseRef = stripWikiMdExt(target.targetRefNorm); wikiTargets.push({ idx, collection: target.targetCollection, baseRef, baseRefMd: `${baseRef}.md`, }); } else { markdownTargets.push({ idx, collection: target.targetCollection, relPath: target.targetRefNorm, }); } } const titleExpr = "lower(trim(d.title))"; const relExpr = "lower(d.rel_path)"; const baseRefExpr = "t.base_ref"; const baseRefMdExpr = "t.base_ref_md"; const wikiWhere = ` ${titleExpr} = ${baseRefExpr} OR ${titleExpr} = ${baseRefMdExpr} OR ${suffixMatch(baseRefExpr, titleExpr)} OR ${suffixMatch(baseRefMdExpr, `${titleExpr} || '.md'`)} OR ${relExpr} = ${baseRefExpr} OR ${relExpr} = ${baseRefMdExpr} OR ${suffixMatch(relExpr, baseRefMdExpr)} OR ${suffixMatch(relExpr, baseRefExpr)} OR ${suffixMatch(baseRefMdExpr, relExpr)} OR ${suffixMatch(baseRefExpr, relExpr)} `; const wikiRank = `CASE WHEN ${titleExpr} = ${baseRefExpr} THEN 1 WHEN ${titleExpr} = ${baseRefMdExpr} THEN 2 WHEN ${suffixMatch(baseRefExpr, titleExpr)} THEN 3 WHEN ${suffixMatch(baseRefMdExpr, `${titleExpr} || '.md'`)} THEN 4 WHEN ${relExpr} = ${baseRefExpr} THEN 5 WHEN ${relExpr} = ${baseRefMdExpr} THEN 6 WHEN ${suffixMatch(relExpr, baseRefMdExpr)} THEN 7 WHEN ${suffixMatch(relExpr, baseRefExpr)} THEN 8 WHEN ${suffixMatch(baseRefMdExpr, relExpr)} THEN 9 WHEN ${suffixMatch(baseRefExpr, relExpr)} THEN 10 ELSE 99 END`; const wikiBatchSize = Math.max(1, Math.floor(MAX_SQL_PARAMS / 4)); for (const batch of chunkArray(wikiTargets, wikiBatchSize)) { const valuesClause = batch.map(() => "(?, ?, ?, ?)").join(", "); const params = batch.flatMap((target) => [ target.idx, target.collection, target.baseRef, target.baseRefMd, ]); const rows = db .query< { idx: number; target_id: number; target_docid: string; match_rank: number; match_count: number; }, (string | number)[] >( `WITH targets(idx, collection, base_ref, base_ref_md) AS ( VALUES ${valuesClause} ), candidates AS ( SELECT t.idx, d.id AS target_id, d.docid AS target_docid, ${wikiRank} AS match_rank FROM targets t JOIN documents d ON d.active = 1 AND d.collection = t.collection WHERE ${wikiWhere} ), best_candidates AS ( SELECT candidates.* FROM candidates JOIN ( SELECT idx, MIN(match_rank) AS best_rank FROM candidates GROUP BY idx ) best ON best.idx = candidates.idx AND best.best_rank = candidates.match_rank ), ranked AS ( SELECT *, COUNT(*) OVER (PARTITION BY idx) AS match_count, ROW_NUMBER() OVER (PARTITION BY idx ORDER BY target_id) AS rn FROM best_candidates ) SELECT idx, target_id, target_docid, match_rank, match_count FROM ranked WHERE rn = 1` ) .all(...params); for (const row of rows) { results[row.idx] = { targetId: row.target_id, targetDocid: row.target_docid, matchRank: row.match_rank, matchCount: row.match_count, }; } } const markdownBatchSize = Math.max(1, Math.floor(MAX_SQL_PARAMS / 3)); for (const batch of chunkArray(markdownTargets, markdownBatchSize)) { const valuesClause = batch.map(() => "(?, ?, ?)").join(", "); const params = batch.flatMap((target) => [ target.idx, target.collection, target.relPath, ]); const rows = db .query< { idx: number; target_id: number; target_docid: string }, (string | number)[] >( `WITH targets(idx, collection, rel_path) AS ( VALUES ${valuesClause} ), ranked AS ( SELECT t.idx, d.id AS target_id, d.docid AS target_docid, ROW_NUMBER() OVER (PARTITION BY t.idx ORDER BY d.id) AS rn FROM targets t JOIN documents d ON d.active = 1 AND d.collection = t.collection AND d.rel_path = t.rel_path ) SELECT idx, target_id, target_docid FROM ranked WHERE rn = 1` ) .all(...params); for (const row of rows) { results[row.idx] = { targetId: row.target_id, targetDocid: row.target_docid, matchRank: 5, matchCount: 1, }; } } return results; }; export type GraphLinkResolutionStrategy = "auto" | "sql" | "bulk"; /** * Resolve all targets with one contract for every graph consumer: plain wiki * links inside a link workspace use workspace ranking; explicit, markdown and * non-workspace links keep collection-scoped resolution. `strategy` forces * the SQL or bulk candidate path (parity tests); results are identical. */ export function resolveGraphLinkTargets( db: Database, targets: GraphLinkTarget[], strategy: GraphLinkResolutionStrategy = "auto" ): Array { const results: Array = Array.from( { length: targets.length }, () => null ); const memberships = targets.some( (target) => target.linkType === "wiki" && target.source && !target.source.explicit ) ? loadLinkWorkspaceMemberships(db) : new Map(); const workspaceInputs: WorkspaceTargetInput[] = []; const workspaceIndexes: number[] = []; const legacyTargets: GraphLinkTarget[] = []; const legacyIndexes: number[] = []; for (const [index, target] of targets.entries()) { const source = target.source; const wsKey = target.linkType === "wiki" && source && !source.explicit ? workspaceKeyForDocument( memberships, source.collection, source.relPath ) : null; if (wsKey !== null && source) { workspaceInputs.push({ targetRefNorm: target.targetRefNorm, source: { collection: source.collection, relPath: source.relPath }, wsKey, }); workspaceIndexes.push(index); } else { legacyTargets.push(target); legacyIndexes.push(index); } } const legacyResults = resolveCollectionScopedTargets( db, legacyTargets, strategy ); // Results are shared per unique resolution (as the resolvers dedupe // targets) so large inventories do not allocate one object per link. const legacyShared = new Map< ResolvedGraphLinkTarget, Map >(); for (const [position, index] of legacyIndexes.entries()) { const resolution = legacyResults[position] ?? null; if (!resolution) continue; const target = targets[index]!; const scope = target.source ? target.source.explicit ? ("explicit-collection" as const) : ("same-collection" as const) : undefined; const variantKey = `${target.targetCollection}\0${scope ?? ""}`; let variants = legacyShared.get(resolution); if (!variants) { variants = new Map(); legacyShared.set(resolution, variants); } let shared = variants.get(variantKey); if (!shared) { shared = { ...resolution, targetCollection: target.targetCollection, ...(scope ? { scope } : {}), }; variants.set(variantKey, shared); } results[index] = shared; } if (workspaceInputs.length > 0) { // Only whether the unique count passes the threshold matters. const unique = new Set(); for (const input of workspaceInputs) { unique.add( `${input.wsKey}\0${input.source.collection}\0${input.source.relPath}\0${input.targetRefNorm}` ); if (unique.size > BULK_RESOLUTION_THRESHOLD) break; } const uniqueCount = unique.size; const bulk = strategy === "bulk" || (strategy === "auto" && uniqueCount > BULK_RESOLUTION_THRESHOLD && countActiveDocuments( db, workspaceMemberCollections( memberships, new Set(workspaceInputs.map((input) => input.wsKey)) ) ) <= GRAPH_LINK_BULK_MAX_DOCUMENTS); const workspaceResults = resolveWorkspaceTargets( db, memberships, workspaceInputs, { bulk } ); const workspaceShared = new Map(); for (const [position, index] of workspaceIndexes.entries()) { const resolution = workspaceResults[position] ?? null; const source = targets[index]!.source!; // One resolution object per (workspace, source collection, folder, // target), so its scope is fixed and the converted result is shared. const cached = resolution ? workspaceShared.get(resolution) : undefined; if (cached) { results[index] = cached; continue; } results[index] = resolution ? { targetId: resolution.target.id, targetDocid: resolution.target.docid, matchRank: resolution.titleRank ?? 0, matchCount: resolution.traversable ? 1 : resolution.tied.length, targetCollection: resolution.target.collection, scope: resolution.target.collection === source.collection ? "same-collection" : "cross-collection", reason: resolution.reason, traversable: resolution.traversable, ...(resolution.traversable ? {} : { candidates: resolution.tied.map( ({ id, docid, collection }) => ({ id, docid, collection }) ), }), } : null; if (resolution && results[index]) { workspaceShared.set(resolution, results[index]!); } } } return results; } const countActiveDocuments = ( db: Database, collections: readonly string[] ): number => collections.length === 0 ? 0 : (db .query<{ count: number }, string[]>( `SELECT COUNT(*) AS count FROM documents WHERE active = 1 AND collection IN (${collections.map(() => "?").join(",")})` ) .get(...collections)?.count ?? 0); /** Collection-scoped (pre-workspace) resolution, deduplicated per target. */ function resolveCollectionScopedTargets( db: Database, targets: GraphLinkTarget[], strategy: GraphLinkResolutionStrategy ): Array { const uniqueTargets: GraphLinkTarget[] = []; const uniqueIndexByKey = new Map(); const originalToUniqueIndex: number[] = []; for (const target of targets) { const key = JSON.stringify([ target.linkType, target.targetCollection, target.targetRefNorm, ]); let uniqueIndex = uniqueIndexByKey.get(key); if (uniqueIndex === undefined) { uniqueIndex = uniqueTargets.length; uniqueIndexByKey.set(key, uniqueIndex); uniqueTargets.push(target); } originalToUniqueIndex.push(uniqueIndex); } if ( strategy === "bulk" || (strategy === "auto" && uniqueTargets.length > BULK_RESOLUTION_THRESHOLD) ) { const bulkResults = resolveGraphLinkTargetsBulk(db, uniqueTargets); if (bulkResults) { return originalToUniqueIndex.map( (uniqueIndex) => bulkResults[uniqueIndex] ?? null ); } } const uniqueResults = resolveGraphLinkTargetsSql(db, uniqueTargets); return originalToUniqueIndex.map( (uniqueIndex) => uniqueResults[uniqueIndex] ?? null ); } /** * Capture one bounded, read-only link inventory using set-oriented SQL and the * same target resolver as graph expansion. No query is issued per finding. */ export function captureAuditLinkSnapshot( db: Database, options: AuditLinkSnapshotOptions = {} ): AuditLinkSnapshot { const maxDocuments = Math.max( 1, Math.min( AUDIT_LINK_SNAPSHOT_MAX_DOCUMENTS, options.maxDocuments ?? AUDIT_LINK_SNAPSHOT_MAX_DOCUMENTS ) ); const maxLinks = Math.max( 1, Math.min( AUDIT_LINK_SNAPSHOT_MAX_LINKS, options.maxLinks ?? AUDIT_LINK_SNAPSHOT_MAX_LINKS ) ); const conditions = ["d.active = 1"]; const params: string[] = []; const collections = [...new Set(options.collections ?? [])] .map((value) => value.normalize("NFC").trim()) .filter(Boolean) .sort(); if (collections.length > 0) { conditions.push( `d.collection IN (${collections.map(() => "?").join(",")})` ); params.push(...collections); } const prefixes = [...new Set(options.pathPrefixes ?? [])] .map((value) => value.normalize("NFC").trim().replace(/^\/+/, "")) .filter(Boolean) .sort(); if (prefixes.length > 0) { conditions.push( `(${prefixes.map(() => "COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) = ? OR COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) LIKE ? ESCAPE '\\'").join(" OR ")})` ); for (const prefix of prefixes) { const escaped = prefix .replaceAll("\\", "\\\\") .replaceAll("%", "\\%") .replaceAll("_", "\\_"); params.push(prefix, `${escaped}/%`); } } const where = conditions.join(" AND "); const totalDocuments = db .query<{ count: number }, string[]>( `SELECT COUNT(*) AS count FROM documents d WHERE ${where}` ) .get(...params)?.count ?? 0; const documentRows = db .query< { id: number; docid: string; uri: string; collection: string; rel_path: string; record_source_path: string | null; title: string | null; mirror_hash: string | null; }, (string | number)[] >( `SELECT d.id, d.docid, d.uri, d.collection, d.rel_path, d.record_source_path, d.title, d.mirror_hash FROM documents d WHERE ${where} ORDER BY d.id LIMIT ?` ) .all(...params, maxDocuments); const totalLinks = db .query<{ count: number }, string[]>( `SELECT COUNT(*) AS count FROM doc_links dl JOIN documents d ON d.id = dl.source_doc_id WHERE ${where} AND dl.source = 'parsed'` ) .get(...params)?.count ?? 0; const rawLinks = db .query< { source_id: number; source_docid: string; source_uri: string; source_collection: string; source_rel_path: string; source_doc_rel_path: string; target_ref: string; target_ref_norm: string; target_anchor: string | null; target_collection: string | null; link_type: "wiki" | "markdown"; start_line: number; start_col: number; end_line: number; end_col: number; }, (string | number)[] >( `SELECT d.id AS source_id, d.docid AS source_docid, d.uri AS source_uri, d.collection AS source_collection, COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) AS source_rel_path, d.rel_path AS source_doc_rel_path, dl.target_ref, dl.target_ref_norm, dl.target_anchor, dl.target_collection, dl.link_type, dl.start_line, dl.start_col, dl.end_line, dl.end_col FROM doc_links dl JOIN documents d ON d.id = dl.source_doc_id WHERE ${where} AND dl.source = 'parsed' ORDER BY d.id, dl.start_line, dl.start_col, dl.id LIMIT ?` ) .all(...params, maxLinks); const targets = rawLinks.map((row) => ({ targetRefNorm: row.target_ref_norm, targetCollection: row.target_collection ?? row.source_collection, linkType: row.link_type, source: linkSourceIdentity({ source_collection: row.source_collection, source_rel_path: row.source_doc_rel_path, target_collection: row.target_collection, }), })); const resolutions = resolveGraphLinkTargets(db, targets); const uniqueTargetsResolved = new Set( targets.map((target) => JSON.stringify([ target.linkType, target.targetCollection, target.targetRefNorm, ]) ) ).size; return { documents: documentRows.map((row) => ({ id: row.id, docid: row.docid, uri: row.uri, collection: row.collection, relPath: row.rel_path, recordSourcePath: row.record_source_path, title: row.title, mirrorHash: row.mirror_hash, })), links: rawLinks.map((row, index) => ({ sourceId: row.source_id, sourceDocid: row.source_docid, sourceUri: row.source_uri, sourceCollection: row.source_collection, sourceRelPath: row.source_rel_path, targetRef: row.target_ref, targetRefNorm: row.target_ref_norm, targetAnchor: row.target_anchor, targetCollection: row.target_collection ?? row.source_collection, explicitCollection: row.target_collection !== null, linkType: row.link_type, startLine: row.start_line, startCol: row.start_col, endLine: row.end_line, endCol: row.end_col, resolved: resolutions[index] ?? null, })), totals: { documents: totalDocuments, links: totalLinks }, truncated: { documents: totalDocuments > documentRows.length, links: totalLinks > rawLinks.length, }, metrics: { documentRowsExamined: documentRows.length, linkRowsExamined: rawLinks.length, uniqueTargetsResolved, batchedResolution: true, }, }; }