/** * Shared capture planning, provenance, and receipt contracts. * * Server-side capture adapters call this before transport-specific write/sync. * * @module src/core/capture */ // node:path has no Bun equivalent import { posix as pathPosix } from "node:path"; import { buildUri } from "../app/constants"; import { browserClipProvenanceSchema, type BrowserClipProvenance, } from "./browser-clip-provenance"; import { resolveNoteCreatePlan, type NoteCollisionPolicy, } from "./note-creation"; import { getNotePreset, resolveNotePreset, type NotePresetId, } from "./note-presets"; import { normalizeTag, validateTag } from "./tags"; import { validateRelPath } from "./validation"; export const CAPTURE_MAX_TEXT_BYTES = 1024 * 1024; export type CaptureSourceKind = | "direct" | "web" | "email" | "meeting" | "chat" | "file" | "api" | "unknown"; export type CaptureStatus = | "not_requested" | "pending" | "running" | "completed" | "skipped" | "failed" | "unknown"; export type CaptureCollisionPolicyResult = | "created" | "opened_existing" | "created_with_suffix" | "overwritten" | "conflict"; export interface CaptureSource { kind: CaptureSourceKind; title?: string; url?: string; uri?: string; docid?: string; mime?: string; ext?: string; author?: string; canonicalUrl?: string; site?: string; publishedAt?: string; observedAt?: string; capturedAt: string; externalId?: string; browserClip?: BrowserClipProvenance; } export interface CaptureIndexStatus { status: CaptureStatus; jobId?: string | null; reason?: string; error?: string; } export interface CaptureReceipt { uri: string; docid?: string; collection: string; relPath: string; absPath?: string; created: boolean; openedExisting: boolean; createdWithSuffix: boolean; overwritten?: boolean; contentHash: string; source: CaptureSource; tags: string[]; sync: CaptureIndexStatus; embed: CaptureIndexStatus; collisionPolicyResult: CaptureCollisionPolicyResult; serverInstanceId?: string; } export interface CaptureInput { collection: string; content?: string; title?: string; relPath?: string; folderPath?: string; collisionPolicy?: NoteCollisionPolicy; presetId?: NotePresetId; tags?: string[]; source?: Partial; overwrite?: boolean; } export type PublicCaptureInput = Omit; export interface CapturePlan { collection: string; relPath: string; filename: string; content: string; body: string; contentHash: string; title: string; tags: string[]; source: CaptureSource; openedExisting: boolean; createdWithSuffix: boolean; provenanceConflict: boolean; collisionPolicy: NoteCollisionPolicy; collisionPolicyResult: CaptureCollisionPolicyResult; overwrite: boolean; } export interface PlanCaptureOptions { input: CaptureInput; existingRelPaths: Iterable; diskRelPaths?: Iterable; existingProvenanceByRelPath?: ReadonlyMap; now?: Date; } const FRONTMATTER_REGEX = /^---\r?\n([\s\S]*?)(?:\r?\n)?---(?:\r?\n|$)/; const VALID_SOURCE_KINDS = new Set([ "direct", "web", "email", "meeting", "chat", "file", "api", "unknown", ]); const VALID_COLLISION_POLICIES = new Set([ "error", "open_existing", "create_with_suffix", ]); const URL_SOURCE_FIELDS = new Set(["url", "uri", "canonicalUrl"]); const LEGACY_SOURCE_FIELD_MAP: Record = { gno_source_docid: "docid", gno_source_uri: "uri", gno_source_mime: "mime", gno_source_ext: "ext", }; const CAPTURE_SOURCE_STRING_KEYS = new Set([ "title", "url", "uri", "docid", "mime", "ext", "author", "canonicalUrl", "site", "publishedAt", "externalId", ]); export const CAPTURE_PROVENANCE_REQUIRED_FIELDS = [ "kind", "capturedAt", ] as const; export interface CaptureProvenanceIssue { field: string; reason: "missing" | "invalid"; } /** Validate only fields declared by the CaptureSource contract. */ export const validateDeclaredCaptureProvenance = ( source: Partial ): CaptureProvenanceIssue[] => { const issues: CaptureProvenanceIssue[] = []; if (!source.kind) issues.push({ field: "source.kind", reason: "missing" }); else if (!VALID_SOURCE_KINDS.has(source.kind)) issues.push({ field: "source.kind", reason: "invalid" }); const capturedAt = source.capturedAt as unknown; if (capturedAt === undefined || capturedAt === null || capturedAt === "") issues.push({ field: "source.capturedAt", reason: "missing" }); else if ( typeof capturedAt !== "string" || Number.isNaN(new Date(capturedAt).getTime()) ) issues.push({ field: "source.capturedAt", reason: "invalid" }); for (const field of ["observedAt", "publishedAt"] as const) { const value = source[field]; if (value === undefined) continue; if (typeof value !== "string" || Number.isNaN(new Date(value).getTime())) { issues.push({ field: `source.${field}`, reason: "invalid" }); } } for (const field of URL_SOURCE_FIELDS) { const value = source[field as keyof CaptureSource]; if (value === undefined) continue; if (typeof value !== "string") { issues.push({ field: `source.${field}`, reason: "invalid" }); continue; } try { new URL(value); } catch { issues.push({ field: `source.${field}`, reason: "invalid" }); } } for (const field of CAPTURE_SOURCE_STRING_KEYS) { if (URL_SOURCE_FIELDS.has(field) || field === "publishedAt") continue; const value = source[field as keyof CaptureSource]; if (value !== undefined && value !== null && typeof value !== "string") { issues.push({ field: `source.${field}`, reason: "invalid" }); } } if ( source.browserClip !== undefined && !browserClipProvenanceSchema.safeParse(source.browserClip).success ) { issues.push({ field: "source.browserClip", reason: "invalid" }); } return issues.sort((left, right) => left.field < right.field ? -1 : left.field > right.field ? 1 : 0 ); }; function normalizeContentForHash(content: string): string { return content.replace(/\r\n/g, "\n").trim(); } export function hashCaptureContent(content: string): string { return new Bun.CryptoHasher("sha256") .update(normalizeContentForHash(content)) .digest("hex"); } export async function listCaptureDiskRelPaths( collectionPath: string ): Promise { const paths: string[] = []; const glob = new Bun.Glob("**/*"); for await (const relPath of glob.scan({ cwd: collectionPath, onlyFiles: true, followSymlinks: false, })) { try { paths.push(validateRelPath(relPath.split("\\").join("/"))); } catch { // Ignore unsafe/unrepresentable disk paths for collision planning. } } return paths; } function validateTextContent(content: string): void { if (content.includes("\0")) { throw new Error("Capture content contains a NUL byte."); } for (let index = 0; index < content.length; index += 1) { const code = content.charCodeAt(index); const isAllowedWhitespace = code === 9 || code === 10 || code === 13; if ((code < 32 || code === 127) && !isAllowedWhitespace) { throw new Error("Capture content must be text, not binary-like data."); } } if (new TextEncoder().encode(content).byteLength > CAPTURE_MAX_TEXT_BYTES) { throw new Error( `Capture content exceeds ${CAPTURE_MAX_TEXT_BYTES} byte limit.` ); } } function normalizeCollisionPolicy( policy: CaptureInput["collisionPolicy"] | undefined, fallback: NoteCollisionPolicy ): NoteCollisionPolicy { if (policy === undefined) { return fallback; } if (!VALID_COLLISION_POLICIES.has(policy)) { throw new Error( "collisionPolicy must be one of: error, open_existing, create_with_suffix" ); } return policy; } function isPlainObject(value: unknown): value is Record { return ( typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype ); } function normalizeIsoDate(value: string, field: string): string { const parsed = new Date(value); if (Number.isNaN(parsed.getTime())) { throw new Error(`${field} must be an ISO-like date/time.`); } return parsed.toISOString(); } function normalizeSource( source: Partial | undefined, capturedAt: string ): CaptureSource { if (source !== undefined && !isPlainObject(source)) { throw new Error("source must be an object."); } const kind = source?.kind ?? "direct"; if (!VALID_SOURCE_KINDS.has(kind)) { throw new Error(`Unsupported source.kind: ${kind}`); } const normalized: CaptureSource = { kind, capturedAt, }; for (const [key, value] of Object.entries(source ?? {})) { if (value === undefined || value === null || key === "kind") { continue; } if (key === "capturedAt") { if (typeof value !== "string") { throw new Error("source.capturedAt must be a string."); } normalized.capturedAt = normalizeIsoDate(value, "source.capturedAt"); continue; } if (key === "observedAt") { if (typeof value !== "string") { throw new Error("source.observedAt must be a string."); } normalized.observedAt = normalizeIsoDate(value, "source.observedAt"); continue; } if (key === "publishedAt") { if (typeof value !== "string") { throw new Error("source.publishedAt must be a string."); } normalized.publishedAt = /^\d{4}-\d{2}-\d{2}$/.test(value) ? value : normalizeIsoDate(value, "source.publishedAt"); continue; } if (key === "browserClip") { normalized.browserClip = browserClipProvenanceSchema.parse(value); continue; } if (URL_SOURCE_FIELDS.has(key)) { if (typeof value !== "string") { throw new Error(`source.${key} must be a string.`); } try { new URL(value); } catch { throw new Error(`source.${key} must be a valid URL.`); } } if (CAPTURE_SOURCE_STRING_KEYS.has(key)) { if (typeof value !== "string") { throw new Error(`source.${key} must be a string.`); } normalized[ key as keyof Pick< CaptureSource, | "title" | "url" | "uri" | "docid" | "mime" | "ext" | "author" | "canonicalUrl" | "site" | "externalId" > ] = value; } } return normalized; } function normalizeCaptureTags(tags: string[] | undefined): string[] { if (tags !== undefined && !Array.isArray(tags)) { throw new Error("tags must be an array of strings."); } const normalized: string[] = []; for (const tag of tags ?? []) { if (typeof tag !== "string") { throw new Error("tags must be an array of strings."); } const value = normalizeTag(tag); if (!validateTag(value)) { throw new Error( `Invalid tag "${tag}". Tags must be lowercase, alphanumeric with hyphens/dots/slashes.` ); } normalized.push(value); } return [...new Set(normalized)]; } function chooseTitle(input: CaptureInput, fallback: string): string { return ( input.title?.trim() || input.source?.title?.trim() || pathPosix.basename(input.relPath ?? "").replace(/\.[^.]+$/u, "") || fallback ); } function generatedCaptureRelPath( capturedAt: string, contentHash: string ): string { const day = capturedAt.slice(0, 10); return `inbox/${day}/capture-${contentHash.slice(0, 12)}.md`; } function buildExistingSet( indexedRelPaths: Iterable, diskRelPaths: Iterable | undefined ): Set { const existing = new Set(); for (const relPath of indexedRelPaths) { existing.add(validateRelPath(relPath)); } for (const relPath of diskRelPaths ?? []) { existing.add(validateRelPath(relPath)); } return existing; } function splitFrontmatter(source: string): { lines: string[]; body: string; hasFrontmatter: boolean; } { const match = FRONTMATTER_REGEX.exec(source); if (!match) { return { lines: [], body: source, hasFrontmatter: false }; } return { lines: (match[1] ?? "").split("\n"), body: source.slice(match[0].length), hasFrontmatter: true, }; } function stripYamlString(value: string): string { const trimmed = value.trim(); if ( (trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'")) ) { try { return JSON.parse(trimmed); } catch { return trimmed.slice(1, -1); } } return trimmed; } function parseInlineTagList(value: string): string[] { const trimmed = value.trim(); if (!trimmed || trimmed === "[]") { return []; } if (!(trimmed.startsWith("[") && trimmed.endsWith("]"))) { return [stripYamlString(trimmed)]; } return trimmed .slice(1, -1) .split(",") .map((tag) => stripYamlString(tag)) .filter((tag) => tag.length > 0); } function readFrontmatterTags(lines: string[]): string[] { const tags: string[] = []; for (let index = 0; index < lines.length; index += 1) { const line = lines[index]; if (!line?.startsWith("tags:")) { continue; } tags.push(...parseInlineTagList(line.slice("tags:".length))); for ( let nestedIndex = index + 1; nestedIndex < lines.length; nestedIndex += 1 ) { const nested = lines[nestedIndex]; if (!nested?.startsWith(" ")) { break; } const item = nested.trim(); if (item.startsWith("-")) { const tag = stripYamlString(item.slice(1)); if (tag) { tags.push(tag); } } } } return normalizeCaptureTags(tags); } function shouldSkipNestedFrontmatterLine(line: string): boolean { return line.startsWith(" ") || line.trim() === ""; } const parseFrontmatterScalar = (rawValue: string): unknown => { try { return (Bun.YAML.parse(`value: ${rawValue}`) as { value?: unknown }).value; } catch { return stripYamlString(rawValue); } }; export function extractCaptureSourceFromFrontmatter( content: string ): Partial { const { lines } = splitFrontmatter(content); const source: Partial = {}; let parsedSourceMapping = false; try { const parsed = Bun.YAML.parse(lines.join("\n")) as { source?: unknown }; if ( parsed.source !== null && typeof parsed.source === "object" && !Array.isArray(parsed.source) ) { parsedSourceMapping = true; for (const [key, value] of Object.entries(parsed.source)) { source[key as keyof CaptureSource] = value as never; } } } catch { // Invalid YAML falls through to the declaration-preserving parser below. } for (let index = 0; index < lines.length; index += 1) { const line = lines[index]; if (line === undefined) { continue; } const colonIndex = line.indexOf(":"); if (colonIndex <= 0) { continue; } const key = line.slice(0, colonIndex).trim(); const rawValue = line.slice(colonIndex + 1).trim(); const legacyKey = LEGACY_SOURCE_FIELD_MAP[key]; if (legacyKey && rawValue) { switch (legacyKey) { case "docid": source.docid = stripYamlString(rawValue); break; case "uri": source.uri = stripYamlString(rawValue); break; case "mime": source.mime = stripYamlString(rawValue); break; case "ext": source.ext = stripYamlString(rawValue); break; } continue; } if (key !== "source") { continue; } if (parsedSourceMapping) { continue; } if (rawValue) { try { const parsed = Bun.YAML.parse(`source: ${rawValue}`) as { source?: unknown; }; if ( parsed.source !== null && typeof parsed.source === "object" && !Array.isArray(parsed.source) ) { for (const [nestedKey, nestedValue] of Object.entries( parsed.source )) { source[nestedKey as keyof CaptureSource] = nestedValue as never; } } } catch { // Invalid inline YAML remains declaration-visible to the audit. } continue; } for ( let nestedIndex = index + 1; nestedIndex < lines.length; nestedIndex += 1 ) { const nested = lines[nestedIndex]; if (!nested?.startsWith(" ")) { break; } const nestedColon = nested.indexOf(":"); if (nestedColon <= 0) { continue; } const nestedKey = nested .slice(0, nestedColon) .trim() as keyof CaptureSource; const nestedValue = nested.slice(nestedColon + 1).trim(); if (nestedValue) { if (nestedKey === "browserClip") { try { const parsed = JSON.parse(nestedValue) as unknown; const provenance = browserClipProvenanceSchema.safeParse(parsed); // Retain invalid declarations so provenance audits can report them. // Runtime consumers only inspect known fields via optional chaining. source.browserClip = provenance.success ? provenance.data : (parsed as BrowserClipProvenance); } catch { source.browserClip = {} as BrowserClipProvenance; } continue; } source[nestedKey] = parseFrontmatterScalar(nestedValue) as never; } } } return source; } /** Whether a note explicitly declares the CaptureSource frontmatter contract. */ export const hasDeclaredCaptureSource = (content: string): boolean => { const { lines } = splitFrontmatter(content); const captureKeys = new Set([ "kind", "capturedAt", "url", "docid", "uri", "mime", "ext", "title", "author", "canonicalUrl", "site", "publishedAt", "observedAt", "externalId", "browserClip", ]); try { const parsed = Bun.YAML.parse(lines.join("\n")) as { source?: unknown }; if ( parsed.source !== null && typeof parsed.source === "object" && !Array.isArray(parsed.source) ) { const keys = Object.keys(parsed.source); if (keys.length === 0 || keys.some((key) => captureKeys.has(key))) { return true; } } } catch { // Fall through to the declaration-preserving line parser below. } for (let index = 0; index < lines.length; index += 1) { const line = lines[index]; if (line === undefined) continue; const inlineSource = /^source\s*:\s*(\{.*\})\s*$/u.exec(line)?.[1]; if (inlineSource !== undefined) { if (/^\{\s*\}$/u.test(inlineSource)) return true; try { const parsed = Bun.YAML.parse(`source: ${inlineSource}`) as { source?: unknown; }; if ( parsed.source !== null && typeof parsed.source === "object" && !Array.isArray(parsed.source) && Object.keys(parsed.source).some((key) => captureKeys.has(key)) ) { return true; } } catch { return true; } continue; } if (!/^source\s*:\s*$/u.test(line)) continue; for ( let nestedIndex = index + 1; nestedIndex < lines.length; nestedIndex += 1 ) { const nested = lines[nestedIndex]; if (!nested?.startsWith(" ")) break; const colonIndex = nested.indexOf(":"); if (colonIndex <= 0) continue; const nestedKey = nested .slice(0, colonIndex) .trim() as keyof CaptureSource; if (captureKeys.has(nestedKey)) return true; } } return false; }; function sourceFrontmatterLines(source: CaptureSource): string[] { const lines = ["source:"]; for (const [key, value] of Object.entries(source)) { if (value === undefined || value === "") { continue; } lines.push(` ${key}: ${JSON.stringify(value)}`); } return lines; } export function mergeCaptureFrontmatter(input: { content: string; source: CaptureSource; tags: string[]; title?: string; }): string { return mergeCaptureFrontmatterAndTags(input).content; } function mergeCaptureFrontmatterAndTags(input: { content: string; source: CaptureSource; tags: string[]; title?: string; }): { content: string; tags: string[] } { const { lines, body, hasFrontmatter } = splitFrontmatter(input.content); const nextLines: string[] = []; let skippingSource = false; let skippingTags = false; let hasTitle = false; const mergedTags = [ ...new Set([...readFrontmatterTags(lines), ...input.tags]), ]; for (const line of lines) { if (skippingSource) { if (shouldSkipNestedFrontmatterLine(line)) { continue; } skippingSource = false; } if (skippingTags) { if (shouldSkipNestedFrontmatterLine(line)) { continue; } skippingTags = false; } if (line.startsWith("source:")) { skippingSource = true; continue; } if (line.startsWith("tags:")) { skippingTags = true; continue; } if (line.startsWith("title:")) { hasTitle = true; } nextLines.push(line); } if (input.title && !hasTitle) { nextLines.unshift(`title: ${JSON.stringify(input.title)}`); } if (mergedTags.length > 0) { nextLines.push("tags:"); for (const tag of mergedTags) { nextLines.push(` - ${JSON.stringify(tag)}`); } } nextLines.push(...sourceFrontmatterLines(input.source)); const normalizedBody = hasFrontmatter ? body : input.content; return { content: `---\n${nextLines.join("\n")}\n---\n\n${normalizedBody.trimStart()}`.trimEnd() + "\n", tags: mergedTags, }; } function buildCaptureContent(input: { captureInput: CaptureInput; title: string; tags: string[]; source: CaptureSource; }): { content: string; body: string; tags: string[] } { const presetId = input.captureInput.presetId; if (presetId && !getNotePreset(presetId)) { throw new Error(`Unknown presetId: ${presetId}`); } const body = input.captureInput.content; if (body !== undefined) { validateTextContent(body); } if (!presetId && (!body || body.trim().length === 0)) { throw new Error( "Capture content is required unless presetId scaffolds it." ); } const resolvedPreset = resolveNotePreset({ presetId, title: input.title, tags: input.tags, frontmatter: { source: [], }, body, }); const rawContent = resolvedPreset?.content ?? body ?? `# ${input.title || "Untitled"}\n`; const merged = mergeCaptureFrontmatterAndTags({ content: rawContent, source: input.source, tags: resolvedPreset?.tags ?? input.tags, title: input.title, }); return { content: merged.content, body: body ?? resolvedPreset?.body ?? "", tags: merged.tags, }; } export function planCapture(options: PlanCaptureOptions): CapturePlan { const capturedAt = (options.now ?? new Date()).toISOString(); const source = normalizeSource(options.input.source, capturedAt); const title = chooseTitle(options.input, "Captured Note"); const tags = normalizeCaptureTags(options.input.tags); const { content, body, tags: contentTags, } = buildCaptureContent({ captureInput: options.input, title, tags, source, }); const contentHash = hashCaptureContent(body || content); const generatedRelPath = !options.input.relPath && !options.input.folderPath && !options.input.title ? generatedCaptureRelPath(source.capturedAt, contentHash) : undefined; const existing = buildExistingSet( options.existingRelPaths, options.diskRelPaths ); const collisionPolicy = normalizeCollisionPolicy( options.input.collisionPolicy, generatedRelPath ? "open_existing" : "error" ); const overwrite = options.input.overwrite === true; const createPlan = resolveNoteCreatePlan( { collection: options.input.collection, relPath: options.input.relPath ?? generatedRelPath, title, folderPath: options.input.folderPath, collisionPolicy: overwrite ? "error" : collisionPolicy, }, overwrite ? [] : existing ); const overwritten = overwrite && existing.has(createPlan.relPath); const clipIdentity = source.browserClip?.clipIdentity; const provenanceConflict = createPlan.openedExisting && clipIdentity !== undefined && options.existingProvenanceByRelPath?.get(createPlan.relPath) !== clipIdentity; const openedExisting = createPlan.openedExisting && !provenanceConflict; return { collection: options.input.collection, relPath: createPlan.relPath, filename: createPlan.filename, content, body, contentHash, title, tags: contentTags, source, openedExisting, createdWithSuffix: createPlan.createdWithSuffix, provenanceConflict, collisionPolicy, collisionPolicyResult: overwritten ? "overwritten" : provenanceConflict ? "conflict" : openedExisting ? "opened_existing" : createPlan.createdWithSuffix ? "created_with_suffix" : "created", overwrite, }; } export function buildCaptureReceipt(input: { plan: CapturePlan; absPath?: string; docid?: string; sync?: CaptureIndexStatus; embed?: CaptureIndexStatus; overwritten?: boolean; serverInstanceId?: string; }): CaptureReceipt { const overwritten = input.overwritten ?? false; return { uri: buildUri(input.plan.collection, input.plan.relPath), docid: input.docid, collection: input.plan.collection, relPath: input.plan.relPath, absPath: input.absPath, created: !input.plan.openedExisting && !input.plan.provenanceConflict && !overwritten, openedExisting: input.plan.openedExisting, createdWithSuffix: input.plan.createdWithSuffix, overwritten, contentHash: input.plan.contentHash, source: input.plan.source, tags: input.plan.tags, sync: input.sync ?? { status: "not_requested" }, embed: input.embed ?? { status: "not_requested", reason: "Capture does not embed automatically.", }, collisionPolicyResult: overwritten ? "overwritten" : input.plan.collisionPolicyResult, serverInstanceId: input.serverInstanceId, }; } export function buildLegacyEditableCopySource(input: { docid: string; uri: string; mime: string; ext: string; capturedAt?: string; }): CaptureSource { return { kind: "file", docid: input.docid, uri: input.uri, mime: input.mime, ext: input.ext, capturedAt: input.capturedAt ?? new Date().toISOString(), }; } export function serializeCaptureReceipt(receipt: CaptureReceipt): string { return JSON.stringify(receipt, null, 2); } export { CAPTURE_SYNC_FAILED_CODE, type CaptureSyncPaths, CaptureSyncError, ensureCapturedFileIndexed, type SyncCapturedFileInput, type SyncCapturedFileResult, syncCapturedFile, } from "./capture-sync";