/** * Shared resident capture planning and write execution. * * Browser clipping and the ordinary REST capture route both use this service so * collision, provenance, atomic-write, sync, and receipt semantics cannot drift. */ // node:fs/promises structure operations have no Bun equivalent. import { mkdir } from "node:fs/promises"; // node:path has no Bun path utilities. import { basename, dirname, join as pathJoin } from "node:path"; import type { Collection, Config } from "../config/types"; import type { PreparedBrowserClip } from "../core/browser-clip"; import type { CaptureInput, CapturePlan, CaptureSource } from "../core/capture"; import type { JobManager } from "../core/job-manager"; import type { SqliteAdapter } from "../store/sqlite/adapter"; import type { ClipperIdempotencyPlan } from "../store/sqlite/clipper-store-types"; import type { DocumentEventBus } from "./doc-events"; import type { EmbedScheduler } from "./embed-scheduler"; import type { CollectionWatchService } from "./watch-service"; import { getIndexDbPath } from "../app/constants"; import { buildCaptureReceipt, CaptureSyncError, type CaptureSyncPaths, ensureCapturedFileIndexed, extractCaptureSourceFromFrontmatter, hashCaptureContent, listCaptureDiskRelPaths, planCapture, syncCapturedFile, } from "../core/capture"; import { writeCapturePlanFile } from "../core/capture-write"; import { MCP_ERRORS } from "../core/errors"; import { withWriteLock } from "../core/file-lock"; import { recordContentMutation } from "../core/mutation-generations"; import { DEFAULT_LOCK_WAIT_MS, writeLeasePath } from "../core/write-lease"; import { type CollectionSyncResult, defaultSyncService, type SyncResult, withContentTypeRules, } from "../ingestion"; import { stripFrontmatter } from "../ingestion/frontmatter"; import { startJob } from "./jobs"; export interface ResidentCaptureContext { config: Config; /** Resident server context; only the index name is read (lease path). */ current?: { indexName?: string }; scheduler: EmbedScheduler | null; eventBus: DocumentEventBus | null; watchService: CollectionWatchService | null; jobManager?: JobManager; markContentMutation?: () => void; } export interface ResidentCaptureDependencies { /** * `await-sync` (the `/api/capture` contract): write + lexical sync complete * under the shared write lease before the response, `201` on create. * `job` (browser clipper): write, then `202` with a sync job to poll. */ mode?: "await-sync" | "job"; syncPaths?: CaptureSyncPaths; syncCollection?: typeof defaultSyncService.syncCollection; /** Shared `.mcp-write.lock` path; defaults to the resident index's lease. */ lockPath?: string; lockWaitMs?: number; } export interface ResidentCaptureErrorShape { code: string; message: string; status: number; details?: Record; } const HTTP_OK = 200; const HTTP_CREATED = 201; const HTTP_ACCEPTED = 202; const HTTP_CONFLICT = 409; const HTTP_INTERNAL = 500; /** * Map a capture execution failure to its wire shape: lease busy is `LOCKED` * (409, the MCP write-lock code), a written-but-unsynced capture is * `CAPTURE_SYNC_FAILED` (500) carrying the write half of the receipt, and * anything else is `RUNTIME` (500). */ export const classifyResidentCaptureError = ( error: unknown, planned?: Extract ): ResidentCaptureErrorShape => { const message = error instanceof Error ? error.message : String(error); if (error instanceof CaptureSyncError) { return { code: error.code, message, status: HTTP_INTERNAL, details: { absPath: error.absPath, relPath: error.relPath, ...(planned ? { uri: `gno://${planned.collection.name}/${planned.plan.relPath}`, contentHash: planned.plan.contentHash, } : {}), }, }; } if (message.startsWith(`${MCP_ERRORS.LOCKED.code}:`)) { return { code: MCP_ERRORS.LOCKED.code, message, status: HTTP_CONFLICT }; } return { code: "RUNTIME", message: `Failed to capture document: ${message}`, status: HTTP_INTERNAL, }; }; const resolveCaptureLockPath = ( context: ResidentCaptureContext, dependencies: ResidentCaptureDependencies ): string => dependencies.lockPath ?? writeLeasePath(getIndexDbPath(context.current?.indexName)); export type ResidentCapturePlanResult = | { ok: true; collection: Collection; fullPath: string; plan: CapturePlan; } | { ok: false; code: "NOT_FOUND" | "RUNTIME" | "VALIDATION"; message: string; status: number; }; const listCollectionRelPaths = async ( store: Pick, collection: string ): Promise => { const result = await store.listDocuments(collection); if (!result.ok) throw new Error(result.error.message); return result.value.map((entry) => entry.relPath); }; const readCandidateClipIdentity = async ( collection: Collection, relPath: string ): Promise => { const file = Bun.file(pathJoin(collection.path, relPath)); if (!(await file.exists())) return null; try { return ( extractCaptureSourceFromFrontmatter(await file.text()).browserClip ?.clipIdentity ?? null ); } catch { return null; } }; export const planResidentCapture = async ( context: ResidentCaptureContext, store: SqliteAdapter, input: CaptureInput, now?: Date ): Promise => { const collection = context.config.collections.find( (candidate) => candidate.name.toLowerCase() === input.collection.toLowerCase() ); if (!collection) { return { ok: false, code: "NOT_FOUND", message: `Collection not found: ${input.collection}`, status: 404, }; } let existingRelPaths: string[]; let diskRelPaths: string[]; try { existingRelPaths = await listCollectionRelPaths(store, collection.name); diskRelPaths = await listCaptureDiskRelPaths(collection.path); } catch (error) { return { ok: false, code: "RUNTIME", message: error instanceof Error ? error.message : String(error), status: 500, }; } try { const normalizedInput = { ...input, collection: collection.name }; let plan = planCapture({ input: normalizedInput, existingRelPaths, diskRelPaths, now, }); if (plan.provenanceConflict && plan.source.browserClip !== undefined) { const identity = await readCandidateClipIdentity( collection, plan.relPath ); plan = planCapture({ input: normalizedInput, existingRelPaths, diskRelPaths, existingProvenanceByRelPath: identity === null ? new Map() : new Map([[plan.relPath, identity]]), now, }); } return { ok: true, collection, fullPath: pathJoin(collection.path, plan.relPath), plan, }; } catch (error) { return { ok: false, code: "VALIDATION", message: error instanceof Error ? error.message : String(error), status: 409, }; } }; const syncResidentCollection = async ( context: ResidentCaptureContext, collection: Collection, store: SqliteAdapter, syncCollection: typeof defaultSyncService.syncCollection ): Promise => { const result = await syncCollection( collection, store, withContentTypeRules({ runUpdateCmd: false }, context.config) ); recordContentMutation(result, context.markContentMutation); return result; }; const emitCaptureCreated = ( context: ResidentCaptureContext, collection: Collection, relPath: string ): void => { context.scheduler?.notifySyncComplete([relPath]); context.eventBus?.emit({ type: "document-changed", uri: `gno://${collection.name}/${relPath}`, collection: collection.name, relPath, origin: "create", changedAt: new Date().toISOString(), }); }; /** * Write the planned capture, then start the legacy sync job (browser * clipper contract: `202` + `sync.status: "pending"`). */ const executeCaptureAsJob = async ( context: ResidentCaptureContext, store: SqliteAdapter, planned: Extract, dependencies: ResidentCaptureDependencies ): Promise<{ body: unknown; status: number }> => { const { collection, fullPath, plan } = planned; await mkdir(dirname(fullPath), { recursive: true }); context.watchService?.suppress(fullPath); await writeCapturePlanFile(plan, fullPath); const syncCollection = dependencies.syncCollection ?? defaultSyncService.syncCollection.bind(defaultSyncService); const jobResult = await startJob( "sync", async (): Promise => { const result = await syncResidentCollection( context, collection, store, syncCollection ); emitCaptureCreated(context, collection, plan.relPath); return { collections: [result], totalDurationMs: result.durationMs, totalFilesProcessed: result.filesProcessed, totalFilesAdded: result.filesAdded, totalFilesUpdated: result.filesUpdated, totalFilesErrored: result.filesErrored, totalFilesSkipped: result.filesSkipped, }; }, context.jobManager ); return { body: buildCaptureReceipt({ plan, absPath: fullPath, sync: jobResult.ok ? { status: "pending", jobId: jobResult.jobId, reason: "Sync job started; poll /api/jobs/:id for status.", } : { status: "skipped", jobId: jobResult.activeJobId, reason: "Sync skipped because another job is running.", error: jobResult.error, }, }), status: HTTP_ACCEPTED, }; }; /** * Write + lexical sync under the shared write lease; the response is sent * only once the capture is retrievable. Throws `CaptureSyncError` when the * file landed but sync failed, and the `LOCKED` error when the lease stays * busy past `lockWaitMs`. */ const executeCaptureAwaitingSync = async ( context: ResidentCaptureContext, store: SqliteAdapter, planned: Extract, dependencies: ResidentCaptureDependencies ): Promise<{ body: unknown; status: number }> => { const { collection, fullPath, plan } = planned; return withWriteLock( resolveCaptureLockPath(context, dependencies), async () => { await mkdir(dirname(fullPath), { recursive: true }); context.watchService?.suppress(fullPath); await writeCapturePlanFile(plan, fullPath); const synced = await syncCapturedFile({ collection, store, relPath: plan.relPath, absPath: fullPath, config: context.config, syncPaths: dependencies.syncPaths, }); if (synced.result) { recordContentMutation(synced.result, context.markContentMutation); } emitCaptureCreated(context, collection, plan.relPath); return { body: buildCaptureReceipt({ plan, absPath: fullPath, docid: synced.docid, sync: synced.sync, }), status: HTTP_CREATED, }; }, dependencies.lockWaitMs ?? DEFAULT_LOCK_WAIT_MS ); }; /** * `open_existing`: an indexed file needs no lease; a disk-only file is synced * under the lease so opening it is also a retrievable success. */ const openExistingCapture = async ( context: ResidentCaptureContext, store: SqliteAdapter, planned: Extract, dependencies: ResidentCaptureDependencies ): Promise<{ body: unknown; status: number }> => { const { collection, fullPath, plan } = planned; const syncInput = { collection, store, relPath: plan.relPath, absPath: fullPath, config: context.config, syncPaths: dependencies.syncPaths, }; const existingDocument = await store.getDocument( collection.name, plan.relPath ); if (!existingDocument.ok) { throw new Error(existingDocument.error.message); } const synced = existingDocument.value ? await ensureCapturedFileIndexed(syncInput) : await withWriteLock( resolveCaptureLockPath(context, dependencies), () => ensureCapturedFileIndexed(syncInput), dependencies.lockWaitMs ?? DEFAULT_LOCK_WAIT_MS ); if (synced.result) { recordContentMutation(synced.result, context.markContentMutation); } return { body: buildCaptureReceipt({ plan, absPath: fullPath, docid: synced.docid, sync: synced.sync, }), status: HTTP_OK, }; }; export const executeResidentCapturePlan = async ( context: ResidentCaptureContext, store: SqliteAdapter, planned: Extract, dependencies: ResidentCaptureDependencies = {} ): Promise<{ body: unknown; status: number }> => { const { fullPath, plan } = planned; if (plan.provenanceConflict) { return { body: buildCaptureReceipt({ plan, absPath: fullPath, sync: { status: "skipped", reason: "Existing capture has absent or different browser provenance.", }, }), status: HTTP_CONFLICT, }; } if (plan.openedExisting) { return openExistingCapture(context, store, planned, dependencies); } return (dependencies.mode ?? "job") === "job" ? executeCaptureAsJob(context, store, planned, dependencies) : executeCaptureAwaitingSync(context, store, planned, dependencies); }; export const browserClipIdempotencyPlan = ( planned: Extract ): ClipperIdempotencyPlan => { const browserClip = planned.plan.source.browserClip; if (!browserClip) { throw new Error("Browser capture plan is missing browser provenance"); } return { collection: planned.plan.collection, relPath: planned.plan.relPath, collisionPolicyResult: planned.plan.collisionPolicyResult, contentHash: planned.plan.contentHash, clipIdentity: browserClip.clipIdentity, }; }; export const browserClipIdempotencyPlansMatch = ( left: ClipperIdempotencyPlan, right: ClipperIdempotencyPlan ): boolean => left.collection === right.collection && left.relPath === right.relPath && left.collisionPolicyResult === right.collisionPolicyResult && left.contentHash === right.contentHash && left.clipIdentity === right.clipIdentity; export type ResidentCaptureRecoveryResult = | { status: "recovered"; body: unknown; statusCode: number } | { status: "execute"; planned: Extract; } | { status: "conflict"; message: string }; const recoveredCapturePlan = ( prepared: PreparedBrowserClip, persisted: ClipperIdempotencyPlan, storedSource: CaptureSource ): CapturePlan => ({ collection: persisted.collection, relPath: persisted.relPath, filename: basename(persisted.relPath), content: prepared.captureInput.content ?? prepared.preview.body, body: prepared.preview.body, contentHash: persisted.contentHash, title: prepared.payload.title, tags: prepared.preview.tags, source: storedSource, openedExisting: persisted.collisionPolicyResult === "opened_existing", createdWithSuffix: persisted.collisionPolicyResult === "created_with_suffix", provenanceConflict: persisted.collisionPolicyResult === "conflict", collisionPolicy: prepared.payload.destination.collisionPolicy, collisionPolicyResult: persisted.collisionPolicyResult, overwrite: persisted.collisionPolicyResult === "overwritten", }); /** * Reconcile a pending clipper claim without ever choosing a new destination. * A matching exact file proves the atomic write landed; otherwise execution is * allowed only when a fresh plan is byte-for-byte identical to the saved plan. */ export const recoverPendingResidentBrowserClip = async ( context: ResidentCaptureContext, store: SqliteAdapter, prepared: PreparedBrowserClip, persisted: ClipperIdempotencyPlan ): Promise => { const collection = context.config.collections.find( (candidate) => candidate.name === persisted.collection ); if (!collection) { return { status: "conflict", message: "Saved browser capture collection is no longer configured", }; } const fullPath = pathJoin(collection.path, persisted.relPath); const file = Bun.file(fullPath); if (await file.exists()) { try { const storedContent = await file.text(); const source = extractCaptureSourceFromFrontmatter(storedContent); const browserClip = source.browserClip; if ( source.kind === "web" && typeof source.capturedAt === "string" && browserClip?.clipIdentity === persisted.clipIdentity && browserClip.finalBodyHash === persisted.contentHash && browserClip.previewDigest === prepared.preview.digest && hashCaptureContent(stripFrontmatter(storedContent)) === persisted.contentHash ) { const storedSource: CaptureSource = { ...source, kind: source.kind, capturedAt: source.capturedAt, browserClip, }; const plan = recoveredCapturePlan(prepared, persisted, storedSource); return { status: "recovered", body: buildCaptureReceipt({ plan, absPath: fullPath, sync: { status: "skipped", reason: "Recovered a completed atomic write after receipt persistence was interrupted.", }, }), statusCode: persisted.collisionPolicyResult === "opened_existing" ? 200 : 202, }; } } catch { // A malformed or unrelated exact file is handled as plan drift below. } } const replanned = await planResidentCapture( context, store, prepared.captureInput ); if ( !replanned.ok || !browserClipIdempotencyPlansMatch( browserClipIdempotencyPlan(replanned), persisted ) ) { return { status: "conflict", message: "Pending browser capture cannot be recovered because its destination changed", }; } return { status: "execute", planned: replanned }; };