import { createHash, randomBytes } from "node:crypto"; import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; import { basename, join } from "node:path"; import { getAgentDir } from "@earendil-works/pi-coding-agent"; import type { FetchContentKind } from "./fetch-formats.ts"; const SEARCH_ID_RE = /^ws_[a-f0-9]{24}$/; const FETCH_ID_RE = /^wf_[a-f0-9]{24}$/; const DEFAULT_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1_000; const DEFAULT_MAX_TOTAL_BYTES = 256 * 1024 * 1024; export type ArtifactKind = "search" | "fetch"; export type FetchArtifactFormat = "source" | "text" | "markdown" | "html"; export interface SearchArtifactMetadata { kind: "search"; id: string; query: string; createdAt: number; reportBytes: number; reportSha256: string; rawOutputBytes?: number; rawOutputSha256?: string; referenceIds?: string[]; resultCount?: number; summaryTimeouts?: number; provider?: string; model?: string; pipeline?: string; summaryProvider?: string; summaryModel?: string; usage?: Record; } export interface FetchArtifactFiles { raw: string; source?: string; text?: string; markdown?: string; } export interface FetchArtifactMetadata { kind: "fetch"; id: string; requestedUrl: string; finalUrl: string; createdAt: number; status: number; title: string; contentType: string; contentKind?: FetchContentKind; sourceFormat?: string; extension?: string; charset: string; headers: Record; redirects: string[]; files?: FetchArtifactFiles; parseWarning?: string; rawBytes: number; rawSha256: string; sourceBytes?: number; sourceSha256?: string; textBytes?: number; textSha256?: string; markdownBytes?: number; markdownSha256?: string; } export type ArtifactMetadata = SearchArtifactMetadata | FetchArtifactMetadata; export interface FetchArtifactInput { requestedUrl: string; finalUrl: string; status: number; title: string; contentType: string; contentKind: FetchContentKind; sourceFormat: string; extension: string; charset: string; headers: Record; redirects: string[]; rawBody: Uint8Array; sourceText?: string; text?: string; markdown?: string; parseWarning?: string; } export interface ArtifactPaths { directory: string; metadata: string; report?: string; rawSearch?: string; raw?: string; source?: string; text?: string; markdown?: string; /** Legacy v0.3 fetch paths. */ rawHtml?: string; htmlView?: string; } export interface ArtifactCleanupOptions { maxAgeMs?: number; maxTotalBytes?: number; now?: number; } function sha256(value: string | Uint8Array): string { return createHash("sha256").update(value).digest("hex"); } function sessionDirectoryName(sessionId: string): string { return createHash("sha256").update(sessionId).digest("hex").slice(0, 24); } function createArtifactId(kind: ArtifactKind): string { const prefix = kind === "search" ? "ws" : "wf"; return `${prefix}_${randomBytes(12).toString("hex")}`; } function assertArtifactId(id: string, kind?: ArtifactKind): void { const valid = kind === "search" ? SEARCH_ID_RE.test(id) : kind === "fetch" ? FETCH_ID_RE.test(id) : SEARCH_ID_RE.test(id) || FETCH_ID_RE.test(id); if (!valid) throw new Error(`Invalid ${kind ?? "web"} artifact id: ${id}`); } function artifactFilePath(directory: string, filename: string): string { if (!filename || basename(filename) !== filename || /[\\/]/.test(filename)) { throw new Error(`Invalid fetch artifact filename: ${filename}`); } return join(directory, filename); } function artifactExtension(value: string): string { const normalized = value.toLowerCase().replace(/^\./, ""); return /^[a-z0-9]{1,12}$/.test(normalized) ? normalized : "bin"; } async function writeAtomic(path: string, content: string | Uint8Array): Promise { const temporary = `${path}.${randomBytes(6).toString("hex")}.tmp`; await writeFile(temporary, content, { mode: 0o600 }); await rename(temporary, path); } async function directorySize(path: string): Promise { let total = 0; for (const entry of await readdir(path, { withFileTypes: true })) { const child = join(path, entry.name); if (entry.isDirectory()) total += await directorySize(child); else if (entry.isFile()) total += (await stat(child)).size; } return total; } export class ArtifactStore { readonly root: string; constructor(sessionId: string, cacheRoot = join(getAgentDir(), "cache", "pi-web")) { this.root = join(cacheRoot, sessionDirectoryName(sessionId)); } private async ensureRoot(): Promise { await mkdir(this.root, { recursive: true, mode: 0o700 }); } paths(id: string): ArtifactPaths { assertArtifactId(id); const directory = join(this.root, id); if (SEARCH_ID_RE.test(id)) { return { directory, metadata: join(directory, "metadata.json"), report: join(directory, "report.md"), rawSearch: join(directory, "raw-search.txt"), }; } return { directory, metadata: join(directory, "metadata.json"), raw: join(directory, "original.bin"), source: join(directory, "source.txt"), text: join(directory, "content.txt"), markdown: join(directory, "content.md"), rawHtml: join(directory, "original.html"), htmlView: join(directory, "original.view.html"), }; } async saveSearch( query: string, report: string, extra: Pick< SearchArtifactMetadata, "provider" | "model" | "pipeline" | "summaryProvider" | "summaryModel" | "referenceIds" | "resultCount" | "summaryTimeouts" | "usage" > = {}, rawOutput?: string, ): Promise<{ id: string; metadata: SearchArtifactMetadata; paths: ArtifactPaths }> { await this.ensureRoot(); const id = createArtifactId("search"); const paths = this.paths(id); await mkdir(paths.directory, { recursive: false, mode: 0o700 }); const metadata: SearchArtifactMetadata = { kind: "search", id, query, createdAt: Date.now(), reportBytes: Buffer.byteLength(report, "utf8"), reportSha256: sha256(report), ...(rawOutput === undefined ? {} : { rawOutputBytes: Buffer.byteLength(rawOutput, "utf8"), rawOutputSha256: sha256(rawOutput) }), ...extra, }; const writes = [ writeAtomic(paths.report as string, report), writeAtomic(paths.metadata, `${JSON.stringify(metadata, null, 2)}\n`), ]; if (rawOutput !== undefined) writes.push(writeAtomic(paths.rawSearch as string, rawOutput)); await Promise.all(writes); return { id, metadata, paths }; } async saveFetch(input: FetchArtifactInput): Promise<{ id: string; metadata: FetchArtifactMetadata; paths: ArtifactPaths }> { await this.ensureRoot(); const id = createArtifactId("fetch"); const basePaths = this.paths(id); await mkdir(basePaths.directory, { recursive: false, mode: 0o700 }); const files: FetchArtifactFiles = { raw: `original.${artifactExtension(input.extension)}` }; if (input.sourceText !== undefined) files.source = `source.${artifactExtension(input.extension)}`; if (input.text !== undefined) files.text = input.text === input.sourceText && files.source ? files.source : "content.txt"; if (input.markdown !== undefined) files.markdown = "content.md"; const paths: ArtifactPaths = { directory: basePaths.directory, metadata: basePaths.metadata, raw: artifactFilePath(basePaths.directory, files.raw), ...(files.source ? { source: artifactFilePath(basePaths.directory, files.source) } : {}), ...(files.text ? { text: artifactFilePath(basePaths.directory, files.text) } : {}), ...(files.markdown ? { markdown: artifactFilePath(basePaths.directory, files.markdown) } : {}), }; const metadata: FetchArtifactMetadata = { kind: "fetch", id, requestedUrl: input.requestedUrl, finalUrl: input.finalUrl, createdAt: Date.now(), status: input.status, title: input.title, contentType: input.contentType, contentKind: input.contentKind, sourceFormat: input.sourceFormat, extension: artifactExtension(input.extension), charset: input.charset, headers: input.headers, redirects: input.redirects, files, ...(input.parseWarning ? { parseWarning: input.parseWarning } : {}), rawBytes: input.rawBody.byteLength, rawSha256: sha256(input.rawBody), ...(input.sourceText === undefined ? {} : { sourceBytes: Buffer.byteLength(input.sourceText, "utf8"), sourceSha256: sha256(input.sourceText) }), ...(input.text === undefined ? {} : { textBytes: Buffer.byteLength(input.text, "utf8"), textSha256: sha256(input.text) }), ...(input.markdown === undefined ? {} : { markdownBytes: Buffer.byteLength(input.markdown, "utf8"), markdownSha256: sha256(input.markdown) }), }; const writes: Promise[] = [ writeAtomic(paths.raw as string, input.rawBody), writeAtomic(paths.metadata, `${JSON.stringify(metadata, null, 2)}\n`), ]; if (input.sourceText !== undefined) writes.push(writeAtomic(paths.source as string, input.sourceText)); if (input.text !== undefined && paths.text !== paths.source) writes.push(writeAtomic(paths.text as string, input.text)); if (input.markdown !== undefined) writes.push(writeAtomic(paths.markdown as string, input.markdown)); await Promise.all(writes); return { id, metadata, paths }; } async readMetadata(id: string): Promise { assertArtifactId(id); const text = await readFile(this.paths(id).metadata, "utf8").catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") throw new Error(`Web artifact not found: ${id}`); throw error; }); const metadata = JSON.parse(text) as ArtifactMetadata; if (metadata.id !== id || (metadata.kind !== "search" && metadata.kind !== "fetch")) { throw new Error(`Invalid metadata for web artifact: ${id}`); } return metadata; } async readSearch(id: string): Promise<{ text: string; metadata: SearchArtifactMetadata; paths: ArtifactPaths }> { assertArtifactId(id, "search"); const paths = this.paths(id); const [text, metadata] = await Promise.all([ readFile(paths.report as string, "utf8").catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") throw new Error(`Search artifact not found: ${id}`); throw error; }), this.readMetadata(id), ]); if (metadata.kind !== "search") throw new Error(`Artifact ${id} is not a search report`); return { text, metadata, paths }; } async readFetch( id: string, format: FetchArtifactFormat, ): Promise<{ text: string; metadata: FetchArtifactMetadata; paths: ArtifactPaths }> { assertArtifactId(id, "fetch"); const metadata = await this.readMetadata(id); if (metadata.kind !== "fetch") throw new Error(`Artifact ${id} is not fetched web content`); const basePaths = this.paths(id); const paths: ArtifactPaths = metadata.files ? { directory: basePaths.directory, metadata: basePaths.metadata, raw: artifactFilePath(basePaths.directory, metadata.files.raw), ...(metadata.files.source ? { source: artifactFilePath(basePaths.directory, metadata.files.source) } : {}), ...(metadata.files.text ? { text: artifactFilePath(basePaths.directory, metadata.files.text) } : {}), ...(metadata.files.markdown ? { markdown: artifactFilePath(basePaths.directory, metadata.files.markdown) } : {}), } : basePaths; const target = metadata.files ? format === "source" || format === "html" ? paths.source : format === "text" ? paths.text ?? paths.source : paths.markdown ?? paths.text ?? paths.source : format === "source" || format === "html" ? paths.htmlView : paths.markdown; let text: string; if (target) { text = await readFile(target, "utf8").catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") throw new Error(`Fetch artifact view not found: ${id} (${format})`); throw error; }); } else { const rawPath = paths.raw ?? paths.rawHtml; const label = metadata.contentKind === "image" ? "Image" : metadata.contentKind === "document" ? "Document" : "Binary file"; text = [ `[${label} downloaded without OCR]`, `Source format: ${metadata.sourceFormat ?? metadata.contentType}`, `Content type: ${metadata.contentType}`, `Local path: ${rawPath}`, ...(metadata.parseWarning ? [`Warning: ${metadata.parseWarning}`] : []), ].join("\n"); } return { text, metadata, paths }; } async cleanup(options: ArtifactCleanupOptions = {}): Promise { await this.ensureRoot(); const now = options.now ?? Date.now(); const maxAgeMs = options.maxAgeMs ?? DEFAULT_MAX_AGE_MS; const maxTotalBytes = options.maxTotalBytes ?? DEFAULT_MAX_TOTAL_BYTES; const records: Array<{ path: string; modifiedAt: number; bytes: number }> = []; for (const entry of await readdir(this.root, { withFileTypes: true })) { if (!entry.isDirectory() || (!SEARCH_ID_RE.test(entry.name) && !FETCH_ID_RE.test(entry.name))) continue; const path = join(this.root, entry.name); const info = await stat(path); if (now - info.mtimeMs > maxAgeMs) { await rm(path, { recursive: true, force: true }); continue; } records.push({ path, modifiedAt: info.mtimeMs, bytes: await directorySize(path) }); } let total = records.reduce((sum, record) => sum + record.bytes, 0); for (const record of records.sort((left, right) => left.modifiedAt - right.modifiedAt)) { if (total <= maxTotalBytes) break; await rm(record.path, { recursive: true, force: true }); total -= record.bytes; } } } export function isSearchArtifactId(value: string): boolean { return SEARCH_ID_RE.test(value); } export function isFetchArtifactId(value: string): boolean { return FETCH_ID_RE.test(value); }