/** * Authenticated operation registry — mirrors papyrus/src/service.ts's * EXPECTED_OPERATION_NAMES + typed OperationInputs/OperationOutputs pattern. * * `cache.list` and `cache.search` are the first two real operations, * proving the full path: HTTP → auth → SQLite → typed response, and * preserving the grep/offset/limit/query semantics of today's pi-extension * handleCacheListing/handleCacheSearch. Later tasks (fetch/crawl/search) * add operations here without touching the auth/transport shape. */ import { randomUUID } from "node:crypto"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { VehicleRegistry } from "@danypops/vehicle-server"; import { type DaemonIdentity, type DaemonLifecycleEvent, type DaemonLifecycleLog, diagnoseDaemon, } from "@danypops/vehicle-server/daemon-lifecycle"; import { createVehicleHttpApp } from "@danypops/vehicle-server/http"; import { createLogger, type Logger } from "@danypops/vehicle-server/logging"; import { errorResponse, healthResponse, jsonResponse, readyResponse, requireBearerToken } from "@danypops/vehicle-server/rpc-http"; import { createDefaultHttpClient, createSsrfGuard, DomainThrottle, type IHttpClient, type ISsrfGuard, PlaywrightHttpClient, RobotsCache, type SearchEngine, } from "@danypops/web-spider"; import type { CacheStore } from "./cache/cache-store.ts"; import type { CachedPageListFilter, CachedPageListResult, CachedPageSearchResult, CategoryAssignmentResult, CategoryListResult, CategoryRenameResult, } from "./cache/page.ts"; import { SQLiteCacheStore } from "./cache/sqlite-cache-store.ts"; import { SEARCH_ENGINE_USAGE_LIST_DEFAULT_LIMIT, SERVICE_MAX_BODY_BYTES, SESSION_DOWNLOADS_DIRECTORY_NAME, SQLITE_SCHEMA_VERSION, } from "./constants.ts"; import { resolveCurrentIdentityOrWait } from "./daemon-lifecycle.ts"; import { openWebSpiderDb, schemaVersion } from "./db.ts"; import { type CrawlOperationInput, type CrawlOperationOutput, CrawlService } from "./fetch/crawl-service.ts"; import { type FetchOperationInput, type FetchOperationOutput, FetchService } from "./fetch/fetch-service.ts"; import { type QuotesOperationInput, type QuotesOperationOutput, QuotesService } from "./fetch/quotes-service.ts"; import { registerCacheVehicleOperations } from "./handlers/cache.ts"; import { registerCategoryVehicleOperations } from "./handlers/category.ts"; import { registerDaemonVehicleOperations } from "./handlers/daemon.ts"; import { registerFetchVehicleOperations } from "./handlers/fetch.ts"; import { registerQuotesVehicleOperations } from "./handlers/quotes.ts"; import { registerSearchVehicleOperations } from "./handlers/search.ts"; import { registerSessionVehicleOperations } from "./handlers/session.ts"; import { importLegacyJsonCache, type LegacyImportResult } from "./migrate-legacy-cache.ts"; import { createEngineResolver, type KeyTestResult, testProviderKeys, type WebSearchInput, type WebSearchOutput, WebSearchService, } from "./search/search-service.ts"; import type { SearchEngineUsageEntry } from "./search/search-usage.ts"; import type { SearchUsageJournal } from "./search/search-usage-journal.ts"; import { SQLiteSearchUsageJournal } from "./search/sqlite-search-usage-journal.ts"; import type { DownloadStoragePolicy } from "./session/download-store.ts"; import { PlaywrightSessionRegistry } from "./session/playwright-session-registry.ts"; import type { SessionInfo } from "./session/session.ts"; import { isSessionAction, SESSION_ACTIONS, type SessionAction } from "./session/session-audit.ts"; import type { SessionFinalizationReport } from "./session/session-registry.ts"; import { type SessionActInput, type SessionActOutput, type SessionCloseInput, SessionNotFoundError, SessionService, StaleSnapshotError, } from "./session/session-service.ts"; import { SQLiteSessionAuditJournal } from "./session/sqlite-session-audit-journal.ts"; import { VERSION } from "./version.ts"; export const EXPECTED_OPERATION_NAMES = [ "daemon.diagnose", "cache.list", "cache.search", "search", "search.usage", "search.testKeys", "fetch", "crawl", "quotes", "session.create", "session.list", "session.close", "session.act", "category.assign", "category.remove", "category.rename", "category.list", ] as const; export type OperationName = (typeof EXPECTED_OPERATION_NAMES)[number]; export interface OperationInputs { "daemon.diagnose": { historyLimit?: number }; "cache.list": CachedPageListFilter; "cache.search": { query: string; limit?: number }; search: WebSearchInput; "search.usage": { engine?: string; limit?: number }; "search.testKeys": { engine: string }; fetch: FetchOperationInput; crawl: CrawlOperationInput; quotes: QuotesOperationInput; "session.create": { name: string; forceChromeChannel?: boolean; headed?: boolean }; "session.list": Record; "session.close": SessionCloseInput; "session.act": SessionActInput; "category.assign": { url: string; category: string }; "category.remove": { url: string; category: string }; "category.rename": { category: string; newName: string }; "category.list": Record; } export interface OperationOutputs { "daemon.diagnose": Awaited>; "cache.list": CachedPageListResult; "cache.search": CachedPageSearchResult; search: WebSearchOutput; "search.usage": { entries: SearchEngineUsageEntry[] }; "search.testKeys": { engine: string; results: KeyTestResult[] }; fetch: FetchOperationOutput; crawl: CrawlOperationOutput; quotes: QuotesOperationOutput; "session.create": SessionInfo; "session.list": { sessions: SessionInfo[] }; "session.close": { name: string; closed: true; finalization: SessionFinalizationReport }; "session.act": SessionActOutput; "category.assign": CategoryAssignmentResult; "category.remove": { url: string; category: string; removed: true }; "category.rename": CategoryRenameResult; "category.list": CategoryListResult; } export type OperationInput = Record; type OperationHandler = (input: OperationInput) => unknown | Promise; export class UnknownOperationError extends Error {} export class PayloadTooLargeError extends Error {} export function requireString(input: OperationInput, key: string): string { const value = input[key]; if (typeof value !== "string") throw new Error(`${key} is required`); return value; } export function optionalString(input: OperationInput, key: string): string | undefined { const value = input[key]; if (value === undefined) return undefined; if (typeof value !== "string") throw new Error(`${key} must be a string`); return value; } export function optionalNumber(input: OperationInput, key: string): number | undefined { const value = input[key]; if (value === undefined) return undefined; if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${key} must be a number`); return value; } export function optionalBoolean(input: OperationInput, key: string): boolean | undefined { const value = input[key]; if (value === undefined) return undefined; if (typeof value !== "boolean") throw new Error(`${key} must be a boolean`); return value; } export function optionalStringArray(input: OperationInput, key: string): string[] | undefined { const value = input[key]; if (value === undefined) return undefined; if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) throw new Error(`${key} must be an array of strings`); return value; } /** * Builds the full WebSearchInput, including siteFilter/wantFullContent -- both declared on * WebSearchInput and sent by the pi-extension's handleSearch(). Shared by the legacy * /api/v1/ops handler and the search Vehicle operation so both stay identical by construction. */ export function searchInput(input: OperationInput): WebSearchInput { return { query: requireString(input, "query"), numResults: optionalNumber(input, "numResults"), timeRange: optionalString(input, "timeRange") as WebSearchInput["timeRange"], topic: optionalString(input, "topic") as WebSearchInput["topic"], searchEngine: optionalString(input, "searchEngine") as WebSearchInput["searchEngine"], siteFilter: optionalString(input, "siteFilter"), wantFullContent: optionalBoolean(input, "wantFullContent"), }; } export function quotesInput(input: OperationInput): QuotesOperationInput { return { query: requireString(input, "query"), urls: optionalStringArray(input, "urls") ?? [], maxQuotesPerUrl: optionalNumber(input, "maxQuotesPerUrl"), maxQuotesTotal: optionalNumber(input, "maxQuotesTotal"), timeoutMs: optionalNumber(input, "timeoutMs"), enhanced: optionalBoolean(input, "enhanced"), ignoreRobots: optionalBoolean(input, "ignoreRobots"), sources: optionalStringArray(input, "sources"), maxCacheAgeMs: optionalNumber(input, "maxCacheAgeMs"), }; } export function fetchInput(input: OperationInput): FetchOperationInput { return { url: requireString(input, "url"), format: optionalString(input, "format") as FetchOperationInput["format"], rootSelector: optionalString(input, "rootSelector"), excludeSelectors: optionalString(input, "excludeSelectors"), tokenBudget: optionalNumber(input, "tokenBudget"), pdfPageStart: optionalNumber(input, "pdfPageStart"), pdfPageEnd: optionalNumber(input, "pdfPageEnd"), enhanced: optionalBoolean(input, "enhanced"), timeoutMs: optionalNumber(input, "timeoutMs"), query: optionalString(input, "query"), path: optionalString(input, "path"), topN: optionalNumber(input, "topN"), ignoreRobots: optionalBoolean(input, "ignoreRobots"), sources: optionalStringArray(input, "sources"), maxCacheAgeMs: optionalNumber(input, "maxCacheAgeMs"), }; } const TAB_OPERATIONS = new Set(["list", "new", "close", "select"]); const LOAD_STATES = new Set(["load", "domcontentloaded", "networkidle"]); const ELEMENT_STATES = new Set(["visible", "hidden", "attached", "detached"]); const SCREENSHOT_SCALES = new Set(["css", "device"]); const SNAPSHOT_MODES = new Set(["ai", "default"]); export function sessionActInput(input: OperationInput): SessionActInput { const name = requireString(input, "name"); const snapshotVersionRaw = input.snapshotVersion; if (typeof snapshotVersionRaw !== "number" || !Number.isInteger(snapshotVersionRaw) || snapshotVersionRaw < 0) { throw new Error("snapshotVersion is required and must be a non-negative integer"); } const action = requireString(input, "action"); if (!isSessionAction(action)) { throw new Error(`action must be one of ${[...SESSION_ACTIONS].map((a) => `"${a}"`).join(", ")}`); } const loadState = optionalString(input, "loadState"); if (loadState !== undefined && !LOAD_STATES.has(loadState)) { throw new Error('loadState must be one of "load", "domcontentloaded", "networkidle"'); } const state = optionalString(input, "state"); if (state !== undefined && !ELEMENT_STATES.has(state)) { throw new Error('state must be one of "visible", "hidden", "attached", "detached"'); } const scale = optionalString(input, "scale"); if (scale !== undefined && !SCREENSHOT_SCALES.has(scale)) { throw new Error('scale must be one of "css", "device"'); } const mode = optionalString(input, "mode"); if (mode !== undefined && !SNAPSHOT_MODES.has(mode)) { throw new Error('mode must be one of "ai", "default"'); } const tabOperation = optionalString(input, "tabOperation"); if (tabOperation !== undefined && !TAB_OPERATIONS.has(tabOperation)) { throw new Error('tabOperation must be one of "list", "new", "close", "select"'); } return { name, snapshotVersion: snapshotVersionRaw, action: action as SessionAction, url: optionalString(input, "url"), selector: optionalString(input, "selector"), script: optionalString(input, "script"), timeoutMs: optionalNumber(input, "timeoutMs"), text: optionalString(input, "text"), clear: optionalBoolean(input, "clear"), value: optionalString(input, "value"), label: optionalString(input, "label"), loadState: loadState as SessionActInput["loadState"], state: state as SessionActInput["state"], fullPage: optionalBoolean(input, "fullPage"), scale: scale as SessionActInput["scale"], depth: optionalNumber(input, "depth"), boxes: optionalBoolean(input, "boxes"), mode: mode as SessionActInput["mode"], accept: optionalBoolean(input, "accept"), promptText: optionalString(input, "promptText"), key: optionalString(input, "key"), includeStatic: optionalBoolean(input, "includeStatic"), tabOperation: tabOperation as SessionActInput["tabOperation"], tabIndex: optionalNumber(input, "tabIndex"), }; } function handlers( store: CacheStore, webSearch: WebSearchService, fetchService: FetchService, crawlService: CrawlService, quotesService: QuotesService, sessionService: SessionService, searchUsage: SearchUsageJournal, loadSearchKeys: (engine: string) => string[], lifecycleLog: DaemonLifecycleLog, getCurrentIdentity: () => DaemonIdentity | undefined, ): Record { return { "daemon.diagnose": async (input) => { // See resolveCurrentIdentityOrWait's own doc comment (daemon-lifecycle.ts): a real, // narrow race, not a hypothetical -- the daemon handle a real client polls for becomes // visible before this daemon's own onListen populates getCurrentIdentity(). const current = await resolveCurrentIdentityOrWait(getCurrentIdentity); return diagnoseDaemon({ lifecycleLog, current, historyLimit: optionalNumber(input, "historyLimit") }); }, "cache.list": (input) => store.list({ grep: optionalString(input, "grep"), domain: optionalString(input, "domain"), tag: optionalString(input, "tag"), category: optionalString(input, "category"), fetchedAfter: optionalNumber(input, "fetchedAfter"), fetchedBefore: optionalNumber(input, "fetchedBefore"), publishedAfter: optionalString(input, "publishedAfter"), publishedBefore: optionalString(input, "publishedBefore"), sortBy: optionalString(input, "sortBy") as CachedPageListFilter["sortBy"], sortOrder: optionalString(input, "sortOrder") as CachedPageListFilter["sortOrder"], offset: optionalNumber(input, "offset"), limit: optionalNumber(input, "limit"), }), "cache.search": (input) => store.search(requireString(input, "query"), { topN: optionalNumber(input, "limit"), }), search: (input) => webSearch.search(searchInput(input)), "search.usage": (input) => ({ entries: searchUsage.recent({ engine: optionalString(input, "engine"), limit: optionalNumber(input, "limit") ?? SEARCH_ENGINE_USAGE_LIST_DEFAULT_LIMIT, }), }), "search.testKeys": async (input) => { const engine = requireString(input, "engine"); return { engine, results: await testProviderKeys(engine as SearchEngine, loadSearchKeys(engine)) }; }, fetch: (input) => fetchService.fetch(fetchInput(input)), crawl: (input) => crawlService.crawl({ ...fetchInput(input), format: optionalString(input, "format") as CrawlOperationInput["format"], depth: optionalNumber(input, "depth"), maxPages: optionalNumber(input, "maxPages"), sameDomain: optionalBoolean(input, "sameDomain"), discoverOnly: optionalBoolean(input, "discoverOnly"), crawlUrls: optionalStringArray(input, "crawlUrls"), maxTotalChars: optionalNumber(input, "maxTotalChars"), deadlineMs: optionalNumber(input, "deadlineMs"), excludeDomains: optionalStringArray(input, "excludeDomains"), includeDomains: optionalStringArray(input, "includeDomains"), maxCacheAgeMs: optionalNumber(input, "maxCacheAgeMs"), }), quotes: (input) => quotesService.quotes(quotesInput(input)), "session.create": (input) => sessionService.create({ name: requireString(input, "name"), forceChromeChannel: optionalBoolean(input, "forceChromeChannel"), headed: optionalBoolean(input, "headed"), }), "session.list": () => ({ sessions: sessionService.list() }), "session.close": (input) => sessionService.close({ name: requireString(input, "name") }), "session.act": (input) => sessionService.act(sessionActInput(input)), "category.assign": (input) => store.assignCategory(requireString(input, "url"), requireString(input, "category")), "category.remove": (input) => { const url = requireString(input, "url"); const category = requireString(input, "category"); store.removeCategory(url, category); return { url, category, removed: true as const }; }, "category.rename": (input) => store.renameCategory(requireString(input, "category"), requireString(input, "newName")), "category.list": () => store.listCategories(), }; } export interface SchemaState { current: number; required: number; } export interface AsyncDisposableHttpClient extends IHttpClient { close(): Promise; } export interface WebSpiderServiceDependencies { logger?: Logger; env?: Record; /** BYOK key stacking: extra API keys per search provider beyond the single one (if any) already merged into `env` -- see resolveAdditionalSearchKeys() (search-env.ts) and RotatingKeySearchEngine (@danypops/web-spider). Absent/empty preserves exact single-key behavior. */ additionalSearchKeys?: Partial>; /** Every locally stored key for one provider (regardless of count), for search.testKeys -- see search-secrets.ts's SearchKeyStore.loadAll(). Defaults to reporting nothing stored, for callers (most tests) that don't care about this feature. */ loadSearchKeys?: (engine: string) => string[]; /** Lazy factory for the daemon-owned enhanced fetch client. Tests inject a fake; production constructs Playwright. */ enhancedClientFactory?: () => AsyncDisposableHttpClient; /** Persistent, durable-across-restarts structured event log for daemon.diagnose (see daemon-lifecycle.ts). Defaults to an in-memory, this-process-only log for callers (most tests) that don't care about restart history -- daemon.diagnose still works out of the box, it just has nothing from a prior instance to report. Production wiring (daemon.ts) passes a real file-backed one. */ lifecycleLog?: DaemonLifecycleLog; /** Lazily resolves this daemon's own current identity for daemon.diagnose. Defaults to one fixed identity minted at construction time -- real production wiring (daemon.ts) passes a getter reading a ref populated once the real daemon has actually bound (see vehicle-server daemon.ts's own onListen doc comment on why identity isn't known any earlier). May return undefined during that narrow pre-bind window; the handler fails safely rather than crashing when it does. */ getCurrentIdentity?: () => DaemonIdentity | undefined; /** Overrides the default SSRF guard (see WEB_SPIDER_SSRF_ALLOW_RANGES below). Tests that monkey-patch globalThis.fetch directly, with no real DNS involved, pass a permissive stub here. */ ssrfGuard?: ISsrfGuard; /** Finite browser-download ceilings. Omitted fields retain the secure per-file, per-session, and daemon-wide defaults. */ downloadPolicy?: Partial; } /** This-process-only default for callers who don't care about lifecycle history surviving a restart (most tests, and any daemon.ts that hasn't wired a real file-backed one yet) -- daemon.diagnose still works, it just always reports empty history. */ function createInMemoryLifecycleLog(): DaemonLifecycleLog { const events: DaemonLifecycleEvent[] = []; return { async record(event) { const full = { ...event, at: new Date().toISOString() }; events.push(full); return full; }, async recent(limit) { return limit === undefined ? [...events] : events.slice(-limit); }, }; } export interface WebSpiderService { operationNames(): OperationName[]; schemaState(): SchemaState; execute(operation: string, input?: OperationInput): Promise; /** Every operation migrated onto the real Vehicle protocol so far -- see handlers/category.ts and handlers/cache.ts. Served alongside (not replacing) the /api/v1/ops route above. */ vehicleRegistry: VehicleRegistry; /** Best-effort, one-time import of a pre-daemon JSON DiskCache. No-op once the store already has rows. */ importLegacyCacheIfEmpty(jsonPath: string): LegacyImportResult; checkpoint(): void; optimize(): void; close(): Promise; } export function createWebSpiderService(path: string, deps: WebSpiderServiceDependencies = {}): WebSpiderService { const db = openWebSpiderDb(path); // :memory: databases (tests) have no sibling directory to spill large images into — // use an isolated temp directory instead of guessing a path relative to cwd. const ownsTemporaryDirectories = path === ":memory:"; const imagesDir = ownsTemporaryDirectories ? mkdtempSync(join(tmpdir(), "web-spider-images-")) : join(dirname(path), "images"); // Same derivation as imagesDir above — a sibling of the database, or an // isolated temp directory for :memory: (test) databases with no sibling // directory to spill downloaded files into. const downloadsBaseDir = ownsTemporaryDirectories ? mkdtempSync(join(tmpdir(), "web-spider-downloads-")) : join(dirname(path), SESSION_DOWNLOADS_DIRECTORY_NAME); const store = new SQLiteCacheStore(db, { imagesDir }); const logger = deps.logger ?? createLogger("web-spider-daemon"); // Provider API keys are read from this (daemon) process's own environment only — // never accepted as operation input, never logged; onEngineFailure logs only // the engine name and error message, never a key. deps.env is the Enigma-augmented // environment resolveSearchEnv() built at startup (see daemon.ts); defaults to the // raw process environment for callers (tests) that construct this directly. const searchUsage = new SQLiteSearchUsageJournal(db); const webSearch = new WebSearchService( createEngineResolver( deps.env ?? process.env, (engineName, error, reason) => { logger.warn("web_search_engine_degraded", { engine: engineName, reason, error: error instanceof Error ? error.message : String(error), }); }, (engineName, usage) => { searchUsage.record({ engine: engineName, observedAt: Date.now(), ...usage }); logger.debug("web_search_engine_usage", { engine: engineName, ...usage }); }, undefined, deps.additionalSearchKeys, ), ); // Daemon-process-wide throttle/robots singletons — replaces the pi-extension's // per-session instances with per-daemon ones, a more correct scope since the // daemon is now the sole process performing fetches. const throttle = new DomainThrottle({ minDelayMs: 500 }); // SSRF guard: default-deny in production. WEB_SPIDER_SSRF_ALLOW_RANGES (comma- // separated CIDRs) is the audited opt-in escape hatch -- test daemons targeting a // local fixture server set it; production wiring (daemon.ts) leaves it unset. const env = deps.env ?? process.env; const ssrfAllowRanges = env.WEB_SPIDER_SSRF_ALLOW_RANGES?.split(",") .map((range) => range.trim()) .filter(Boolean); const ssrfGuard = deps.ssrfGuard ?? (ssrfAllowRanges?.length ? createSsrfGuard({ allowRanges: ssrfAllowRanges }) : undefined); const defaultHttpClient = ssrfGuard ? createDefaultHttpClient(ssrfGuard) : undefined; const robotsCache = new RobotsCache(undefined, ssrfGuard); // The composition root owns the enhanced client as a separate async-disposable // capability; IHttpClient remains segregated for adapters that own no resource. const enhancedClientFactory = deps.enhancedClientFactory ?? (() => { const executablePath = process.env.WEB_SPIDER_PLAYWRIGHT_EXECUTABLE; return new PlaywrightHttpClient(executablePath ? { executablePath } : undefined); }); let enhancedClient: AsyncDisposableHttpClient | undefined; const getPlaywrightClient = (): IHttpClient => { enhancedClient ??= enhancedClientFactory(); return enhancedClient; }; const fetchService = new FetchService({ cache: store, throttle, robotsCache, getPlaywrightClient, logger, defaultHttpClient }); const crawlService = new CrawlService({ cache: store, throttle, robotsCache, getPlaywrightClient, logger, defaultHttpClient }); const quotesService = new QuotesService({ cache: store, throttle, robotsCache, getPlaywrightClient, logger, defaultHttpClient }); const sessionRegistry = new PlaywrightSessionRegistry({ downloadsBaseDir, downloadPolicy: deps.downloadPolicy, logger }); const sessionAuditJournal = new SQLiteSessionAuditJournal(db); const sessionService = new SessionService(sessionRegistry, sessionAuditJournal, Date.now, logger); const defaultIdentity: DaemonIdentity = { instanceId: randomUUID(), pid: process.pid, startedAt: new Date().toISOString(), provenance: "unknown", }; const lifecycleLog: DaemonLifecycleLog = deps.lifecycleLog ?? createInMemoryLifecycleLog(); const getCurrentIdentity = deps.getCurrentIdentity ?? (() => defaultIdentity); const registry = handlers( store, webSearch, fetchService, crawlService, quotesService, sessionService, searchUsage, deps.loadSearchKeys ?? (() => []), lifecycleLog, getCurrentIdentity, ); const vehicleRegistry = new VehicleRegistry({ name: "web-spider", packageJsonUrl: new URL("../package.json", import.meta.url), description: "Web fetch/search/crawl, curated page categories, and a disk-backed cache, behind a supervised daemon.", }); // The default open-world policy protects future operations. Reviewed common web reads // opt out individually; explicit robots bypass and browser mutation descriptors opt in. vehicleRegistry.configureApprovals(); // withVehicleErrorParity() already converts every real handler error into a well-formed // VehicleError, so this only affects a genuine registration/binding bug. vehicleRegistry.setExposeHandlerFailureDetails(true); registerDaemonVehicleOperations(vehicleRegistry, lifecycleLog, getCurrentIdentity); registerCategoryVehicleOperations(vehicleRegistry, store); registerCacheVehicleOperations(vehicleRegistry, store); registerSearchVehicleOperations(vehicleRegistry, webSearch, searchUsage, deps.loadSearchKeys ?? (() => [])); registerFetchVehicleOperations(vehicleRegistry, fetchService, crawlService); registerQuotesVehicleOperations(vehicleRegistry, quotesService); registerSessionVehicleOperations(vehicleRegistry, sessionService); let closePromise: Promise | undefined; const closeOwnedResources = async (): Promise => { const enhancedClientClosed = enhancedClient?.close().catch((error) => { logger.warn("playwright_close_failed", { error: String(error) }); }); const sessionsClosed = sessionRegistry.closeAll().catch((error) => { logger.warn("session_close_failed", { error: String(error) }); }); await Promise.all([enhancedClientClosed, sessionsClosed]); db.exec("PRAGMA optimize"); db.close(); if (ownsTemporaryDirectories) { rmSync(imagesDir, { recursive: true, force: true }); rmSync(downloadsBaseDir, { recursive: true, force: true }); } }; return { operationNames: () => [...EXPECTED_OPERATION_NAMES], schemaState: () => ({ current: schemaVersion(db), required: SQLITE_SCHEMA_VERSION }), vehicleRegistry, async execute(operation, input = {}) { const handler = registry[operation as OperationName]; if (!handler) throw new UnknownOperationError(`unknown operation "${operation}"`); return await handler(input); }, importLegacyCacheIfEmpty(jsonPath) { const { total } = store.list({ limit: 1 }); if (total > 0) return { imported: 0, skipped: true }; return importLegacyJsonCache(store, jsonPath); }, checkpoint: () => { db.exec("PRAGMA wal_checkpoint(PASSIVE)"); }, optimize: () => { db.exec("PRAGMA optimize"); }, close: () => { closePromise ??= closeOwnedResources(); return closePromise; }, }; } async function readOperationBody(request: Request): Promise<{ op?: unknown; input?: unknown }> { const declared = Number(request.headers.get("content-length")); if (Number.isFinite(declared) && declared > SERVICE_MAX_BODY_BYTES) { throw new PayloadTooLargeError(`request exceeds ${SERVICE_MAX_BODY_BYTES} bytes`); } if (!request.body) return {}; const reader = request.body.getReader(); const chunks: Uint8Array[] = []; let size = 0; for (;;) { const { done, value } = await reader.read(); if (done) break; size += value.byteLength; if (size > SERVICE_MAX_BODY_BYTES) { await reader.cancel(); throw new PayloadTooLargeError(`request exceeds ${SERVICE_MAX_BODY_BYTES} bytes`); } chunks.push(value); } const bytes = new Uint8Array(size); let offset = 0; for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } return JSON.parse(new TextDecoder().decode(bytes)) as { op?: unknown; input?: unknown }; } export function createApp( deps: { service: WebSpiderService; token: string }, options: { logger?: Logger } = {}, ): { fetch(request: Request): Promise } { // A real second transport for whatever operations createCategoryVehicleRegistry // has migrated so far -- composed here rather than replacing /api/v1/ops, so the // rest of this daemon's operations keep working unchanged while the migration // proceeds operation-by-operation (see handlers/category.ts's own doc comment). const vehicleApp = createVehicleHttpApp({ registry: deps.service.vehicleRegistry, token: deps.token, logger: options.logger }); return { async fetch(request: Request): Promise { const url = new URL(request.url); if (url.pathname.startsWith("/vehicle/")) return vehicleApp.fetch(request); if (!requireBearerToken(request, deps.token)) { return errorResponse("missing or invalid bearer token", 401); } if (request.method === "GET" && url.pathname === "/health") { return healthResponse(VERSION, { schema: deps.service.schemaState() }); } if (request.method === "GET" && url.pathname === "/ready") { return readyResponse(true); } if (request.method === "GET" && url.pathname === "/api/v1/ops") { return jsonResponse({ operations: deps.service.operationNames() }); } if (request.method === "POST" && url.pathname === "/api/v1/ops") { try { const body = await readOperationBody(request); if (typeof body.op !== "string") return errorResponse("op is required", 400); const input = body.input === undefined ? {} : body.input; if (typeof input !== "object" || input === null || Array.isArray(input)) { return errorResponse("input must be an object", 400); } return jsonResponse({ result: await deps.service.execute(body.op, input as OperationInput) }); } catch (error) { const status = error instanceof PayloadTooLargeError ? 413 : error instanceof UnknownOperationError || error instanceof SessionNotFoundError ? 404 : error instanceof StaleSnapshotError ? 409 : 400; return errorResponse(error instanceof Error ? error.message : String(error), status); } } return errorResponse("not found", 404); }, }; }