import { chromium } from 'playwright'; import { extractReadableContentSafely } from '../extract/readability.js'; import { resolveBrowserExecutable, type BrowserResolutionResult } from './browser-resolution.js'; import type { WebFetchHeadlessResponse } from '../types.js'; function cleanupRenderedText(text: string): string { let cleaned = text.replace(/(Show more)(\s+\1){1,}/gi, '$1'); cleaned = cleaned.replace(/(Privacy Terms)(\s+\1){1,}/gi, '$1'); cleaned = cleaned.replace(/\s+/g, ' ').trim(); return cleaned; } export async function headlessFetch( url: string, { configuredPath, resolveBrowser = (options?: { configuredPath?: string }) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless }: { executablePath?: string; headless: true }) => chromium.launch(executablePath ? { executablePath, headless } : { headless }), now = () => Date.now() }: { configuredPath?: string; resolveBrowser?: (options?: { configuredPath?: string }) => Promise; launchBrowser?: (options: { executablePath?: string; headless: true }) => Promise<{ newContext: () => Promise<{ newPage: () => Promise; close: () => Promise }>; close: () => Promise; }>; now?: () => number; } = {} ): Promise { const resolved = await resolveBrowser({ configuredPath }); if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') { return { status: 'error', url, metadata: { method: 'headless', cacheHit: false }, error: resolved.error }; } const browserName = resolved.ok ? resolved.browser : 'chromium'; const launchOptions = resolved.ok ? { executablePath: resolved.executablePath, headless: true as const } : { headless: true as const }; let browser: Awaited> | undefined; let context: Awaited>['newContext']>> | undefined; let page: Awaited>['newContext']>>['newPage']>> | undefined; try { browser = await launchBrowser(launchOptions); context = await browser.newContext(); page = await context.newPage(); const startedAt = now(); await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 }); await page.waitForLoadState('load', { timeout: 10000 }); await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined); const html = await page.content(); const finishedAt = now(); const extraction = extractReadableContentSafely(html); const cleanedContent = { ...extraction.content, text: cleanupRenderedText(extraction.content.text) }; if (!cleanedContent.text || cleanedContent.text.length < 40) { return { status: 'blocked', url, metadata: { method: 'headless', cacheHit: false, browser: browserName, navigationMs: finishedAt - startedAt }, error: { code: 'HEADLESS_EXTRACTION_WEAK', message: 'Rendered page did not produce enough readable content.' } }; } return { status: 'ok', url, content: cleanedContent, metadata: { method: 'headless', cacheHit: false, browser: browserName, navigationMs: finishedAt - startedAt, truncated: cleanedContent.text.length >= 4000 } }; } catch (error) { return { status: 'error', url, metadata: { method: 'headless', cacheHit: false, browser: browserName }, error: { code: 'HEADLESS_NAVIGATION_FAILED', message: error instanceof Error ? error.message : 'Unknown headless navigation failure.' } }; } finally { await page?.close?.().catch(() => undefined); await context?.close?.().catch(() => undefined); await browser?.close?.().catch(() => undefined); } }