import crypto from 'node:crypto'; import type { RawElement } from '../screen-reader.js'; export interface CrawlEvidenceFact { type: string; payload: unknown; } export interface CrawlArtifactEnvelope { tenantId: string; projectId: string; jobId?: string; url: string; canonicalUrl: string; screenId?: string; status: 'ok' | 'blocked' | 'auth_expired' | 'error' | 'skipped'; outputs: { screenshot?: string; html?: string; markdown?: string; fitMarkdown?: string; accessibilityTree?: string; structuredJson?: unknown; networkTrace?: string; consoleLog?: string; }; fingerprints: { textHash?: string; domHash?: string; accessibilityHash?: string; urlCanonicalHash: string; }; evidence: CrawlEvidenceFact[]; } export function sha256(input: string): string { return crypto.createHash('sha256').update(input).digest('hex'); } export function buildFitMarkdown(markdown: string, keywords: string[] = []): string { const lines = markdown .split('\n') .map((line) => line.trimEnd()) .filter((line) => line.trim().length > 0); const lowerKeywords = keywords.map((k) => k.toLowerCase()).filter(Boolean); const filtered = lines.filter((line) => { const plain = line.replace(/[#*_`>\-[\]()]/g, '').trim(); if (plain.length < 3) return false; if (/^(cookie|privacy|terms|copyright|all rights reserved|subscribe|newsletter)$/i.test(plain)) return false; if (lowerKeywords.length === 0) return true; const lower = plain.toLowerCase(); return lowerKeywords.some((kw) => lower.includes(kw)) || /^#{1,4}\s/.test(line); }); const candidate = filtered.join('\n'); return candidate.length >= 200 ? candidate : markdown; } export function summarizeStructuredElements(elements: RawElement[]) { const forms = elements.filter((e) => ['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role)); const actions = elements.filter((e) => e.role === 'button' || e.role === 'submit' || e.role === 'link' || e.role === 'a'); return { elementCount: elements.length, formFieldCount: forms.length, actionCount: actions.length, fields: forms.slice(0, 50).map((e) => ({ label: e.label, role: e.role, expectedData: e.expectedData })), actions: actions.slice(0, 50).map((e) => ({ label: e.label, role: e.role })), }; } export function buildCrawlArtifactEnvelope(input: { tenantId: string; projectId: string; jobId?: string; url: string; canonicalUrl: string; screenId?: string; screenshotPath?: string; domPath?: string; domContent?: string; markdown?: string; fitMarkdown?: string; accessibilityElements?: RawElement[]; structuredJson?: unknown; evidence?: CrawlEvidenceFact[]; }): CrawlArtifactEnvelope { const accessibilityTree = input.accessibilityElements ? JSON.stringify(input.accessibilityElements) : undefined; return { tenantId: input.tenantId, projectId: input.projectId, jobId: input.jobId, url: input.url, canonicalUrl: input.canonicalUrl, screenId: input.screenId, status: 'ok', outputs: { screenshot: input.screenshotPath, html: input.domPath, markdown: input.markdown, fitMarkdown: input.fitMarkdown, accessibilityTree, structuredJson: input.structuredJson, }, fingerprints: { textHash: input.markdown ? sha256(input.markdown) : undefined, domHash: input.domContent ? sha256(input.domContent) : undefined, accessibilityHash: accessibilityTree ? sha256(accessibilityTree) : undefined, urlCanonicalHash: sha256(input.canonicalUrl), }, evidence: input.evidence ?? [], }; }