/** * Clone Architect — Script d'extraction Playwright * * Extrait le design RÉEL d'un site via getComputedStyle() : * - Screenshots multi-viewport (desktop 1440px + mobile 390px) * - CSS computed sur tous les éléments clés * - CSS custom properties (--color-*, --font-*, etc.) * - Couleurs dominantes, typo, spacing, layout * - Structure DOM (sections, composants, navigation) */ import { chromium, type Page, type Browser } from 'playwright'; import { mkdir, writeFile } from 'fs/promises'; import { join } from 'path'; // ── Types ──────────────────────────────────────────────────────────── interface ComputedStyles { backgroundColor: string; backgroundImage: string; color: string; fontFamily: string; fontSize: string; fontWeight: string; lineHeight: string; letterSpacing: string; textAlign: string; padding: string; margin: string; borderRadius: string; border: string; boxShadow: string; width: string; height: string; minHeight: string; maxWidth: string; display: string; gap: string; gridTemplateColumns: string; flexDirection: string; alignItems: string; justifyContent: string; position: string; overflow: string; transition: string; opacity: string; textTransform: string; textDecoration: string; fontFeatureSettings: string; fontVariationSettings: string; } interface ElementExtraction { selector: string; tag: string; classes: string[]; text: string; styles: ComputedStyles; children: number; rect: { x: number; y: number; width: number; height: number }; } interface SectionExtraction { index: number; tag: string; classes: string[]; role: string; estimatedPurpose: string; rect: { x: number; y: number; width: number; height: number }; styles: Partial | Record; childCount: number; bgTreatment?: string; isDark?: boolean; aboveFold?: boolean; isFullBleed?: boolean; imgRatio?: number; maxHeadingPx?: number; gridCols?: number; hasAnimation?: boolean; textLen?: number; } interface ComponentVariant { tag: string; classes: string[]; text: string; styles: ComputedStyles; rect: { x: number; y: number; width: number; height: number }; } interface FontFaceDeclaration { family: string; src: string; weight: string; style: string; display: string; } interface ExtractionResult { url: string; domain: string; timestamp: string; viewport: { width: number; height: number }; pageTitle: string; cssCustomProperties: Record; elements: Record; sections: SectionExtraction[]; allColors: string[]; allFontFamilies: string[]; allFontSizes: string[]; allBorderRadii: string[]; allShadows: string[]; allTransitions: string[]; images: { src: string; alt: string; width: number; height: number }[]; widgets?: WidgetExtraction; imageryProfile?: { ogImage: string | null; ogImageWidth: number | null; ogImageHeight: number | null; twitterImage: string | null; heroImage: { src: string; alt: string; width: number; height: number; aspectRatio: number } | null; formats: { png: number; jpg: number; webp: number; svg: number; gif: number; other: number }; totalImages: number; totalAboveFold: number; aspectRatioBuckets: { landscape: number; portrait: number; square: number; ultrawide: number }; illustrationHeavy: boolean; photoHeavy: boolean; avgImageSize: { width: number; height: number }; decorativePatterns?: { multiStopGradients: number; radialGradients: number; largeSvgShapes: number; backgroundImagePatterns: number; hasNoise: boolean; hasGlassmorphism: boolean; }; }; links: { href: string; text: string; isNav: boolean }[]; componentVariants: Record; componentStates: Record; fontFaces: FontFaceDeclaration[]; mediaBreakpoints: string[]; openTypeFeatures: string[]; variableAxes: string[]; displaySignature?: { family: string; fontSize: string; fontWeight: string; isSerif: boolean; isItalic: boolean; sample: string }; // Phase 5 Sprint 80/20 — advanced capture (kept: has downstream consumers) keyframes?: Record>>; zIndexMap?: Array<{ selector: string; z: number; stackingRoot: string }>; // RFC C/D/F/A/B fields removed in v2.4 (Phase 1.2) — zero downstream consumers detected: // transform3DMap, containerQueries, containerTypes, gridLayouts, responsiveSnapshot, pseudoElements // If needed, re-add with explicit consumer; YAGNI removed speculative extraction (saves ~50KB/site, ~10s extract time). } // ── Config ─────────────────────────────────────────────────────────── const VIEWPORTS = { desktop: { width: 1440, height: 900 }, mobile: { width: 390, height: 844 }, } as const; // Phase 5.1.1 — Stricter selectors. AVANT: [class*="header"] matchait // → headerHeight = 12802px (toute la page) sur Attio, Cursor, Airbnb. // APRÈS: tag-first selectors, plus ARIA roles, plus class-strict. const KEY_SELECTORS: Record = { body: 'body', header: 'header, [role="banner"], [data-testid*="header"]:not(body):not(html)', nav: 'nav, [role="navigation"]', main: 'main, [role="main"]', sidebar: 'aside, [role="complementary"]', footer: 'footer, [role="contentinfo"]', hero: 'section[class*="hero" i]:not(body), [class*="hero-section" i]:not(body), [data-section="hero"]', card: 'article, [class*="card" i]:not(body):not(html):not(main):not(section)', button: 'button, [role="button"]:not(body):not(html)', input: 'input[type="text"], input[type="search"], input[type="email"], textarea', heading: 'h1', subheading: 'h2', link: 'a:not([class*="btn"]):not([class*="button"])', badge: '[class*="badge" i]:not(body), [class*="tag" i]:not(body), [class*="chip" i]:not(body)', modal: '[class*="modal" i]:not(body), [role="dialog"]', dropdown: '[role="menu"], [class*="dropdown" i]:not(body)', avatar: 'img[class*="avatar" i], img[class*="profile" i]', logo: 'header a img, header a svg, nav a img, nav a svg, [class*="logo" i]:not(body):not(html)', }; // ── Extraction functions ───────────────────────────────────────────── async function extractComputedStyles(page: Page): Promise> { return page.evaluate((selectors: Record) => { const result: Record = {}; for (const [name, selector] of Object.entries(selectors)) { // For heading selectors (h1/h2), prefer the first VISIBLE element in DOM // (SPAs often have a hidden SEO h1 before the visible one) let el: Element | null = null; const isHeading = /^h[1-6]$/.test(selector.trim()); if (isHeading) { const candidates = Array.from(document.querySelectorAll(selector)); el = candidates.find(c => { const r = c.getBoundingClientRect(); const cs = getComputedStyle(c); return r.width > 0 && r.height > 0 && cs.display !== 'none' && cs.visibility !== 'hidden' && cs.opacity !== '0'; }) || candidates[0] || null; } else { el = document.querySelector(selector); } if (!el) { result[name] = null; continue; } const cs = getComputedStyle(el); const rect = el.getBoundingClientRect(); result[name] = { tag: el.tagName.toLowerCase(), classes: Array.from(el.classList), text: (el as HTMLElement).innerText?.slice(0, 200) || '', ariaLabel: (el as HTMLElement).getAttribute('aria-label') || undefined, dataTestId: (el as HTMLElement).getAttribute('data-testid') || undefined, role: (el as HTMLElement).getAttribute('role') || undefined, children: el.children.length, rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, styles: { backgroundColor: cs.backgroundColor, backgroundImage: cs.backgroundImage, color: cs.color, fontFamily: cs.fontFamily, fontSize: cs.fontSize, fontWeight: cs.fontWeight, lineHeight: cs.lineHeight, letterSpacing: cs.letterSpacing, textAlign: cs.textAlign, padding: cs.padding, margin: cs.margin, borderRadius: cs.borderRadius, border: cs.border, boxShadow: cs.boxShadow, width: cs.width, height: cs.height, minHeight: cs.minHeight, maxWidth: cs.maxWidth, display: cs.display, gap: cs.gap, gridTemplateColumns: cs.gridTemplateColumns, flexDirection: cs.flexDirection, alignItems: cs.alignItems, justifyContent: cs.justifyContent, position: cs.position, overflow: cs.overflow, transition: cs.transition, opacity: cs.opacity, textTransform: cs.textTransform, textDecoration: cs.textDecoration, fontFeatureSettings: cs.fontFeatureSettings, fontVariationSettings: (cs as any).fontVariationSettings || 'normal', }, }; } return result; }, KEY_SELECTORS); } async function extractCSSCustomProperties(page: Page): Promise> { return page.evaluate(() => { const vars: Record = {}; // From :root / html const rootStyles = getComputedStyle(document.documentElement); for (let i = 0; i < rootStyles.length; i++) { const prop = rootStyles[i]; if (prop.startsWith('--')) { vars[prop] = rootStyles.getPropertyValue(prop).trim(); } } // v4-V1-T5: Also scan body + main sections for scoped CSS vars // (Tailwind/shadcn often define --vars on .dark, body, or section scopes) const scopedRoots = [ document.body, document.querySelector('main'), document.querySelector('[data-theme]'), document.querySelector('.dark, [class*="dark"]'), document.querySelector('header, nav'), ].filter(Boolean) as HTMLElement[]; scopedRoots.forEach(el => { const sc = getComputedStyle(el); for (let i = 0; i < sc.length; i++) { const prop = sc[i]; if (prop.startsWith('--') && !(prop in vars)) { vars[prop] = sc.getPropertyValue(prop).trim(); } } }); // Also check stylesheets for CSS variables (extended selectors) try { for (const sheet of document.styleSheets) { try { for (const rule of sheet.cssRules) { if (rule instanceof CSSStyleRule && ( rule.selectorText === ':root' || rule.selectorText === 'html' || rule.selectorText === 'body' || rule.selectorText === '*' || /\.dark\b/.test(rule.selectorText) || /\[data-theme/.test(rule.selectorText) )) { for (let i = 0; i < rule.style.length; i++) { const prop = rule.style[i]; if (prop.startsWith('--') && !(prop in vars)) { vars[prop] = rule.style.getPropertyValue(prop).trim(); } } } } } catch { /* cross-origin stylesheet, skip */ } } } catch { /* no stylesheets accessible */ } return vars; }); } async function extractAllColors(page: Page): Promise { return page.evaluate(() => { const colorSet = new Set(); const MAX = 3000; const all = Array.from(document.querySelectorAll('*')); // Prioritize visible elements, fill remaining with rest — up to MAX const visible = all.filter(el => { const r = (el as HTMLElement).getBoundingClientRect(); return r.width > 0 && r.height > 0; }); const sample = visible.length >= MAX ? visible.slice(0, MAX) : [...visible, ...all.filter(el => !visible.includes(el))].slice(0, MAX); const props = ['backgroundColor', 'color', 'borderColor', 'borderTopColor', 'outlineColor', 'caretColor', 'columnRuleColor', 'fill', 'stroke'] as const; // Phase 5.1.1 — Inline multi-value parsing (avoids tsx __name emission inside page.evaluate) // borderColor can be "rgb(A) rgb(B) rgb(C) rgb(D)" when sides differ. If mixed, drop entirely. const COLOR_REGEX = /rgba?\([^)]+\)|#[0-9a-fA-F]{3,8}/g; sample.forEach(el => { const cs = getComputedStyle(el); props.forEach(prop => { const val = cs[prop as keyof CSSStyleDeclaration] as string; if (!val || val === 'rgba(0, 0, 0, 0)' || val === 'transparent') return; // Parse multi-color inline const matches = val.match(COLOR_REGEX) || []; if (matches.length === 0) { if (val.startsWith('#') || val.startsWith('rgb')) colorSet.add(val); } else if (matches.length === 1) { colorSet.add(matches[0]); } else { const uniq = Array.from(new Set(matches)); if (uniq.length === 1) colorSet.add(uniq[0]); // Mixed-side borderColor → drop (no canonical color) } }); ['::before', '::after'].forEach(pseudo => { const pcs = getComputedStyle(el, pseudo); if (pcs.content && pcs.content !== 'none' && pcs.content !== 'normal') { props.forEach(prop => { const val = pcs[prop as keyof CSSStyleDeclaration] as string; if (!val || val === 'rgba(0, 0, 0, 0)' || val === 'transparent') return; const matches = val.match(COLOR_REGEX) || []; if (matches.length === 0) { if (val.startsWith('#') || val.startsWith('rgb')) colorSet.add(val); } else if (matches.length === 1) { colorSet.add(matches[0]); } else { const uniq = Array.from(new Set(matches)); if (uniq.length === 1) colorSet.add(uniq[0]); } }); } }); }); return [...colorSet]; }); } /** * Phase 5.1.3 — Detect anti-bot challenge pages BEFORE extraction. * Cloudflare, captchas, "Just a moment", and similar interstitials would otherwise * be extracted as if they were the real site, producing nonsense tokens. */ async function detectBotChallenge(page: Page): Promise<{ blocked: boolean; reason?: string }> { return page.evaluate(() => { const title = (document.title || '').toLowerCase(); const bodyText = (document.body?.textContent || '').slice(0, 500).toLowerCase(); const html = (document.documentElement?.outerHTML || '').slice(0, 2000).toLowerCase(); const SIGNATURES = [ { pattern: /just a moment\.\.\.|checking your browser/i, reason: 'Cloudflare interstitial' }, { pattern: /verify you are human|are you a robot/i, reason: 'Human verification challenge' }, { pattern: /captcha|recaptcha|hcaptcha/i, reason: 'CAPTCHA detected' }, { pattern: /access denied|forbidden|403/i, reason: 'Access denied (403)' }, { pattern: /enable javascript and cookies|please enable cookies/i, reason: 'JS/cookies required gate' }, { pattern: /ddos protection|protection by/i, reason: 'DDoS protection page' }, ]; for (const sig of SIGNATURES) { if (sig.pattern.test(title) || sig.pattern.test(bodyText) || sig.pattern.test(html.slice(0, 1000))) { return { blocked: true, reason: sig.reason }; } } // Heuristic: page with <100 chars of visible text AND <5 elements = suspicious const visibleText = (document.body?.innerText || '').trim(); const elementCount = document.body?.children?.length || 0; if (visibleText.length < 100 && elementCount < 5) { return { blocked: true, reason: `Suspicious empty page (${visibleText.length} chars, ${elementCount} elements)` }; } return { blocked: false }; }); } async function extractAllFonts(page: Page): Promise<{ families: string[]; sizes: string[] }> { return page.evaluate(() => { const families = new Set(); const sizes = new Set(); const elements = document.querySelectorAll('*'); const sample = Array.from(elements).slice(0, 3000); sample.forEach(el => { const cs = getComputedStyle(el); families.add(cs.fontFamily); sizes.add(cs.fontSize); }); return { families: [...families], sizes: [...sizes] }; }); } async function extractAllBorderRadii(page: Page): Promise { return page.evaluate(() => { const radii = new Set(); const elements = document.querySelectorAll('*'); const sample = Array.from(elements).slice(0, 3000); sample.forEach(el => { const cs = getComputedStyle(el); if (cs.borderRadius && cs.borderRadius !== '0px') { radii.add(cs.borderRadius); } }); return [...radii]; }); } async function extractAllShadows(page: Page): Promise { return page.evaluate(() => { const shadows = new Set(); const elements = document.querySelectorAll('*'); const sample = Array.from(elements).slice(0, 3000); sample.forEach(el => { const cs = getComputedStyle(el); if (cs.boxShadow && cs.boxShadow !== 'none') { shadows.add(cs.boxShadow); } }); return [...shadows]; }); } async function extractAllTransitions(page: Page): Promise { return page.evaluate(() => { const transitions = new Set(); const elements = document.querySelectorAll('*'); const sample = Array.from(elements).slice(0, 1000); sample.forEach(el => { const cs = getComputedStyle(el); if (cs.transition && cs.transition !== 'all 0s ease 0s' && cs.transition !== 'none 0s ease 0s') { transitions.add(cs.transition); } }); return [...transitions]; }); } // ── Component Variants (all unique styles per component type) ─────── async function extractComponentVariants(page: Page): Promise> { return page.evaluate(() => { const variantSelectors: Record = { // ── Core interactive ── // v4-V1-T1: Extended button selector covers Tailwind utility CSS + ARIA + handlers // (sites without semantic .btn class like utility-first frameworks now detected) buttons: 'button, [class*="btn"], [role="button"], a[class*="button"], input[type="submit"], input[type="button"], a[class*="bg-"][href], a[class*="-cta"], button[class*="bg-"], [onclick]:not(div):not(span):not(li), [data-action="button"], a[aria-label][href][class*="px-"]', inputs: 'input, textarea, select', searchBar: '[role="search"], [class*="search-bar"], [class*="searchbar"], [class*="search-form"]', // ── Content blocks ── cards: '[class*="card"], article, [class*="tile"]', badges: '[class*="badge"], [class*="tag"], [class*="chip"], [class*="label"]:not(label)', statusBadge: '[class*="status"], [class*="pill"], [class*="indicator"], [class*="dot"][class*="color"], [data-status]', // ── Navigation & structure ── navLinks: 'nav a, [class*="nav"] a, [class*="menu"] a, [role="navigation"] a, header a', tabs: '[role="tablist"], [role="tab"], [class*="tabs"], [class*="tab-bar"], [class*="tab-nav"]', footerLinks: 'footer a, [role="contentinfo"] a', // ── Marketing sections ── pricingCard: '[class*="pricing"], [class*="price-card"], [class*="plan-card"], [class*="plan-tile"], [class*="tier"]', ctaBanner: '[class*="cta"], [class*="call-to-action"], [class*="banner-cta"], [class*="promo-banner"]', testimonial: '[class*="testimonial"], [class*="review-card"], [class*="quote-block"], [class*="customer-story"]', logoTile: '[class*="logo-grid"] img, [class*="logos"] img, [class*="customer-logo"], [class*="partner-logo"], [class*="brand-logo"]', // ── Typography roles ── headingH1: 'h1', headingH2: 'h2', headingH3: 'h3', headingH4: 'h4', headingH5: 'h5', headingH6: 'h6', links: 'a', eyebrowLabels: '[class*="eyebrow"], [class*="overline"], [class*="kicker"], [class*="label--small"], [class*="meta"], [class*="subtitle"]', captions: 'figcaption, caption, [class*="caption"], [class*="helper-text"], [class*="supporting"]', tableHeaders: 'th, thead td', // ── Misc UI ── avatar: '[class*="avatar"], [class*="profile-pic"], [class*="user-pic"]', divider: 'hr, [class*="divider"], [class*="separator"]', tooltip: '[role="tooltip"], [class*="tooltip"]', }; // Site-specific selectors — only injected when found in DOM (avoids noise on non-matching sites) const siteSpecificSelectors: Record = { // Travel / booking datePicker: '[class*="date-picker"], [class*="datepicker"], [class*="calendar"], [data-testid*="date"]', reservationCard: '[class*="reservation"], [class*="booking"], [class*="checkout-panel"]', propertyCard: '[class*="property-card"], [class*="listing-card"], [class*="stay-card"]', ratingDisplay: '[class*="rating"], [class*="review-score"], [class*="star-rating"]', hostCard: '[class*="host-card"], [class*="host-info"], [class*="profile-card"]', // SaaS / dev tools codeBlock: 'pre, code, [class*="code-block"], [class*="syntax"], [class*="prism"]', changelogRow: '[class*="changelog"], [class*="release"], [class*="release-note"], [class*="version-row"]', breadcrumb: '[aria-label="breadcrumb"], [class*="breadcrumb"], nav[class*="crumb"]', alert: '[role="alert"], [class*="alert"], [class*="notification-bar"], [class*="toast"]', commandPalette: '[class*="command-palette"], [class*="CommandPalette"], [class*="cmdk-root"], [role="dialog"][class*="search"], [class*="CommandDialog"]', dataTable: 'table[class*="data"], [class*="data-table"], [class*="DataTable"], [class*="DataGrid"], [role="grid"]', accordion: '[class*="accordion"], [class*="Accordion"], details:has(summary), [role="region"][class*="collapsible"]', skeleton: '[class*="skeleton"], [class*="Skeleton"], [class*="shimmer"], [class*="loading-placeholder"]', progressBar: '[role="progressbar"], [class*="progress-bar"], [class*="ProgressBar"], [class*="progress-track"]', emptyState: '[class*="empty-state"], [class*="EmptyState"], [class*="empty-placeholder"], [class*="no-results"]', kpiCard: '[class*="metric-card"], [class*="stat-card"], [class*="KpiCard"], [class*="StatCard"], [class*="stats-card"]', timelinePill: '[class*="timeline"], [class*="Timeline"], [class*="agent-trace"], [class*="AgentTrace"]', // Ecom productCard: '[class*="product-card"], [class*="product-tile"], [class*="sku-card"]', priceTag: '[class*="price"], [class*="cost"], [itemprop="price"]', }; for (const [name, sel] of Object.entries(siteSpecificSelectors)) { if (document.querySelectorAll(sel).length > 0) (variantSelectors as Record)[name] = sel; } const result: Record = {}; for (const [name, selector] of Object.entries(variantSelectors)) { const elems = document.querySelectorAll(selector); const seen = new Set(); const variants: any[] = []; const els = Array.from(elems).slice(0, 50); for (const el of els) { const rect = el.getBoundingClientRect(); if (rect.width < 5 || rect.height < 5) continue; const cs = getComputedStyle(el); // Inline fingerprint const fp = [ cs.backgroundColor, cs.color, cs.fontSize, cs.fontWeight, cs.fontFamily, cs.borderRadius, cs.padding, cs.border, cs.boxShadow, cs.display, cs.height, ].join('|'); if (seen.has(fp)) continue; seen.add(fp); // Inline extractStyles variants.push({ tag: el.tagName.toLowerCase(), classes: Array.from(el.classList), text: (el as HTMLElement).innerText?.trim().slice(0, 100) || '', rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, styles: { backgroundColor: cs.backgroundColor, color: cs.color, fontFamily: cs.fontFamily, fontSize: cs.fontSize, fontWeight: cs.fontWeight, lineHeight: cs.lineHeight, letterSpacing: cs.letterSpacing, padding: cs.padding, margin: cs.margin, borderRadius: cs.borderRadius, border: cs.border, boxShadow: cs.boxShadow, width: cs.width, height: cs.height, maxWidth: cs.maxWidth, display: cs.display, gap: cs.gap, gridTemplateColumns: cs.gridTemplateColumns, flexDirection: cs.flexDirection, alignItems: cs.alignItems, justifyContent: cs.justifyContent, position: cs.position, overflow: cs.overflow, transition: cs.transition, opacity: cs.opacity, textTransform: cs.textTransform, textDecoration: cs.textDecoration, fontFeatureSettings: cs.fontFeatureSettings, fontVariationSettings: (cs as any).fontVariationSettings || 'normal', }, }); if (variants.length >= 10) break; } if (variants.length > 0) { result[name] = variants; } } return result; }); } // ── Component States (:hover, :focus) ────────────────────── // Note: `:active` not supported — UA state unreachable via Playwright dispatchEvent interface ComponentStateStyles { default: Record; hover?: Record; focus?: Record; } const STATE_PROPS = [ 'backgroundColor', 'color', 'border', 'borderColor', 'boxShadow', 'opacity', 'transform', 'outline', 'textDecoration', 'cursor', ] as const; async function extractComponentStates(page: Page): Promise> { const result: Record = {}; // Phase 5 Sprint 80/20 — `:active` retiré (dispatchEvent('mousedown') ne déclenche // PAS l'état UA `:active` — les données étaient identiques au default, inutile. // Voir audit du 2026-04-18. const targets: { name: string; selector: string; states: ('hover' | 'focus')[] }[] = [ { name: 'button', selector: 'button:not([disabled]), [class*="btn"]:not([disabled]), a[class*="button"]', states: ['hover', 'focus'] }, { name: 'input', selector: 'input[type="text"], input[type="search"], input[type="email"]', states: ['focus'] }, { name: 'link', selector: 'a:not([class*="btn"]):not([class*="button"])', states: ['hover'] }, { name: 'card', selector: '[class*="card"]:not(body), article, [class*="tile"]', states: ['hover'] }, { name: 'navLink', selector: 'nav a, header a, [role="navigation"] a', states: ['hover'] }, { name: 'tab', selector: '[role="tab"], [class*="tab-item"], [class*="tab-link"]', states: ['hover'] }, { name: 'badge', selector: '[class*="badge"], [class*="tag"], [class*="chip"]', states: ['hover'] }, { name: 'footerLink', selector: 'footer a, [role="contentinfo"] a', states: ['hover'] }, ]; for (const target of targets) { try { // Get default styles first const defaultStyles = await page.evaluate((selector) => { const el = document.querySelector(selector); if (!el) return null; const cs = getComputedStyle(el); const props = ['backgroundColor', 'color', 'border', 'borderColor', 'boxShadow', 'opacity', 'transform', 'outline', 'textDecoration', 'cursor']; const result: Record = {}; for (const p of props) result[p] = (cs as any)[p]; return result; }, target.selector); if (!defaultStyles) continue; const stateStyles: ComponentStateStyles = { default: defaultStyles }; for (const state of target.states) { try { let stateResult: Record | null = null; if (state === 'hover') { // Use Playwright native hover await page.hover(target.selector, { timeout: 2000 }); stateResult = await page.evaluate((selector) => { const el = document.querySelector(selector); if (!el) return null; const cs = getComputedStyle(el); const props = ['backgroundColor', 'color', 'border', 'borderColor', 'boxShadow', 'opacity', 'transform', 'outline', 'textDecoration', 'cursor']; const result: Record = {}; for (const p of props) result[p] = (cs as any)[p]; return result; }, target.selector); // Move away to reset await page.mouse.move(0, 0); } else if (state === 'focus') { await page.focus(target.selector); stateResult = await page.evaluate((selector) => { const el = document.querySelector(selector) as HTMLElement; if (!el) return null; el.focus(); const cs = getComputedStyle(el); const props = ['backgroundColor', 'color', 'border', 'borderColor', 'boxShadow', 'opacity', 'transform', 'outline', 'textDecoration', 'cursor']; const result: Record = {}; for (const p of props) result[p] = (cs as any)[p]; return result; }, target.selector); await page.evaluate(() => { (document.activeElement as HTMLElement)?.blur?.(); }); } // Note: `:active` state intentionally not extracted — requires real pointer // events not available via Playwright (dispatchEvent doesn't trigger UA state). if (stateResult) { // Only keep properties that CHANGED vs default const changed: Record = {}; for (const [k, v] of Object.entries(stateResult)) { if (v !== defaultStyles[k]) changed[k] = v; } if (Object.keys(changed).length > 0) { stateStyles[state] = stateResult; } } } catch (err) { // State extraction failed — site may block interaction (CSP, intercepted event) if (process.env.CLONE_LOG_LEVEL === 'debug') { console.log(` ⚠️ ${target.name}:${state} failed: ${(err as Error).message.slice(0, 80)}`); } } } result[target.name] = stateStyles; } catch (err) { // Element not found for target.selector — common, site may not have this component if (process.env.CLONE_LOG_LEVEL === 'debug') { console.log(` ⚠️ ${target.name} skipped: ${(err as Error).message.slice(0, 80)}`); } } } return result; } // ── OpenType Features & Variable Font Axes ────────────────────────── async function extractOpenTypeFeatures(page: Page): Promise<{ features: string[]; axes: string[] }> { return page.evaluate(() => { const featureSet = new Set(); const axisSet = new Set(); // Scan all elements for font-feature-settings and font-variation-settings const elements = Array.from(document.querySelectorAll('*')); for (const el of elements) { const cs = getComputedStyle(el); const featureVal = cs.fontFeatureSettings; const variationVal = (cs as any).fontVariationSettings; if (featureVal && featureVal !== 'normal') { // Parse "ss01" on, "kern" 1, "liga" 0 etc. const matches = featureVal.match(/"([a-z0-9]{4})"/gi); if (matches) { for (const m of matches) featureSet.add(m.replace(/"/g, '').toLowerCase()); } } if (variationVal && variationVal !== 'normal') { // Parse "wght" 400, "wdth" 75 etc. const matches = variationVal.match(/"([A-Z]{4})"/gi); if (matches) { for (const m of matches) axisSet.add(m.replace(/"/g, '').toUpperCase()); } } } // Also scan CSS rules in stylesheets try { for (const sheet of Array.from(document.styleSheets)) { try { const rules = Array.from(sheet.cssRules || []); for (const rule of rules) { const text = (rule as CSSStyleRule).cssText || ''; const featureMatches = text.match(/font-feature-settings\s*:\s*([^;]+)/gi); if (featureMatches) { for (const fm of featureMatches) { const tags = fm.match(/"([a-z0-9]{4})"/gi); if (tags) tags.forEach(t => featureSet.add(t.replace(/"/g, '').toLowerCase())); } } const variationMatches = text.match(/font-variation-settings\s*:\s*([^;]+)/gi); if (variationMatches) { for (const vm of variationMatches) { const tags = vm.match(/"([A-Z]{4})"/gi); if (tags) tags.forEach(t => axisSet.add(t.replace(/"/g, '').toUpperCase())); } } } } catch { /* cross-origin sheet */ } } } catch { /* security error */ } return { features: Array.from(featureSet), axes: Array.from(axisSet), }; }); } // ── Font-face declarations ────────────────────────────────────────── async function extractFontFaces(page: Page): Promise { return page.evaluate(() => { const fonts: Array<{ family: string; src: string; weight: string; style: string; display: string }> = []; const seen = new Set(); // Method 1: document.fonts API (if available) try { if ('fonts' in document) { (document as any).fonts.forEach((font: FontFace) => { const key = `${font.family}|${font.weight}|${font.style}`; if (seen.has(key)) return; seen.add(key); // font.src can be a URL string or local() reference let src = ''; try { src = (font as any).src || ''; // Clean up the src — it's often a CSS url() value if (typeof src === 'string') { const urlMatch = src.match(/url\(["']?([^"')]+)["']?\)/); if (urlMatch) src = urlMatch[1]; } } catch { /* ignore */ } fonts.push({ family: font.family.replace(/['"]/g, ''), src, weight: font.weight || 'normal', style: font.style || 'normal', display: (font as any).display || 'auto', }); }); } } catch { /* fonts API not supported */ } // Method 2: Parse @font-face rules from stylesheets try { for (const sheet of document.styleSheets) { try { for (const rule of sheet.cssRules) { if (rule instanceof CSSFontFaceRule) { const family = rule.style.getPropertyValue('font-family').replace(/['"]/g, '').trim(); const src = rule.style.getPropertyValue('src'); const weight = rule.style.getPropertyValue('font-weight') || 'normal'; const style = rule.style.getPropertyValue('font-style') || 'normal'; const display = rule.style.getPropertyValue('font-display') || 'auto'; const key = `${family}|${weight}|${style}`; if (seen.has(key)) continue; seen.add(key); // Extract actual URLs from the src property const urls: string[] = []; const urlMatches = src.matchAll(/url\(["']?([^"')]+)["']?\)/g); for (const m of urlMatches) { urls.push(m[1]); } fonts.push({ family, src: urls.join(', ') || src, weight, style, display, }); } } } catch { /* cross-origin stylesheet, skip */ } } } catch { /* no stylesheets accessible */ } return fonts; }); } // ── Media query breakpoints ───────────────────────────────────────── async function extractMediaBreakpoints(page: Page): Promise { return page.evaluate(() => { const breakpoints = new Set(); // Use a stack instead of recursion to avoid named function declarations try { for (const sheet of document.styleSheets) { try { const ruleStack: CSSRuleList[] = [sheet.cssRules]; while (ruleStack.length > 0) { const rules = ruleStack.pop()!; for (const rule of rules) { if (typeof CSSContainerRule !== 'undefined' && rule instanceof CSSContainerRule) continue; if (rule instanceof CSSMediaRule) { const media = rule.conditionText || rule.media?.mediaText || ''; if (media) { // Only capture viewport width breakpoints (min-width / max-width), not feature queries if (/(min|max)-width\s*:\s*\d/.test(media)) { const matches = media.matchAll(/(\d+(?:\.\d+)?)(px|em|rem)/g); for (const m of matches) { const val = parseFloat(m[1]); const unit = m[2]; // Filter: ignore sub-320px values (print, tiny accessibility queries) const pxVal = unit === 'px' ? val : unit === 'em' ? val * 16 : val * 16; if (pxVal >= 320) breakpoints.add(`${m[1]}${m[2]}`); } breakpoints.add(`@media ${media}`); } } if (rule.cssRules && rule.cssRules.length > 0) { ruleStack.push(rule.cssRules); } } } } } catch { /* cross-origin stylesheet, skip */ } } } catch { /* no stylesheets accessible */ } const numericBPs: string[] = []; const mediaBPs: string[] = []; for (const bp of breakpoints) { if (bp.startsWith('@media')) { mediaBPs.push(bp); } else { numericBPs.push(bp); } } numericBPs.sort((a, b) => parseFloat(a) - parseFloat(b)); // Phase 5.2.3 — STOP synthetic Tailwind defaults. // The previous fallback injected '640px,768px,1024px,1280px' when <2 detected, // presenting Tailwind defaults AS IF they were extracted from the site = factual lie. // If <2 detected, return what we have. Downstream marks "fixed-width / responsive units only". return [...numericBPs, ...mediaBPs]; }); } // ── Sections extraction ───────────────────────────────────────────── // v2.7 A.1 — the hero headline is frequently NOT the largest measured (it's a styled
, // a , or image text), and the §1 narrative was hardcoded to the BODY font — so the single most // recognizable signature (serif vs sans display) was lost on every site. Detect the display face by // VISUAL PROMINENCE: the above-fold text node with the largest fontSize × rendered width, and classify it. async function extractDisplaySignature(page: Page): Promise<{ family: string; fontSize: string; fontWeight: string; isSerif: boolean; isItalic: boolean; sample: string } | null> { return page.evaluate(() => { const vh = window.innerHeight, vw = window.innerWidth; let best: { score: number; family: string; fontSize: string; fontWeight: string; style: string; sample: string } | null = null; const els = document.querySelectorAll('h1,h2,h3,p,a,span,div,[class*="title"],[class*="heading"],[class*="hero"],[class*="headline"]'); els.forEach(el => { const r = el.getBoundingClientRect(); if (r.top > vh * 1.2 || r.top < -80 || r.width < 80 || r.height < 24) return; // above-fold, sizeable // own direct text only (a real headline), not a wrapper of the whole page const own = Array.from(el.childNodes).filter(n => n.nodeType === 3).map(n => n.textContent || '').join('').replace(/\s+/g, ' ').trim(); if (own.length < 3 || own.length > 120) return; const cs = getComputedStyle(el); const fs = parseFloat(cs.fontSize) || 0; if (fs < 22) return; // display scale only const score = fs * Math.min(r.width, vw); if (!best || score > best.score) best = { score, family: cs.fontFamily || '', fontSize: cs.fontSize, fontWeight: cs.fontWeight, style: cs.fontStyle, sample: own.slice(0, 60) }; }); if (!best) return null; const b: { family: string; fontSize: string; fontWeight: string; style: string; sample: string } = best; const first = (b.family.split(',')[0] || '').replace(/["']/g, '').trim(); const firstLow = first.toLowerCase(); const SERIF = ['times', 'georgia', 'garamond', 'playfair', 'canela', 'fraunces', 'tiempos', 'teodor', 'quincy', 'recoleta', 'editorial', 'freight', 'noe', 'reckless', 'domaine', 'ogg', 'signifier', 'lora', 'merriweather', 'source serif', 'dm serif', 'spectral', 'newsreader', 'cormorant', 'gt sectra', 'ppeditorial', 'instrument serif']; const isSerif = SERIF.some(s => firstLow.includes(s)) || (/serif/.test(firstLow) && !/sans/.test(firstLow)); const isItalic = b.style === 'italic' || /italic/.test(firstLow); return { family: first, fontSize: b.fontSize, fontWeight: b.fontWeight, isSerif, isItalic, sample: b.sample }; }); } async function extractSections(page: Page): Promise { return page.evaluate(() => { // Find major page sections const sectionSelectors = [ 'header', 'nav', 'main', 'section', 'aside', 'footer', '[role="banner"]', '[role="navigation"]', '[role="main"]', '[role="complementary"]', '[role="contentinfo"]', 'div[class*="section"]', 'div[class*="container"]', 'div[class*="wrapper"]', 'div[class*="hero"]', ]; const seen = new Set(); const sections: Array<{ index: number; tag: string; classes: string[]; role: string; estimatedPurpose: string; rect: { x: number; y: number; width: number; height: number }; styles: Record; childCount: number; bgTreatment: string; isDark: boolean; aboveFold: boolean; isFullBleed: boolean; imgRatio: number; maxHeadingPx: number; gridCols: number; hasAnimation: boolean; textLen: number; }> = []; // Also get direct children of body that are significant const bodyChildren = (document.body ? Array.from(document.body.children) : []).filter(el => { const rect = el.getBoundingClientRect(); return rect.height > 50 && rect.width > 200; }); const allCandidates = [ ...bodyChildren, ...sectionSelectors.flatMap(s => Array.from(document.querySelectorAll(s))), ]; let idx = 0; for (const el of allCandidates) { if (seen.has(el)) continue; seen.add(el); const rect = el.getBoundingClientRect(); if (rect.height < 30) continue; const cs = getComputedStyle(el); const tag = el.tagName.toLowerCase(); const classes = Array.from(el.classList); const role = el.getAttribute('role') || ''; // v2.7 A.3/A.4 — content-signature classification (not tag/class strings only). // The old classifier defaulted to 'unknown' on Webflow/Shopify/Chakra div-soup, making §13 // a useless stub. We now read real content signals so §13 names real bands. const classStr = classes.join(' ').toLowerCase(); const vw = window.innerWidth, vh = window.innerHeight; const txt = (el.textContent || '').replace(/\s+/g, ' ').trim(); const txtLen = txt.length; const imgEls = el.querySelectorAll('img,picture,svg,video'); let imgArea = 0; imgEls.forEach(im => { const r = im.getBoundingClientRect(); imgArea += Math.max(0, r.width) * Math.max(0, r.height); }); const imgRatio = imgArea / Math.max(1, rect.width * rect.height); let maxHeadingPx = 0; el.querySelectorAll('h1,h2,h3').forEach(h => { const fs = parseFloat(getComputedStyle(h).fontSize) || 0; if (fs > maxHeadingPx) maxHeadingPx = fs; }); const isFullBleed = rect.width >= vw * 0.95; const aboveFold = rect.y < vh * 0.9; const gridCols = cs.gridTemplateColumns && cs.gridTemplateColumns !== 'none' ? cs.gridTemplateColumns.split(' ').filter(Boolean).length : 0; const hasAnimation = !!cs.animationName && cs.animationName !== 'none'; // Background treatment (A.4) — the dominant atmosphere the old extractor was blind to. const bgImg = cs.backgroundImage || 'none'; let bgTreatment = 'flat'; if (bgImg && bgImg !== 'none') { if (/gradient/i.test(bgImg)) bgTreatment = /radial-gradient|conic-gradient/i.test(bgImg) ? 'radial-gradient' : ((bgImg.match(/rgba?\(|#[0-9a-f]{3,8}/gi) || []).length >= 4 ? 'mesh-gradient' : 'linear-gradient'); else if (/url\(/i.test(bgImg)) bgTreatment = 'image'; } const bgM = (cs.backgroundColor || '').match(/(\d+),\s*(\d+),\s*(\d+)/); const isDarkBand = bgM ? (((+bgM[1]) * 0.2126 + (+bgM[2]) * 0.7152 + (+bgM[3]) * 0.0722) / 255 < 0.35) : false; // Classify: structural tags first, then content signature. let purpose = 'unknown'; if (tag === 'header' || role === 'banner' || classStr.includes('header')) purpose = 'header'; else if (tag === 'nav' || role === 'navigation' || classStr.includes('nav')) purpose = 'navigation'; else if (tag === 'footer' || role === 'contentinfo' || classStr.includes('footer')) purpose = 'footer'; else if (tag === 'aside' || role === 'complementary' || classStr.includes('sidebar')) purpose = 'sidebar'; else if (hasAnimation && txtLen > 0 && txtLen < 220 && rect.height < 180 && isFullBleed) purpose = 'marquee'; else if (aboveFold && idx <= 2 && maxHeadingPx >= 30 && isFullBleed) purpose = 'hero'; else if (/pricing|tarif|formule|\bplan/.test(classStr) || (gridCols >= 2 && /€|\$|\/mo\b|\/mois|par mois/i.test(txt.slice(0, 400)))) purpose = 'pricing'; else if (/faq|accordion|question/.test(classStr)) purpose = 'faq'; else if (/testimonial|review|avis|t[ée]moignage/i.test(classStr + ' ' + txt.slice(0, 160))) purpose = 'testimonials'; else if (imgEls.length >= 5 && rect.height < 220 && maxHeadingPx < 24) purpose = 'logo-strip'; else if (imgRatio > 0.4 && gridCols >= 3) purpose = 'gallery-grid'; else if (gridCols >= 2 && el.children.length >= 3 && maxHeadingPx < 30) purpose = 'card-grid'; else if (maxHeadingPx >= 24 && txtLen > 60) purpose = 'feature-section'; else if (txtLen > 220) purpose = 'content-section'; else if (tag === 'main' || role === 'main') purpose = 'main-content'; else if (tag === 'section') purpose = 'section'; sections.push({ index: idx++, tag, classes, role, estimatedPurpose: purpose, rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, styles: { backgroundColor: cs.backgroundColor, color: cs.color, fontFamily: cs.fontFamily, fontSize: cs.fontSize, padding: cs.padding, margin: cs.margin, display: cs.display, gap: cs.gap, gridTemplateColumns: cs.gridTemplateColumns, flexDirection: cs.flexDirection, maxWidth: cs.maxWidth, width: cs.width, height: cs.height, position: cs.position, borderRadius: cs.borderRadius, boxShadow: cs.boxShadow, overflow: cs.overflow, alignItems: cs.alignItems, justifyContent: cs.justifyContent, textTransform: cs.textTransform, textDecoration: cs.textDecoration, border: cs.border, letterSpacing: cs.letterSpacing, lineHeight: cs.lineHeight, fontWeight: cs.fontWeight, opacity: cs.opacity, transition: cs.transition, fontFeatureSettings: cs.fontFeatureSettings, fontVariationSettings: (cs as any).fontVariationSettings || 'normal', }, childCount: el.children.length, bgTreatment, isDark: isDarkBand, aboveFold, isFullBleed, imgRatio: Math.round(imgRatio * 100) / 100, maxHeadingPx: Math.round(maxHeadingPx), gridCols, hasAnimation, textLen: txtLen, }); } // Sort by Y position sections.sort((a, b) => a.rect.y - b.rect.y); return sections; }); } async function extractImages(page: Page): Promise<{ src: string; alt: string; width: number; height: number }[]> { return page.evaluate(() => { return Array.from(document.querySelectorAll('img')).slice(0, 50).map(img => ({ src: img.src, alt: img.alt, width: Math.round(img.getBoundingClientRect().width), height: Math.round(img.getBoundingClientRect().height), })); }); } /** * Extract imagery profile — OG image, hero image, format distribution, illustration vs photo mix. * Used in DESIGN.md §10b to guide LLMs on visual tone (lifestyle photography vs product mockups * vs abstract gradients vs illustration-heavy). */ interface ImageryProfile { ogImage: string | null; ogImageWidth: number | null; ogImageHeight: number | null; twitterImage: string | null; heroImage: { src: string; alt: string; width: number; height: number; aspectRatio: number } | null; formats: { png: number; jpg: number; webp: number; svg: number; gif: number; other: number }; totalImages: number; totalAboveFold: number; aspectRatioBuckets: { landscape: number; portrait: number; square: number; ultrawide: number }; illustrationHeavy: boolean; // svg + png count vs jpg/webp (illustrations vs photos) photoHeavy: boolean; avgImageSize: { width: number; height: number }; decorativePatterns?: { multiStopGradients: number; radialGradients: number; largeSvgShapes: number; backgroundImagePatterns: number; hasNoise: boolean; hasGlassmorphism: boolean; }; } async function extractImageryProfile(page: Page): Promise { return page.evaluate(() => { // OG / Twitter meta — inline lookups to avoid tsx __name() compilation issues const ogImageEl = document.querySelector('meta[property="og:image"]') || document.querySelector('meta[name="og:image"]'); const ogImage = ogImageEl ? ogImageEl.getAttribute('content') : null; const ogWEl = document.querySelector('meta[property="og:image:width"]'); const ogW = ogWEl ? ogWEl.getAttribute('content') : null; const ogHEl = document.querySelector('meta[property="og:image:height"]'); const ogH = ogHEl ? ogHEl.getAttribute('content') : null; const twitterEl = document.querySelector('meta[name="twitter:image"]') || document.querySelector('meta[property="twitter:image"]'); const twitterImage = twitterEl ? twitterEl.getAttribute('content') : null; // All images on page const imgs = Array.from(document.querySelectorAll('img')); const formats = { png: 0, jpg: 0, webp: 0, svg: 0, gif: 0, other: 0 }; const buckets = { landscape: 0, portrait: 0, square: 0, ultrawide: 0 }; let totalW = 0; let totalH = 0; let counted = 0; let aboveFold = 0; const viewportH = window.innerHeight; let heroImage = null; let heroArea = 0; for (const img of imgs.slice(0, 100)) { const rect = img.getBoundingClientRect(); const w = Math.round(rect.width); const h = Math.round(rect.height); if (w < 50 || h < 50) continue; counted++; totalW += w; totalH += h; if (rect.top < viewportH) aboveFold++; const area = w * h; if (area > heroArea && rect.top < viewportH * 1.5) { heroArea = area; heroImage = { src: img.src, alt: img.alt || '', width: w, height: h, aspectRatio: Math.round((w / h) * 100) / 100, }; } const ratio = w / h; if (ratio > 2.3) buckets.ultrawide++; else if (ratio > 1.15) buckets.landscape++; else if (ratio < 0.87) buckets.portrait++; else buckets.square++; const src = img.src.toLowerCase(); const srcsetFirst = (img.srcset || '').toLowerCase().split(' ')[0]; const extMatch = src.match(/\.(png|jpe?g|webp|svg|gif)(\?|$)/) || srcsetFirst.match(/\.(png|jpe?g|webp|svg|gif)(\?|$)/); const ext = extMatch ? extMatch[1] : ''; if (ext === 'png') formats.png++; else if (ext === 'jpg' || ext === 'jpeg') formats.jpg++; else if (ext === 'webp') formats.webp++; else if (ext === 'svg') formats.svg++; else if (ext === 'gif') formats.gif++; else formats.other++; } // Count significant SVGs only (skip tiny icons < 60px which dominate counts) const inlineSvgs = Array.from(document.querySelectorAll('svg')); let largeSvgs = 0; let iconSvgs = 0; for (const svg of inlineSvgs.slice(0, 300)) { const rect = svg.getBoundingClientRect(); if (rect.width >= 80 && rect.height >= 80) largeSvgs++; else iconSvgs++; } formats.svg += largeSvgs; // only count significant SVGs as illustrations const totalCounted = counted || 1; const avgImageSize = { width: Math.round(totalW / totalCounted), height: Math.round(totalH / totalCounted), }; // Heuristics: // - photo-heavy: 10+ raster photos (jpg/webp), regardless of icon count // - illustration-heavy: large SVGs dominate AND few raster photos exist // - otherwise: mixed or text-driven const photoCount = formats.jpg + formats.webp; const illustrationCount = formats.svg; // already filtered to >=80px const photoHeavy = photoCount >= 10; const illustrationHeavy = !photoHeavy && illustrationCount > photoCount * 1.5 && illustrationCount >= 5; // Phase 4.4 — Decorative patterns detection (gradient mesh, blobs, glassmorphism) let multiStopGradients = 0; let radialGradients = 0; let backgroundImagePatterns = 0; let hasNoise = false; let hasGlassmorphism = false; const allEls = document.querySelectorAll('*'); const scanLimit = Math.min(allEls.length, 1500); for (let i = 0; i < scanLimit; i++) { const el = allEls[i]; const cs = getComputedStyle(el); const bgImage = cs.backgroundImage; if (bgImage && bgImage !== 'none') { if (bgImage.indexOf('radial-gradient') !== -1) radialGradients++; if (bgImage.indexOf('linear-gradient') !== -1 || bgImage.indexOf('conic-gradient') !== -1) { const m = bgImage.match(/\b(linear|conic)-gradient\([^)]*\)/); if (m && (m[0].match(/,/g) || []).length >= 3) multiStopGradients++; } if (bgImage.indexOf('url(') !== -1 && !/\.(jpe?g|webp|png)/i.test(bgImage)) { backgroundImagePatterns++; } if (/noise|grain|pattern/i.test(bgImage)) hasNoise = true; } const backdrop = (cs as any).backdropFilter || (cs as any).webkitBackdropFilter; if (backdrop && /blur\(\d+/.test(backdrop)) hasGlassmorphism = true; } // Large SVG shapes (already counted above via largeSvgs) const decorativePatterns = { multiStopGradients, radialGradients, largeSvgShapes: largeSvgs, backgroundImagePatterns, hasNoise, hasGlassmorphism, }; return { ogImage, ogImageWidth: ogW ? parseInt(ogW, 10) : null, ogImageHeight: ogH ? parseInt(ogH, 10) : null, twitterImage, heroImage, formats, totalImages: counted, totalAboveFold: aboveFold, aspectRatioBuckets: buckets, illustrationHeavy, photoHeavy, avgImageSize, decorativePatterns, }; }) as Promise; } async function extractLinks(page: Page): Promise<{ href: string; text: string; isNav: boolean }[]> { return page.evaluate(() => { return Array.from(document.querySelectorAll('a')).slice(0, 100).map(a => { const inNav = !!a.closest('nav, header, [role="navigation"], [class*="nav"]'); return { href: a.href, text: a.innerText?.trim().slice(0, 100) || '', isNav: inNav, }; }); }); } // ── Advanced extractors (kept after Phase 1.2 cleanup — only zIndexMap + keyframes have consumers) ──── import { extractKeyframes, extractZIndexMap, } from './extractors/advanced.js'; import { extractWidgets, type WidgetExtraction } from './extractors/widgets.js'; // ── Screenshots ────────────────────────────────────────────────────── async function takeScreenshots(page: Page, outputDir: string, viewport: string): Promise { // Full page (with timeout fallback) — heavy sites need longer timeouts try { await page.screenshot({ path: join(outputDir, `full-page-${viewport}.png`), fullPage: true, timeout: 60000, }); } catch { console.log(` ⚠️ Full-page screenshot timeout, using viewport-only`); try { await page.screenshot({ path: join(outputDir, `full-page-${viewport}.png`), fullPage: false, timeout: 30000, }); } catch { console.log(` ⚠️ Viewport screenshot also timeout, skipping`); } } // Above the fold await page.screenshot({ path: join(outputDir, `above-fold-${viewport}.png`), fullPage: false, }); // Individual sections const sections = await page.evaluate(() => { const candidates = [ ...Array.from(document.querySelectorAll('header, nav, main, section, aside, footer')), ...(document.body ? Array.from(document.body.children) : []).filter(el => { const r = el.getBoundingClientRect(); return r.height > 100 && r.width > 200; }), ]; const seen = new Set(); return candidates.filter(el => { if (seen.has(el)) return false; seen.add(el); return true; }).slice(0, 15).map((el, i) => { const r = el.getBoundingClientRect(); const tag = el.tagName.toLowerCase(); const cls = Array.from(el.classList).join('-').slice(0, 30) || 'no-class'; return { name: `${i}-${tag}-${cls}`, clip: { x: r.x, y: r.y + window.scrollY, width: r.width, height: Math.min(r.height, 2000) }, }; }); }); for (const section of sections) { if (section.clip.width < 10 || section.clip.height < 10) continue; try { await page.screenshot({ path: join(outputDir, `section-${section.name}-${viewport}.png`), clip: section.clip, }); } catch { // Skip sections that fail (out of bounds, etc.) } } } // ── Scroll-and-screenshot (catches lazy-loaded content) ───────────── async function scrollAndScreenshot(page: Page, outputDir: string, viewport: string): Promise { const viewportHeight = page.viewportSize()?.height || 900; // Get total page height const totalHeight = await page.evaluate(() => { return Math.max( document.body?.scrollHeight || 0, document.documentElement.scrollHeight, ); }); // Phase 5 Sprint 80/20 — 5 positions × 300ms au lieu de 10 × 800ms (gain -6.5s/viewport) // Couverture lazy-load préservée par full-page screenshot déjà pris. const MAX_POSITIONS = 5; const WAIT_LAZY = 300; const positions: number[] = []; let currentY = 0; while (currentY < totalHeight && positions.length < MAX_POSITIONS) { positions.push(currentY); currentY += viewportHeight; } for (let i = 0; i < positions.length; i++) { const y = positions[i]; // Scroll to position await page.evaluate((scrollY: number) => { window.scrollTo({ top: scrollY, behavior: 'instant' as ScrollBehavior }); }, y); // Wait for lazy-loaded content to appear await page.waitForTimeout(WAIT_LAZY); // Take screenshot at this scroll position try { await page.screenshot({ path: join(outputDir, `scroll-${i}-${viewport}.png`), fullPage: false, // Only the current viewport }); } catch { // Skip on failure } } // Scroll back to top for subsequent extractions await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' as ScrollBehavior }); }); await page.waitForTimeout(500); } // ── Main extraction pipeline ───────────────────────────────────────── async function extractFromURL(url: string): Promise { const domain = new URL(url).hostname.replace('www.', ''); const baseDir = join(process.cwd(), 'extractions', domain); const screenshotDir = join(baseDir, 'screenshots'); await mkdir(screenshotDir, { recursive: true }); console.log(`\n🔍 Clone Architect — Extracting: ${url}`); console.log(`📁 Output: ${baseDir}\n`); let browser: Browser | null = null; try { browser = await chromium.launch({ headless: true, args: [ '--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled', '--disable-infobars', '--window-size=1440,900', ], }); const results: Record = {}; for (const [vpName, vpSize] of Object.entries(VIEWPORTS)) { console.log(`📐 Viewport: ${vpName} (${vpSize.width}x${vpSize.height})`); // sec-ch-ua Client Hints are Chromium-only — Safari/iOS never sends them. // Sending them with an iPhone UA is a detectable bot fingerprint. const mobileUA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1'; const desktopUA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'; let context = await browser.newContext({ viewport: vpSize, userAgent: vpName === 'mobile' ? mobileUA : desktopUA, deviceScaleFactor: vpName === 'mobile' ? 3 : 2, locale: 'fr-FR', timezoneId: 'Europe/Paris', extraHTTPHeaders: { 'Accept-Language': 'fr-FR,fr;q=0.9,en;q=0.8', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', // Client Hints uniquement pour desktop (Chrome) — jamais pour mobile Safari ...(vpName !== 'mobile' ? { 'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', } : {}), }, }); let page = await context.newPage(); // Masquer les indicateurs d'automatisation await page.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => false }); Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] }); (window as any).chrome = { runtime: {} }; }); // Phase 2.1 — Fast page load: skip networkidle, use domcontentloaded + targeted waits // Why: networkidle waits for tracker scripts (GA, Hotjar) that often never settle on tracker-heavy // sites like Stripe/Coinbase, wasting 30s. domcontentloaded + load + conditional lazy-wait is faster // and still captures full computed styles. console.log(' ⏳ Loading page...'); try { await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 }); // Wait for load event (fonts + critical CSS) but don't block on tracker network await page.waitForLoadState('load', { timeout: 5000 }).catch(() => {}); } catch { console.log(' ⚠️ Initial load failed, falling back to bare domcontentloaded...'); await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); } // Conditional lazy-wait: only wait if the page is long enough to suggest lazy-loaded content const isLongPage = await page.evaluate(() => document.documentElement.scrollHeight > window.innerHeight * 2 ).catch(() => false); if (isLongPage) await page.waitForTimeout(2000); // Dismiss common popups/modals await dismissPopups(page); // Phase 5.1.3 — Bot challenge detection (Cloudflare, CAPTCHA, etc.) const botCheck = await detectBotChallenge(page); if (botCheck.blocked) { console.log(` 🤖 Bot challenge: ${botCheck.reason} — stealth retry...`); await page.close().catch(() => {}); await context.close().catch(() => {}); await browser!.close().catch(() => {}); // Stealth retry via browser-stealth.ts (playwright-extra + puppeteer-stealth) const { launchBrowser } = await import('./browser-stealth.js'); browser = await launchBrowser({ stealth: true }); context = await browser.newContext({ viewport: vpSize, userAgent: vpName === 'mobile' ? mobileUA : desktopUA, deviceScaleFactor: vpName === 'mobile' ? 3 : 2, locale: 'fr-FR', timezoneId: 'Europe/Paris', extraHTTPHeaders: { 'Accept-Language': 'fr-FR,fr;q=0.9,en;q=0.8', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', ...(vpName !== 'mobile' ? { 'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', } : {}), }, }); page = await context.newPage(); await page.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => false }); Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] }); (window as any).chrome = { runtime: {} }; }); console.log(' ⏳ Stealth retry — loading page...'); await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }); await page.waitForLoadState('load', { timeout: 10000 }).catch(() => {}); await dismissPopups(page); const botCheck2 = await detectBotChallenge(page); if (botCheck2.blocked) { throw new Error(`Bot challenge persists after stealth retry: ${botCheck2.reason}`); } console.log(` ✅ Stealth retry succeeded`); } // ── Screenshots ── console.log(' 📸 Taking screenshots...'); await takeScreenshots(page, screenshotDir, vpName); // ── Scroll-and-screenshot (catches lazy-loaded content) ── console.log(' 📜 Scroll-and-screenshot for lazy-loaded content...'); await scrollAndScreenshot(page, screenshotDir, vpName); // ── Extract everything ── console.log(' 🎨 Extracting computed styles...'); const [ elements, cssVars, allColors, fonts, borderRadii, shadows, transitions, sections, images, imageryProfile, widgets, links, componentVariants, fontFaces, mediaBreakpoints, ] = await Promise.all([ extractComputedStyles(page), extractCSSCustomProperties(page), extractAllColors(page), extractAllFonts(page), extractAllBorderRadii(page), extractAllShadows(page), extractAllTransitions(page), extractSections(page), extractImages(page), extractImageryProfile(page).catch((e) => { console.warn(' ⚠️ imagery profile extraction failed:', (e as Error).message); return null; }), extractWidgets(page).catch((e) => { console.warn(' ⚠️ widgets extraction failed:', (e as Error).message); return null; }), extractLinks(page), extractComponentVariants(page), extractFontFaces(page), extractMediaBreakpoints(page), ]); // ── Advanced design capture (kept: keyframes + zIndexMap have consumers) ── const [keyframes, zIndexMap] = await Promise.all([ extractKeyframes(page).catch((e) => { console.warn(' ⚠️ keyframes extraction failed:', (e as Error).message); return {}; }), extractZIndexMap(page).catch((e) => { console.warn(' ⚠️ z-index extraction failed:', (e as Error).message); return []; }), ]); console.log(` 🎬 Advanced: ${Object.keys(keyframes).length} keyframes, ${zIndexMap.length} z-index`); // ── Component States (hover/focus) — desktop only, sequential ── let componentStates: Record = {}; if (vpName === 'desktop') { console.log(' 🖱️ Extracting component states (hover/focus)...'); componentStates = await extractComponentStates(page); const stateCount = Object.keys(componentStates).length; console.log(` → ${stateCount} component states captured`); } // ── OpenType Features ── const openTypeData = await extractOpenTypeFeatures(page); console.log(` 🔤 OpenType features: [${openTypeData.features.join(', ') || 'none'}] | axes: [${openTypeData.axes.join(', ') || 'none'}]`); const displaySignature = await extractDisplaySignature(page).catch(() => null); if (displaySignature) console.log(` 🅰️ Display signature: ${displaySignature.family} ${displaySignature.fontSize} ${displaySignature.isSerif ? 'SERIF' : 'sans'}${displaySignature.isItalic ? ' italic' : ''}`); const pageTitle = await page.title(); results[vpName] = { url, domain, timestamp: new Date().toISOString(), viewport: vpSize, pageTitle, cssCustomProperties: cssVars, elements, sections, allColors, allFontFamilies: fonts.families, allFontSizes: fonts.sizes, allBorderRadii: borderRadii, allShadows: shadows, allTransitions: transitions, images, imageryProfile: imageryProfile || undefined, widgets: widgets || undefined, links, componentVariants, componentStates, fontFaces, mediaBreakpoints, openTypeFeatures: openTypeData.features, variableAxes: openTypeData.axes, displaySignature: displaySignature || undefined, // Advanced capture (Phase 1.2: only kept fields with consumers) keyframes, zIndexMap, }; const variantCount = Object.values(componentVariants).reduce((sum, v) => sum + v.length, 0); console.log(` ✅ ${vpName} done — ${Object.keys(cssVars).length} CSS vars, ${allColors.length} colors, ${sections.length} sections, ${variantCount} component variants, ${fontFaces.length} font-faces, ${mediaBreakpoints.filter(b => !b.startsWith('@')).length} breakpoints`); await context.close(); } // ── Save results ── console.log('\n💾 Saving extraction data...'); // Phase 1.3 — add schema version so downstream can detect drift const versionedResults = { ...results, version: '2.4.0', }; await writeFile( join(baseDir, 'raw-css.json'), JSON.stringify(versionedResults, null, 2), ); // Save a summary for quick reference const desktop = results.desktop; const summary = { url, domain, extractedAt: new Date().toISOString(), pageTitle: desktop.pageTitle, cssCustomPropertiesCount: Object.keys(desktop.cssCustomProperties).length, colorsFound: desktop.allColors.length, fontFamilies: desktop.allFontFamilies, fontSizes: desktop.allFontSizes, borderRadii: desktop.allBorderRadii, shadows: desktop.allShadows, sectionsFound: desktop.sections.length, elementsFound: Object.entries(desktop.elements).filter(([, v]) => v !== null).map(([k]) => k), imagesCount: desktop.images.length, navLinksCount: desktop.links.filter(l => l.isNav).length, componentVariants: Object.fromEntries( Object.entries(desktop.componentVariants).map(([k, v]) => [k, v.length]) ), fontFaces: desktop.fontFaces.map(f => `${f.family} (${f.weight} ${f.style})`), mediaBreakpoints: desktop.mediaBreakpoints.filter(b => !b.startsWith('@')), }; await writeFile( join(baseDir, 'extraction-summary.json'), JSON.stringify(summary, null, 2), ); console.log(`\n✅ Extraction complete for ${domain}`); console.log(` 📁 ${baseDir}/`); console.log(` 📸 Screenshots: ${screenshotDir}/`); console.log(` 🎨 Raw CSS: ${baseDir}/raw-css.json`); console.log(` 📊 Summary: ${baseDir}/extraction-summary.json`); console.log(`\n Next: run 'npm run analyze -- ${domain}' to analyze layout`); } catch (err) { console.error('❌ Extraction failed:', err); throw err; } finally { if (browser) await browser.close(); } } // ── Popup dismissal ────────────────────────────────────────────────── async function dismissPopups(page: Page): Promise { // ── Couche 1 : Click accept (déclenche le callback consent du site) ── const clickSelectors = [ // Didomi (priorité — CMP le plus courant en FR) '#didomi-notice-agree-button', '[data-testid="didomi-notice-agree-button"]', 'button[aria-label*="Accepter"]', 'button[aria-label*="Accept all"]', 'button[aria-label*="J\'accepte"]', '.didomi-continue-without-agreeing', // OneTrust '#onetrust-accept-btn-handler', 'button[data-testid="ot-sdk-btn-allow-all"]', // Cookiebot '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll', 'a#CybotCookiebotDialogBodyLevelButtonAccept', '#CybotCookiebotDialogBodyButtonAccept', '.cm__btn-allow', '.cm__btn-accept', // Axeptio '[data-axeptio-action="accept"]', '.axeptio__btn-accept', // TarteAuCitron '#tarteaucitronPersonalize2', '.tarteaucitronAllow', '.tarteaucitron-accept', // Quantcast '.qc-cmp2-buttons-primary button', // Google CMP / Funding Choices (utilisé par claude.com, anthropic.com, googleblog, etc.) '.fc-button-label', '.fc-cta-consent', '.qc-cmp2-summary-buttons button:nth-of-type(2)', 'button[mode="primary"][label*="Accept"]', // Usercentrics '#uc-btn-accept-banner', 'button[data-testid="uc-accept-all-button"]', // Pierrot (Shopify) '.pd-banner__button--accept', 'button[data-cookiebanner-action="accept"]', // Native dialog element 'dialog button:has-text("Accept")', 'dialog button:has-text("Accepter")', // Generic aria-label English/French 'button[aria-label*="cookie" i][aria-label*="accept" i]', 'button[aria-label*="cookie" i][aria-label*="allow" i]', 'button[aria-label*="accepter" i]', // Generic consent 'button[class*="cookie"][class*="accept"]', 'button[class*="consent"][class*="accept"]', '[class*="cookie"] button[class*="accept"]', 'button[class*="accept-all"]', '[class*="consent"] button', 'button[class*="cookie"]', 'button[class*="accept"]', 'button[class*="consent"]', // Generic close '[class*="modal"] button[class*="close"]', '[class*="popup"] button[class*="close"]', '[class*="banner"] button[class*="close"]', '[aria-label="Close"]', '[aria-label="Dismiss"]', ]; // ── Helper : essaie de cliquer sur un selector dans un frame donné ── const tryClickInFrame = async (frame: any): Promise => { for (const selector of clickSelectors) { try { const btn = frame.locator(selector).first(); if (await btn.isVisible({ timeout: 600 })) { await btn.click({ timeout: 1500 }); return selector; } } catch { // Continue } } return null; }; // ── Couche 1a : Main frame ── let clicked = false; const mainHit = await tryClickInFrame(page); if (mainHit) { console.log(` 🖱️ Popup fermé via click (main): ${mainHit}`); clicked = true; await page.waitForTimeout(1500); } // ── Couche 1b : Iframes (IAB TCF, Google Funding Choices, etc.) ── if (!clicked) { const frames = page.frames(); for (const frame of frames) { if (frame === page.mainFrame()) continue; try { const url = frame.url(); // Skip about:blank et frames sans URL pertinente if (!url || url === 'about:blank' || url.startsWith('data:')) continue; const hit = await tryClickInFrame(frame); if (hit) { console.log(` 🖱️ Popup fermé via click (iframe ${url.slice(0, 60)}): ${hit}`); clicked = true; await page.waitForTimeout(1500); break; } } catch { // Continue } } } // ── Couche 2 : DOM cleanup (supprimer les overlays restants) ───── const removed = await page.evaluate(() => { let count = 0; const cmpSelectors = [ '#didomi-host', '.didomi-popup-container', '#didomi-consent-popup', '[class*="didomi-popup"]', '[class*="didomi-notice"]', '#onetrust-banner-sdk', '#onetrust-consent-sdk', '.onetrust-pc-dark-filter', '#CookieConsent', '.CookieConsent', '#CybotCookiebotDialog', '.axeptio_container', '[class*="axeptio"]', '.tarteaucitronRoot', '#tarteaucitron', '#tarteaucitronRoot', '.qc-cmp2-container', '[class*="qc-cmp"]', '.pd-cookie-banner-window', '[class*="pd-banner"]', '[id*="cookie-banner"]', '[id*="consent-banner"]', '[class*="cookie-banner"]', '[class*="consent-banner"]', '[class*="cookie-notice"]', '[class*="consent-notice"]', '[class*="gdpr"]', '[id*="gdpr"]', ]; for (const sel of cmpSelectors) { document.querySelectorAll(sel).forEach(el => { el.remove(); count++; }); } // Supprimer les overlays/backdrops à z-index élevé document.querySelectorAll('[style*="position: fixed"], [style*="position:fixed"]').forEach(el => { const cs = window.getComputedStyle(el); const z = parseInt(cs.zIndex || '0'); if (z > 9000 && (cs.backgroundColor.includes('rgba') || cs.opacity < '1')) { (el as HTMLElement).style.display = 'none'; count++; } }); // Débloquer le scroll sur body if (document.body) { document.body.classList.remove( 'didomi-popup-open', 'modal-open', 'no-scroll', 'overflow-hidden', 'cookie-open', 'consent-open', 'noscroll' ); document.body.style.overflow = ''; } document.documentElement.style.overflow = ''; return count; }); if (removed > 0) { console.log(` 🧹 ${removed} overlay(s) nettoyé(s)`); } } // ── CLI ────────────────────────────────────────────────────────────── const url = process.argv[2]; if (!url) { console.error('Usage: npm run extract -- '); console.error('Example: npm run extract -- https://linear.app'); process.exit(1); } // Validate URL try { new URL(url); } catch { console.error(`Invalid URL: ${url}`); process.exit(1); } extractFromURL(url).catch((err) => { console.error('Fatal error:', err); process.exit(1); });