/** * Cookie consent auto-acceptor. * * Detects and clicks "Accept" on common cookie consent banners before the * crawler proceeds. Without this, many EU sites show a consent wall that * blocks content or shows a different layout, making crawl results unreliable. * * Supports: Cookiebot, OneTrust, TrustArc, Usercentrics, GDPR Cookie Compliance, * generic "Accept All" / "Accept Cookies" buttons. * * Strategy: try known selectors first, then fall back to text-based button search. * Non-blocking — if no banner is found or click fails, crawl proceeds normally. */ import type { Page } from 'playwright'; export interface ConsentResult { found: boolean; clicked: boolean; method?: string; // which selector/strategy matched error?: string; } // Known consent platform selectors const CONSENT_SELECTORS = [ // Cookiebot '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll', '#CybotCookiebotDialogBodyButtonAccept', // OneTrust '#onetrust-accept-btn-handler', '.ot-sdk-accept-btn', // TrustArc '.truste_popframe [title*="Accept"]', '#truste-consent-button', // Usercentrics '[data-testid="uc-accept-all-button"]', // Generic high-confidence '[id*="accept"][id*="cookie"]', '[class*="accept"][class*="cookie"]', '[class*="cookie"][class*="accept"]', // GDPR Cookie Compliance '.gdpr-cookie-notice-accept', // Quantcast '.qc-cmp2-summary-buttons button:first-child', ]; // Fallback: button text matching const ACCEPT_TEXT_PATTERNS = [ /^accept all$/i, /^accept all cookies$/i, /^allow all$/i, /^i accept$/i, /^agree$/i, /^ok$/i, /^got it$/i, /^allow cookies$/i, ]; export async function acceptCookieConsent(page: Page, opts: { waitMs?: number } = {}): Promise { const { waitMs = 1_500 } = opts; // Wait briefly for consent banners to appear await page.waitForTimeout(waitMs).catch(() => {}); // Try known selectors first for (const selector of CONSENT_SELECTORS) { try { const el = await page.$(selector); if (el && await el.isVisible()) { await el.click({ timeout: 2_000 }); await page.waitForTimeout(500); return { found: true, clicked: true, method: `selector: ${selector}` }; } } catch { /* try next */ } } // Fallback: find buttons with accept-like text try { const buttons = await page.$$('button, [role="button"], a[class*="cookie"]'); for (const btn of buttons) { const text = ((await btn.textContent()) ?? '').trim(); if (ACCEPT_TEXT_PATTERNS.some(re => re.test(text))) { if (await btn.isVisible()) { await btn.click({ timeout: 2_000 }); await page.waitForTimeout(500); return { found: true, clicked: true, method: `text: "${text}"` }; } } } } catch { /* non-fatal */ } return { found: false, clicked: false }; } /** Check if a consent banner is still visible after attempted dismissal */ export async function isConsentBannerVisible(page: Page): Promise { try { for (const sel of CONSENT_SELECTORS.slice(0, 5)) { const el = await page.$(sel); if (el && await el.isVisible()) return true; } return false; } catch { return false; } }