/** * SEO indexability checker — determines if a page is crawlable and indexable. * * Checks: * 1. X-Robots-Tag response header (noindex, nofollow, noarchive) * 2. tag (noindex, nofollow) * 3. Canonical URL (self-canonical vs cross-canonical) * 4. rel="nofollow" on outbound links * 5. hreflang tags (multilingual) * 6. Open Graph completeness (og:title, og:description, og:image) * 7. Structured data presence (JSON-LD) * 8. Meta description * 9. Title tag length (< 60 chars recommended) * 10. URL structure (query params, fragments, excessive depth) */ import type { Page } from 'playwright'; export interface SeoIssue { rule: string; severity: 'critical' | 'warning' | 'info'; message: string; value?: string; } export interface SeoCheckResult { url: string; isIndexable: boolean; canonicalUrl?: string; isSelfCanonical?: boolean; hreflang: Record; // lang → url issues: SeoIssue[]; score: number; // 0-100 checkedAt: string; } export async function checkSeo(page: Page, url: string, responseHeaders: Record = {}): Promise { const issues: SeoIssue[] = []; const checkedAt = new Date().toISOString(); let isIndexable = true; // 1. X-Robots-Tag const robotsHeader = responseHeaders['x-robots-tag'] ?? ''; if (/noindex/i.test(robotsHeader)) { isIndexable = false; issues.push({ rule: 'x-robots-noindex', severity: 'critical', message: 'X-Robots-Tag: noindex found in response header', value: robotsHeader }); } // 2-9. Page-level checks const pageData = await page.evaluate(() => { const meta = (name: string) => document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ?? ''; const og = (prop: string) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute('content') ?? ''; const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href') ?? ''; const hreflang: Record = {}; document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el: any) => { hreflang[el.getAttribute('hreflang')] = el.getAttribute('href') ?? ''; }); const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s: any) => s.textContent ?? ''); return { robotsMeta: meta('robots'), description: meta('description'), title: document.title ?? '', canonical, hreflang, ogTitle: og('title'), ogDescription: og('description'), ogImage: og('image'), hasJsonLd: jsonLd.length > 0, jsonLd, }; }).catch(() => ({ robotsMeta: '', description: '', title: '', canonical: '', hreflang: {}, ogTitle: '', ogDescription: '', ogImage: '', hasJsonLd: false, jsonLd: [], })); // 2. Meta robots if (/noindex/i.test(pageData.robotsMeta)) { isIndexable = false; issues.push({ rule: 'meta-robots-noindex', severity: 'critical', message: 'Meta robots noindex found', value: pageData.robotsMeta }); } // 3. Canonical let isSelfCanonical: boolean | undefined; if (pageData.canonical) { isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, '') === url.replace(/\/$/, ''); if (!isSelfCanonical) { issues.push({ rule: 'cross-canonical', severity: 'warning', message: 'Cross-canonical points elsewhere', value: pageData.canonical }); } } else { issues.push({ rule: 'missing-canonical', severity: 'warning', message: 'No canonical link tag' }); } // 4. Title if (!pageData.title) { issues.push({ rule: 'missing-title', severity: 'critical', message: 'Page has no title tag' }); } else if (pageData.title.length > 60) { issues.push({ rule: 'title-too-long', severity: 'warning', message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title }); } // 5. Meta description if (!pageData.description) { issues.push({ rule: 'missing-description', severity: 'warning', message: 'Missing meta description' }); } else if (pageData.description.length > 160) { issues.push({ rule: 'description-too-long', severity: 'info', message: `Description ${pageData.description.length} chars (>160)` }); } // 6. OG tags if (!pageData.ogTitle) issues.push({ rule: 'missing-og-title', severity: 'info', message: 'Missing og:title' }); if (!pageData.ogImage) issues.push({ rule: 'missing-og-image', severity: 'info', message: 'Missing og:image' }); // 7. Structured data if (!pageData.hasJsonLd) issues.push({ rule: 'no-structured-data', severity: 'info', message: 'No JSON-LD structured data found' }); // 8. URL depth try { const depth = new URL(url).pathname.split('/').filter(Boolean).length; if (depth > 5) issues.push({ rule: 'deep-url', severity: 'info', message: `URL depth ${depth} (>5 levels)`, value: url }); } catch {} const criticals = issues.filter(i => i.severity === 'critical').length; const warnings = issues.filter(i => i.severity === 'warning').length; const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter(i => i.severity === 'info').length * 2); return { url, isIndexable, canonicalUrl: pageData.canonical || undefined, isSelfCanonical, hreflang: pageData.hreflang, issues, score, checkedAt, }; }