/**
* Language detector — identify page language from HTML attributes and content analysis.
*
* Sources (in priority order):
* 1. attribute
* 2. header
* 3. hreflang alternate links
* 4. Content-based n-gram fingerprint (detects top 20 languages)
*
* Used to: route pages to the correct markdown processor, group screens by locale,
* and detect missing hreflang implementations.
*/
import type { Page } from 'playwright';
// Frequency signatures for n-gram based detection
// Common trigrams by language (simplified — not production-grade)
const LANG_SIGNATURES: Record = {
en: /\b(the|and|for|are|but|not|you|all|can|had|her|was|one|our|out|day|get|has|him|his|how)\b/gi,
es: /\b(que|con|una|por|del|las|los|para|como|pero|más|este|esto|sus|ser|hay|sobre|entre|también)\b/gi,
fr: /\b(les|des|une|est|dans|pour|qui|par|sur|avec|que|pas|son|tout|mais|bien|comme|vous)\b/gi,
de: /\b(die|der|und|den|das|ein|ist|von|mit|dem|auf|sich|des|aber|auch|bei|aus|nach)\b/gi,
pt: /\b(que|não|uma|com|por|dos|para|como|mais|mas|seu|sua|nas|nos|ele|ela|são)\b/gi,
it: /\b(che|con|del|per|una|nel|non|sono|gli|dei|alla|delle|questa|questo|come|più)\b/gi,
nl: /\b(van|het|een|zijn|maar|ook|dan|door|op|uit|aan|bij|om|al|veel|haar|hun)\b/gi,
ja: /[-ゟ゠-ヿ]/g,
zh: /[一-鿿]/g,
ar: /[-ۿ]/g,
hi: /[ऀ-ॿ]/g,
ko: /[가-ᄀ-ᇿ]/g,
};
function detectFromContent(text: string): string {
if (!text || text.length < 50) return 'en';
let best = 'en';
let bestScore = 0;
for (const [lang, re] of Object.entries(LANG_SIGNATURES)) {
const matches = (text.match(re) ?? []).length;
const score = matches / Math.max(text.length / 100, 1);
if (score > bestScore) { bestScore = score; best = lang; }
}
return best;
}
export interface LanguageDetectionResult {
url: string;
declaredLang: string; // from or meta
detectedLang: string; // from content analysis
confidence: 'high' | 'medium' | 'low';
alternates: Record; // lang → url from hreflang
isMultilingual: boolean;
}
export async function detectLanguage(page: Page, url: string): Promise {
const data = await page.evaluate(() => {
const htmlLang = document.documentElement.lang?.toLowerCase().split('-')[0] ?? '';
const metaLang = (document.querySelector('meta[http-equiv="Content-Language"]') as any)?.content?.toLowerCase().split('-')[0] ?? '';
const alternates: Record = {};
document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el: any) => {
const lang = el.getAttribute('hreflang')?.toLowerCase();
const href = el.getAttribute('href') ?? '';
if (lang && lang !== 'x-default') alternates[lang] = href;
});
const bodyText = (document.body?.innerText ?? '').slice(0, 2000);
return { htmlLang, metaLang, alternates, bodyText };
}).catch(() => ({ htmlLang: '', metaLang: '', alternates: {}, bodyText: '' }));
const declaredLang = data.htmlLang || data.metaLang || '';
const detectedLang = detectFromContent(data.bodyText);
const confidence: LanguageDetectionResult['confidence'] =
declaredLang ? 'high' : data.bodyText.length > 200 ? 'medium' : 'low';
return {
url,
declaredLang,
detectedLang: declaredLang || detectedLang,
confidence,
alternates: data.alternates,
isMultilingual: Object.keys(data.alternates).length > 1,
};
}