/** * PII scrubber — detects and redacts Personally Identifiable Information * from crawled markdown, HTML, and extracted text before storage. * * GDPR Article 4(1): "any information relating to an identified or identifiable natural person" */ export interface PiiMatch { type: 'email' | 'phone' | 'ssn' | 'credit-card' | 'ip-address' | 'passport' | 'iban' | 'dob'; value: string; start: number; end: number; } export interface ScrubResult { text: string; matches: PiiMatch[]; scrubbed: boolean; } // Patterns — ordered from most-specific to least-specific const PATTERNS: Array<{ type: PiiMatch['type']; re: RegExp }> = [ // Email — RFC 5321 simplified { type: 'email', re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g }, // US SSN — 9 digits with dashes or spaces { type: 'ssn', re: /\b(?!000|666|9\d\d)\d{3}[-\s](?!00)\d{2}[-\s](?!0000)\d{4}\b/g }, // Credit card — Visa/MC/Amex/Discover with optional spaces/dashes { type: 'credit-card', re: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{3,4}\b/g }, // IBAN — international bank account (15-34 chars) { type: 'iban', re: /\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b/g }, // Phone — international or US formats { type: 'phone', re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b|\+\d{1,3}[-.\s]\d{2,4}[-.\s]\d{3,4}[-.\s]\d{3,4}/g }, // Date of birth — common formats { type: 'dob', re: /\b(?:0?[1-9]|1[0-2])\/(?:0?[1-9]|[12]\d|3[01])\/(?:19|20)\d{2}\b/g }, // IPv4 private/internal addresses — only flag private-looking ones in content { type: 'ip-address', re: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/g }, ]; const REDACT_MAP: Record = { email: '[EMAIL]', phone: '[PHONE]', ssn: '[SSN]', 'credit-card': '[CARD]', 'ip-address': '[IP]', passport: '[PASSPORT]', iban: '[IBAN]', dob: '[DOB]', }; export function detectPii(text: string): PiiMatch[] { const matches: PiiMatch[] = []; const seen = new Set(); for (const { type, re } of PATTERNS) { const r = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g'); let m: RegExpExecArray | null; while ((m = r.exec(text)) !== null) { const key = `${type}:${m.index}`; if (!seen.has(key)) { seen.add(key); matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length }); } } } return matches.sort((a, b) => a.start - b.start); } export function scrubPii(text: string): ScrubResult { const matches = detectPii(text); if (matches.length === 0) return { text, matches: [], scrubbed: false }; // Replace from end to start so indices stay valid let scrubbed = text; for (let i = matches.length - 1; i >= 0; i--) { const { type, start, end } = matches[i]; scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end); } return { text: scrubbed, matches, scrubbed: true }; } export function hasPii(text: string): boolean { for (const { re } of PATTERNS) { const r = new RegExp(re.source, re.flags); if (r.test(text)) return true; } return false; } export function scrubObject>(obj: T, fields: (keyof T)[]): T { const result = { ...obj }; for (const field of fields) { if (typeof result[field] === 'string') { (result as any)[field] = scrubPii(result[field] as string).text; } } return result; }