/** * Secret scanner for detecting sensitive data before sync * Focused on Bitcoin/BSV WIF private keys */ import type { TimelineEventMetadata } from "../types/timeline.js"; export interface ScanResult { hasSecrets: boolean; findings: SecretFinding[]; } export interface SecretFinding { type: "wif_key" | "hex_key" | "mnemonic" | "api_key" | "connection_string"; field: string; preview: string; // Redacted preview eventId: string; } // WIF (Wallet Import Format) patterns for Bitcoin/BSV // Mainnet uncompressed: starts with '5', 51 chars base58 // Mainnet compressed: starts with 'K' or 'L', 52 chars base58 // Testnet: starts with '9' or 'c' const WIF_PATTERNS = [ // Mainnet uncompressed (starts with 5) /\b5[HJK][1-9A-HJ-NP-Za-km-z]{49}\b/g, // Mainnet compressed (starts with K or L) /\b[KL][1-9A-HJ-NP-Za-km-z]{51}\b/g, // Testnet /\b[9c][1-9A-HJ-NP-Za-km-z]{50,51}\b/g, ]; // Raw hex private key (64 hex chars) const HEX_KEY_PATTERN = /\b[0-9a-fA-F]{64}\b/g; // API Keys and tokens const API_KEY_PATTERNS = [ /sk-[a-zA-Z0-9]{48}/g, // OpenAI /sk-ant-[a-zA-Z0-9-]{95}/g, // Anthropic /sk-ant-api[a-zA-Z0-9-]{90,}/g, // Anthropic (newer format) /AKIA[0-9A-Z]{16}/g, // AWS Access Key /ghp_[a-zA-Z0-9]{36}/g, // GitHub PAT /gho_[a-zA-Z0-9]{36}/g, // GitHub OAuth /github_pat_[a-zA-Z0-9_]{22,}/g, // GitHub fine-grained PAT /xoxb-[0-9]{11}-[0-9]{11}-[a-zA-Z0-9]{24}/g, // Slack Bot /xoxp-[0-9]{11}-[0-9]{11}-[a-zA-Z0-9]{24}/g, // Slack User /Bearer\s+[a-zA-Z0-9._-]{40,}/g, // Generic Bearer tokens /glpat-[a-zA-Z0-9_-]{20,}/g, // GitLab PAT /npm_[a-zA-Z0-9]{36}/g, // NPM token /pypi-[a-zA-Z0-9]{60,}/g, // PyPI token ]; // Connection strings with credentials const CONNECTION_PATTERNS = [ /postgres:\/\/[^:]+:[^@]+@[^\s]+/g, // PostgreSQL /postgresql:\/\/[^:]+:[^@]+@[^\s]+/g, // PostgreSQL alt /mongodb(\+srv)?:\/\/[^:]+:[^@]+@[^\s]+/g, // MongoDB /redis:\/\/:[^@]+@[^\s]+/g, // Redis /mysql:\/\/[^:]+:[^@]+@[^\s]+/g, // MySQL ]; // BIP39 mnemonic seed phrases (12 or 24 words) const MNEMONIC_WORDS = [ "abandon", "ability", "able", "about", "above", "absent", "absorb", "abstract", "absurd", "abuse", "access", "accident", "account", "accuse", "achieve", "acid", "acoustic", "acquire", "across", "act", "action", "actor", "actress", "actual", // ... abbreviated - in practice would include all 2048 BIP39 words "zoo", "zone", "zero", "wheat", "winter", "witness", "wolf", "woman", ]; // Simple mnemonic detection (looks for sequences of known BIP39 words) function detectMnemonic(text: string): boolean { const words = text.toLowerCase().split(/\s+/); let consecutiveBip39 = 0; for (const word of words) { if (MNEMONIC_WORDS.includes(word)) { consecutiveBip39++; if (consecutiveBip39 >= 12) return true; } else { consecutiveBip39 = 0; } } return false; } /** * Scan a string for potential secrets */ export function scanText( text: string, ): Array<{ type: SecretFinding["type"]; match: string }> { const findings: Array<{ type: SecretFinding["type"]; match: string }> = []; // Check WIF patterns for (const pattern of WIF_PATTERNS) { const matches = text.match(pattern); if (matches) { for (const match of matches) { findings.push({ type: "wif_key", match }); } } } // Check hex keys (64 chars of hex) const hexMatches = text.match(HEX_KEY_PATTERN); if (hexMatches) { for (const match of hexMatches) { // Filter out common false positives (git hashes, etc) // Real private keys are less common in commit hashes if (!isLikelyGitHash(text, match)) { findings.push({ type: "hex_key", match }); } } } // Check for mnemonics if (detectMnemonic(text)) { findings.push({ type: "mnemonic", match: "[seed phrase detected]" }); } // Check API keys and tokens for (const pattern of API_KEY_PATTERNS) { const matches = text.match(pattern); if (matches) { for (const match of matches) { findings.push({ type: "api_key", match }); } } } // Check connection strings with embedded credentials for (const pattern of CONNECTION_PATTERNS) { const matches = text.match(pattern); if (matches) { for (const match of matches) { findings.push({ type: "connection_string", match }); } } } return findings; } /** * Check if a 64-char hex string is likely a false positive (not a private key) * False positives include: git hashes, transaction IDs, public key hashes in filenames */ function isLikelyGitHash(context: string, hex: string): boolean { const lowerContext = context.toLowerCase(); // Git context keywords const gitKeywords = ["commit", "merge", "parent", "tree", "blob", "ref"]; if (gitKeywords.some((keyword) => lowerContext.includes(keyword))) { return true; } // Check if hex appears in a file path (likely a txid or public key hash) // Pattern: hex string followed by file extension or preceded by path separator const hexInPath = new RegExp( `[/\\\\]${hex}\\.|${hex}\\.(hex|json|txt|dat|bin)`, "i", ); if (hexInPath.test(context)) { return true; } // Check if hex is part of a filename (surrounded by path-like context) if (lowerContext.includes(`/${hex}`) || lowerContext.includes(`\\${hex}`)) { return true; } // Check for data/ordinals/inscriptions directories (common for blockchain data) const dataPathKeywords = [ "src/data", "data/", "ordinals", "inscriptions", "txid", "txhash", ]; if (dataPathKeywords.some((keyword) => lowerContext.includes(keyword))) { return true; } return false; } /** * Redact a secret for safe display */ export function redactSecret(secret: string): string { if (secret.length <= 8) return "***"; return `${secret.slice(0, 4)}...${secret.slice(-4)}`; } /** * Scan an event's metadata for secrets */ export function scanEventMetadata( eventId: string, metadata: TimelineEventMetadata | null, ): SecretFinding[] { if (!metadata) return []; const findings: SecretFinding[] = []; // Recursively scan all string values in metadata function scanObject(obj: unknown, path: string) { if (typeof obj === "string") { const textFindings = scanText(obj); for (const finding of textFindings) { findings.push({ type: finding.type, field: path, preview: redactSecret(finding.match), eventId, }); } } else if (Array.isArray(obj)) { for (let i = 0; i < obj.length; i++) { scanObject(obj[i], `${path}[${i}]`); } } else if (obj && typeof obj === "object") { for (const [key, value] of Object.entries(obj)) { scanObject(value, path ? `${path}.${key}` : key); } } } scanObject(metadata, ""); return findings; } /** * Redact secrets in an object (mutates in place) * Returns the paths and original values that were redacted */ export function redactSecretsInObject( obj: unknown, path = "", ): Array<{ path: string; original: string; redacted: string }> { const redactions: Array<{ path: string; original: string; redacted: string; }> = []; if (typeof obj === "string") { // Can't mutate primitive - handled by parent return redactions; } if (Array.isArray(obj)) { for (let i = 0; i < obj.length; i++) { if (typeof obj[i] === "string") { const findings = scanText(obj[i]); if (findings.length > 0) { let redacted = obj[i]; for (const finding of findings) { redacted = redacted.replace(finding.match, "[REDACTED]"); redactions.push({ path: `${path}[${i}]`, original: redactSecret(finding.match), redacted: "[REDACTED]", }); } obj[i] = redacted; } } else { redactions.push(...redactSecretsInObject(obj[i], `${path}[${i}]`)); } } } else if (obj && typeof obj === "object") { const record = obj as Record; for (const [key, value] of Object.entries(record)) { const currentPath = path ? `${path}.${key}` : key; if (typeof value === "string") { const findings = scanText(value); if (findings.length > 0) { let redacted = value; for (const finding of findings) { redacted = redacted.replace(finding.match, "[REDACTED]"); redactions.push({ path: currentPath, original: redactSecret(finding.match), redacted: "[REDACTED]", }); } record[key] = redacted; } } else { redactions.push(...redactSecretsInObject(value, currentPath)); } } } return redactions; } /** * Batch scan multiple events */ export function scanEvents( events: Array<{ id: string; metadata: TimelineEventMetadata | null }>, onProgress?: (scanned: number, total: number, findings: number) => void, ): ScanResult { const allFindings: SecretFinding[] = []; for (let i = 0; i < events.length; i++) { const event = events[i]; const eventFindings = scanEventMetadata(event.id, event.metadata); allFindings.push(...eventFindings); if (onProgress && (i % 1000 === 0 || i === events.length - 1)) { onProgress(i + 1, events.length, allFindings.length); } } return { hasSecrets: allFindings.length > 0, findings: allFindings, }; } /** * Scan and redact secrets in events (mutates metadata in place) * Returns scan result with findings for reporting */ export function scanAndRedactEvents( events: Array<{ id: string; metadata: TimelineEventMetadata | null }>, onProgress?: (scanned: number, total: number, findings: number) => void, ): ScanResult { const allFindings: SecretFinding[] = []; for (let i = 0; i < events.length; i++) { const event = events[i]; if (event.metadata) { // First scan to get findings for reporting const eventFindings = scanEventMetadata(event.id, event.metadata); allFindings.push(...eventFindings); // Then redact in place if there were findings if (eventFindings.length > 0) { redactSecretsInObject(event.metadata); } } if (onProgress && (i % 1000 === 0 || i === events.length - 1)) { onProgress(i + 1, events.length, allFindings.length); } } return { hasSecrets: allFindings.length > 0, findings: allFindings, }; }