/** * Free proxy auto-selection for bot-protection bypass. * * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN. * These are datacenter proxies — they bypass simple IP blocks but NOT * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites, * users need a residential proxy (Bright Data, Oxylabs, etc.). */ import { createConnection } from 'net'; const PROXY_LIST_URL = 'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json'; interface ProxyEntry { ip: string; port: number; protocols: string[]; upTime?: number; speed?: number; } /** Fetch and filter proxy list. Returns http proxies sorted by speed (fastest first). */ async function fetchCandidates(): Promise { const res = await fetch(PROXY_LIST_URL, { signal: AbortSignal.timeout(10_000), headers: { 'Accept': 'application/json' }, }); if (!res.ok) throw new Error(`proxy list fetch failed: ${res.status}`); const raw: ProxyEntry[] = await res.json(); return raw .filter((p) => p.protocols?.includes('http') && p.port > 0 && (p.upTime ?? 0) >= 30) .sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999)) .slice(0, 50); // test at most 50 candidates } /** TCP-level proxy liveness check via HTTP CONNECT tunnel. No extra dependencies. */ function testProxy(ip: string, port: number, timeoutMs: number): Promise { return new Promise((resolve) => { let settled = false; const settle = (ok: boolean) => { if (settled) return; settled = true; resolve(ok); try { sock.destroy(); } catch { /* ignore */ } }; const timer = setTimeout(() => settle(false), timeoutMs); const sock = createConnection({ host: ip, port, timeout: timeoutMs }); sock.on('connect', () => { sock.write('CONNECT example.com:80 HTTP/1.1\r\nHost: example.com:80\r\n\r\n'); sock.once('data', (data) => { clearTimeout(timer); const ok = /^HTTP\/1\.[01] 200/.test(data.toString()); settle(ok); }); }); sock.on('error', () => { clearTimeout(timer); settle(false); }); sock.on('timeout', () => { clearTimeout(timer); settle(false); }); }); } /** Test `batchSize` proxies concurrently and return first working one. */ async function testBatch( candidates: ProxyEntry[], start: number, batchSize: number, timeoutMs: number, ): Promise { const batch = candidates.slice(start, start + batchSize); if (batch.length === 0) return null; const results = await Promise.all( batch.map(async (p) => { const ok = await testProxy(p.ip, p.port, timeoutMs); return ok ? `http://${p.ip}:${p.port}` : null; }), ); return results.find((r) => r !== null) ?? null; } /** * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout. * Returns null if no working proxy found (not an error — crawl can proceed without proxy). */ export async function pickFreeProxy(): Promise { let candidates: ProxyEntry[]; try { candidates = await fetchCandidates(); } catch (err: any) { console.warn('[proxy] failed to fetch proxy list:', err?.message ?? err); return null; } const BATCH = 10; const TIMEOUT = 4_000; for (let i = 0; i < candidates.length; i += BATCH) { const found = await testBatch(candidates, i, BATCH, TIMEOUT); if (found) { console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`); return found; } } console.warn('[proxy] no working free proxy found in list'); return null; } // ── Tiered Proxy Rotation ───────────────────────────────────────────────────── export type ProxyTier = 'free' | 'datacenter' | 'residential' | 'premium'; export interface ProxyTierConfig { /** List of proxy URLs for this tier. E.g. ['http://user:pass@dc1.proxy:8080'] */ urls: string[]; tier: ProxyTier; } interface DomainHistory { tier: ProxyTier; consecutiveFailures: number; lastEscalatedAt: number; totalRequests: number; successRate: number; } const TIER_ORDER: ProxyTier[] = ['free', 'datacenter', 'residential', 'premium']; function nextTier(current: ProxyTier): ProxyTier { const idx = TIER_ORDER.indexOf(current); return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)]; } /** * Tiered proxy manager with per-domain error histograms. * * Tracks success/failure per domain and automatically escalates to higher proxy tiers * on consecutive failures. Applies time-decay to probe returning to lower tiers. * * Usage: * const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]); * const proxy = await mgr.selectProxy('api.salesforce.com'); * // ... make request ... * mgr.recordResult('api.salesforce.com', success); */ export class TieredProxyManager { private tiers: Map = new Map(); private domainHistory: Map = new Map(); private readonly ESCALATE_AFTER = 3; // consecutive failures before escalation private readonly DECAY_AFTER_MS = 30 * 60 * 1000; // 30 min: probe tier downgrade constructor(configs: ProxyTierConfig[]) { for (const cfg of configs) { if (cfg.urls.length > 0) this.tiers.set(cfg.tier, cfg.urls); } } private getHistory(domain: string): DomainHistory { if (!this.domainHistory.has(domain)) { this.domainHistory.set(domain, { tier: this.lowestAvailableTier(), consecutiveFailures: 0, lastEscalatedAt: 0, totalRequests: 0, successRate: 1, }); } return this.domainHistory.get(domain)!; } private lowestAvailableTier(): ProxyTier { for (const t of TIER_ORDER) { if (this.tiers.has(t)) return t; } return 'free'; } /** Pick a proxy URL for this domain, based on its tier history. */ selectProxy(domain: string): string | null { const history = this.getHistory(domain); // Time-decay: if we've been on elevated tier for DECAY_AFTER_MS, try stepping down if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) { const elapsed = Date.now() - history.lastEscalatedAt; if (elapsed > this.DECAY_AFTER_MS) { const currentIdx = TIER_ORDER.indexOf(history.tier); if (currentIdx > 0) { const lowerTier = TIER_ORDER[currentIdx - 1]; if (this.tiers.has(lowerTier)) { history.tier = lowerTier; history.lastEscalatedAt = 0; } } } } const urls = this.tiers.get(history.tier); if (!urls || urls.length === 0) return null; // Round-robin within tier const idx = history.totalRequests % urls.length; return urls[idx]; } /** Record a request result for a domain. Triggers tier escalation on repeated failures. */ recordResult(domain: string, success: boolean): void { const history = this.getHistory(domain); history.totalRequests++; if (success) { history.consecutiveFailures = 0; // Update EMA success rate history.successRate = history.successRate * 0.9 + 0.1; } else { history.consecutiveFailures++; history.successRate = history.successRate * 0.9; if (history.consecutiveFailures >= this.ESCALATE_AFTER) { const elevated = nextTier(history.tier); if (elevated !== history.tier && this.tiers.has(elevated)) { console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}→${elevated} after ${history.consecutiveFailures} failures`); history.tier = elevated; history.lastEscalatedAt = Date.now(); history.consecutiveFailures = 0; } } } } stats(): Array<{ domain: string; tier: ProxyTier; successRate: number; totalRequests: number }> { return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({ domain, tier: h.tier, successRate: Math.round(h.successRate * 100) / 100, totalRequests: h.totalRequests, })); } }