import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; import { IpIntelligenceDoc, SecurityBlockRuleDoc, SecurityPolicyAuditDoc } from '../db/index.js'; import type { IIpIntelligenceRecord, ISecurityBlockRule, ISecurityCompiledPolicy, ISecurityPolicyAuditEvent, TSecurityBlockRuleMatchMode, TSecurityBlockRuleType, } from '../../ts_interfaces/data/security-policy.js'; export interface ISecurityPolicyManagerOptions { intelligenceRefreshMs?: number; retentionSweepIntervalMs?: number; now?: () => number; onPolicyChanged?: () => void | Promise; } export interface IRemoteIngressFirewallSnapshot { blockedIps: string[]; } const OBSERVED_IP_QUEUE_LIMIT = 512; const OBSERVED_IP_BATCH_LIMIT = 20; const OBSERVED_IP_QUEUE_CONCURRENCY = 2; const OBSERVED_IP_REQUEUE_THROTTLE_MS = 60_000; export const IP_INTELLIGENCE_LIST_LIMIT = 500; export const IP_INTELLIGENCE_RETENTION_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; export const IP_INTELLIGENCE_RETENTION_HIGH_WATER = 10_000; export const IP_INTELLIGENCE_RETENTION_LOW_WATER = 9_000; export const IP_INTELLIGENCE_RETENTION_BATCH_SIZE = 500; export const IP_INTELLIGENCE_RETENTION_SWEEP_INTERVAL_MS = 60 * 60 * 1000; export class SecurityPolicyManager { private readonly smartNetwork = new plugins.smartnetwork.SmartNetwork({ cacheTtl: 24 * 60 * 60 * 1000, ipIntelligenceTimeout: 5_000, }); private readonly intelligenceRefreshMs: number; private readonly retentionSweepIntervalMs: number; private readonly now: () => number; private retentionSweepTimer?: ReturnType; private retentionSweepPromise?: Promise; private retentionMutationTail: Promise = Promise.resolve(); private persistedIntelligenceCount?: number; private readonly inFlightObservations = new Map>(); private readonly queuedObservations = new Set(); private readonly observationQueue: string[] = []; private readonly lastQueuedAt = new Map(); private activeQueuedObservations = 0; private queueDrainScheduled = false; private isStopping = false; private readonly onPolicyChanged?: () => void | Promise; constructor(options: ISecurityPolicyManagerOptions = {}) { this.intelligenceRefreshMs = options.intelligenceRefreshMs ?? 24 * 60 * 60 * 1000; this.retentionSweepIntervalMs = options.retentionSweepIntervalMs ?? IP_INTELLIGENCE_RETENTION_SWEEP_INTERVAL_MS; this.now = options.now ?? Date.now; this.onPolicyChanged = options.onPolicyChanged; } public async start(): Promise { this.isStopping = false; await this.requestRetentionSweep(); this.retentionSweepTimer = setInterval(() => { void this.requestRetentionSweep(); }, this.retentionSweepIntervalMs); this.retentionSweepTimer.unref?.(); logger.log('info', 'SecurityPolicyManager started'); } public async stop(): Promise { this.isStopping = true; if (this.retentionSweepTimer) { clearInterval(this.retentionSweepTimer); this.retentionSweepTimer = undefined; } this.observationQueue.length = 0; this.queuedObservations.clear(); await this.smartNetwork.stop(); await Promise.allSettled([...this.inFlightObservations.values()]); await this.retentionSweepPromise; await this.retentionMutationTail; } public async observeIps(ips: string[]): Promise { const uniqueIps = [...new Set(ips.map((ip) => this.normalizeIp(ip)).filter(Boolean) as string[])]; await Promise.allSettled(uniqueIps.map((ip) => this.observeIp(ip))); } public queueObservedIps(ips: string[]): void { if (this.isStopping) return; const now = Date.now(); const uniqueIps = [...new Set(ips.map((ip) => this.normalizeIp(ip)).filter(Boolean) as string[])]; for (const ip of uniqueIps.slice(0, OBSERVED_IP_BATCH_LIMIT)) { if (!this.isPublicIp(ip)) continue; if (this.inFlightObservations.has(ip) || this.queuedObservations.has(ip)) continue; const lastQueuedAt = this.lastQueuedAt.get(ip); if (lastQueuedAt && now - lastQueuedAt < OBSERVED_IP_REQUEUE_THROTTLE_MS) continue; if (this.observationQueue.length >= OBSERVED_IP_QUEUE_LIMIT) { const droppedIp = this.observationQueue.shift(); if (droppedIp) this.queuedObservations.delete(droppedIp); } this.observationQueue.push(ip); this.queuedObservations.add(ip); this.lastQueuedAt.set(ip, now); } this.pruneQueuedIpMemory(now); this.scheduleQueueDrain(); } public async observeIp(ipAddress: string, options: { force?: boolean } = {}): Promise { if (this.isStopping) return; const ip = this.normalizeIp(ipAddress); if (!ip || !this.isPublicIp(ip)) { return; } const existingObservation = this.inFlightObservations.get(ip); if (existingObservation) { await existingObservation; if (!options.force) return; } const observationPromise = this.performObserveIp(ip, options).finally(() => { if (this.inFlightObservations.get(ip) === observationPromise) { this.inFlightObservations.delete(ip); } }); this.inFlightObservations.set(ip, observationPromise); await observationPromise; } private async performObserveIp(ip: string, options: { force?: boolean } = {}): Promise { try { const now = this.now(); let doc = await IpIntelligenceDoc.findByIp(ip); if (doc && !options.force && now - doc.updatedAt < this.intelligenceRefreshMs) { if (now - doc.lastSeenAt > 60_000) { doc.lastSeenAt = now; doc.seenCount = (doc.seenCount || 0) + 1; await doc.save(); } return; } const intelligence = await this.smartNetwork.getIpIntelligence(ip); if (doc) { Object.assign(doc, intelligence); doc.lastSeenAt = now; doc.updatedAt = now; doc.seenCount = (doc.seenCount || 0) + 1; await doc.save(); } else { doc = await this.persistNewIntelligence(ip, intelligence, now); if (!doc) { return; } } if (await this.matchesAnyReactiveRule(doc)) { await this.notifyPolicyChanged(); } } catch (err) { logger.log('warn', `Failed to enrich IP ${ip}: ${(err as Error).message}`); } } public async listBlockRules(): Promise { return (await SecurityBlockRuleDoc.findAll()).map((doc) => this.ruleFromDoc(doc)); } public async listIpIntelligence(options: { ipAddresses?: string[]; limit?: number } = {}): Promise { const limit = Number.isInteger(options.limit) && options.limit! > 0 ? Math.min(options.limit!, IP_INTELLIGENCE_LIST_LIMIT) : IP_INTELLIGENCE_LIST_LIMIT; let ipAddresses: string[] | undefined; if (options.ipAddresses !== undefined) { if (!Array.isArray(options.ipAddresses)) { throw new Error('ipAddresses must be an array'); } const normalizedIps = options.ipAddresses.map((ipAddress, index) => { if (typeof ipAddress !== 'string') { throw new Error('ipAddresses[' + index + '] must be a string'); } const normalizedIp = this.normalizeIp(ipAddress); if (!normalizedIp) { throw new Error('ipAddresses[' + index + '] is not a valid IP address'); } return normalizedIp; }); ipAddresses = [...new Set(normalizedIps)]; if (ipAddresses.length > IP_INTELLIGENCE_LIST_LIMIT) { throw new Error( 'ipAddresses may contain at most ' + IP_INTELLIGENCE_LIST_LIMIT + ' unique addresses', ); } if (ipAddresses.length === 0) { return []; } } const docs = await IpIntelligenceDoc.findRecent({ ipAddresses, limit }); return docs.map((doc) => this.intelligenceFromDoc(doc)); } private async persistNewIntelligence( ip: string, intelligence: Awaited>, now: number, ): Promise { return await this.withRetentionMutation(async () => { if (this.isStopping) return null; let doc = await IpIntelligenceDoc.findByIp(ip); let isNewDocument = false; let verifiedCount = this.persistedIntelligenceCount; if (!doc) { try { verifiedCount ??= await this.pruneIpIntelligence({ forCapacity: true }); } catch (error: unknown) { logger.log( 'warn', 'Skipping persistence for new IP ' + ip + ': cache capacity could not be verified: ' + (error as Error).message, ); return null; } if (verifiedCount >= IP_INTELLIGENCE_RETENTION_HIGH_WATER) { logger.log( 'warn', 'Skipping persistence for new IP ' + ip + ': cache remains at capacity (' + verifiedCount + ')', ); return null; } doc = new IpIntelligenceDoc(); doc.ipAddress = ip; doc.firstSeenAt = now; isNewDocument = true; } Object.assign(doc, intelligence); doc.lastSeenAt = now; doc.updatedAt = now; doc.seenCount = (doc.seenCount || 0) + 1; await doc.save(); if (isNewDocument) { this.persistedIntelligenceCount = verifiedCount! + 1; } return doc; }); } private async requestRetentionSweep(): Promise { if (this.retentionSweepPromise) { return await this.retentionSweepPromise; } const sweepPromise = this.withRetentionMutation(async () => { await this.pruneIpIntelligence({ forCapacity: false }); }).catch((error: unknown) => { logger.log( 'warn', 'IP intelligence retention sweep failed: ' + (error as Error).message, ); }).finally(() => { if (this.retentionSweepPromise === sweepPromise) { this.retentionSweepPromise = undefined; } }); this.retentionSweepPromise = sweepPromise; return await sweepPromise; } private async pruneIpIntelligence(options: { forCapacity: boolean }): Promise { this.persistedIntelligenceCount = undefined; let deletedCount = 0; const expiryCutoff = this.now() - IP_INTELLIGENCE_RETENTION_MAX_AGE_MS; while (!this.isStopping) { const expiredIds = await IpIntelligenceDoc.findOldestIds({ lastSeenBefore: expiryCutoff, limit: IP_INTELLIGENCE_RETENTION_BATCH_SIZE, }); if (expiredIds.length === 0) break; deletedCount += await IpIntelligenceDoc.deleteByDocumentIds(expiredIds); if (expiredIds.length < IP_INTELLIGENCE_RETENTION_BATCH_SIZE) break; } let currentCount = await IpIntelligenceDoc.countAll(); const shouldPruneExcess = currentCount > IP_INTELLIGENCE_RETENTION_HIGH_WATER || (options.forCapacity && currentCount >= IP_INTELLIGENCE_RETENTION_HIGH_WATER); if (shouldPruneExcess) { let remainingToDelete = currentCount - IP_INTELLIGENCE_RETENTION_LOW_WATER; while (remainingToDelete > 0 && !this.isStopping) { const batchLimit = Math.min( remainingToDelete, IP_INTELLIGENCE_RETENTION_BATCH_SIZE, ); const oldestIds = await IpIntelligenceDoc.findOldestIds({ limit: batchLimit }); if (oldestIds.length === 0) break; const deletedThisBatch = await IpIntelligenceDoc.deleteByDocumentIds(oldestIds); deletedCount += deletedThisBatch; currentCount -= deletedThisBatch; remainingToDelete -= deletedThisBatch; if (deletedThisBatch === 0 || oldestIds.length < batchLimit) break; } } if (deletedCount > 0) { logger.log('info', 'IP intelligence retention removed ' + deletedCount + ' cached record(s)'); } this.persistedIntelligenceCount = currentCount; return currentCount; } private async withRetentionMutation(operation: () => Promise): Promise { const previous = this.retentionMutationTail; let release!: () => void; this.retentionMutationTail = new Promise((resolve) => { release = resolve; }); await previous; try { return await operation(); } finally { release(); } } public async refreshIpIntelligence(ipAddress: string): Promise { const ip = this.normalizeIp(ipAddress); if (!ip || !this.isPublicIp(ip)) { return null; } await this.observeIp(ip, { force: true }); const doc = await IpIntelligenceDoc.findByIp(ip); return doc ? this.intelligenceFromDoc(doc) : null; } private scheduleQueueDrain(): void { if (this.queueDrainScheduled || this.isStopping) return; this.queueDrainScheduled = true; setTimeout(() => { this.queueDrainScheduled = false; this.drainObservationQueue(); }, 0); } private drainObservationQueue(): void { if (this.isStopping) return; while ( this.activeQueuedObservations < OBSERVED_IP_QUEUE_CONCURRENCY && this.observationQueue.length > 0 ) { const ip = this.observationQueue.shift()!; this.queuedObservations.delete(ip); this.activeQueuedObservations++; void this.observeIp(ip) .catch(() => undefined) .finally(() => { this.activeQueuedObservations--; if (this.observationQueue.length > 0) { this.scheduleQueueDrain(); } }); } } private pruneQueuedIpMemory(now: number): void { if (this.lastQueuedAt.size <= OBSERVED_IP_QUEUE_LIMIT * 2) return; for (const [ip, lastQueuedAt] of this.lastQueuedAt) { if (now - lastQueuedAt > OBSERVED_IP_REQUEUE_THROTTLE_MS * 2) { this.lastQueuedAt.delete(ip); } } } public async listAuditEvents(limit = 100): Promise { return (await SecurityPolicyAuditDoc.findRecent(limit)).map((doc) => ({ id: doc.id, action: doc.action, actor: doc.actor, details: doc.details, createdAt: doc.createdAt, })); } private intelligenceFromDoc(doc: IpIntelligenceDoc): IIpIntelligenceRecord { return { ipAddress: doc.ipAddress, asn: doc.asn, asnOrg: doc.asnOrg, registrantOrg: doc.registrantOrg, registrantCountry: doc.registrantCountry, networkRange: doc.networkRange, networkCidrs: doc.networkCidrs, abuseContact: doc.abuseContact, country: doc.country, countryCode: doc.countryCode, city: doc.city, latitude: doc.latitude, longitude: doc.longitude, accuracyRadius: doc.accuracyRadius, timezone: doc.timezone, firstSeenAt: doc.firstSeenAt, lastSeenAt: doc.lastSeenAt, updatedAt: doc.updatedAt, seenCount: doc.seenCount, }; } public async createBlockRule(input: { type: TSecurityBlockRuleType; value: string; matchMode?: TSecurityBlockRuleMatchMode; reason?: string; enabled?: boolean; }, actor = 'system'): Promise { const now = Date.now(); const doc = new SecurityBlockRuleDoc(); doc.id = plugins.uuid.v4(); doc.type = input.type; doc.value = input.value.trim(); doc.matchMode = input.matchMode; doc.reason = input.reason; doc.enabled = input.enabled ?? true; doc.createdAt = now; doc.updatedAt = now; doc.createdBy = actor; await doc.save(); await this.writeAudit('createBlockRule', actor, { rule: this.ruleFromDoc(doc) }); await this.notifyPolicyChanged(); return this.ruleFromDoc(doc); } public async updateBlockRule(id: string, patch: Partial>, actor = 'system'): Promise { const doc = await SecurityBlockRuleDoc.findById(id); if (!doc) { return null; } if (patch.value !== undefined) doc.value = patch.value.trim(); if (patch.matchMode !== undefined) doc.matchMode = patch.matchMode; if (patch.reason !== undefined) doc.reason = patch.reason; if (patch.enabled !== undefined) doc.enabled = patch.enabled; doc.updatedAt = Date.now(); await doc.save(); await this.writeAudit('updateBlockRule', actor, { id, patch }); await this.notifyPolicyChanged(); return this.ruleFromDoc(doc); } public async deleteBlockRule(id: string, actor = 'system'): Promise { const doc = await SecurityBlockRuleDoc.findById(id); if (!doc) { return false; } await doc.delete(); await this.writeAudit('deleteBlockRule', actor, { id }); await this.notifyPolicyChanged(); return true; } public async compilePolicy(): Promise { const rules = await SecurityBlockRuleDoc.findEnabled(); const blockedIps = new Set(); const blockedCidrs = new Set(); const reactiveRules: SecurityBlockRuleDoc[] = []; for (const rule of rules) { const normalizedValue = rule.value.trim(); if (!normalizedValue) continue; if (rule.type === 'ip') { const ip = this.normalizeIp(normalizedValue); if (ip && plugins.net.isIP(ip)) blockedIps.add(ip); continue; } if (rule.type === 'cidr') { for (const cidr of this.normalizeNetworkEntries(normalizedValue)) { blockedCidrs.add(cidr); } continue; } if (this.getReactiveRuleQuery(rule)) { reactiveRules.push(rule); } } if (reactiveRules.length === 0) { return this.compiledPolicyFromSets(blockedIps, blockedCidrs); } const candidateDocs = new Map(); for (const rule of reactiveRules) { const query = this.getReactiveRuleQuery(rule)!; const docs = query.type === 'asn' ? await IpIntelligenceDoc.findByAsn(query.asn) : await IpIntelligenceDoc.findByOrganization(query.organization, query.matchMode); for (const doc of docs) { const documentId = (doc as IpIntelligenceDoc & { _id?: unknown })._id; const candidateKey = documentId === undefined ? `ip:${doc.ipAddress}` : `id:${String(documentId)}`; candidateDocs.set(candidateKey, doc); } } for (const rule of reactiveRules) { for (const doc of candidateDocs.values()) { if (!this.ruleMatchesIntelligence(rule, doc)) continue; const networkEntries = this.normalizeNetworkEntryList([ ...(doc.networkCidrs || []), doc.networkRange, ]); if (networkEntries.length > 0) { for (const cidr of networkEntries) { blockedCidrs.add(cidr); } } else if (this.normalizeIp(doc.ipAddress)) { blockedIps.add(this.normalizeIp(doc.ipAddress)!); } } } return this.compiledPolicyFromSets(blockedIps, blockedCidrs); } private compiledPolicyFromSets( blockedIps: Set, blockedCidrs: Set, ): ISecurityCompiledPolicy { return { blockedIps: [...blockedIps].sort(), blockedCidrs: [...blockedCidrs].sort(), }; } private getReactiveRuleQuery(rule: SecurityBlockRuleDoc): | { type: 'asn'; asn: number } | { type: 'organization'; organization: string; matchMode: TSecurityBlockRuleMatchMode } | undefined { const value = rule.value.trim(); if (!value) return undefined; if (rule.type === 'asn') { const normalizedAsn = value.replace(/^as/i, ''); if (!/^\d+$/.test(normalizedAsn)) return undefined; const asn = Number(normalizedAsn); if (!Number.isSafeInteger(asn) || String(asn) !== normalizedAsn) return undefined; return { type: 'asn', asn }; } if (rule.type === 'organization') { return { type: 'organization', organization: value, matchMode: rule.matchMode === 'exact' ? 'exact' : 'contains', }; } return undefined; } public async compileSmartProxyPolicy(): Promise { return await this.compilePolicy(); } public async compileRemoteIngressFirewall(): Promise { const policy = await this.compilePolicy(); const blockedIps = [ ...policy.blockedIps.filter((ip) => plugins.net.isIP(ip) === 4), ...policy.blockedCidrs.filter((cidr) => plugins.net.isIP(cidr.split('/')[0]) === 4), ]; return { blockedIps }; } private async matchesAnyReactiveRule(doc: IpIntelligenceDoc): Promise { const rules = await SecurityBlockRuleDoc.findEnabled(); return rules.some((rule) => rule.type === 'asn' || rule.type === 'organization' ? this.ruleMatchesIntelligence(rule, doc) : false); } private ruleMatchesIntelligence(rule: SecurityBlockRuleDoc, doc: IpIntelligenceDoc): boolean { const value = rule.value.trim().toLowerCase(); if (!value) return false; if (rule.type === 'asn') { return String(doc.asn ?? '') === value.replace(/^as/i, ''); } if (rule.type === 'organization') { const candidates = [doc.asnOrg, doc.registrantOrg] .filter(Boolean) .map((candidate) => candidate!.toLowerCase()); if (rule.matchMode === 'exact') { return candidates.some((candidate) => candidate === value); } return candidates.some((candidate) => candidate.includes(value)); } return false; } private normalizeIp(ipAddress: string): string | undefined { const ip = ipAddress.trim(); const normalizedIp = ip.startsWith('::ffff:') ? ip.slice('::ffff:'.length) : ip; return plugins.net.isIP(normalizedIp) ? normalizedIp : undefined; } private normalizeCidr(value: string): string | undefined { const [rawIp, rawPrefix] = value.trim().split('/'); if (!rawIp || !rawPrefix) return undefined; const ip = this.normalizeIp(rawIp); if (!ip) return undefined; const prefix = Number(rawPrefix); const maxPrefix = plugins.net.isIP(ip) === 4 ? 32 : 128; if (!Number.isInteger(prefix) || prefix < 0 || prefix > maxPrefix) return undefined; return `${ip}/${prefix}`; } private normalizeNetworkEntries(value: string): string[] { const trimmed = value.trim(); if (!trimmed) return []; const cidr = this.normalizeCidr(trimmed); if (cidr) return [cidr]; const rangeParts = trimmed.split(/\s+-\s+/); if (rangeParts.length === 2) { return this.ipv4RangeToCidrs(rangeParts[0], rangeParts[1]); } return []; } private normalizeNetworkEntryList(values: Array): string[] { const cidrs = new Set(); for (const value of values) { if (!value) continue; for (const entry of value.split(',').map((part) => part.trim()).filter(Boolean)) { for (const cidr of this.normalizeNetworkEntries(entry)) { cidrs.add(cidr); } } } return [...cidrs]; } private ipv4RangeToCidrs(startIp: string, endIp: string): string[] { const start = this.ipv4ToBigInt(startIp); const end = this.ipv4ToBigInt(endIp); if (start === undefined || end === undefined || start > end) return []; const cidrs: string[] = []; let current = start; while (current <= end) { let maxBlockSize = current === 0n ? 1n << 32n : current & -current; const remaining = end - current + 1n; while (maxBlockSize > remaining) { maxBlockSize = maxBlockSize / 2n; } const prefixLength = 32 - this.powerOfTwoExponent(maxBlockSize); cidrs.push(`${this.numberToIpv4(current)}/${prefixLength}`); current += maxBlockSize; } return cidrs; } private ipv4ToBigInt(ip: string): bigint | undefined { const normalized = this.normalizeIp(ip); if (!normalized || plugins.net.isIP(normalized) !== 4) return undefined; return normalized .split('.') .reduce((sum, part) => (sum * 256n) + BigInt(Number(part)), 0n); } private numberToIpv4(value: bigint): string { return [ Number((value >> 24n) & 255n), Number((value >> 16n) & 255n), Number((value >> 8n) & 255n), Number(value & 255n), ].join('.'); } private powerOfTwoExponent(value: bigint): number { let exponent = 0; let remaining = value; while (remaining > 1n) { remaining >>= 1n; exponent++; } return exponent; } private isPublicIp(ip: string): boolean { const family = plugins.net.isIP(ip); if (family === 4) { const parts = ip.split('.').map((part) => Number(part)); const [a, b] = parts; if (a === 10 || a === 127 || a === 0 || a >= 224) return false; if (a === 100 && b >= 64 && b <= 127) return false; if (a === 169 && b === 254) return false; if (a === 172 && b >= 16 && b <= 31) return false; if (a === 192 && b === 168) return false; return true; } if (family === 6) { const lower = ip.toLowerCase(); if (lower === '::1' || lower === '::') return false; if (lower.startsWith('fe80:') || lower.startsWith('fc') || lower.startsWith('fd')) return false; return true; } return false; } private ruleFromDoc(doc: SecurityBlockRuleDoc): ISecurityBlockRule { return { id: doc.id, type: doc.type, value: doc.value, matchMode: doc.matchMode, enabled: doc.enabled, reason: doc.reason, createdAt: doc.createdAt, updatedAt: doc.updatedAt, createdBy: doc.createdBy, }; } private async writeAudit(action: string, actor: string, details: Record): Promise { const doc = new SecurityPolicyAuditDoc(); doc.id = plugins.uuid.v4(); doc.action = action; doc.actor = actor; doc.details = details; doc.createdAt = Date.now(); await doc.save(); } private async notifyPolicyChanged(): Promise { if (this.onPolicyChanged) { await this.onPolicyChanged(); } } }