import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; import type { IRemoteIngressPerformanceConfig, IRemoteIngressStatus } from '../../ts_interfaces/data/remoteingress.js'; import type { IEmailOutboundEgressStatus } from '../../ts_interfaces/data/email-settings.js'; import type { RemoteIngressManager } from './classes.remoteingress-manager.js'; export interface ITunnelManagerConfig { tunnelPort?: number; targetHost?: string; tls?: { certPem?: string; keyPem?: string; }; performance?: IRemoteIngressPerformanceConfig; } export type TRemoteIngressEgressProxyRequest = Omit & { edgeId?: string; /** Restrict selection to edges whose id or tags match any entry. */ edgeFilter?: string[]; }; export type TRemoteIngressHubStatus = Awaited['getStatus'] >>; type TRemoteIngressHubEdgeStatus = TRemoteIngressHubStatus['connectedEdges'][number]; /** Cooldown applied to an edge after a failed egress open before it is deprioritized. */ const egressFailureCooldownMs = 30_000; const egressStatusFreshnessMs = 45_000; /** * Manages the RemoteIngressHub instance and tracks connected edge statuses. */ export class TunnelManager { private hub: InstanceType; private manager: RemoteIngressManager; private config: ITunnelManagerConfig; private edgeStatuses: Map = new Map(); private reconcileInterval: ReturnType | null = null; private syncChain: Promise = Promise.resolve(); private reconcileChain: Promise = Promise.resolve(); private stopped = true; private egressProxyEdgeByProxyId: Map = new Map(); private activeEgressProxiesByEdge: Map = new Map(); private egressFailureCooldownUntil: Map = new Map(); private onTopologyChanged?: (reason: string) => void; constructor(manager: RemoteIngressManager, config: ITunnelManagerConfig = {}) { this.manager = manager; this.config = config; this.hub = new plugins.remoteingress.RemoteIngressHub(); // Listen for edge connect/disconnect events this.hub.on('edgeConnected', (data: { edgeId: string; peerAddr: string }) => { this.edgeStatuses.set(data.edgeId, { edgeId: data.edgeId, connected: true, publicIp: data.peerAddr || null, activeTunnels: 0, lastHeartbeat: Date.now(), connectedAt: Date.now(), }); this.onTopologyChanged?.(`RemoteIngress edge connected: ${data.edgeId}`); }); this.hub.on('edgeDisconnected', (data: { edgeId: string }) => { this.edgeStatuses.delete(data.edgeId); this.onTopologyChanged?.(`RemoteIngress edge disconnected: ${data.edgeId}`); }); this.hub.on('streamSummary', (data: { edgeId: string; activeStreams: number; streamsOpenedTotal: number; streamsClosedTotal: number; }) => { const existing = this.edgeStatuses.get(data.edgeId); if (existing) { existing.activeTunnels = data.activeStreams; existing.lastHeartbeat = Date.now(); if (existing.traffic) { existing.traffic.streamsOpenedTotal = data.streamsOpenedTotal; existing.traffic.streamsClosedTotal = data.streamsClosedTotal; } } }); this.hub.on('egressConnection', (data: { proxyId: string; edgeId: string; logicalHost: string; port: number; outcome: string; resolvedIp?: string; error?: string; }) => { const message = `RemoteIngress egress ${data.outcome}: edge=${data.edgeId} proxy=${data.proxyId} target=${data.logicalHost}:${data.port}${data.resolvedIp ? ` resolvedIp=${data.resolvedIp}` : ''}${data.error ? ` error=${data.error}` : ''}`; const isExpectedLifecycleEvent = data.outcome === 'connected' || data.outcome === 'closed'; logger.log(isExpectedLifecycleEvent ? 'info' : 'warn', message); if (!isExpectedLifecycleEvent) { // Connection-level failures steer subsequent selections to other edges. this.egressFailureCooldownUntil.set(data.edgeId, Date.now() + egressFailureCooldownMs); } }); this.hub.on('egressIdentityChanged', (data) => { this.onTopologyChanged?.(`RemoteIngress egress identity changed: ${data.edgeId}`); }); } public setOnTopologyChanged(callback: (reason: string) => void): void { this.onTopologyChanged = callback; } /** * Start the tunnel hub and load allowed edges. */ public async start(): Promise { this.stopped = false; try { await this.hub.start({ tunnelPort: this.config.tunnelPort ?? 8443, targetHost: this.config.targetHost ?? '127.0.0.1', tls: this.config.tls, ...(this.config.performance ? { performance: this.config.performance } : {}), streamEventMode: 'summary', } as any); if (this.stopped) return; // Send allowed edges to the hub await this.syncAllowedEdges(); if (this.stopped) return; // Periodically reconcile with authoritative Rust hub status this.reconcileInterval = setInterval(() => { this.reconcileChain = this.reconcileChain .catch(() => {}) .then(() => this.reconcile()); this.reconcileChain.catch(() => {}); }, 15_000); } catch (err) { await this.stop(); throw err; } } /** * Stop the tunnel hub. */ public async stop(): Promise { if (this.stopped) { return; } this.stopped = true; if (this.reconcileInterval) { clearInterval(this.reconcileInterval); this.reconcileInterval = null; } await Promise.all([ this.syncChain.catch(() => {}), this.reconcileChain.catch(() => {}), ]); // Remove event listeners before stopping to prevent leaks this.hub.removeAllListeners(); await this.hub.stop(); this.edgeStatuses.clear(); this.egressProxyEdgeByProxyId.clear(); this.activeEgressProxiesByEdge.clear(); this.egressFailureCooldownUntil.clear(); } /** * Reconcile TS-side edge statuses with the authoritative Rust hub status. * Overwrites event-derived activeTunnels with the real activeStreams count. */ private async reconcile(): Promise { if (this.stopped) return; const hubStatus = await this.hub.getStatus(); if (this.stopped) return; if (!hubStatus || !hubStatus.connectedEdges) return; const before = this.getTopologySignature(); const rustEdgeIds = new Set(); for (const rustEdge of hubStatus.connectedEdges) { rustEdgeIds.add(rustEdge.edgeId); const existing = this.edgeStatuses.get(rustEdge.edgeId); if (existing) { existing.activeTunnels = rustEdge.activeStreams; existing.lastHeartbeat = Date.now(); this.applyRustStatus(existing, rustEdge); // Update peer address if available from Rust hub if (rustEdge.peerAddr) { existing.publicIp = rustEdge.peerAddr; } } else { // Missed edgeConnected event — add entry const status: IRemoteIngressStatus = { edgeId: rustEdge.edgeId, connected: true, publicIp: rustEdge.peerAddr || null, activeTunnels: rustEdge.activeStreams, lastHeartbeat: Date.now(), connectedAt: rustEdge.connectedAt * 1000, }; this.applyRustStatus(status, rustEdge); this.edgeStatuses.set(rustEdge.edgeId, status); } } // Remove entries for edges no longer connected in Rust (missed edgeDisconnected) for (const edgeId of this.edgeStatuses.keys()) { if (!rustEdgeIds.has(edgeId)) { this.edgeStatuses.delete(edgeId); } } const after = this.getTopologySignature(); if (before !== after) { this.onTopologyChanged?.('RemoteIngress mail egress topology changed'); } } /** * Sync allowed edges from the manager to the hub. * Call this after creating/deleting/updating edges. */ public async syncAllowedEdges(): Promise { const run = this.syncChain.catch(() => {}).then(async () => { if (this.stopped) return; const edges = this.manager.getAllowedEdges(); if (this.stopped) return; await this.hub.updateAllowedEdges(edges as any); }); this.syncChain = run; await run; } public async startEgressTcpProxy( request: TRemoteIngressEgressProxyRequest, ): Promise { if (this.stopped) { throw new Error('RemoteIngress hub is not running'); } const status = await this.hub.getStatus(); if (this.stopped) { throw new Error('RemoteIngress hub is not running'); } const connectedEdges = Array.isArray(status?.connectedEdges) ? status.connectedEdges : []; const candidates = this.selectEgressCandidates(connectedEdges, request); if (candidates.length === 0) { throw new Error(`No eligible RemoteIngress QUIC egress edge is connected${this.describeEgressScope(request)}`); } const { edgeId: _ignored, edgeFilter: _alsoIgnored, ...proxyRequest } = request; let lastError: Error | undefined; for (const candidate of candidates) { try { const proxy = await this.hub.startEgressTcpProxy({ ...proxyRequest, edgeId: candidate.edgeId, }); this.egressProxyEdgeByProxyId.set(proxy.proxyId, candidate.edgeId); this.activeEgressProxiesByEdge.set( candidate.edgeId, (this.activeEgressProxiesByEdge.get(candidate.edgeId) || 0) + 1, ); return proxy; } catch (error: unknown) { lastError = error as Error; this.egressFailureCooldownUntil.set(candidate.edgeId, Date.now() + egressFailureCooldownMs); logger.log('warn', `RemoteIngress egress open failed on edge ${candidate.edgeId}, trying next candidate: ${lastError.message}`); } } throw new Error( `All eligible RemoteIngress egress edges failed${this.describeEgressScope(request)}: ${lastError?.message || 'unknown error'}`, ); } /** Authoritative hub state used by the source-bound mail identity adapter. */ public async getAuthoritativeHubStatus(): Promise { if (this.stopped) return undefined; return await this.hub.getStatus(); } /** * Eligibility (connected + egressEnabled + egressTcpV1 + native QUIC), scoped by * edgeId/edgeFilter, capped by per-edge maxConcurrentStreams, ordered deterministically: * not-in-cooldown first, then fewest active tunnels, then edgeId. */ private selectEgressCandidates( connectedEdges: TRemoteIngressHubEdgeStatus[], request: TRemoteIngressEgressProxyRequest, ): Array<{ edgeId: string; activeStreams: number }> { const eligible = connectedEdges.filter((edge: any) => { const hasEgressCapability = Array.isArray(edge.capabilities) && edge.capabilities.includes('egressTcpV1'); const usesQuic = edge.transportMode === 'quic' || (edge.transportMode === 'quicWithFallback' && edge.fallbackUsed === false); const observed = this.edgeStatuses.get(edge.edgeId); const fresh = observed?.connected && observed.lastHeartbeat !== null && Date.now() - observed.lastHeartbeat <= egressStatusFreshnessMs; return edge.egressEnabled === true && hasEgressCapability && usesQuic && fresh; }); let scoped = eligible; if (request.edgeId) { scoped = eligible.filter((edge) => edge.edgeId === request.edgeId); } else if (request.edgeFilter && request.edgeFilter.length > 0) { const allowedIds = new Set( this.manager.resolveEdgesByFilter(request.edgeFilter).map((edge) => edge.id), ); scoped = eligible.filter((edge) => allowedIds.has(edge.edgeId)); } const underCap = scoped.filter((edge) => { const cap = this.manager.getEdge(edge.edgeId)?.egress?.maxConcurrentStreams; if (!cap) return true; return (this.activeEgressProxiesByEdge.get(edge.edgeId) || 0) < cap; }); const now = Date.now(); const inCooldown = (edgeId: string) => (this.egressFailureCooldownUntil.get(edgeId) || 0) > now; return underCap .map((edge) => ({ edgeId: edge.edgeId, activeStreams: Number(edge.activeStreams) || 0 })) .sort((a, b) => { const cooldownDelta = Number(inCooldown(a.edgeId)) - Number(inCooldown(b.edgeId)); if (cooldownDelta !== 0) return cooldownDelta; if (a.activeStreams !== b.activeStreams) return a.activeStreams - b.activeStreams; return a.edgeId.localeCompare(b.edgeId); }); } private describeEgressScope(request: TRemoteIngressEgressProxyRequest): string { if (request.edgeId) return ` (pinned edge: ${request.edgeId})`; if (request.edgeFilter && request.edgeFilter.length > 0) return ` (filter: ${request.edgeFilter.join(', ')})`; return ''; } /** * Readiness of the outbound egress path for a given edge scope — * consumed by the ops API and the email-setup preflight. */ public getEgressReadiness(edgeFilter: string[]): IEmailOutboundEgressStatus { const allowedIds = new Set( this.manager.resolveEdgesByFilter(edgeFilter).map((edge) => edge.id), ); let eligibleEdgeCount = 0; for (const status of this.edgeStatuses.values()) { if (!status.connected) continue; if (!allowedIds.has(status.edgeId)) continue; const hasEgressCapability = status.capabilities?.includes('egressTcpV1') ?? false; const usesQuic = status.transportMode === 'quic' || (status.transportMode === 'quicWithFallback' && status.fallbackUsed === false); if (status.egressEnabled === true && hasEgressCapability && usesQuic) { eligibleEdgeCount++; } } if (eligibleEdgeCount > 0) { return { ready: true, eligibleEdgeCount }; } const reason = allowedIds.size === 0 ? `no enabled edge matches filter [${edgeFilter.join(', ')}]` : 'no connected matching edge with native QUIC egress capability'; return { ready: false, eligibleEdgeCount: 0, reason }; } /** * Delegate source-bound connection evidence to the owned RemoteIngress hub. */ public async waitForEgressConnectionEvidence( proxyId: string, edgeId: string, timeoutMs?: number, ): Promise { if (this.stopped) { return null; } return await this.hub.waitForEgressConnectionEvidence(proxyId, edgeId, timeoutMs); } public async stopEgressTcpProxy(proxyId: string): Promise { if (this.stopped) { return false; } const edgeId = this.egressProxyEdgeByProxyId.get(proxyId); if (edgeId) { this.egressProxyEdgeByProxyId.delete(proxyId); const current = this.activeEgressProxiesByEdge.get(edgeId) || 0; if (current <= 1) { this.activeEgressProxiesByEdge.delete(edgeId); } else { this.activeEgressProxiesByEdge.set(edgeId, current - 1); } } return await this.hub.stopEgressTcpProxy(proxyId); } private applyRustStatus(status: IRemoteIngressStatus, rustEdge: TRemoteIngressHubEdgeStatus): void { status.transportMode = rustEdge.transportMode; status.fallbackUsed = rustEdge.fallbackUsed; status.capabilities = rustEdge.capabilities; status.egressEnabled = rustEdge.egressEnabled; status.performance = rustEdge.performance; status.flowControl = rustEdge.flowControl; status.queues = rustEdge.queues; status.traffic = rustEdge.traffic; status.udp = rustEdge.udp; } private getTopologySignature(): string { return [...this.edgeStatuses.values()] .map((status) => [ status.edgeId, status.connected, status.egressEnabled, status.transportMode, status.fallbackUsed, [...(status.capabilities || [])].sort().join(','), ].join('|')) .sort() .join(';'); } /** * Get runtime statuses for all known edges. */ public getEdgeStatuses(): IRemoteIngressStatus[] { return Array.from(this.edgeStatuses.values()); } /** * Get status for a specific edge. */ public getEdgeStatus(edgeId: string): IRemoteIngressStatus | undefined { return this.edgeStatuses.get(edgeId); } /** * Get the count of connected edges. */ public getConnectedCount(): number { let count = 0; for (const status of this.edgeStatuses.values()) { if (status.connected) count++; } return count; } /** * Get the public IPs of all connected edges. */ public getConnectedEdgeIps(): string[] { const ips: string[] = []; for (const status of this.edgeStatuses.values()) { if (status.connected && status.publicIp) { ips.push(status.publicIp); } } return ips; } /** * Get the total number of active tunnels across all edges. */ public getTotalActiveTunnels(): number { let total = 0; for (const status of this.edgeStatuses.values()) { total += status.activeTunnels; } return total; } }