import WebSocket from 'ws' import { fetch, type Dispatcher, type RequestInit, type Response } from 'undici' import { getDefaultDispatcher } from './default-dispatcher' import { IdCounter, RequestMatcher, buildAuthRequest, ErrorCode, NetworkError, DNS_ERROR_CODES, DNS_TERMINAL_CODE, isAuthTerminalCode, OAUTH_TERMINAL_CODE, } from './types' import type { TrueConfAccountConfig, TrueConfResponse, TrueConfRequest, OAuthTokenResponse, Logger, } from './types' import { readHeartbeatIntervalMs, readHeartbeatPongTimeoutMs, readOauthTimeoutMs, readWsHandshakeTimeoutMs, readDnsFailLimit, readOauthFailLimit, readTcpKeepaliveMs, } from './env-config.js' // Each tunable is read once at module load. Defaults match TrueConf's // python-trueconf-bot SDK production semantics; override via the matching // TRUECONF_* env vars for corporate-NAT idle timeouts, hung reverse-proxy // auth, the WS handshake budget, and DNS / OAuth terminal thresholds. const HEARTBEAT_INTERVAL_MS = readHeartbeatIntervalMs() const HEARTBEAT_PONG_TIMEOUT_MS = readHeartbeatPongTimeoutMs() const OAUTH_TIMEOUT_MS = readOauthTimeoutMs() const WS_HANDSHAKE_TIMEOUT_MS = readWsHandshakeTimeoutMs() const DNS_FAIL_LIMIT = readDnsFailLimit() const OAUTH_FAIL_LIMIT = readOauthFailLimit() // SO_KEEPALIVE is sent by the OS kernel independently of the Node.js event // loop, so it survives long synchronous blocks (e.g. openclaw LLM preprocessing // that pegs the loop for 60-90s). Probe data on a corporate TrueConf-on-prem // network showed the server (or upstream proxy) graceful-FIN'd idle TCP // after ~41s — the WS-protocol heartbeat could not run during the loop block, // so the close went unnoticed. Default 15s sits well below the observed // 41s server idle window. const TCP_KEEPALIVE_MS = readTcpKeepaliveMs() export function hostPort(config: { serverUrl: string; useTls: boolean; port?: number }): string { if (typeof config.serverUrl !== 'string' || config.serverUrl.length === 0) { throw new Error('hostPort: serverUrl must be a non-empty string') } if (config.port !== undefined) { if (typeof config.port !== 'number' || !Number.isInteger(config.port) || config.port < 1 || config.port > 65535) { throw new Error(`hostPort: invalid port ${JSON.stringify(config.port)}`) } } const port = config.port ?? (config.useTls ? 443 : 4309) if (config.useTls && port === 443) return config.serverUrl if (!config.useTls && port === 80) return config.serverUrl return `${config.serverUrl}:${port}` } export function buildWsUrl(config: TrueConfAccountConfig): string { return `${config.useTls ? 'wss' : 'ws'}://${hostPort(config)}/websocket/chat_bot/` } export function buildTokenUrl(config: TrueConfAccountConfig): string { return `${config.useTls ? 'https' : 'http'}://${hostPort(config)}/bridge/api/client/v1/oauth/token` } // Pull libuv-style metadata off `err.cause`. fetch wraps a TypeError around // the underlying ENOTFOUND/ECONNREFUSED/etc. — we want code/syscall/hostname // to flow through to NetworkError so callers (DNS retry policy, telemetry) // can branch on them without parsing message strings. function extractFetchCauseMeta(err: unknown): { code?: string; syscall?: string; hostname?: string } { const cause = err instanceof Error ? err.cause : undefined if (!cause || typeof cause !== 'object') return {} const obj = cause as Record return { code: 'code' in obj && typeof obj.code !== 'undefined' ? String(obj.code) : undefined, syscall: 'syscall' in obj && typeof obj.syscall !== 'undefined' ? String(obj.syscall) : undefined, hostname: 'hostname' in obj && typeof obj.hostname !== 'undefined' ? String(obj.hostname) : undefined, } } export async function acquireToken( config: TrueConfAccountConfig, options?: { dispatcher?: Dispatcher }, ): Promise { // Defence-in-depth at the runtime boundary: TrueConfAccountConfig.password // is typed as `string | { useEnv: string }`, but if an upstream caller // forgets to resolve the useEnv indirection, JSON.stringify would silently // serialise `"password":{"useEnv":"X"}`, the server would 401, and the // operator would chase bogus 'bad creds' instead of the real type bug. if (typeof config.password !== 'string') { throw new Error( `OAuth: password must be resolved to a string before acquireToken (got ${typeof config.password})`, ) } let response: Response try { response = await fetch(buildTokenUrl(config), { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ client_id: config.clientId ?? 'chat_bot', client_secret: config.clientSecret ?? '', grant_type: 'password', username: config.username, password: config.password, }), dispatcher: options?.dispatcher ?? getDefaultDispatcher(), signal: AbortSignal.timeout(OAUTH_TIMEOUT_MS), } as RequestInit) } catch (err) { // AbortSignal.timeout fires as DOMException with name='TimeoutError' // (per WHATWG; undici uses globalThis.AbortSignal which surfaces this // shape). Promote it to a NetworkError with a synthetic code so the // lifecycle catch can classify it as retryable, distinct from auth // failure (which carries the HTTP status as code). if (err instanceof Error && err.name === 'TimeoutError') { throw new NetworkError( `OAuth token request timed out after ${OAUTH_TIMEOUT_MS}ms`, 'oauth', err, 'OAUTH_TIMEOUT', ) } const meta = extractFetchCauseMeta(err) const outerMsg = err instanceof Error ? err.message : String(err) const causeMsg = err instanceof Error && err.cause instanceof Error ? err.cause.message : '' const detail = `${outerMsg}${meta.code ? ` (${meta.code}: ${causeMsg || meta.code})` : causeMsg ? ` (${causeMsg})` : ''}` throw new NetworkError( `OAuth token request failed: ${detail}`, 'oauth', err instanceof Error ? err : undefined, meta.code, meta.syscall, meta.hostname, ) } if (!response.ok) { let detail = response.statusText try { const err = (await response.json()) as Record if (typeof err.error_description === 'string') detail = err.error_description } catch { // non-JSON error body (reverse-proxy HTML etc.) } throw new NetworkError( `OAuth token acquisition failed (${response.status}): ${detail}`, 'oauth', undefined, String(response.status), ) } const json = (await response.json()) as Record if (typeof json.access_token !== 'string' || typeof json.expires_at !== 'number') { // Promote to NetworkError(phase='oauth') so scheduleReconnect's catch // classifies it the same way as every other oauth-phase failure. A plain // Error fell into the else branch, reset oauthFailCount, and would loop // silently forever on a misconfigured bridge that returns malformed token // bodies. OAUTH_INVALID_RESPONSE is a non-numeric code, so the // isAuthTerminalError parseInt guard treats it as transient (NaN), same // safe behavior as OAUTH_TIMEOUT. throw new NetworkError( 'Invalid OAuth response: missing access_token or expires_at', 'oauth', undefined, 'OAUTH_INVALID_RESPONSE', ) } return json as unknown as OAuthTokenResponse } interface Deferred { promise: Promise resolve: () => void reject: (err: Error) => void } export interface WsClientOptions { ca?: Buffer tlsVerify?: boolean // Optional reconnect adapter. When sendRequest sees errorCode=203 // CREDENTIALS_EXPIRED it calls back here to drive a full close → fresh-token // reconnect, then retries the original request once. Without an adapter // wired, the 203 response surfaces to the caller as-is. forceReconnect?: (reason: string) => Promise } export class WsClient { private ws: WebSocket | null = null private idCounter = new IdCounter() private matcher = new RequestMatcher() // Per-fileId listener for uploadFileProgress notifications pushed by the // server after subscribeFileProgress. Cleared on WS close. private progressHandlers = new Map void>() public botUserId: string | null = null public onInboundMessage: ((msg: TrueConfRequest) => void | Promise) | null = null public onClose: ((code: number, reason: string) => void) | null = null public onPong: (() => void) | null = null public logger: Logger | null = null // Custom CA bundle for WebSocket TLS (downloaded by the setup wizard and // written to caPath). Passed straight through to ws's `ca` option. public ca: Buffer | undefined = undefined // When false, the WebSocket handshake skips cert verification — the // operator-acknowledged insecure mode for self-signed TrueConf Servers. // Defaults to true so any code path that forgets to thread the flag stays // safe-by-default. public tlsVerify = true private pushListeners: Array<(method: string, payload: Record) => void> = [] private authListeners: Array<() => void> = [] private readonly forceReconnect?: (reason: string) => Promise // Awaitable barrier between connect-start and auth-success. sendRequest // gates on this so callers can fire requests during a reconnect window // and have them automatically queue until the next auth completes. private authBarrier: Deferred = this.makeDeferred() constructor(options?: WsClientOptions) { if (options?.ca) this.ca = options.ca if (options?.tlsVerify === false) this.tlsVerify = false this.forceReconnect = options?.forceReconnect } // Returns the option bag passed to `new WebSocket(..., options)`. When // tlsVerify=false, the insecure flag wins over a stale ca pin so the // outcome is unambiguous. When neither knob is set, returns undefined so // ws falls back to the system trust store (default behavior). buildClientOptions(): { rejectUnauthorized: false } | { ca: Buffer } | undefined { if (this.tlsVerify === false) return { rejectUnauthorized: false } if (this.ca) return { ca: this.ca } return undefined } onFileProgress(fileId: string, handler: (progress: number) => void): void { this.progressHandlers.set(fileId, handler) } offFileProgress(fileId: string): void { this.progressHandlers.delete(fileId) } onPush(listener: (method: string, payload: Record) => void): () => void { this.pushListeners.push(listener) return () => { this.pushListeners = this.pushListeners.filter((l) => l !== listener) } } onAuth(listener: () => void): () => void { this.authListeners.push(listener) return () => { this.authListeners = this.authListeners.filter((l) => l !== listener) } } private makeDeferred(): Deferred { let resolve!: () => void let reject!: (err: Error) => void const promise = new Promise((res, rej) => { resolve = res reject = rej }) // Pre-attach a no-op catch so a barrier rejected before anyone called // waitAuthenticated() does not surface as an unhandled-rejection warning. promise.catch(() => undefined) return { promise, resolve, reject } } // resetAuthBarrier MUST reject the previous deferred before swapping so any // pending waitAuthenticated() callers fail fast instead of hanging until // their per-call timeout. Otherwise concurrent senders can hold a // resolved-old reference past close(). resetAuthBarrier(reason: string = 'reset'): void { const old = this.authBarrier this.authBarrier = this.makeDeferred() old.reject(NetworkError.parkable(`auth barrier reset: ${reason}`)) } markAuthenticated(): void { this.authBarrier.resolve() } markAuthFailed(err: Error): void { this.authBarrier.reject(err) } async waitAuthenticated(timeoutMs = 30_000): Promise { let timer: ReturnType | undefined try { return await Promise.race([ this.authBarrier.promise, new Promise((_, reject) => { timer = setTimeout(() => { reject(NetworkError.parkable(`waitAuthenticated timed out after ${timeoutMs}ms`)) }, timeoutMs) timer.unref?.() }), ]) } finally { if (timer) clearTimeout(timer) } } connect(config: TrueConfAccountConfig, token: string): Promise { this.matcher.rejectAll(NetworkError.parkable('New connection started')) this.progressHandlers.clear() this.idCounter.reset() const ws = new WebSocket( buildWsUrl(config), 'json.v1', this.buildClientOptions(), ) this.ws = ws return new Promise((resolve, reject) => { let settled = false let handshakeTimer: ReturnType | undefined const settle = (fn: () => void) => { if (settled) return settled = true if (handshakeTimer) { clearTimeout(handshakeTimer) handshakeTimer = undefined } fn() } // Bound the wall-clock budget from socket construction to the first // 'open' event (TLS + WS upgrade). If the upgrade never arrives // (silent reverse-proxy, misconfigured firewall), terminate the // socket and surface a NetworkError so the lifecycle backoff loop // runs. Cleared in 'open' so the auth round-trip runs under // RequestMatcher's own 30s timeout, not under this budget. handshakeTimer = setTimeout(() => { try { ws.terminate() } catch { /* ws may already be torn down */ } settle(() => reject(new NetworkError( `WS handshake timed out after ${WS_HANDSHAKE_TIMEOUT_MS}ms`, 'ws-handshake', undefined, 'WS_HANDSHAKE_TIMEOUT', ))) }, WS_HANDSHAKE_TIMEOUT_MS) handshakeTimer.unref?.() ws.on('open', () => { // Handshake completed — release the handshake budget so a slow // auth round-trip is not punished by a timer that was meant to // bound the upgrade alone. if (handshakeTimer) { clearTimeout(handshakeTimer) handshakeTimer = undefined } // Enable SO_KEEPALIVE on the underlying TCP socket so the kernel // keeps the connection alive across long Node event-loop blocks // (openclaw LLM preprocessing). The WS-protocol heartbeat would // not run during such blocks, allowing an upstream idle-killer // (server or proxy) to graceful-FIN the session unnoticed. const underlying = (ws as unknown as { _socket?: { setKeepAlive?: (enable: boolean, initialDelay: number) => void } })._socket try { underlying?.setKeepAlive?.(true, TCP_KEEPALIVE_MS) } catch (err) { this.logger?.warn(`[trueconf] setKeepAlive failed: ${err instanceof Error ? err.message : String(err)}`) } const authId = this.idCounter.next() const authPromise = this.matcher.track(authId) ws.send(JSON.stringify(buildAuthRequest(authId, token))) authPromise .then((response: TrueConfResponse) => { const errorCode = response.payload?.errorCode if (errorCode !== undefined && errorCode !== 0) { const desc = response.payload?.errorDescription ?? '' settle(() => reject(new Error(`Auth failed: errorCode ${errorCode}${desc ? ' - ' + desc : ''}`))) } else { this.botUserId = (response.payload?.userId as string) ?? null for (const l of this.authListeners) { try { l() } catch (err) { this.logger?.error( `[trueconf] auth listener error: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`, ) } } settle(() => resolve()) } }) .catch((err: Error) => settle(() => reject(err))) }) // Auto-ack and inbound dispatch close over the LOCAL `ws` reference so a // mid-frame reconnect that reassigns `this.ws` cannot misroute the ack // (or push) to the new socket. This mirrors python-trueconf-bot's // per-connection task model. ws.on('message', (data: Buffer | string) => { let msg: { type?: number; id?: number; method?: string; payload?: unknown } try { msg = JSON.parse(data.toString()) as typeof msg } catch (err) { this.logger?.warn( `[trueconf] Malformed JSON in message handler: ${err instanceof Error ? err.message : String(err)}`, ) return } // Auto-ack every server-originated request so we never miss the // protocol's mandatory reply. Use the captured ws (not this.ws) and // skip if the socket already closed underneath us. if (msg?.type === 1 && typeof msg.id === 'number') { if (ws.readyState === WebSocket.OPEN) { try { ws.send(JSON.stringify({ type: 2, id: msg.id })) } catch (err) { this.logger?.warn( `[trueconf] auto-ack failed: ${err instanceof Error ? err.message : String(err)}`, ) } } } if (msg?.type === 2 && typeof msg.id === 'number') { this.matcher.resolve(msg.id, msg as TrueConfResponse) return } if (msg?.type === 1) { // Docs name the event `uploadingProgress` but the server actually // sends `uploadFileProgress`. Route it to the registered // per-fileId handler and don't forward to onInboundMessage. if (msg.method === 'uploadFileProgress') { const payload = (msg.payload ?? {}) as { fileId?: unknown; progress?: unknown } const fileId = payload.fileId const progress = payload.progress if (typeof fileId === 'string' && typeof progress === 'number') { this.progressHandlers.get(fileId)?.(progress) } return } // sendMessage is delivered via onInboundMessage only — push // listeners are for non-message events (chat lifecycle, member role // changes, presence, etc.). Mirrors python-trueconf-bot routing. if (msg.method !== 'sendMessage') { for (const l of this.pushListeners) { try { l(msg.method!, (msg.payload ?? {}) as Record) } catch (err) { this.logger?.error( `[trueconf] push listener error for method=${msg.method}: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`, ) } } } if (this.onInboundMessage) { void Promise.resolve(this.onInboundMessage(msg as TrueConfRequest)).catch((err) => { this.logger?.error( `[trueconf] inbound message handler error: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`, ) }) } } }) ws.on('error', (err: Error) => { // `'code' in err` returns true even when the property is defined-but- // undefined; String(undefined) is the literal string 'undefined' which // would propagate into NetworkError.code and mislead any downstream // telemetry that branches on err.code != null. Mirror the // extractFetchCauseMeta idiom (===undefined) so the WS error path // surfaces real codes only. const codeRaw = (err as { code?: unknown }).code const code = codeRaw === undefined ? undefined : String(codeRaw) const syscallRaw = (err as { syscall?: unknown }).syscall const syscall = syscallRaw === undefined ? undefined : String(syscallRaw) const hostnameRaw = (err as { hostname?: unknown }).hostname const hostname = hostnameRaw === undefined ? undefined : String(hostnameRaw) settle(() => reject(new NetworkError( `WebSocket error: ${err.message}`, 'websocket', err, code, syscall, hostname, ))) }) ws.on('close', (code: number, reason: Buffer) => { // Stale close from an old socket (e.g., a delayed close event arriving // after a forced reconnect already swapped this.ws to a new socket). // Ignoring is correct: rejecting matcher pendings or clearing progress // handlers would clobber the new socket's in-flight state, and bubbling // up to lifecycle.handleClose would schedule a redundant reconnect. if (this.ws !== ws) { this.logger?.info(`[trueconf] stale close from old socket (code=${code}); ignoring`) return } this.matcher.rejectAll(NetworkError.parkable( 'WebSocket closed: ' + code + ' ' + (reason?.toString() ?? ''), )) this.progressHandlers.clear() this.onClose?.(code, reason?.toString() ?? '') }) ws.on('pong', () => this.onPong?.()) }) } // Public sendRequest gates on the auth barrier (so requests fired during a // reconnect window queue until auth lands) and recovers from // CREDENTIALS_EXPIRED (203) by forcing a fresh-token reconnect once. async sendRequest( method: string, payload: Record, traceId?: string, ): Promise { const c = payload.chatId const chatIdSeg = typeof c === 'string' ? ` chatId=${c}` : '' const log = (event: 'wait_auth' | 'ack', extra = ''): void => { if (traceId === undefined) return this.logger?.info(`[trueconf] outbound ${event}: qid=${traceId} method=${method}${chatIdSeg}${extra}`) } await this.waitAuthenticated() log('wait_auth') const response = await this.sendRequestInternal(method, payload, traceId) log('ack', ` errorCode=${response.payload?.errorCode}`) if (response.payload?.errorCode === ErrorCode.CREDENTIALS_EXPIRED) { this.logger?.warn( `[trueconf] ${method} returned 203 CREDENTIALS_EXPIRED; forcing reconnect with fresh token`, ) if (!this.forceReconnect) { // Fail-soft: surface the 203 instead of hanging forever. ConnectionLifecycle // is the only entity allowed to drive reconnects; if it never wired the // callback there is nothing safe we can do here. this.logger?.error( '[trueconf] 203 received but no forceReconnect callback wired; surfacing original response', ) return response } await this.forceReconnect('203_credentials_expired') await this.waitAuthenticated() log('wait_auth') const retried = await this.sendRequestInternal(method, payload, traceId) log('ack', ` errorCode=${retried.payload?.errorCode}`) return retried } return response } private sendRequestInternal( method: string, payload: Record, traceId?: string, ): Promise { const id = this.idCounter.next() const request: TrueConfRequest = { type: 1, id, method, payload } const tracked = this.matcher.track(id) try { this.send(request) if (traceId !== undefined) { const c = payload.chatId const chatIdSeg = typeof c === 'string' ? ` chatId=${c}` : '' this.logger?.info(`[trueconf] outbound wire_send: qid=${traceId} method=${method}${chatIdSeg}`) } } catch (err) { this.matcher.reject(id, err instanceof Error ? err : new Error(String(err))) } return tracked } send(msg: object): void { if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { throw NetworkError.parkable('WebSocket is not connected') } this.ws.send(JSON.stringify(msg)) } close(): void { this.ws?.close(1000, 'Client closing') } ping(): void { if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.ping() } terminate(): void { this.ws?.terminate() } } // Discriminated union over the bounded set of terminal lifecycle outcomes. // Receivers branch on `kind` instead of parsing message strings; the original // Error survives on `cause` for logging. export type TerminalCause = | { kind: 'shutdown'; cause: Error } | { kind: 'dns_exhausted'; retries: number; cause: NetworkError } | { kind: 'auth_exhausted'; retries: number; cause: NetworkError } interface LifecycleOptions { onConnectionClosed?: (code: number, reason: string) => void onConnected?: () => void onDisconnected?: () => void // Fires once on terminal lifecycle outcome (not on transient close events, // which go through onConnectionClosed and reconnect). onTerminalFailure?: (terminal: TerminalCause) => void } // Liveness is governed by WebSocket protocol ping/pong (opcode 0x9/0xA) on a // 30s/10s schedule, mirroring python-trueconf-bot's websockets.connect // configuration. Pong timeout escalates via terminate() so the normal // close → scheduleReconnect path runs. export class ConnectionLifecycle { private heartbeatTimer: ReturnType | null = null private tokenRefreshTimer: ReturnType | null = null private reconnectTimer: ReturnType | null = null private lastPingAt = 0 private lastPongAt = 0 private backoffMs = 1000 private shuttingDown = false private reconnecting = false // Re-entry guard for start() itself, distinct from `reconnecting` (which // handleClose sets to gate its own re-entry while a reconnect timer is // armed). Prevents the boot-path race where the handshake-timeout-driven // ws.terminate() emits a close event that handleClose would otherwise // turn into a second scheduleReconnect() while start()'s catch is still // re-throwing to the bootstrap caller. private startInFlight = false private dnsRetryCount = 0 private oauthFailCount = 0 private reconnectInflight: Promise | null = null private suppressNextCloseReconnect = false constructor( private wsClient: WsClient, private config: TrueConfAccountConfig, private logger: Logger, private options?: LifecycleOptions & { dispatcher?: Dispatcher }, ) {} async start(): Promise { // Synchronous re-entry guard: a second start() while the first is still // running would race the auth barrier and the connect()/acquireToken // promises. The boot path is the exposed seam — see startInFlight // declaration. Returning silently is safe because the in-flight call // will resolve or reject on its own. if (this.startInFlight) return this.startInFlight = true try { // Re-arm the auth barrier at the top of every start() so requests issued // during the reconnect window queue on the new attempt rather than seeing // the stale resolved promise from the previous session. this.wsClient.resetAuthBarrier('lifecycle.start') let tokenResponse: OAuthTokenResponse try { tokenResponse = await acquireToken(this.config, { dispatcher: this.options?.dispatcher }) } catch (err) { // Mark parkable so OutboundQueue items waiting on the auth barrier // park instead of rejecting; scheduleReconnect's catch decides // terminal-vs-retry and fires onTerminalFailure -> failAll for the // give-up cases. this.wsClient.markAuthFailed(NetworkError.asParkable(err)) throw err } // Register lifecycle handlers BEFORE connect so a close event that fires // between auth completion and the first post-connect line still routes // through handleClose → scheduleReconnect. this.wsClient.onClose = (code, reason) => this.handleClose(code, reason) this.wsClient.onPong = () => { this.lastPongAt = Date.now() } try { await this.wsClient.connect(this.config, tokenResponse.access_token) } catch (err) { this.wsClient.markAuthFailed(NetworkError.asParkable(err)) throw err } this.backoffMs = 1000 this.reconnecting = false this.dnsRetryCount = 0 this.oauthFailCount = 0 this.wsClient.markAuthenticated() this.logger.info('[trueconf] Connected and authenticated') try { this.options?.onConnected?.() } catch (err) { this.logger.warn(`[trueconf] onConnected callback failed: ${err instanceof Error ? err.message : String(err)}`) } this.startTimers(tokenResponse.expires_at) } finally { this.startInFlight = false } } shutdown(): void { this.logger.info('[trueconf] Shutting down connection') this.shuttingDown = true // Reject the auth barrier with an explicit reason so any pending // waitAuthenticated() callers fail fast on shutdown instead of waiting // out their per-call timeout. const cause = new Error('lifecycle shutting down') this.wsClient.markAuthFailed(cause) try { this.options?.onTerminalFailure?.({ kind: 'shutdown', cause }) } catch (err) { this.logger.warn(`[trueconf] onTerminalFailure callback failed: ${err instanceof Error ? err.message : String(err)}`) } this.stopTimers() this.cancelReconnect() this.wsClient.close() } // forceReconnect tears down the current connection and brings up a fresh // one, used when sendRequest sees CREDENTIALS_EXPIRED (203). Sequencing is // load-bearing: SYNC barrier-reject → SYNC suppress flag → SYNC cancel // pending retry timer → close (async). The suppress flag prevents the // close handler from racing us with its own scheduleReconnect. async forceReconnect(reason: string): Promise { if (this.reconnectInflight) return this.reconnectInflight this.logger.info(`[trueconf] Forced reconnect: ${reason}`) this.wsClient.resetAuthBarrier(`forced reconnect: ${reason}`) this.suppressNextCloseReconnect = true if (this.reconnectTimer) { clearTimeout(this.reconnectTimer) this.reconnectTimer = null } this.wsClient.close() this.reconnectInflight = (async () => { try { await this.start() } finally { this.reconnectInflight = null this.suppressNextCloseReconnect = false } })() return this.reconnectInflight } private startTimers(expiresAt: number): void { this.stopTimers() const now = Date.now() this.lastPingAt = now this.lastPongAt = now this.heartbeatTimer = setInterval(() => this.heartbeatTick(), HEARTBEAT_INTERVAL_MS) const delayMs = (expiresAt - 3600) * 1000 - Date.now() if (delayMs <= 0) { this.refreshAndReconnect() return } this.tokenRefreshTimer = setTimeout(() => this.refreshAndReconnect(), Math.min(delayMs, 2_147_483_647)) } private stopTimers(): void { if (this.heartbeatTimer) { clearInterval(this.heartbeatTimer); this.heartbeatTimer = null } if (this.tokenRefreshTimer) { clearTimeout(this.tokenRefreshTimer); this.tokenRefreshTimer = null } } private handleClose(code: number, reason: string): void { this.stopTimers() try { this.options?.onDisconnected?.() } catch (err) { this.logger.warn(`[trueconf] onDisconnected callback failed: ${err instanceof Error ? err.message : String(err)}`) } if (this.shuttingDown || this.reconnecting || this.startInFlight) return if (this.suppressNextCloseReconnect) { // forceReconnect owns the next start() — do not race it with our own // scheduleReconnect. One-shot: cleared by the inflight finally block. this.suppressNextCloseReconnect = false return } this.logger.info(`[trueconf] Connection closed (code: ${code}, reason: "${reason}"), scheduling reconnect`) try { this.options?.onConnectionClosed?.(code, reason) } catch (err) { this.logger.warn(`[trueconf] onConnectionClosed callback failed: ${err instanceof Error ? err.message : String(err)}`) } this.reconnecting = true this.scheduleReconnect() } private heartbeatTick(): void { if (this.lastPingAt - this.lastPongAt > HEARTBEAT_PONG_TIMEOUT_MS) { this.logger.warn('[trueconf] Heartbeat pong timeout, terminating socket') this.escalateDeadConnection('heartbeat pong timeout') return } try { this.wsClient.ping() this.lastPingAt = Date.now() } catch (err) { this.logger.warn(`[trueconf] Heartbeat ping threw: ${err instanceof Error ? err.message : String(err)}`) this.escalateDeadConnection('heartbeat ping throw') } } private escalateDeadConnection(cause: string): void { this.logger.warn(`[trueconf] Terminating dead connection (${cause})`) this.wsClient.terminate() } private isDnsError(err: unknown): boolean { return err instanceof NetworkError && typeof err.code === 'string' && DNS_ERROR_CODES.has(err.code) } private isAuthTerminalError(err: unknown): boolean { if (!(err instanceof NetworkError)) return false if (err.phase !== 'oauth') return false if (typeof err.code !== 'string') return false const status = Number.parseInt(err.code, 10) if (!Number.isFinite(status)) return false return isAuthTerminalCode(status) } private scheduleReconnect(): void { this.reconnectTimer = setTimeout(async () => { try { await this.start() this.dnsRetryCount = 0 this.oauthFailCount = 0 } catch (err) { this.logger.warn(`[trueconf] Reconnect attempt failed: ${err instanceof Error ? err.message : String(err)}`) if (this.isDnsError(err)) { this.dnsRetryCount++ if (this.dnsRetryCount >= DNS_FAIL_LIMIT) { this.logger.error( `[trueconf] DNS resolve failed ${this.dnsRetryCount} times; check serverUrl. Giving up.`, ) try { this.options?.onConnectionClosed?.(0, 'dns_unreachable') } catch { /* swallow */ } // Reject the auth barrier so pending senders see the actionable // dns_unreachable reason immediately instead of timing out silently. // Use DNS_TERMINAL_CODE (paired with the transient DNS_ERROR_CODES // set in types.ts) so consumers branching on `err instanceof // NetworkError` can distinguish a terminal DNS failure from a // retryable one without spinning further. const cause = new NetworkError( `dns_unreachable: gave up after ${this.dnsRetryCount} attempts`, 'websocket', undefined, DNS_TERMINAL_CODE, ) this.wsClient.markAuthFailed(cause) try { this.options?.onTerminalFailure?.({ kind: 'dns_exhausted', retries: this.dnsRetryCount, cause }) } catch (cbErr) { this.logger.warn(`[trueconf] onTerminalFailure callback failed: ${cbErr instanceof Error ? cbErr.message : String(cbErr)}`) } this.wsClient.close() return } } else if (this.isAuthTerminalError(err)) { this.oauthFailCount++ if (this.oauthFailCount >= OAUTH_FAIL_LIMIT) { this.logger.error( `[trueconf] OAuth authentication failed ${this.oauthFailCount} times; check bot credentials (username/password) on TrueConf Server. Giving up.`, ) try { this.options?.onConnectionClosed?.(0, 'oauth_unauthorized') } catch { /* swallow */ } const cause = new NetworkError( `oauth_unauthorized: gave up after ${this.oauthFailCount} consecutive 401/403`, 'oauth', err instanceof Error ? err : undefined, OAUTH_TERMINAL_CODE, ) this.wsClient.markAuthFailed(cause) try { this.options?.onTerminalFailure?.({ kind: 'auth_exhausted', retries: this.oauthFailCount, cause }) } catch (cbErr) { this.logger.warn(`[trueconf] onTerminalFailure callback failed: ${cbErr instanceof Error ? cbErr.message : String(cbErr)}`) } this.wsClient.close() return } } else { // D-07 consecutive-only counter: any non-401/403 outcome (network // error, 500, timeout, etc.) resets the OAuth fail counter to 0. // dnsRetryCount stays cumulative — that's the existing DNS contract. this.oauthFailCount = 0 } this.backoffMs = Math.min(this.backoffMs * 2, 60_000) this.scheduleReconnect() } }, this.backoffMs + Math.random() * 1000) this.reconnectTimer.unref?.() } private cancelReconnect(): void { if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); this.reconnectTimer = null } this.reconnecting = false } // ws library dispatches close events async; handleClose() sees // shuttingDown=false and fires scheduleReconnect → start() with a fresh token. private refreshAndReconnect(): void { this.wsClient.close() } }