/** * Cloud Sync Service * * Singleton service that replaces the WebSocket Relay pairing model with a * simpler heartbeat + polling architecture. All devices under the same Cloud * account are automatically visible and can exchange messages. * * Responsibilities: * - Heartbeat: periodically uploads device status to Cloud * - Device poll: periodically fetches the device list for the account * - Message poll: periodically fetches pending messages for this device * - Send: posts messages to specific devices via Cloud * * @see docs/cloud-sync-design.md * @module services/cloud/cloud-sync.service */ import { EventEmitter } from 'events'; import type { CloudSyncConfig, CloudSyncHealth, CloudSyncState, SyncDevice, MessageType } from './cloud-sync.types.js'; /** * Gap before the next queue registration after `failures` consecutive * failures: REGISTER_RETRY_BACKOFF_MS in order, then its last value forever. * * @param failures - Consecutive failures so far (>= 1) * @returns Delay in ms */ export declare function registerRetryDelay(failures: number): number; /** The relay answered 429 with a Retry-After: per-account rate or daily limit. */ export declare class RelayRateLimitedError extends Error { readonly retryAfterMs: number; /** * @param message - English description including the status and body * @param retryAfterMs - How long the relay asked us to wait (0 when unknown) */ constructor(message: string, retryAfterMs: number); } export declare class CloudSyncService extends EventEmitter { private static instance; private readonly logger; /** Current service state */ private state; /** Configuration (set on start) */ private config; /** Cached device list from last poll */ private devices; /** Cached Crewly version */ private version; /** Heartbeat timer handle */ private heartbeatTimer; /** Device poll timer handle */ private devicePollTimer; /** Message poll timer handle (self-rescheduling long-poll loop). */ private messagePollTimer; /** Guards against overlapping self-scheduled message-poll cycles. */ private messagePollRunning; /** When the last message-poll cycle finished (watchdog input). */ private lastMessagePollAt; /** * Next queue registration (self-scheduling): every REGISTER_INTERVAL_MS * while registered (lets the relay evict stale Portal pairs), on the * REGISTER_RETRY_BACKOFF_MS schedule while registration fails. */ private registerTimer; /** Consecutive failed queue registrations */ private queueFailures; /** Epoch ms of the first failure in the current registration failure streak */ private queueFailingSince; /** * When this machine began having NO queue at all (epoch ms); null while it * has one. Unlike {@link queueFailingSince}, a failed re-registration while * an earlier queue is still being polled does not start a gap — nothing is * missed then. */ private inboundGapSince; /** The current inbound gap was already announced (`queue-outage`). */ private queueOutageNotified; /** Epoch ms of the next scheduled registration */ private nextQueueAttemptAt; /** * Bumped by start() and stop(). An attempt that started under an older * generation must not touch state or files — a stop() (or a test's * cleanup) can land while a registration is still in flight. */ private generation; /** * Crewly home, resolved once at start(). Resolving it at write time let a * write that outlived its caller land in a different home (a test that * had already restored CREWLY_HOME wrote a random id into the owner's * real ~/.crewly — 2026-10-02). */ private crewlyHome; /** Peer reported by the last registration, so a repeat is not logged at info. */ private lastLoggedPeer; /** Consecutive heartbeat failure count */ private heartbeatFailures; /** Consecutive device poll failure count */ private devicePollFailures; /** Consecutive message poll failure count */ private messagePollFailures; /** Error recovery timer handle */ private errorRecoveryTimer; /** Consecutive error recovery attempts (reset on success or stop) */ private errorRecoveryAttempts; /** Our assigned queue ID from Cloud relay queue registration (for message polling) */ private queueId; /** Why the last queue registration failed, for /cloud/status. */ private queueError; /** * Recently processed message IDs to prevent re-delivery when ackMessages fails. * Keeps the last 500 IDs; pruned on overflow. */ private processedMessageIds; /** Epoch ms of the last request Cloud answered successfully (0 = never) */ private lastContactAt; /** Epoch ms of the last start() (0 = never started) */ private startedAt; /** True when Cloud's last answer was 401/403 and the token refresh failed */ private authRejected; /** Maximum size of the processedMessageIds dedup set */ private static readonly MAX_DEDUP_IDS; private constructor(); /** * Get the singleton instance. * * @returns CloudSyncService instance */ static getInstance(): CloudSyncService; /** * Reset the singleton (for testing). */ static resetInstance(): void; /** * Start the Cloud Sync service. * * Begins heartbeat, device polling, and message polling timers. * Performs an immediate heartbeat and device poll on start. * * @param config - Cloud connection configuration */ start(config: CloudSyncConfig): void; /** * Stop the Cloud Sync service. * * Clears all timers and resets state. Safe to call multiple times. */ stop(): void; /** * Check if the sync service has been started. * * @returns True if the service is currently syncing */ isStarted(): boolean; /** * Get the current service state. * * @returns Current CloudSyncState */ getState(): CloudSyncState; /** * Connection health for the disconnect monitor: the state, when Cloud last * answered, when sync was started and whether Cloud refused the sign-in. * * @returns Health snapshot (times are epoch ms, null when never) */ getHealth(): CloudSyncHealth; /** Record a request Cloud answered successfully. */ private markContact; /** * Get all cached devices for this Cloud account. * * @returns Array of SyncDevice objects (may include offline devices) */ getDevices(): SyncDevice[]; /** * Update the authentication token used for Cloud API requests. * * Called by CloudClientService after a successful token refresh * so that heartbeat, device poll, and message poll use the new token. * * @param newToken - New JWT access token */ updateToken(newToken: string): void; /** * Get only online devices (excluding this device). * * @returns Array of online SyncDevice objects */ getOnlineDevices(): SyncDevice[]; /** * Send a message to another device via the Cloud message queue. * * @param toDeviceId - Target device identifier * @param type - Message type * @param payload - Message payload (must be JSON-serializable) * @throws Error when not started or API call fails * * @example * ```typescript * await sync.sendMessage('dev-456', 'command', { * action: 'delegate-task', * content: 'Implement feature X' * }); * ``` */ sendMessage(toDeviceId: string, type: MessageType, payload: unknown): Promise; /** * Send a message directly to a peer queueId, bypassing the device cache. * * Used by `ChatV2RelayAdapter` for routing `chat_response` back to a * Portal browser whose deviceId may not yet have propagated to the * 30s-interval device poll. The production relay's `/queue/send` * endpoint accepts a raw `peerQueueId`, so when the target identifier * IS already a queueId (which Portal's wrapper.fromDeviceName field * carries — see `chat-v2.relay-adapter.service.ts:handleRequest`), * we can skip the cache lookup entirely and avoid the race. * * Falls back to `sendMessage(...)` semantics on the wire (same JSON * wrapper) so the receiver's CloudSyncService normalizes it identically. * * @param peerQueueId - The relay queueId to route to (no cache lookup) * @param type - MessageType for the wrapper * @param payload - JSON-serializable inner data * @throws Error when not started or the HTTP send fails */ sendToPeerQueueId(peerQueueId: string, type: MessageType, payload: unknown): Promise; /** * One registration attempt, then schedule the next: the keep-alive * interval after a success, the backoff after a failure. Never gives up — * a machine without a queue is deaf to Cloud and Slack, and on 2026-10-02 * one sat like that for half an hour after a single 429 at boot. * * @param gen - Generation this cycle belongs to (a stale cycle exits) */ private runRegisterCycle; /** * Outage bookkeeping after a failed registration. Only a machine with no * queue at all is deaf (Slack events pile up in Cloud); `queue-outage` fires * once when that outlasts QUEUE_OUTAGE_NOTIFY_AFTER_MS. * * Events: `queue-outage` ({ since, error }), `inbound-gap-recovered` ({ since }). */ private noteQueueFailed; /** * The first success after an inbound gap: tell the Slack backfill where the * gap began so it can read what Cloud could not deliver. */ private noteQueueRecovered; /** * Schedule the next registration attempt. * * @param gen - Generation the attempt belongs to * @param delayMs - Gap before it */ private scheduleRegister; /** * Register with the Cloud relay message queue. * * Calls POST /api/v1/relay/queue/register with a deterministic pairing code * derived from the JWT user ID. This allows both devices of the same user * to auto-discover each other's queueId for message routing. * * Which id to claim, in order: * 1. the queue this process already registered — a running machine never * switches queues (each switch left the old queue behind and used up * the relay's per-user quota: f68e1995 → a8eeab1d → d5c4ebc2 in one * day, all on one MacBook, when something rewrote the id file); * 2. the id remembered in `/cloud/relay-queue.json`; * 3. the device id. * * @param gen - Generation of the calling cycle * @throws Error when registration fails (the caller retries) */ private registerQueue; /** * Keep the id file in step with the queue we actually hold, so the next * boot claims the same queue instead of creating another: rewrite it when * it names a different queue, remove it when we hold the device-id queue * (the file is only for the fallback case). * * @param registered - The id just registered * @param remembered - What the file said before this attempt * @param deviceId - This machine's device id * @param gen - Generation of the calling cycle */ private syncFallbackFile; /** * One registration attempt against a specific queue id. * * @param queueId - The id to claim * @param pairingCode - Deterministic code derived from the signed-in user * @param gen - Generation of the calling cycle * @throws {QueueForbiddenError} When the relay says the queue is someone else's * @throws Error on any other failure, or when Cloud Sync stopped meanwhile */ private registerQueueAs; /** * The id the relay knows this machine by: the queue we actually poll. * * The relay attaches handshake metadata (name, teams) to the queue named by * `deviceId`, and the portal lists machines from that. After a 403 the * machine polls a fresh queue, so a handshake still naming the device id * kept the OLD queue looking alive — with all our teams — while nothing * read it. The portal picked it and every request went unanswered * ("Couldn't load your agents", 2026-09-27). Until registration has * settled, the device id is all we have. * * @returns The polled queue id, else the device id */ private relayDeviceId; /** Where a queue id taken after a 403 is remembered (home resolved at start()). */ private fallbackQueueFile; /** * The queue id this machine fell back to previously, if any. * * @returns The stored id, or null when we have never needed one */ private readFallbackQueueId; /** * Remember a queue id so later boots reuse it rather than minting one each * time — the relay caps queues per user. * * @param queueId - The id that was accepted * @param gen - Generation of the calling cycle (a stale one writes nothing) */ private writeFallbackQueueId; /** * Derive a deterministic pairing code from the JWT token's user ID. * Both devices of the same user will produce the same code. */ private derivePairingCode; /** * Get the queue ID assigned by Cloud registration. * Returns null if not yet registered. */ getQueueId(): string | null; /** * Why the last queue registration failed, or null when it succeeded. * * Surfaced in Slack's `/cloud/status`: a missing queue id silently skips * the Slack instance heartbeat, and that machine then looks healthy while * receiving nothing. * * @returns The failure message, or null */ getQueueError(): string | null; /** * Send a heartbeat to the Cloud server with current device state. * Called periodically by the heartbeat timer. * * Posts to the Cloud Relay handshake endpoint (/api/v1/relay/handshake) * which registers/updates the device with enriched metadata. */ sendHeartbeat(): Promise; /** * Poll the Cloud server for the current device list. * Updates the cached device list and emits events on changes. * * The Cloud server returns devices in relay format: * `{ sessionId, role, state, pairedWith, registeredAt, lastHeartbeatAt, name }` * which we normalize to the SyncDevice shape used by the frontend. */ pollDevices(): Promise; /** * Poll the Cloud server for pending messages addressed to this device. * Emits 'message' event for each received message, then acknowledges. * * The Cloud Sync message poll endpoint uses `deviceId` query param. * Cloud returns messages as: `{ id, from, fromDeviceName, type, payload, encrypted, sentAt }`. */ pollMessages(): Promise; /** * One iteration of the self-rescheduling message-poll loop. Runs a * (possibly long-held) {@link pollMessages}, then schedules the next cycle * with a gap chosen from the outcome: * * - **error** (`count < 0`) → back off to {@link CLOUD_SYNC_CONSTANTS.MESSAGE_POLL_INTERVAL_MS}. * - **long-poll disabled** → fixed interval (legacy cadence). * - **message received** (`count > 0`) → re-open right away to drain more. * - **held then empty** (elapsed ≥ half the wait) → short gap; this is the * normal continuous long-poll. * - **empty too fast** (elapsed < half the wait) → the relay likely ignores * `wait=`; degrade to the fixed interval so we don't hot-loop against an * un-upgraded relay. * * Re-entrancy is guarded so a delayed timer can never overlap an in-flight * cycle. No-op once stopped. */ private runMessagePollCycle; /** * Restart the message-poll loop if it has stopped completing cycles. * * 2026-09-23: the loop went silent for ~50 minutes while heartbeats and * queue re-registration carried on, so Cloud kept pushing Slack events * into this machine's relay queue and nothing picked them up — agents * "didn't reply". A cycle ends within the long-poll timeout; one that * hasn't in {@link CLOUD_SYNC_CONSTANTS.MESSAGE_POLL_STALL_MS} is stuck * (a hung request or a lost timer), so start a fresh loop. * * @returns True when the loop was restarted */ checkMessagePollAlive(now?: number): boolean; /** * Schedule the next message-poll cycle. Stores the (unref'd) timer handle so * `stop()` can cancel a pending cycle. No-op once stopped. * * @param delayMs - Delay before the next {@link runMessagePollCycle} */ private scheduleNextMessagePoll; /** * Acknowledge processed messages so Cloud can remove them from the queue. * Uses `queueId` in body to identify the target message queue (fallback * when JWT has no deviceId claim). * * @param messageIds - Array of message IDs to acknowledge */ private ackMessages; /** * Check if any failure counter has exceeded the threshold. * If so, transition to error state and schedule a recovery attempt. */ private checkErrorThreshold; /** * Schedule periodic recovery attempts when in error state. * * Tries a single heartbeat every ERROR_RECOVERY_INTERVAL_MS (60s by default). * If the heartbeat succeeds, resets all failure counters and resumes normal * polling. After MAX_ERROR_RECOVERY_ATTEMPTS consecutive auth failures (401/403), * transitions to 'auth_expired' terminal state and emits 'auth_expired'. */ private scheduleErrorRecovery; /** * Transition to the terminal auth_expired state. * Clears all timers and emits 'auth_expired' for frontend notification. */ private enterAuthExpiredState; /** Guard to prevent concurrent token refresh attempts */ private tokenRefreshInProgress; /** * Check if an HTTP status code indicates an authentication failure (401/403). * If so, attempt to refresh the token via CloudClientService. * On success, updates this service's config with the new token. * * @param status - HTTP status code from a Cloud API response * @returns true if the token was refreshed successfully */ private handleAuthError; /** * Build authorization headers for Cloud API requests. * * @returns Headers with Bearer token and Content-Type */ private authHeaders; } //# sourceMappingURL=cloud-sync.service.d.ts.map