import { Encryption_Type, TrackInfo } from '@livekit/protocol'; import { EventEmitter } from 'events'; import type TypedEventEmitter from 'typed-emitter'; import type { FrameMetadata } from '../frameMetadata/types'; import { getPublishFrameMetadata, hasFrameMetadataPublishOptions } from '../frameMetadata/utils'; import { LogLevel, LoggerNames, getLogger, onWorkerLogLevelChanged, workerLogger } from '../logger'; import type { CoreRoom } from '../room/CoreRoom'; import { ConnectionState } from '../room/CoreRoom'; import type RTCEngine from '../room/RTCEngine'; import { DeviceUnsupportedError } from '../room/errors'; import { EngineEvent, ParticipantEvent, RoomEvent } from '../room/events'; import type RemoteTrack from '../room/track/RemoteTrack'; import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; import type { Track } from '../room/track/Track'; import type { TrackPublication } from '../room/track/TrackPublication'; import type { TrackPublishOptions, VideoCodec } from '../room/track/options'; import { mimeTypeToVideoCodecString } from '../room/track/utils'; import { Future, isLocalTrack, isSafariBased, isScriptTransformSupportedForWorker, isVideoTrack, } from '../room/utils'; import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays'; import type { BaseKeyProvider } from './KeyProvider'; import { E2EE_FLAG, E2EE_TRACK_ID } from './constants'; import { CryptorError, CryptorErrorReason } from './errors'; import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events'; import type { DecryptDataRequestMessage, DecryptDataResponseMessage, E2EEManagerOptions, E2EEWorkerMessage, EnableMessage, EncodeMessage, EncryptDataRequestMessage, EncryptDataResponseMessage, InitMessage, KeyInfo, RTPVideoMapMessage, RatchetRequestMessage, RemoveTransformMessage, ScriptTransformOptions, SetKeyMessage, SifTrailerMessage, UpdateCodecMessage, } from './types'; import { isE2EESupported } from './utils'; export interface BaseE2EEManager { /** @internal */ setup(room: CoreRoom): void; setupEngine(engine: RTCEngine): void; isEnabled: boolean; isDataChannelEncryptionEnabled: boolean; /** * @internal * @deprecated use `setLocalParticipantCryptorEnabled` */ setParticipantCryptorEnabled(enabled: boolean, participantIdentity: string): void; /** * Sets whether the local participant encrypts. Optional so that existing managers still * satisfy the interface: without it, `setParticipantCryptorEnabled` is called. * @internal */ setLocalParticipantCryptorEnabled?(enabled: boolean): void; /** @internal */ setSifTrailer(trailer: NonSharedUint8Array): void; encryptData(data: NonSharedUint8Array): Promise; handleEncryptedData( payload: NonSharedUint8Array, iv: NonSharedUint8Array, participantIdentity: string, keyIndex: number, ): Promise; on(event: E, listener: E2EEManagerCallbacks[E]): this; off(event: E, listener: E2EEManagerCallbacks[E]): this; dispose?(): void; } /** * @experimental */ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter) implements BaseE2EEManager { protected worker: Worker; protected room?: CoreRoom; private encryptionEnabled: boolean; private keyProvider: BaseKeyProvider; private keysPostedToWorker = new WeakSet(); private decryptDataRequests: Map> = new Map(); private encryptDataRequests: Map> = new Map(); private dataChannelEncryptionEnabled: boolean; private unsubscribeLogLevel?: () => void; private log = getLogger(LoggerNames.E2EE, () => this.logContext); get logContext() { return { room: this.room?.name, participant: this.room?.localParticipant.identity, }; } /** * Runs a cleanup callback once this manager is garbage collected. Lets the * log-level listener (held in a module-global Set on the main-thread logger) * fall out of scope even when the consumer forgets to call `dispose()`. */ private static disposeRegistry = typeof FinalizationRegistry !== 'undefined' && typeof WeakRef !== 'undefined' && new FinalizationRegistry((cleanup: () => void) => { cleanup(); }); constructor(options: E2EEManagerOptions, dcEncryptionEnabled: boolean) { super(); this.keyProvider = options.keyProvider; this.worker = options.worker; this.encryptionEnabled = false; this.dataChannelEncryptionEnabled = dcEncryptionEnabled; } get isEnabled(): boolean { return this.encryptionEnabled; } get isDataChannelEncryptionEnabled(): boolean { return this.isEnabled && this.dataChannelEncryptionEnabled; } /** * @internal */ setup(room: CoreRoom) { if (!isE2EESupported()) { throw new DeviceUnsupportedError( 'tried to setup end-to-end encryption on an unsupported browser', ); } this.log.info('setting up e2ee'); if (room !== this.room) { this.room = room; this.setupEventListeners(room, this.keyProvider); // this.worker = new Worker(''); const msg: InitMessage = { kind: 'init', data: { keyProviderOptions: this.keyProvider.getOptions(), loglevel: workerLogger.getLevel() as LogLevel, }, }; if (this.worker) { this.log.info(`initializing worker`, { worker: this.worker }); this.worker.onmessage = this.onWorkerMessage; this.worker.onerror = this.onWorkerError; this.worker.postMessage(msg); this.subscribeToLogLevelChanges(); } } } /** * Subscribe the current worker to main-thread `workerLogger` level changes, * without strongly retaining `this` or `this.worker` from the module-global * listener Set on the logger. See {@link disposeRegistry}. */ private subscribeToLogLevelChanges() { // Guard against duplicate registration on re-setup. this.unsubscribeLogLevel?.(); let unsub: (() => void) | undefined; if (E2EEManager.disposeRegistry) { // Modern engines: hold the worker weakly so the module-global listener Set // on the logger can't retain this manager, and clean up the entry on GC. const workerRef = new WeakRef(this.worker); unsub = onWorkerLogLevelChanged((level) => { const worker = workerRef.deref(); if (!worker) { unsub?.(); return; } worker.postMessage({ kind: 'setLogLevel', data: { level } }); }); E2EEManager.disposeRegistry.register(this, unsub, this); } else { // Safari <14.1 and similar: no WeakRef. Fall back to a strong reference; // the leak lives until the consumer calls `dispose()`. const worker = this.worker; unsub = onWorkerLogLevelChanged((level) => { worker.postMessage({ kind: 'setLogLevel', data: { level } }); }); } this.unsubscribeLogLevel = unsub; } /** * @internal * Release the log-level subscription, reject any pending encrypt/decrypt * futures, and detach the worker message handlers. The worker itself is * caller-owned and is not terminated. Idempotent. */ dispose() { this.unsubscribeLogLevel?.(); this.unsubscribeLogLevel = undefined; if (E2EEManager.disposeRegistry) { E2EEManager.disposeRegistry.unregister(this); } // Reject pending futures BEFORE detaching worker handlers, so any late // response can't resolve one after we've cut the pipe. Each future's // `onFinally` deletes its own map entry, so both maps drain themselves. // Snapshot before iterating in case a rejection handler mutates the map. const disposalError = new CryptorError( 'E2EEManager disposed', CryptorErrorReason.InternalError, ); for (const future of [...this.encryptDataRequests.values()]) { future.reject?.(disposalError); } for (const future of [...this.decryptDataRequests.values()]) { future.reject?.(disposalError); } if (this.worker) { this.worker.onmessage = null; this.worker.onerror = null; } this.removeAllListeners(); } /** * Sets whether the local participant encrypts. * @internal */ setLocalParticipantCryptorEnabled(enabled: boolean) { this.log.debug(`set e2ee to ${enabled} for the local participant`); this.postEnable(enabled, this.room?.localParticipant.identity ?? '', true); } /** * @internal * @deprecated use `setLocalParticipantCryptorEnabled`. The identity is ignored: this always * sets the state of the local participant. */ setParticipantCryptorEnabled(enabled: boolean, _participantIdentity: string) { this.setLocalParticipantCryptorEnabled(enabled); } /** * @internal */ setSifTrailer(trailer: NonSharedUint8Array) { if (!trailer || trailer.length === 0) { this.log.warn("ignoring server sent trailer as it's empty"); } else { this.postSifTrailer(trailer); } } private onWorkerMessage = (ev: MessageEvent) => { const { kind, data } = ev.data; switch (kind) { case 'error': // If error has uuid, it's from an async operation (encrypt/decrypt). // Reject the corresponding future and let the caller decide how to log/handle; // logging here would duplicate whatever the caller does. if (data.uuid) { const decryptFuture = this.decryptDataRequests.get(data.uuid); if (decryptFuture?.reject) { decryptFuture.reject(data.error); break; } const encryptFuture = this.encryptDataRequests.get(data.uuid); if (encryptFuture?.reject) { encryptFuture.reject(data.error); break; } } this.log.error(data.error.message); this.emit(EncryptionEvent.EncryptionError, data.error, data.participantIdentity); break; case 'initAck': if (data.enabled) { this.keyProvider.getKeys().forEach((keyInfo) => { this.postKey(keyInfo, false); }); } break; case 'enable': if (data.local) { if (this.encryptionEnabled !== data.enabled) { this.emit( EncryptionEvent.ParticipantEncryptionStatusChanged, data.enabled, this.room!.localParticipant, ); this.encryptionEnabled = data.enabled; } } else if (data.participantIdentity) { const participant = this.room?.getParticipantByIdentity(data.participantIdentity); if (!participant) { throw TypeError( `couldn't set encryption status, participant not found${data.participantIdentity}`, ); } this.emit(EncryptionEvent.ParticipantEncryptionStatusChanged, data.enabled, participant); } break; case 'ratchetKey': this.keyProvider.emit( KeyProviderEvent.KeyRatcheted, data.ratchetResult, data.participantIdentity, data.keyIndex, ); break; case 'decryptDataResponse': const decryptFuture = this.decryptDataRequests.get(data.uuid); if (decryptFuture?.resolve) { decryptFuture.resolve(data); } break; case 'encryptDataResponse': const encryptFuture = this.encryptDataRequests.get(data.uuid); if (encryptFuture?.resolve) { encryptFuture.resolve(data as EncryptDataResponseMessage['data']); } break; case 'packetTrailerMetadata': this.handleFrameMetadata(data.trackId, data.rtpTimestamp, data.ssrc, data.metadata); break; case 'log': workerLogger[data.level](data.msg, data.context); break; default: break; } }; private onWorkerError = (ev: ErrorEvent) => { this.log.error('e2ee worker encountered an error:', { error: ev.error }); this.emit(EncryptionEvent.EncryptionError, ev.error, undefined); }; private handleFrameMetadata( trackId: string, rtpTimestamp: number, ssrc: number, metadata: FrameMetadata, ) { if (!this.room) { return; } for (const participant of [ this.room.localParticipant, ...this.room.remoteParticipants.values(), ]) { for (const pub of participant.trackPublications.values()) { if ( pub.track && pub.track.mediaStreamID === trackId && pub.track instanceof RemoteVideoTrack && pub.track.frameMetadataExtractor ) { pub.track.frameMetadataExtractor.storeMetadata(rtpTimestamp, ssrc, metadata); return; } } } } public setupEngine(engine: RTCEngine) { engine.on(EngineEvent.RTPVideoMapUpdate, (rtpMap) => { this.postRTPMap(rtpMap); }); } private setupEventListeners(room: CoreRoom, keyProvider: BaseKeyProvider) { room.on(RoomEvent.TrackPublished, (pub, participant) => this.setParticipantCryptorEnabledForPublication(pub, participant.identity), ); room .on(RoomEvent.ConnectionStateChanged, (state) => { if (state === ConnectionState.Connected) { room.remoteParticipants.forEach((participant) => { participant.trackPublications.forEach((pub) => { this.setParticipantCryptorEnabledForPublication(pub, participant.identity); }); }); } }) .on(RoomEvent.TrackUnsubscribed, (track, _, participant) => { const msg: RemoveTransformMessage = { kind: 'removeTransform', data: { participantIdentity: participant.identity, trackId: track.mediaStreamID, }, }; this.worker?.postMessage(msg); }) .on(RoomEvent.TrackSubscribed, (track, pub, participant) => { this.setupE2EEReceiver(track, participant.identity, pub.trackInfo); }) .on(RoomEvent.SignalConnected, () => { if (!this.room) { throw new TypeError(`expected room to be present on signal connect`); } const latestKeyIndex = keyProvider.getLatestManuallySetKeyIndex(); keyProvider .getKeys() .filter((keyInfo) => !this.keysPostedToWorker.has(keyInfo)) .forEach((keyInfo) => { this.postKey(keyInfo, latestKeyIndex === (keyInfo.keyIndex ?? 0)); }); this.setLocalParticipantCryptorEnabled(this.room.localParticipant.isE2EEEnabled); }); room.localParticipant.on(ParticipantEvent.LocalSenderCreated, async (sender, track) => { this.setupE2EESender(track, sender); }); room.localParticipant.on(ParticipantEvent.LocalTrackPublished, (publication) => { // Safari doesn't support retrieving payload information on RTCEncodedVideoFrame, so we need to update the codec manually once we have the trackInfo from the server if (!isVideoTrack(publication.track) || !isSafariBased()) { return; } const msg: UpdateCodecMessage = { kind: 'updateCodec', data: { trackId: publication.track!.mediaStreamID, codec: mimeTypeToVideoCodecString(publication.trackInfo!.codecs[0].mimeType), participantIdentity: this.room!.localParticipant.identity, hasPacketTrailer: false, operation: 'encode', }, }; this.worker.postMessage(msg); }); keyProvider .on(KeyProviderEvent.SetKey, (keyInfo, updateCurrentKeyIndex) => this.postKey(keyInfo, updateCurrentKeyIndex ?? true), ) .on(KeyProviderEvent.RatchetRequest, (participantId, keyIndex) => this.postRatchetRequest(participantId, keyIndex), ); } async encryptData(data: NonSharedUint8Array): Promise { if (!this.worker) { throw Error('could not encrypt data, worker is missing'); } const uuid = crypto.randomUUID(); const msg: EncryptDataRequestMessage = { kind: 'encryptDataRequest', data: { uuid, payload: data, participantIdentity: this.room!.localParticipant.identity, }, }; const future = new Future(); future.onFinally = () => { this.encryptDataRequests.delete(uuid); }; this.encryptDataRequests.set(uuid, future); this.worker.postMessage(msg); return future!.promise!; } handleEncryptedData( payload: NonSharedUint8Array, iv: NonSharedUint8Array, participantIdentity: string, keyIndex: number, ) { if (!this.worker) { throw Error('could not handle encrypted data, worker is missing'); } const uuid = crypto.randomUUID(); const msg: DecryptDataRequestMessage = { kind: 'decryptDataRequest', data: { uuid, payload, iv, participantIdentity, keyIndex, }, }; const future = new Future(); future.onFinally = () => { this.decryptDataRequests.delete(uuid); }; this.decryptDataRequests.set(uuid, future); this.worker.postMessage(msg); return future.promise; } private postRatchetRequest(participantIdentity?: string, keyIndex?: number) { if (!this.worker) { throw Error('could not ratchet key, worker is missing'); } const msg: RatchetRequestMessage = { kind: 'ratchetRequest', data: { participantIdentity: participantIdentity, keyIndex, }, }; this.worker.postMessage(msg); } private postKey(keyInfo: KeyInfo, updateCurrentKeyIndex: boolean) { if (!this.worker) { throw Error('could not set key, worker is missing'); } const { key, participantIdentity, keyIndex } = keyInfo; const msg: SetKeyMessage = { kind: 'setKey', data: { participantIdentity: participantIdentity, isPublisher: participantIdentity === this.room?.localParticipant.identity, key, keyIndex, updateCurrentKeyIndex, }, }; this.worker.postMessage(msg); this.keysPostedToWorker.add(keyInfo); } private postEnable(enabled: boolean, participantIdentity: string, local = false) { if (this.worker) { const enableMsg: EnableMessage = { kind: 'enable', data: { enabled, participantIdentity, local, }, }; this.worker.postMessage(enableMsg); } else { throw new ReferenceError('failed to enable e2ee, worker is not ready'); } } private postRTPMap(map: Map) { if (!this.worker) { throw TypeError('could not post rtp map, worker is missing'); } if (!this.room?.localParticipant.identity) { throw TypeError('could not post rtp map, local participant identity is missing'); } const msg: RTPVideoMapMessage = { kind: 'setRTPMap', data: { map, participantIdentity: this.room.localParticipant.identity, }, }; this.worker.postMessage(msg); } private postSifTrailer(trailer: NonSharedUint8Array) { if (!this.worker) { throw Error('could not post SIF trailer, worker is missing'); } const msg: SifTrailerMessage = { kind: 'setSifTrailer', data: { trailer, }, }; this.worker.postMessage(msg); } /** * Derive the participant's encryption state from one of its publications. * Publications without trackInfo are skipped rather than thrown on: throwing * inside the forEach above would leave every remaining participant without an * encryption state, and a cryptor with unknown state can no longer decrypt. */ private setParticipantCryptorEnabledForPublication( pub: TrackPublication, participantIdentity: string, ) { if (!pub.trackInfo) { this.log.warn('skipping e2ee enabled update for publication without trackInfo', { trackSid: pub.trackSid, }); return; } this.postEnable(pub.trackInfo.encryption !== Encryption_Type.NONE, participantIdentity); } private setupE2EEReceiver(track: RemoteTrack, remoteId: string, trackInfo?: TrackInfo) { if (!track.receiver) { return; } if (!trackInfo?.mimeType || trackInfo.mimeType === '') { throw new TypeError('MimeType missing from trackInfo, cannot set up E2EE cryptor'); } const hasPacketTrailer = track.kind === 'video' && !!trackInfo.packetTrailerFeatures && trackInfo.packetTrailerFeatures.length > 0; this.handleReceiver( track.receiver, track.mediaStreamID, remoteId, track.kind === 'video' ? mimeTypeToVideoCodecString(trackInfo.mimeType) : undefined, hasPacketTrailer, trackInfo.encryption !== Encryption_Type.NONE, ); } private setupE2EESender(track: Track, sender: RTCRtpSender) { if (!isLocalTrack(track) || !sender) { if (!sender) this.log.warn('early return because sender is not ready'); return; } const isVideo = isVideoTrack(track); // Safari exposes no per-frame codec info until trackInfo arrives. Seed only h264/h265: // a wrong vp8 seed would skip the NALU path, a wrong NALU seed only costs a logged fallback. const publishedCodec = isVideo ? track.publishOptions?.videoCodec : undefined; this.handleSender( sender, track.mediaStreamID, publishedCodec === 'h264' || publishedCodec === 'h265' ? publishedCodec : undefined, isVideo ? getPublishFrameMetadata(track.publishOptions) : undefined, ); } /** * Handles the given {@code RTCRtpReceiver} by creating a {@code TransformStream} which will inject * a frame decoder. * */ private async handleReceiver( receiver: RTCRtpReceiver, trackId: string, participantIdentity: string, codec: VideoCodec | undefined, hasPacketTrailer: boolean, encrypted: boolean, ) { if (!this.worker) { return; } if (isScriptTransformSupportedForWorker()) { const options: ScriptTransformOptions = { kind: 'decode', participantIdentity, trackId, codec, hasPacketTrailer, encrypted, }; receiver.transform = new RTCRtpScriptTransform(this.worker, options); } else { if (E2EE_FLAG in receiver && E2EE_TRACK_ID in receiver) { // The transceiver is being reused for a new track. We cannot set up a new // pipeline from here: this receiver's encoded streams were transferred to // the worker on first setup and a transferred stream can't be transferred // again, while createEncodedStreams() may only be called once per receiver. // So hand the worker the new track/participant and let it re-point (and if // necessary rebuild) the cryptor that still owns those streams. const msg: UpdateCodecMessage = { kind: 'updateCodec', data: { trackId, previousTrackId: receiver[E2EE_TRACK_ID] as string, codec, participantIdentity, hasPacketTrailer, encrypted, }, }; this.worker.postMessage(msg); receiver[E2EE_TRACK_ID] = trackId; return; } // @ts-ignore let writable: WritableStream = receiver.writableStream; // @ts-ignore let readable: ReadableStream = receiver.readableStream; if (!writable || !readable) { // @ts-ignore const receiverStreams = receiver.createEncodedStreams(); // @ts-ignore receiver.writableStream = receiverStreams.writable; writable = receiverStreams.writable; // @ts-ignore receiver.readableStream = receiverStreams.readable; readable = receiverStreams.readable; } const msg: EncodeMessage = { kind: 'decode', data: { readableStream: readable, writableStream: writable, trackId: trackId, codec, participantIdentity: participantIdentity, hasPacketTrailer, encrypted, }, }; this.worker.postMessage(msg, [readable, writable]); } // @ts-ignore receiver[E2EE_FLAG] = true; // @ts-ignore receiver[E2EE_TRACK_ID] = trackId; } /** * Handles the given {@code RTCRtpSender} by creating a {@code TransformStream} which will inject * a frame encoder. * */ private handleSender( sender: RTCRtpSender, trackId: string, codec?: VideoCodec, frameMetadata?: TrackPublishOptions['frameMetadata'], ) { if (E2EE_FLAG in sender || !this.worker) { return; } if (!this.room?.localParticipant.identity || this.room.localParticipant.identity === '') { throw TypeError('local identity needs to be known in order to set up encrypted sender'); } if (isScriptTransformSupportedForWorker()) { this.log.info('initialize script transform'); const options: ScriptTransformOptions = { kind: 'encode', participantIdentity: this.room.localParticipant.identity, trackId, codec, hasPacketTrailer: hasFrameMetadataPublishOptions(frameMetadata), packetTrailer: frameMetadata, }; // @ts-ignore sender.transform = new RTCRtpScriptTransform(this.worker, options); } else { this.log.info('initialize encoded streams'); // @ts-ignore const senderStreams = sender.createEncodedStreams(); const msg: EncodeMessage = { kind: 'encode', data: { readableStream: senderStreams.readable, writableStream: senderStreams.writable, codec, trackId, participantIdentity: this.room.localParticipant.identity, hasPacketTrailer: hasFrameMetadataPublishOptions(frameMetadata), packetTrailer: frameMetadata, }, }; this.worker.postMessage(msg, [senderStreams.readable, senderStreams.writable]); } // @ts-ignore sender[E2EE_FLAG] = true; } }