{"version":3,"file":"FrameMetadataManager.mjs","sources":["../../../src/frameMetadata/FrameMetadataManager.ts"],"sourcesContent":["import type { TrackInfo } from '@livekit/protocol';\nimport log from '../logger';\nimport type { CoreRoom } from '../room/CoreRoom';\nimport { RoomEvent } from '../room/events';\nimport { FrameMetadataExtractor } from '../room/track/FrameMetadataExtractor';\nimport type RemoteTrack from '../room/track/RemoteTrack';\nimport RemoteVideoTrack from '../room/track/RemoteVideoTrack';\nimport type { PTDecodeMessage, PTUpdateTrackIdMessage, PTWorkerMessage } from './types';\nimport { isFrameMetadataSupported, shouldUseFrameMetadataScriptTransform } from './utils';\n\nexport interface FrameMetadataOptions {\n  /**\n   * Dedicated worker for extracting packet trailers off the main thread.\n   *\n   * Encoded video streams are transferred to the worker for processing, which\n   * avoids per-frame work on the main thread.\n   */\n  worker: Worker;\n}\n\n/**\n * Manages packet trailer extraction for received video tracks.\n *\n * When a track's TrackInfo indicates packet trailer features, the manager\n * wires up an encoded frame transform to strip the trailer from encoded frames\n * and cache the metadata for lookup.\n *\n * Packet trailer extraction is worker-only. If no worker is configured, the\n * SDK does not advertise packet trailer support and skips extraction.\n *\n * When E2EE is active, the E2EE FrameCryptor worker handles trailer\n * extraction directly (before decryption), so this manager only creates\n * the extractor/metadata cache — no separate pipeline is installed.\n *\n * @experimental\n */\nexport class FrameMetadataManager {\n  private worker?: Worker;\n\n  private room?: CoreRoom;\n\n  private extractors = new Map<string, FrameMetadataExtractor>();\n\n  /**\n   * Tracks the trackId associated with each receiver that has had its\n   * encoded streams handed off to the worker. Used to detect receiver\n   * reuse (transceiver recycling) so we can remap trackIds instead of\n   * re-transferring already-consumed streams.\n   */\n  private workerPipelines = new Map<RTCRtpReceiver, string>();\n\n  constructor(options?: FrameMetadataOptions) {\n    this.worker = options?.worker;\n  }\n\n  /** @internal */\n  setup(room: CoreRoom) {\n    if (room === this.room) {\n      return;\n    }\n    this.room = room;\n\n    if (this.worker) {\n      this.worker.onmessage = this.onWorkerMessage;\n      this.worker.onerror = this.onWorkerError;\n      this.worker.postMessage({ kind: 'init' });\n    }\n\n    room\n      .on(RoomEvent.TrackSubscribed, (track, pub, _participant) => {\n        if (track.kind !== 'video') {\n          return;\n        }\n        this.setupReceiver(track as unknown as RemoteVideoTrack, pub.trackInfo);\n      })\n      .on(RoomEvent.TrackUnsubscribed, (track) => {\n        this.teardownTrack(track);\n      })\n      .on(RoomEvent.Disconnected, () => {\n        this.cleanup();\n      });\n  }\n\n  private setupReceiver(track: RemoteVideoTrack, trackInfo?: TrackInfo) {\n    const receiver = track.receiver;\n    if (!receiver) {\n      return;\n    }\n\n    // Only install a pipeline for tracks that actually advertise packet\n    // trailer features. This keeps us out of the way for tracks published by\n    // clients on older protocols or that don't opt into the feature.\n    const hasFeatures =\n      !!trackInfo?.packetTrailerFeatures && trackInfo.packetTrailerFeatures.length > 0;\n    if (!hasFeatures) {\n      if (!this.room?.hasE2EESetup) {\n        this.setupPassthroughReceiver(receiver, track.mediaStreamID);\n      }\n      return;\n    }\n\n    if (\n      !isFrameMetadataSupported(this.worker ? { worker: this.worker } : undefined) &&\n      !this.room?.hasE2EESetup\n    ) {\n      log.warn('frame metadata transform not supported; skipping extraction');\n      return;\n    }\n\n    const extractor = new FrameMetadataExtractor();\n    const trackId = track.mediaStreamID;\n\n    this.extractors.set(trackId, extractor);\n    track.frameMetadataExtractor = extractor;\n\n    if (this.room?.hasE2EESetup) {\n      // E2EE worker strips the trailer and injects metadata directly into\n      // the extractor via E2eeManager; no pipeline is needed here.\n      return;\n    }\n\n    this.setupWorkerReceiver(receiver, trackId, true);\n  }\n\n  private setupPassthroughReceiver(receiver: RTCRtpReceiver, trackId: string) {\n    if (shouldUseFrameMetadataScriptTransform()) {\n      if ('transform' in receiver) {\n        // @ts-ignore\n        receiver.transform = null;\n      }\n      return;\n    }\n\n    if (\n      this.worker &&\n      isFrameMetadataSupported({ worker: this.worker }) &&\n      !this.workerPipelines.has(receiver)\n    ) {\n      this.setupWorkerReceiver(receiver, trackId, false);\n      return;\n    }\n\n    if (this.worker && this.workerPipelines.has(receiver)) {\n      this.setupWorkerReceiver(receiver, trackId, false);\n    }\n  }\n\n  private setupWorkerReceiver(\n    receiver: RTCRtpReceiver,\n    newTrackId: string,\n    hasPacketTrailer = true,\n  ) {\n    const worker = this.worker;\n    if (!worker) {\n      return;\n    }\n\n    if (shouldUseFrameMetadataScriptTransform()) {\n      // @ts-ignore\n      receiver.transform = new RTCRtpScriptTransform(worker, {\n        kind: 'decode',\n        trackId: newTrackId,\n      });\n      return;\n    }\n\n    const existingTrackId = this.workerPipelines.get(receiver);\n\n    if (existingTrackId) {\n      // Receiver is reused (transceiver recycled). The worker already owns\n      // the encoded streams — just remap the trackId so metadata is keyed\n      // correctly and re-activate processing.\n      const msg: PTUpdateTrackIdMessage = {\n        kind: 'updateTrackId',\n        data: { oldTrackId: existingTrackId, newTrackId, hasPacketTrailer },\n      };\n      worker.postMessage(msg);\n      this.workerPipelines.set(receiver, newTrackId);\n      return;\n    }\n\n    if (!('createEncodedStreams' in receiver)) {\n      log.warn('createEncodedStreams not supported');\n      return;\n    }\n\n    let streams: { readable: ReadableStream; writable: WritableStream };\n    try {\n      // @ts-ignore — createEncodedStreams is not in standard typings\n      streams = receiver.createEncodedStreams();\n    } catch (err) {\n      log.warn('failed to create encoded streams', { error: err });\n      return;\n    }\n\n    const msg: PTDecodeMessage = {\n      kind: 'decode',\n      data: {\n        readableStream: streams.readable,\n        writableStream: streams.writable,\n        trackId: newTrackId,\n        hasPacketTrailer,\n      },\n    };\n    worker.postMessage(msg, [streams.readable, streams.writable]);\n    this.workerPipelines.set(receiver, newTrackId);\n  }\n\n  private teardownTrack(track: RemoteTrack) {\n    const trackId = track.mediaStreamID;\n    const extractor = this.extractors.get(trackId);\n    if (extractor) {\n      extractor.dispose();\n      this.extractors.delete(trackId);\n    }\n\n    if (track instanceof RemoteVideoTrack) {\n      track.frameMetadataExtractor = undefined;\n    }\n\n    // The receiver pipeline is intentionally left running. If the receiver is\n    // reused for a new track, `setupReceiver` will remap it. If the room\n    // disconnects, `cleanup` drops all state. Any metadata produced in the\n    // meantime is harmless — the extractor above has already been disposed and\n    // is no longer reachable from any track.\n  }\n\n  /** Drops the per-connection state. The worker stays, because the room can connect again. */\n  private cleanup() {\n    for (const extractor of this.extractors.values()) {\n      extractor.dispose();\n    }\n    this.extractors.clear();\n    this.workerPipelines.clear();\n  }\n\n  /** Terminates the worker. Called from `Room.dispose()`; the room cannot connect afterwards. */\n  dispose() {\n    this.cleanup();\n    this.worker?.terminate();\n  }\n\n  private onWorkerMessage = (ev: MessageEvent<PTWorkerMessage>) => {\n    const msg = ev.data;\n    if (msg.kind === 'metadata') {\n      const extractor = this.extractors.get(msg.data.trackId);\n      if (extractor) {\n        extractor.storeMetadata(msg.data.rtpTimestamp, msg.data.ssrc, msg.data.metadata);\n      }\n    }\n  };\n\n  private onWorkerError = (ev: ErrorEvent) => {\n    log.error('frame metadata worker encountered an error:', { error: ev.error });\n  };\n}\n\n/** @deprecated Use {@link FrameMetadataManager} instead. */\nexport const PacketTrailerManager = FrameMetadataManager;\n\n/** @deprecated Use {@link FrameMetadataOptions} instead. */\nexport type PacketTrailerOptions = FrameMetadataOptions;\n"],"names":["FrameMetadataManager","constructor","options","extractors","Map","workerPipelines","onWorkerMessage","ev","msg","data","kind","extractor","get","trackId","storeMetadata","rtpTimestamp","ssrc","metadata","onWorkerError","log","error","worker","setup","room","onmessage","onerror","postMessage","on","RoomEvent","TrackSubscribed","track","pub","_participant","setupReceiver","trackInfo","TrackUnsubscribed","teardownTrack","Disconnected","cleanup","receiver","hasFeatures","packetTrailerFeatures","length","_a","hasE2EESetup","setupPassthroughReceiver","mediaStreamID","isFrameMetadataSupported","undefined","_b","warn","FrameMetadataExtractor","set","frameMetadataExtractor","_c","setupWorkerReceiver","shouldUseFrameMetadataScriptTransform","transform","has","newTrackId","hasPacketTrailer","arguments","RTCRtpScriptTransform","existingTrackId","oldTrackId","streams","createEncodedStreams","err","readableStream","readable","writableStream","writable","dispose","delete","RemoteVideoTrack","values","clear","terminate"],"mappings":"4TAoBA;;;;;;;;;;;;;;;AAeG;MACUA,oBAAoB,CAAA;EAe/BC,WAAAA,CAAYC,OAA8B,EAAA;AAVlC,IAAA,IAAA,CAAAC,UAAU,GAAG,IAAIC,GAAG,EAAkC;AAE9D;;;;;AAKG;AACK,IAAA,IAAA,CAAAC,eAAe,GAAG,IAAID,GAAG,EAA0B;AAiMnD,IAAA,IAAA,CAAAE,eAAe,GAAIC,EAAiC,IAAI;AAC9D,MAAA,MAAMC,GAAG,GAAGD,EAAE,CAACE,IAAI;AACnB,MAAA,IAAID,GAAG,CAACE,IAAI,KAAK,UAAU,EAAE;AAC3B,QAAA,MAAMC,SAAS,GAAG,IAAI,CAACR,UAAU,CAACS,GAAG,CAACJ,GAAG,CAACC,IAAI,CAACI,OAAO,CAAC;AACvD,QAAA,IAAIF,SAAS,EAAE;UACbA,SAAS,CAACG,aAAa,CAACN,GAAG,CAACC,IAAI,CAACM,YAAY,EAAEP,GAAG,CAACC,IAAI,CAACO,IAAI,EAAER,GAAG,CAACC,IAAI,CAACQ,QAAQ,CAAC;AAClF,QAAA;AACF,MAAA;IACF,CAAC;AAEO,IAAA,IAAA,CAAAC,aAAa,GAAIX,EAAc,IAAI;AACzCY,MAAAA,aAAG,CAACC,KAAK,CAAC,6CAA6C,EAAE;QAAEA,KAAK,EAAEb,EAAE,CAACa;AAAK,OAAE,CAAC;IAC/E,CAAC;AA1MC,IAAA,IAAI,CAACC,MAAM,GAAGnB,OAAO,KAAA,IAAA,IAAPA,OAAO,KAAA,MAAA,GAAA,MAAA,GAAPA,OAAO,CAAEmB,MAAM;AAC/B,EAAA;AAEA;EACAC,KAAKA,CAACC,IAAc,EAAA;AAClB,IAAA,IAAIA,IAAI,KAAK,IAAI,CAACA,IAAI,EAAE;AACtB,MAAA;AACF,IAAA;IACA,IAAI,CAACA,IAAI,GAAGA,IAAI;IAEhB,IAAI,IAAI,CAACF,MAAM,EAAE;AACf,MAAA,IAAI,CAACA,MAAM,CAACG,SAAS,GAAG,IAAI,CAAClB,eAAe;AAC5C,MAAA,IAAI,CAACe,MAAM,CAACI,OAAO,GAAG,IAAI,CAACP,aAAa;AACxC,MAAA,IAAI,CAACG,MAAM,CAACK,WAAW,CAAC;AAAEhB,QAAAA,IAAI,EAAE;AAAM,OAAE,CAAC;AAC3C,IAAA;AAEAa,IAAAA,IAAI,CACDI,EAAE,CAACC,SAAS,CAACC,eAAe,EAAE,CAACC,KAAK,EAAEC,GAAG,EAAEC,YAAY,KAAI;AAC1D,MAAA,IAAIF,KAAK,CAACpB,IAAI,KAAK,OAAO,EAAE;AAC1B,QAAA;AACF,MAAA;MACA,IAAI,CAACuB,aAAa,CAACH,KAAoC,EAAEC,GAAG,CAACG,SAAS,CAAC;IACzE,CAAC,CAAC,CACDP,EAAE,CAACC,SAAS,CAACO,iBAAiB,EAAGL,KAAK,IAAI;AACzC,MAAA,IAAI,CAACM,aAAa,CAACN,KAAK,CAAC;IAC3B,CAAC,CAAC,CACDH,EAAE,CAACC,SAAS,CAACS,YAAY,EAAE,MAAK;MAC/B,IAAI,CAACC,OAAO,EAAE;AAChB,IAAA,CAAC,CAAC;AACN,EAAA;AAEQL,EAAAA,aAAaA,CAACH,KAAuB,EAAEI,SAAqB,EAAA;;AAClE,IAAA,MAAMK,QAAQ,GAAGT,KAAK,CAACS,QAAQ;IAC/B,IAAI,CAACA,QAAQ,EAAE;AACb,MAAA;AACF,IAAA;AAEA;AACA;AACA;IACA,MAAMC,WAAW,GACf,CAAC,EAACN,SAAS,KAAA,IAAA,IAATA,SAAS,KAAA,MAAA,GAAA,MAAA,GAATA,SAAS,CAAEO,qBAAqB,CAAA,IAAIP,SAAS,CAACO,qBAAqB,CAACC,MAAM,GAAG,CAAC;IAClF,IAAI,CAACF,WAAW,EAAE;MAChB,IAAI,EAAC,CAAAG,EAAA,GAAA,IAAI,CAACpB,IAAI,MAAA,IAAA,IAAAoB,EAAA,KAAA,MAAA,GAAA,MAAA,GAAAA,EAAA,CAAEC,YAAY,CAAA,EAAE;QAC5B,IAAI,CAACC,wBAAwB,CAACN,QAAQ,EAAET,KAAK,CAACgB,aAAa,CAAC;AAC9D,MAAA;AACA,MAAA;AACF,IAAA;AAEA,IAAA,IACE,CAACC,wBAAwB,CAAC,IAAI,CAAC1B,MAAM,GAAG;MAAEA,MAAM,EAAE,IAAI,CAACA;KAAQ,GAAG2B,SAAS,CAAC,IAC5E,EAAC,MAAA,IAAI,CAACzB,IAAI,MAAA,IAAA,IAAA0B,EAAA,KAAA,MAAA,GAAA,MAAA,GAAAA,EAAA,CAAEL,YAAY,CAAA,EACxB;AACAzB,MAAAA,aAAG,CAAC+B,IAAI,CAAC,6DAA6D,CAAC;AACvE,MAAA;AACF,IAAA;AAEA,IAAA,MAAMvC,SAAS,GAAG,IAAIwC,sBAAsB,EAAE;AAC9C,IAAA,MAAMtC,OAAO,GAAGiB,KAAK,CAACgB,aAAa;IAEnC,IAAI,CAAC3C,UAAU,CAACiD,GAAG,CAACvC,OAAO,EAAEF,SAAS,CAAC;IACvCmB,KAAK,CAACuB,sBAAsB,GAAG1C,SAAS;IAExC,IAAI,MAAA,IAAI,CAACY,IAAI,MAAA,IAAA,IAAA+B,EAAA,KAAA,MAAA,GAAA,MAAA,GAAAA,EAAA,CAAEV,YAAY,EAAE;AAC3B;AACA;AACA,MAAA;AACF,IAAA;IAEA,IAAI,CAACW,mBAAmB,CAAChB,QAAQ,EAAE1B,OAAO,EAAE,IAAI,CAAC;AACnD,EAAA;AAEQgC,EAAAA,wBAAwBA,CAACN,QAAwB,EAAE1B,OAAe,EAAA;IACxE,IAAI2C,qCAAqC,EAAE,EAAE;MAC3C,IAAI,WAAW,IAAIjB,QAAQ,EAAE;AAC3B;QACAA,QAAQ,CAACkB,SAAS,GAAG,IAAI;AAC3B,MAAA;AACA,MAAA;AACF,IAAA;AAEA,IAAA,IACE,IAAI,CAACpC,MAAM,IACX0B,wBAAwB,CAAC;MAAE1B,MAAM,EAAE,IAAI,CAACA;KAAQ,CAAC,IACjD,CAAC,IAAI,CAAChB,eAAe,CAACqD,GAAG,CAACnB,QAAQ,CAAC,EACnC;MACA,IAAI,CAACgB,mBAAmB,CAAChB,QAAQ,EAAE1B,OAAO,EAAE,KAAK,CAAC;AAClD,MAAA;AACF,IAAA;AAEA,IAAA,IAAI,IAAI,CAACQ,MAAM,IAAI,IAAI,CAAChB,eAAe,CAACqD,GAAG,CAACnB,QAAQ,CAAC,EAAE;MACrD,IAAI,CAACgB,mBAAmB,CAAChB,QAAQ,EAAE1B,OAAO,EAAE,KAAK,CAAC;AACpD,IAAA;AACF,EAAA;AAEQ0C,EAAAA,mBAAmBA,CACzBhB,QAAwB,EACxBoB,UAAkB,EACK;AAAA,IAAA,IAAvBC,gBAAgB,GAAAC,SAAA,CAAAnB,MAAA,GAAA,CAAA,IAAAmB,SAAA,CAAA,CAAA,CAAA,KAAAb,SAAA,GAAAa,SAAA,CAAA,CAAA,CAAA,GAAG,IAAI;AAEvB,IAAA,MAAMxC,MAAM,GAAG,IAAI,CAACA,MAAM;IAC1B,IAAI,CAACA,MAAM,EAAE;AACX,MAAA;AACF,IAAA;IAEA,IAAImC,qCAAqC,EAAE,EAAE;AAC3C;AACAjB,MAAAA,QAAQ,CAACkB,SAAS,GAAG,IAAIK,qBAAqB,CAACzC,MAAM,EAAE;AACrDX,QAAAA,IAAI,EAAE,QAAQ;AACdG,QAAAA,OAAO,EAAE8C;AACV,OAAA,CAAC;AACF,MAAA;AACF,IAAA;IAEA,MAAMI,eAAe,GAAG,IAAI,CAAC1D,eAAe,CAACO,GAAG,CAAC2B,QAAQ,CAAC;AAE1D,IAAA,IAAIwB,eAAe,EAAE;AACnB;AACA;AACA;AACA,MAAA,MAAMvD,GAAG,GAA2B;AAClCE,QAAAA,IAAI,EAAE,eAAe;AACrBD,QAAAA,IAAI,EAAE;AAAEuD,UAAAA,UAAU,EAAED,eAAe;UAAEJ,UAAU;AAAEC,UAAAA;AAAgB;OAClE;AACDvC,MAAAA,MAAM,CAACK,WAAW,CAAClB,GAAG,CAAC;MACvB,IAAI,CAACH,eAAe,CAAC+C,GAAG,CAACb,QAAQ,EAAEoB,UAAU,CAAC;AAC9C,MAAA;AACF,IAAA;AAEA,IAAA,IAAI,EAAE,sBAAsB,IAAIpB,QAAQ,CAAC,EAAE;AACzCpB,MAAAA,aAAG,CAAC+B,IAAI,CAAC,oCAAoC,CAAC;AAC9C,MAAA;AACF,IAAA;AAEA,IAAA,IAAIe,OAA+D;IACnE,IAAI;AACF;AACAA,MAAAA,OAAO,GAAG1B,QAAQ,CAAC2B,oBAAoB,EAAE;IAC3C,CAAC,CAAC,OAAOC,GAAG,EAAE;AACZhD,MAAAA,aAAG,CAAC+B,IAAI,CAAC,kCAAkC,EAAE;AAAE9B,QAAAA,KAAK,EAAE+C;AAAG,OAAE,CAAC;AAC5D,MAAA;AACF,IAAA;AAEA,IAAA,MAAM3D,GAAG,GAAoB;AAC3BE,MAAAA,IAAI,EAAE,QAAQ;AACdD,MAAAA,IAAI,EAAE;QACJ2D,cAAc,EAAEH,OAAO,CAACI,QAAQ;QAChCC,cAAc,EAAEL,OAAO,CAACM,QAAQ;AAChC1D,QAAAA,OAAO,EAAE8C,UAAU;AACnBC,QAAAA;AACD;KACF;AACDvC,IAAAA,MAAM,CAACK,WAAW,CAAClB,GAAG,EAAE,CAACyD,OAAO,CAACI,QAAQ,EAAEJ,OAAO,CAACM,QAAQ,CAAC,CAAC;IAC7D,IAAI,CAAClE,eAAe,CAAC+C,GAAG,CAACb,QAAQ,EAAEoB,UAAU,CAAC;AAChD,EAAA;EAEQvB,aAAaA,CAACN,KAAkB,EAAA;AACtC,IAAA,MAAMjB,OAAO,GAAGiB,KAAK,CAACgB,aAAa;IACnC,MAAMnC,SAAS,GAAG,IAAI,CAACR,UAAU,CAACS,GAAG,CAACC,OAAO,CAAC;AAC9C,IAAA,IAAIF,SAAS,EAAE;MACbA,SAAS,CAAC6D,OAAO,EAAE;AACnB,MAAA,IAAI,CAACrE,UAAU,CAACsE,MAAM,CAAC5D,OAAO,CAAC;AACjC,IAAA;IAEA,IAAIiB,KAAK,YAAY4C,gBAAgB,EAAE;MACrC5C,KAAK,CAACuB,sBAAsB,GAAGL,SAAS;AAC1C,IAAA;AAEA;AACA;AACA;AACA;AACA;AACF,EAAA;AAEA;AACQV,EAAAA,OAAOA,GAAA;IACb,KAAK,MAAM3B,SAAS,IAAI,IAAI,CAACR,UAAU,CAACwE,MAAM,EAAE,EAAE;MAChDhE,SAAS,CAAC6D,OAAO,EAAE;AACrB,IAAA;AACA,IAAA,IAAI,CAACrE,UAAU,CAACyE,KAAK,EAAE;AACvB,IAAA,IAAI,CAACvE,eAAe,CAACuE,KAAK,EAAE;AAC9B,EAAA;AAEA;AACAJ,EAAAA,OAAOA,GAAA;;IACL,IAAI,CAAClC,OAAO,EAAE;IACd,CAAAK,EAAA,GAAA,IAAI,CAACtB,MAAM,MAAA,IAAA,IAAAsB,EAAA,KAAA,MAAA,GAAA,MAAA,GAAAA,EAAA,CAAEkC,SAAS,EAAE;AAC1B,EAAA;AAeD"}