import { Mutex } from '@livekit/mutex'; import { EventEmitter } from 'events'; import { parse, write } from 'sdp-transform'; import type { MediaAttributes, MediaDescription, SessionDescription } from 'sdp-transform'; import type TypedEmitter from 'typed-emitter'; import log, { LoggerNames, getLogger } from '../logger'; import { debounce } from './debounce'; import { NegotiationError, UnexpectedConnectionState } from './errors'; import type { LoggerOptions } from './types'; import { ddExtensionURI, isSVCCodec, isSafari } from './utils'; interface TrackBitrateInfo { cid?: string; transceiver?: RTCRtpTransceiver; codec: string; maxbr: number; isScreenShare?: boolean; } /* * Video codecs use a very low bitrate at the beginning and increase slowly by * the bandwidth estimator until they reach the target bitrate. The process commonly * costs more than 10 seconds causing subscribers to get blurry video at the first * few seconds. We use x-google-start-bitrate to hint the BWE to start higher. * * Why 90%: Gives ~10% headroom for bandwidth estimation while starting close to target. * Why same for all codecs: Target bitrate already accounts for codec efficiency * (e.g., users set lower targets for VP9/AV1 knowing they're more efficient). * Why cap at 1 Mbps: Prevents BWE from starting too aggressively on high bitrate tracks. * The cap is lowered further for connections that were slow to set up, see * `computeStartBitrateCap`, or that the browser reports as slow, see * `connectionEffectiveTypeStartBitrateCap`. */ const startBitrateMultiplier = 0.9; /** Maximum x-google-start-bitrate in kbps. 1 Mbps prevents BWE from starting too aggressively. */ const maxStartBitrateKbps = 1000; /** * Minimum x-google-start-bitrate in kbps: libwebrtc's own default starting estimate. A target * below this gets no hint, since seeding above the real capacity costs more than the ramp it * saves, and a slow connection is seeded no higher than this. */ const minTargetBitrateKbps = 300; /** * Connection setup times (ms) bounding the ramp in `computeStartBitrateCap`: at or below the * first the hint is capped at `maxStartBitrateKbps`, at or above the second at * `minTargetBitrateKbps`. Measured on shaped links (CLT-3380): healthy links set up in under * 860 ms, a 1 Mbps link with 150 ms RTT took 1.1–1.7 s, a 500 kbps link ~2.3 s at the median, * and a 300 kbps link never under 3 s. The first sits above the 1 Mbps link, which a lower seed * only slows down, and the second above the 500 kbps link's median, so only links that cannot * carry more land at the floor. */ const setupTimeForMaxStartBitrateMs = 1500; const setupTimeForMinStartBitrateMs = 3500; /** Cap on the start bitrate hint, in kbps, when the browser reports a `3g` connection. */ const effectiveType3gStartBitrateKbps = 500; /** * The part of the Network Information API (`navigator.connection`) read here. It is Chromium * only, so TypeScript's DOM typings do not declare it. */ interface NetworkInformation { readonly effectiveType?: 'slow-2g' | '2g' | '3g' | '4g'; } const debounceInterval = 20; /** * Codec payload for `codec` in this media section, when the section carries `cid`. * * Returns `undefined` when the section does not belong to the track, `0` when it does but * does not offer the requested codec, and the codec payload otherwise. * * @internal */ export function findTrackCodecPayload( media: MediaDescription, cid: string, codec: string, ): number | undefined { if (!media.msid?.includes(cid)) { return undefined; } return media.rtp.find((rtp) => rtp.codec.toUpperCase() === codec.toUpperCase())?.payload ?? 0; } /** * Start bitrate hinted for a single track, or `undefined` when its target is too low to * be worth seeding. * * 90% of the target leaves ~10% headroom for the estimator to settle. The same multiplier * is used for every codec because the target already reflects the codec's efficiency. * Camera is capped at 1 Mbps so the estimator does not open too aggressively on a * high-bitrate track; screen share is exempt, because its content needs the bitrate * immediately to stay legible. * * The cap is lowered for a connection that was slow to set up (`computeStartBitrateCap`), and * for one the browser reports as slow (`connectionEffectiveTypeStartBitrateCap`), whichever is * lower. Screen share is exempt from both, however slow the connection. * * @internal */ export function computeTrackStartBitrate( trackbr: TrackBitrateInfo, connectionSetupTimeMs?: number, ): number | undefined { if (trackbr.maxbr < minTargetBitrateKbps) { return undefined; } const calculated = Math.min(Math.round(trackbr.maxbr * startBitrateMultiplier), trackbr.maxbr); if (trackbr.isScreenShare) { return calculated; } const cap = Math.min( computeStartBitrateCap(connectionSetupTimeMs), connectionEffectiveTypeStartBitrateCap(), ); return Math.min(calculated, cap); } /** * Cap on the start bitrate hint, in kbps, from the browser's effective connection type * (`navigator.connection.effectiveType`, Chromium only): `slow-2g` and `2g` cap at * `minTargetBitrateKbps`, `3g` at `effectiveType3gStartBitrateKbps`. * * It catches links that have low bandwidth but low latency, which set up quickly and so slip * past the setup time ramp. `4g`, and browsers without the API, give no guidance and stay at the * `maxStartBitrateKbps` ceiling, as `computeStartBitrateCap` does without a setup time. The * connection RTT the API also reports is not used: it lags behind network changes, and the setup * time already reflects latency. */ function connectionEffectiveTypeStartBitrateCap(): number { // A `typeof` guard, since `globalThis` is missing from some supported browsers (Chrome < 71, // Firefox < 65, Safari < 12.1) and the build does not polyfill it. const connection = typeof navigator !== 'undefined' ? (navigator as Navigator & { connection?: NetworkInformation }).connection : undefined; switch (connection?.effectiveType) { case 'slow-2g': case '2g': return minTargetBitrateKbps; case '3g': return effectiveType3gStartBitrateKbps; default: return maxStartBitrateKbps; } } /** * Cap on the start bitrate hint, in kbps, from how long the connection took to set up. * * Connection setup time (signaling join plus ICE/DTLS) is the network signal every platform has * before the first video offer, since libwebrtc cannot probe the path until a video sender * exists. It grows with round-trip time and loss, which also mark the links where a 1 Mbps * seed overshoots, so the cap ramps linearly from `maxStartBitrateKbps` at * `setupTimeForMaxStartBitrateMs` down to `minTargetBitrateKbps` at * `setupTimeForMinStartBitrateMs`. A video offer built while the connection is still being set * up (publishing during connect) passes the time elapsed so far, a lower bound on the setup time. * Without any timing the cap stays at the 1 Mbps ceiling. */ function computeStartBitrateCap(connectionSetupTimeMs?: number): number { if (connectionSetupTimeMs === undefined) { return maxStartBitrateKbps; } const ramp = Math.min( Math.max( (connectionSetupTimeMs - setupTimeForMaxStartBitrateMs) / (setupTimeForMinStartBitrateMs - setupTimeForMaxStartBitrateMs), 0, ), 1, ); return Math.round(maxStartBitrateKbps - ramp * (maxStartBitrateKbps - minTargetBitrateKbps)); } /** * The single start bitrate for this peer connection: the largest hint among the video * m-sections of `media` that map to a published track. * * libwebrtc reads `x-google-start-bitrate` per m-section but applies it to the shared * `Call` (`WebRtcVideoSendChannel::ApplyChangedParams` -> `SetSdpBitrateParameters`), where * `RtpBitrateConfigurator` holds one config for the whole connection. Differing per-section * values are therefore last-writer-wins, decided by m-section order, so every video section * gets the same number instead. * * Only sections that can send local media are considered. `trackBitrates` is append-only and an * unpublished section keeps its `a=msid`, so matching on msid alone would still pair a stale * entry with its old section — letting an uncapped screen-share target seed a connection that * now carries only a camera, or consuming the one-shot hint on a section that sends nothing. * * The direction is the discriminator, as an exclusion rather than a match: `recvonly` and * `inactive` are the only directions that cannot carry local media, and they are exactly the * two a dead section lands on — unpublish sets `inactive` (explicitly in * `LocalParticipant.unpublishTrack`, and via `removeTrack`'s sendonly -> inactive transition for * the simulcast senders), `removeTrack` on a `sendrecv` transceiver leaves `recvonly`, and the * pre-populated placeholders are `recvonly`. Everything else sends: `sendonly` from the * `addTransceiver` path, `sendrecv` from the legacy `addTrack` fallback (which reuses a * transceiver rather than creating a sendonly one), and a section with no direction attribute, * which SDP defaults to `sendrecv`. * * @internal */ export function computeConnectionStartBitrate( media: MediaDescription[], trackBitrates: TrackBitrateInfo[], connectionSetupTimeMs?: number, ): number | undefined { let connectionStartBitrate: number | undefined; for (const m of media) { if (m.type !== 'video' || m.direction === 'recvonly' || m.direction === 'inactive') { continue; } for (const trackbr of trackBitrates) { if (!trackbr.cid) { continue; } const codecPayload = findTrackCodecPayload(m, trackbr.cid, trackbr.codec); if (codecPayload === undefined) { continue; } const startBitrate = codecPayload > 0 ? computeTrackStartBitrate(trackbr, connectionSetupTimeMs) : undefined; if ( startBitrate !== undefined && (connectionStartBitrate === undefined || startBitrate > connectionStartBitrate) ) { connectionStartBitrate = startBitrate; } break; } } return connectionStartBitrate; } /** * Declares `x-google-start-bitrate` on `codecPayload`'s fmtp. This SDP munging is used for * a bitrate setting that cannot be applied through `RTCRtpEncodingParameters`. * * Returns whether the section now carries the hint. * * @internal */ export function applyVideoStartBitrate( media: MediaDescription, codecPayload: number, startBitrate: number, ): boolean { const fmtp = media.fmtp.find((entry) => entry.payload === codecPayload); if (fmtp) { // A payload type is shared across the bundle, so a value written for one section is // already the connection-level one; leave it rather than rewrite it. if (!fmtp.config.includes('x-google-start-bitrate')) { fmtp.config += `;x-google-start-bitrate=${startBitrate}`; } } else { // VP8 and some codecs may not have an existing fmtp line. media.fmtp.push({ payload: codecPayload, config: `x-google-start-bitrate=${startBitrate}`, }); } return true; } export const PCEvents = { NegotiationStarted: 'negotiationStarted', NegotiationComplete: 'negotiationComplete', // Fired with the offerId for every successful publisher answer application, // including answers that immediately recurse into another offer via // `renegotiate`. Use this rather than NegotiationComplete to know that a // specific offer has been negotiated end-to-end. OfferAnswered: 'offerAnswered', RTPVideoPayloadTypes: 'rtpVideoPayloadTypes', } as const; export interface PCTransportOptions { singlePeerConnection?: boolean; } export default class PCTransport extends (EventEmitter as new () => TypedEmitter) { private _pc: RTCPeerConnection | null; private get pc() { if (!this._pc) { this._pc = this.createPC(); } return this._pc; } private config?: RTCConfiguration; private log = log; private iceLog = log; private loggerOptions: LoggerOptions; private ddExtID = 0; latestOfferId: number = 0; private answeredOfferId: number = 0; /** offerId (0 if unnumbered) of the answer being applied right now */ private answerInFlight: number | undefined; latestAcknowledgedOfferId: number = 0; private offerLock: Mutex; private pendingInitialOffer?: { original: RTCSessionDescriptionInit; munged: string }; private readonly offeredAudioNackMids: string[]; pendingCandidates: RTCIceCandidateInit[] = []; restartingIce: boolean = false; renegotiate: boolean = false; trackBitrates: TrackBitrateInfo[] = []; /** * Whether an offer carrying the connection-level `x-google-start-bitrate` has been * accepted locally. The hint is written once per peer connection: libwebrtc retains * `start_bitrate_bps` in `RtpBitrateConfigurator` and re-applies it on network route * changes, so a later rewrite is at best a no-op and at worst restarts a converged * bandwidth estimator. A new peer connection (full reconnect) seeds a new estimator. */ private hasAppliedVideoStartBitrate = false; /** * How long this peer connection took to set up, in ms, recorded once the initial connect * succeeds. Lowers the start bitrate hint for a slow connection; see `computeStartBitrateCap`. */ private connectionSetupTimeMs?: number; /** * When the connection attempt that built this peer connection began (`performance.now()`). * Until `connectionSetupTimeMs` is known, the time elapsed since then is used as a lower bound * on the setup time, so a video published during connect is still seeded from it rather than * waiting for the connection, which would delay the offer. */ private connectionStartedAt?: number; remoteStereoMids: string[] = []; remoteNackMids: string[] = []; onOffer?: (offer: RTCSessionDescriptionInit, offerId: number) => void; onIceCandidate?: (candidate: RTCIceCandidate) => void; onIceCandidateError?: (ev: Event) => void; onConnectionStateChange?: (state: RTCPeerConnectionState) => void; onIceConnectionStateChange?: (state: RTCIceConnectionState) => void; onSignalingStatechange?: (state: RTCSignalingState) => void; onDataChannel?: (ev: RTCDataChannelEvent) => void; onTrack?: (ev: RTCTrackEvent) => void; constructor( config?: RTCConfiguration, loggerOptions: LoggerOptions = {}, options: PCTransportOptions = {}, ) { super(); this.offeredAudioNackMids = options.singlePeerConnection ? ['all'] : []; this.loggerOptions = loggerOptions; this.log = getLogger( loggerOptions.loggerName ?? LoggerNames.PCTransport, () => this.logContext, ); this.iceLog = getLogger(LoggerNames.ICE, () => this.logContext); this.config = config; this._pc = this.createPC(); this.offerLock = new Mutex(); } private createPC() { const pc = new RTCPeerConnection(this.config); pc.onicecandidate = (ev) => { if (!ev.candidate) return; this.iceLog.debug('local ICE candidate gathered', { candidate: ev.candidate.candidate }); this.onIceCandidate?.(ev.candidate); }; pc.onicecandidateerror = (ev) => { this.iceLog.debug('ICE candidate error', { event: ev }); this.onIceCandidateError?.(ev); }; pc.oniceconnectionstatechange = () => { this.iceLog.debug(`ICE connection state: ${pc.iceConnectionState}`); this.onIceConnectionStateChange?.(pc.iceConnectionState); }; pc.onsignalingstatechange = () => { this.log.debug(`signaling state: ${pc.signalingState}`); this.onSignalingStatechange?.(pc.signalingState); }; pc.onconnectionstatechange = () => { this.log.debug(`connection state: ${pc.connectionState}`); this.onConnectionStateChange?.(pc.connectionState); }; pc.ondatachannel = (ev) => { this.log.debug('data channel opened by peer', { label: ev.channel.label, id: ev.channel.id, }); this.onDataChannel?.(ev); }; pc.ontrack = (ev) => { this.onTrack?.(ev); }; return pc; } private get logContext() { return { ...this.loggerOptions.loggerContextCb?.(), }; } get isICEConnected(): boolean { return ( this._pc !== null && (this.pc.iceConnectionState === 'connected' || this.pc.iceConnectionState === 'completed') ); } async addIceCandidate(candidate: RTCIceCandidateInit): Promise { if (this.pc.remoteDescription && !this.restartingIce) { return this.pc.addIceCandidate(candidate); } this.iceLog.debug('queuing remote ICE candidate until remote description applied', { pendingCount: this.pendingCandidates.length + 1, }); this.pendingCandidates.push(candidate); } /** Whether an answer for `offerId` answers an offer older than the latest one sent. */ isStaleAnswer(offerId: number): boolean { return this.latestOfferId > 0 && offerId > 0 && offerId !== this.latestOfferId; } async setRemoteDescription(sd: RTCSessionDescriptionInit, offerId: number): Promise { if (sd.type === 'answer' && this.isStaleAnswer(offerId)) { this.log.warn('ignoring answer for old offer', { offerId, latestOfferId: this.latestOfferId, }); return false; } if (sd.type === 'answer') { if ( isDuplicateAnswer( offerId, this.answerInFlight, this.answeredOfferId, this._pc?.signalingState, this.pendingInitialOffer !== undefined, ) ) { // a repeat of an answer already applied or being applied (e.g. // duplicated by a proxy): there is no local offer left to answer this.log.warn('ignoring duplicate answer', { offerId }); return false; } // claimed before any await, so a copy arriving mid-apply is caught too; // released however the apply ends, so a genuine retry is still possible this.answerInFlight = offerId; } try { return await this.applyRemoteDescription(sd, offerId); } finally { if (sd.type === 'answer') this.answerInFlight = undefined; } } private async applyRemoteDescription( sd: RTCSessionDescriptionInit, offerId: number, ): Promise { let mungedSDP: string | undefined = undefined; if (sd.type === 'offer') { let { stereoMids, nackMids } = extractStereoAndNackAudioFromOffer(sd); this.remoteStereoMids = stereoMids; this.remoteNackMids = nackMids; } else if (sd.type === 'answer') { if (this.pendingInitialOffer && this._pc) { const { original, munged } = this.pendingInitialOffer; this.pendingInitialOffer = undefined; this.log.debug('setting pending initial offer before processing answer'); await this.setMungedSDP(original, munged); } const sdpParsed = parse(sd.sdp ?? ''); sdpParsed.media.forEach((media) => { const mid = getMidString(media.mid!); if (media.type === 'audio') { // munge sdp for opus bitrate settings this.trackBitrates.some((trackbr): boolean => { if (!trackbr.transceiver || mid != trackbr.transceiver.mid) { return false; } let codecPayload = 0; media.rtp.some((rtp): boolean => { if (rtp.codec.toUpperCase() === trackbr.codec.toUpperCase()) { codecPayload = rtp.payload; return true; } return false; }); if (codecPayload === 0) { return true; } let fmtpFound = false; for (const fmtp of media.fmtp) { if (fmtp.payload === codecPayload) { fmtp.config = fmtp.config .split(';') .filter((attr) => !attr.includes('maxaveragebitrate')) .join(';'); if (trackbr.maxbr > 0) { fmtp.config += `;maxaveragebitrate=${trackbr.maxbr * 1000}`; } fmtpFound = true; break; } } if (!fmtpFound) { if (trackbr.maxbr > 0) { media.fmtp.push({ payload: codecPayload, config: `maxaveragebitrate=${trackbr.maxbr * 1000}`, }); } } return true; }); } }); // The server's answer sets per-track codec params (e.g. opus `usedtx`) on // the sections mapped to a published track, but leaves the pre-populated // recvonly placeholder sections with defaults. Conform the placeholders so // each shared payload type is consistent across the bundle, otherwise // libwebrtc flags a "bundled payload type collision". const placeholderMids = this.getPlaceholderMids(); if (placeholderMids.size > 0) { conformBundledCodecFmtp(sdpParsed.media, (media) => placeholderMids.has(getMidString(media.mid!)), ); } mungedSDP = write(sdpParsed); } await this.setMungedSDP(sd, mungedSDP, true); // only a successfully applied answer makes later copies duplicates if (sd.type === 'answer' && offerId > 0) this.answeredOfferId = offerId; if (this.pendingCandidates.length > 0) { this.iceLog.debug('flushing queued ICE candidates', { count: this.pendingCandidates.length, }); } this.pendingCandidates.forEach((candidate) => { this.pc.addIceCandidate(candidate); }); this.pendingCandidates = []; this.restartingIce = false; // Fire OfferAnswered for every successfully applied answer, including the // ones that recurse into another offer via `renegotiate`. Callers waiting // on a specific offerId can resolve as soon as their offer's answer is in. if (sd.type === 'answer') { this.latestAcknowledgedOfferId = offerId; this.emit(PCEvents.OfferAnswered, offerId); } if (this.renegotiate) { this.renegotiate = false; await this.createAndSendOffer(); } else if (sd.type === 'answer') { this.emit(PCEvents.NegotiationComplete); if (sd.sdp) { const sdpParsed = parse(sd.sdp); sdpParsed.media.forEach((media) => { if (media.type === 'video') { this.emit(PCEvents.RTPVideoPayloadTypes, media.rtp); } }); } } return true; } // debounced negotiate interface negotiate = debounce(async (onError?: (e: Error) => void) => { this.emit(PCEvents.NegotiationStarted); try { await this.createAndSendOffer(); } catch (e) { if (onError) { onError(e as Error); } else { throw e; } } }, debounceInterval); async createInitialOffer() { const unlock = await this.offerLock.lock(); try { if (this.pc.signalingState !== 'stable') { this.log.warn('signaling state is not stable, cannot create initial offer'); return; } const offerId = this.latestOfferId + 1; this.latestOfferId = offerId; const offer = await this.pc.createOffer(); const original = { sdp: offer.sdp, type: offer.type }; const sdpParsed = parse(offer.sdp ?? ''); sdpParsed.media.forEach((media) => { ensureIPAddrMatchVersion(media); if (media.type === 'audio') { ensureAudioNackAndStereo(media, [], this.offeredAudioNackMids); } }); offer.sdp = write(sdpParsed); this.pendingInitialOffer = { original, munged: offer.sdp }; return { offer, offerId }; } finally { unlock(); } } async createAndSendOffer(options?: RTCOfferOptions) { const unlock = await this.offerLock.lock(); try { if (this.onOffer === undefined) { return; } if (options?.iceRestart) { this.iceLog.debug('restarting ICE'); this.restartingIce = true; } if ( this._pc && (this._pc.signalingState === 'have-local-offer' || this.pendingInitialOffer) ) { // we're waiting for the peer to accept our offer, so we'll just wait // the only exception to this is when ICE restart is needed const currentSD = this._pc.remoteDescription; if (options?.iceRestart && currentSD) { // roll the remote description back in so createOffer produces a valid // ICE-restart offer on top of the already-negotiated state await this._pc.setRemoteDescription(currentSD); } else if (options?.iceRestart) { // ICE restart with no remote description to restart on: `renegotiate` would stall // (the pending offer is never answered), so throw for the caller to recreate the PC. throw new NegotiationError( 'ICE restart requested without a remote description, peer connection must be recreated', ); } else { this.renegotiate = true; this.log.debug('requesting renegotiation'); return; } } else if (!this._pc || this._pc.signalingState === 'closed') { this.log.warn('could not createOffer with closed peer connection'); return; } // actually negotiate this.log.debug('starting to negotiate'); // increase the offer id at the start to ensure the offer is always > 0 so that we can use 0 as a default value for legacy behavior const offerId = this.latestOfferId + 1; this.latestOfferId = offerId; const offer = await this.pc.createOffer(options); this.log.debug('original offer', { sdp: offer.sdp }); const sdpParsed = parse(offer.sdp ?? ''); // One value for every video m-section, written only on the first offer that carries // local video: the hint is connection-level in libwebrtc, so differing per-section // values would be last-writer-wins on m-section order. Offers before any video is // published (data channel or audio only) find no target and leave the latch unset. const connectionStartBitrate = this.hasAppliedVideoStartBitrate ? undefined : computeConnectionStartBitrate( sdpParsed.media, this.trackBitrates, this.connectionSetupTimeEstimateMs(), ); let appliedVideoStartBitrate = false; sdpParsed.media.forEach((media) => { ensureIPAddrMatchVersion(media); if (media.type === 'audio') { ensureAudioNackAndStereo(media, ['all'], this.offeredAudioNackMids); } else if (media.type === 'video') { this.trackBitrates.some((trackbr): boolean => { if (!trackbr.cid) { return false; } const codecPayload = findTrackCodecPayload(media, trackbr.cid, trackbr.codec); if (codecPayload === undefined) { return false; } if (codecPayload > 0) { if (connectionStartBitrate !== undefined) { appliedVideoStartBitrate = applyVideoStartBitrate(media, codecPayload, connectionStartBitrate) || appliedVideoStartBitrate; } if (isSVCCodec(trackbr.codec) && !isSafari()) { this.ddExtID = ensureVideoDDExtension(media, sdpParsed, this.ddExtID); } } return true; }); } }); // Conform the placeholder sections (pre-populated, or reverted after an // unpublish on a server below protocol 19, which keeps the transceiver; from // protocol 19 unpublish stops it) so every shared payload type carries // identical fmtp across the bundle, otherwise libwebrtc flags a "bundled // payload type collision". // Detection is by transceiver (mids are stable across renegotiations) since // an unpublished section keeps its `a=msid`. const placeholderMids = this.getPlaceholderMids(); if (placeholderMids.size > 0) { conformBundledCodecFmtp(sdpParsed.media, (media) => placeholderMids.has(getMidString(media.mid!)), ); } if (this.latestOfferId > offerId) { this.log.warn('latestOfferId mismatch', { latestOfferId: this.latestOfferId, offerId, }); return; } const mungedSdp = write(sdpParsed); await this.setMungedSDP(offer, mungedSdp); // setMungedSDP falls back to the unmunged SDP on rejection. Only consume the // one-shot hint once the SDP carrying it has been accepted locally. if (appliedVideoStartBitrate && offer.sdp === mungedSdp) { this.hasAppliedVideoStartBitrate = true; this.log.info('applied x-google-start-bitrate', { startBitrateKbps: connectionStartBitrate, connectionSetupTimeMs: this.connectionSetupTimeEstimateMs(), }); } this.onOffer(offer, this.latestOfferId); } finally { unlock(); } } async createAndSetAnswer(): Promise { const answer = await this.pc.createAnswer(); const sdpParsed = parse(answer.sdp ?? ''); sdpParsed.media.forEach((media) => { ensureIPAddrMatchVersion(media); if (media.type === 'audio') { ensureAudioNackAndStereo(media, this.remoteStereoMids, this.remoteNackMids); } }); await this.setMungedSDP(answer, write(sdpParsed)); return answer; } /** * Returns the mids of transceivers that carry no outgoing track on this * (publisher) connection: the pre-populated placeholders added by * `RTCEngine.applyInitialPublisherLayout`, plus any transceiver that was used * for a track and reverted on unpublish (below server protocol 19; from protocol * 19 unpublish stops it). Their codec fmtp is conformed to the published tracks * so a shared payload type stays consistent across the bundle. */ private getPlaceholderMids(): Set { return placeholderMidsFromTransceivers(this._pc?.getTransceivers() ?? []); } createDataChannel(label: string, dataChannelDict: RTCDataChannelInit) { return this.pc.createDataChannel(label, dataChannelDict); } addTransceiver(mediaStreamTrack: MediaStreamTrack, transceiverInit: RTCRtpTransceiverInit) { return this.pc.addTransceiver(mediaStreamTrack, transceiverInit); } addTransceiverOfKind(kind: 'audio' | 'video', transceiverInit: RTCRtpTransceiverInit) { return this.pc.addTransceiver(kind, transceiverInit); } addTrack(track: MediaStreamTrack) { if (!this._pc) { throw new UnexpectedConnectionState('PC closed, cannot add track'); } return this._pc.addTrack(track); } setTrackCodecBitrate(info: TrackBitrateInfo) { this.trackBitrates.push(info); } /** * Record how long this peer connection took to set up. Called once, when the initial connect * succeeds; resumes and ICE restarts keep the estimator they have and never call this. */ setConnectionSetupTime(ms: number) { this.connectionSetupTimeMs = ms; } /** Record when the connection attempt that built this peer connection began. */ setConnectionStartedAt(startedAt: number) { this.connectionStartedAt = startedAt; } /** The measured setup time, or while still connecting, the time elapsed so far. */ private connectionSetupTimeEstimateMs(): number | undefined { if (this.connectionSetupTimeMs !== undefined) { return this.connectionSetupTimeMs; } if (this.connectionStartedAt === undefined) { return undefined; } return Math.round(performance.now() - this.connectionStartedAt); } setConfiguration(rtcConfig: RTCConfiguration) { if (!this._pc) { throw new UnexpectedConnectionState('PC closed, cannot configure'); } return this._pc?.setConfiguration(rtcConfig); } canRemoveTrack(): boolean { return !!this._pc?.removeTrack; } /** * Removes `sender` from the connection. With `stop`, it then stops the transceiver of `sender`. */ removeTrack(sender: RTCRtpSender, stop = false) { this._pc?.removeTrack(sender); if (stop) { this.stopTransceivers([sender]); } } /** * Stops the transceivers of `senders`. `removeTrack` keeps the transceiver and its encoder, * and each publish adds a new transceiver. A transceiver that is not stopped stays until the * connection closes. Call this after `removeTrack`, as `removeTrack(sender, true)` does. * Returns true if a transceiver stopped. * * The last live transceiver of each kind stays. The server cannot follow two changes that some * browsers make after all transceivers of a kind stop. The browser drops the ICE and DTLS * transport when all m-sections are rejected. Some browsers also give the next transceiver new * header extension ids. */ stopTransceivers(senders: RTCRtpSender[]): boolean { let stopped = false; const transceivers = this.getTransceivers(); const isLive = (t: RTCRtpTransceiver) => t.direction !== 'stopped' && t.currentDirection !== 'stopped'; for (const transceiver of transceivers) { // RTCRtpTransceiver.stop() is not available before Chrome 88 if (!senders.includes(transceiver.sender) || typeof transceiver.stop !== 'function') { continue; } // the sender has no track after removeTrack, the receiver always has one const { kind } = transceiver.receiver.track; if ( !transceivers.some((t) => t !== transceiver && isLive(t) && t.receiver.track.kind === kind) ) { continue; } try { transceiver.stop(); stopped = true; } catch (e) { this.log.warn('could not stop transceiver', { error: e }); } } return stopped; } getConnectionState() { return this._pc?.connectionState ?? 'closed'; } getICEConnectionState() { return this._pc?.iceConnectionState ?? 'closed'; } getSignallingState() { return this._pc?.signalingState ?? 'closed'; } getTransceivers() { return this._pc?.getTransceivers() ?? []; } getSenders() { return this._pc?.getSenders() ?? []; } getLocalDescription() { return this._pc?.localDescription; } getRemoteDescription() { return this.pc?.remoteDescription; } /** stats of the underlying connection, `undefined` when there is none */ getStats() { return this._pc?.getStats(); } getMaxMessageSize() { return this._pc?.sctp?.maxMessageSize; } async getConnectedAddress(): Promise { if (!this._pc) { return; } let selectedCandidatePairId = ''; const candidatePairs = new Map(); // id -> candidate ip const candidates = new Map(); const stats: RTCStatsReport = await this._pc.getStats(); stats.forEach((v) => { switch (v.type) { case 'transport': selectedCandidatePairId = v.selectedCandidatePairId; break; case 'candidate-pair': if (selectedCandidatePairId === '' && v.selected) { selectedCandidatePairId = v.id; } candidatePairs.set(v.id, v); break; case 'remote-candidate': candidates.set(v.id, `${v.address}:${v.port}`); break; default: } }); if (selectedCandidatePairId === '') { return undefined; } const selectedID = candidatePairs.get(selectedCandidatePairId)?.remoteCandidateId; if (selectedID === undefined) { return undefined; } return candidates.get(selectedID); } close = () => { if (!this._pc) { return; } this.log.debug('closing peer connection'); this.pendingInitialOffer = undefined; this._pc.close(); this._pc.onconnectionstatechange = null; this._pc.oniceconnectionstatechange = null; this._pc.onicegatheringstatechange = null; this._pc.ondatachannel = null; this._pc.onnegotiationneeded = null; this._pc.onsignalingstatechange = null; this._pc.onicecandidate = null; this._pc.ondatachannel = null; this._pc.ontrack = null; this._pc.onconnectionstatechange = null; this._pc.oniceconnectionstatechange = null; this._pc = null; }; private async setMungedSDP(sd: RTCSessionDescriptionInit, munged?: string, remote?: boolean) { const originalSdp = sd.sdp; if (munged) { sd.sdp = munged; try { this.log.debug(`setting munged ${remote ? 'remote' : 'local'} description`); if (remote) { await this.pc.setRemoteDescription(sd); } else { await this.pc.setLocalDescription(sd); } return; } catch (e) { this.log.warn(`not able to set ${sd.type}, falling back to unmodified sdp`, { error: e, mungedSdp: munged, originalSdp, }); sd.sdp = originalSdp; } } try { if (remote) { await this._pc?.setRemoteDescription(sd); } else { await this._pc?.setLocalDescription(sd); } } catch (e) { let msg = 'unknown error'; if (e instanceof Error) { msg = e.message; } else if (typeof e === 'string') { msg = e; } const fields: any = { error: msg, sdp: sd.sdp, }; if (munged && munged !== originalSdp) { fields.mungedSdp = munged; } if (!remote && this.pc.remoteDescription) { fields.remoteSdp = this.pc.remoteDescription; } this.log.error(`unable to set ${sd.type}`, { fields }); throw new NegotiationError(msg); } } } /** * Adds the AV1 dependency descriptor extension to `media` unless it is already there, and * returns the id it is mapped to so callers can pass it back in as `ddExtID` (0 when no id has * been chosen yet). * * A bundle has to map one URI to one id, so an id already in use for the extension anywhere in * `sdp` wins over both the cached one and a fresh one: Chrome advertises the extension itself on * sections it can send on, and an earlier offer may have munged it into others. * @internal */ export function ensureVideoDDExtension( media: { type: string; port: number; protocol: string; payloads?: string | undefined; } & MediaDescription, sdp: SessionDescription, ddExtID: number, ): number { const id = ddExtensionIDFor(sdp, ddExtID); if (id === undefined) { return ddExtID; } if (!media.ext?.some((ext) => ext.uri === ddExtensionURI)) { media.ext ??= []; media.ext.push({ value: id, uri: ddExtensionURI, }); } return id; } /** * The id to map the dependency descriptor to throughout `sdp`, or undefined when no id would be * consistent for the whole bundle and the extension therefore has to be left out. */ function ddExtensionIDFor(sdp: SessionDescription, cachedID: number): number | undefined { const mapped = mappedExtensionID(sdp, ddExtensionURI); if (mapped !== undefined) { // Adopting an id that also stands for another URI is what the browser rejects the bundle // over, and its own half of the map is not ours to renumber, so give up on this offer. return usedForOtherURI(sdp, mapped, ddExtensionURI) ? undefined : mapped; } // Reusing the id from the last offer keeps the mapping stable across renegotiations, but only // while nothing else has taken it: the browser assigns ids to its own extensions without // knowing about ours, so an id that was free when we picked it can since have been claimed — // typically by the fuller extension set that arrives with the first section we send on. if (cachedID !== 0 && !usedForOtherURI(sdp, cachedID, ddExtensionURI)) { return cachedID; } return unusedExtensionID(sdp); } /** The id `uri` is mapped to in `sdp`, if any section maps it. */ function mappedExtensionID(sdp: SessionDescription, uri: string): number | undefined { for (const media of sdp.media) { const ext = media.ext?.find((candidate) => candidate.uri === uri); if (ext) { return ext.value; } } return undefined; } /** Whether `id` stands for anything in `sdp` other than `uri`. */ function usedForOtherURI(sdp: SessionDescription, id: number, uri: string): boolean { return sdp.media.some((media) => media.ext?.some((ext) => ext.value === id && ext.uri !== uri)); } /** * An id no extension in `sdp` uses. Stays above every id in use rather than filling gaps, so it * is less likely to be an id the browser goes on to allocate to another extension, and steps * over 15, which RFC 8285 reserves. */ function unusedExtensionID(sdp: SessionDescription): number { let maxID = 0; sdp.media.forEach((media) => { media.ext?.forEach((ext) => { if (ext.value > maxID) { maxID = ext.value; } }); }); return maxID + 1 === 15 ? 16 : maxID + 1; } /** * Checks whether an fmtp config declares `param` as an exact, `;`-delimited * token. A plain substring check conflates distinct opus parameters — e.g. * `stereo=1` is a substring of `sprop-stereo=1` — so `param` must match a whole * parameter, not appear anywhere within the config string. * @internal */ export function fmtpConfigHasParam(config: string, param: string): boolean { return config.split(';').some((entry) => entry.trim() === param); } /** @internal */ export function ensureAudioNackAndStereo( media: { type: string; port: number; protocol: string; payloads?: string | undefined; } & MediaDescription, stereoMids: string[], nackMids: string[], ) { // sdp-transform types don't include number however the parser outputs mids as numbers in some cases const mid = getMidString(media.mid!); const includesThisSection = (mids: string[]) => mids.includes(mid) || (mids.length === 1 && mids[0] === 'all'); // found opus codec to add nack fb let opusPayload = 0; media.rtp.some((rtp): boolean => { // rtpmap encoding names are case-insensitive (RFC 4855) if (rtp.codec.toLowerCase() === 'opus') { opusPayload = rtp.payload; return true; } return false; }); // add nack rtcpfb if not exist if (opusPayload > 0) { if (!media.rtcpFb) { media.rtcpFb = []; } if ( includesThisSection(nackMids) && !media.rtcpFb.some((fb) => fb.payload === opusPayload && fb.type === 'nack') ) { media.rtcpFb.push({ payload: opusPayload, type: 'nack', }); } if (includesThisSection(stereoMids)) { media.fmtp.some((fmtp): boolean => { if (fmtp.payload === opusPayload) { if (!fmtpConfigHasParam(fmtp.config, 'stereo=1')) { fmtp.config += ';stereo=1'; } return true; } return false; }); } } } /** * Returns the mids of transceivers that carry no outgoing track: the * pre-populated placeholders added by `RTCEngine.applyInitialPublisherLayout`, * plus any transceiver that was used for a track and reverted on unpublish (below * server protocol 19; from protocol 19 unpublish stops it). The `sender.track` * check is the reliable signal — an unpublished section keeps its `a=msid` (and * its stale send-derived fmtp), so it can't be told apart from a real send by SDP * alone. Transceiver mids are stable across renegotiations, so * this works for every offer/answer after the first. * @internal */ export function placeholderMidsFromTransceivers( transceivers: readonly RTCRtpTransceiver[], ): Set { const mids = new Set(); for (const transceiver of transceivers) { if (transceiver.mid && !transceiver.sender.track) { mids.add(transceiver.mid); } } return mids; } /** * Within a BUNDLE group a payload type must map to identical codec parameters * across every m-line. When the same payload type carries different fmtp between * sections — e.g. opus `usedtx=1` on the published microphone but not on the * pre-populated recvonly placeholders, or H.265 with different `level-id` between * a published video track and a placeholder — libwebrtc flags a "bundled payload * type collision". * * Rewrite the placeholder sections so every shared payload type carries the * same fmtp. Real (non-placeholder) sections always win the canonical value, so * a published track's encoder parameters are never altered. When no real * section declares a payload type — e.g. a placeholder that was reused for a * track and then reverted to recvonly keeps its send-derived `level-id` while * fresh placeholders use the default — the placeholders still converge on the * first value seen, so two placeholders can't disagree either. Only placeholder * sections are ever rewritten, and it is codec-agnostic (opus, H.265, ...). * A codec without an `a=fmtp` line has empty parameters. Real sections that * disagree on a payload type are not reconciled: the placeholders follow the * last real section, and the real sections still differ. * `isPlaceholder` identifies the sections to conform. * @internal */ export function conformBundledCodecFmtp( media: MediaDescription[], isPlaceholder: (media: MediaDescription) => boolean, ) { // Canonical fmtp per payload type. Payload types are unique within a BUNDLE // group, so keying by payload alone (across audio and video) is safe. A real // section's value always takes precedence; otherwise the first placeholder // value seen is used so divergent placeholders still converge. const canonicalByPayload = new Map(); for (const m of media) { const placeholder = isPlaceholder(m); for (const { payload } of m.rtp ?? []) { const config = m.fmtp?.find((fmtp) => fmtp.payload === payload)?.config ?? ''; if (!placeholder || !canonicalByPayload.has(payload)) { canonicalByPayload.set(payload, config); } } } // Conform placeholder sections to the canonical fmtp for each shared payload. for (const m of media) { if (!isPlaceholder(m)) { continue; } for (const { payload } of m.rtp ?? []) { // The first loop added every payload type a placeholder lists. const config = canonicalByPayload.get(payload)!; const fmtp = m.fmtp?.find((entry) => entry.payload === payload); if (config === '') { m.fmtp = (m.fmtp ?? []).filter((entry) => entry.payload !== payload); } else if (fmtp) { fmtp.config = config; } else { m.fmtp = [...(m.fmtp ?? []), { payload, config }]; } } } } /** @internal */ /** * Whether an incoming answer repeats one already applied or being applied. * Any answer for the offer whose answer is in flight is a duplicate, numbered * or not. With an offerId (current servers) an answer is also a duplicate if * that offer's answer was already applied. Without one, fall back to the * signaling state: in `stable` no local offer is pending, unless the initial * offer is still deferred (it is set locally only once its answer arrives). */ export function isDuplicateAnswer( offerId: number, answerInFlight: number | undefined, answeredOfferId: number, signalingState: RTCSignalingState | undefined, hasPendingInitialOffer: boolean, ): boolean { if (answerInFlight === offerId) return true; if (offerId > 0) return offerId === answeredOfferId; return signalingState === 'stable' && !hasPendingInitialOffer; } export function extractStereoAndNackAudioFromOffer(offer: RTCSessionDescriptionInit): { stereoMids: string[]; nackMids: string[]; } { const stereoMids: string[] = []; const nackMids: string[] = []; const sdpParsed = parse(offer.sdp ?? ''); let opusPayload = 0; sdpParsed.media.forEach((media) => { const mid = getMidString(media.mid!); if (media.type === 'audio') { media.rtp.some((rtp): boolean => { // rtpmap encoding names are case-insensitive (RFC 4855) if (rtp.codec.toLowerCase() === 'opus') { opusPayload = rtp.payload; return true; } return false; }); if (media.rtcpFb?.some((fb) => fb.payload === opusPayload && fb.type === 'nack')) { nackMids.push(mid); } media.fmtp.some((fmtp): boolean => { if (fmtp.payload === opusPayload) { if (fmtpConfigHasParam(fmtp.config, 'sprop-stereo=1')) { stereoMids.push(mid); } return true; } return false; }); } }); return { stereoMids, nackMids }; } function ensureIPAddrMatchVersion(media: MediaDescription) { // Chrome could generate sdp with c = IN IP4 // in edge case and return error when set sdp.This is not a // sdk error but correct it if the issue detected. if (media.connection) { const isV6 = media.connection.ip.indexOf(':') >= 0; if ((media.connection.version === 4 && isV6) || (media.connection.version === 6 && !isV6)) { // fallback to dummy address media.connection.ip = '0.0.0.0'; media.connection.version = 4; } } } function getMidString(mid: string | number) { return typeof mid === 'number' ? mid.toFixed(0) : mid; } type PCTransportEventCallbacks = { negotiationStarted: () => void; negotiationComplete: () => void; offerAnswered: (offerId: number) => void; rtpVideoPayloadTypes: (attributes: MediaAttributes['rtp']) => void; };