/* eslint-disable class-methods-use-this */ import { LocalCameraStream, LocalMicrophoneStream, LocalDisplayStream, LocalSystemAudioStream, RemoteStream, } from '@webex/media-helpers'; import {parse} from '@webex/ts-sdp'; import {ClientEvent} from '@webex/internal-plugin-metrics'; import {throttle} from 'lodash'; import Metrics from '../metrics'; import {MEETINGS, QUALITY_LEVELS} from '../constants'; import LoggerProxy from '../common/logs/logger-proxy'; import MediaConnectionAwaiter from './MediaConnectionAwaiter'; import BEHAVIORAL_METRICS from '../metrics/constants'; export type MediaDirection = { sendAudio: boolean; sendVideo: boolean; sendShare: boolean; receiveAudio: boolean; receiveVideo: boolean; receiveShare: boolean; }; export type IPVersion = ClientEvent['payload']['ipVersion']; /** * @class MediaProperties */ export default class MediaProperties { audioStream?: LocalMicrophoneStream; mediaDirection: MediaDirection; mediaSettings: any; webrtcMediaConnection: any; remoteAudioStream: RemoteStream; remoteQualityLevel: any; remoteShareStream: RemoteStream; remoteVideoStream: RemoteStream; shareVideoStream?: LocalDisplayStream; shareAudioStream?: LocalSystemAudioStream; videoDeviceId: any; videoStream?: LocalCameraStream; namespace = MEETINGS; mediaIssueCounters: {[key: string]: number} = {}; throttledSendMediaIssueMetric: ReturnType; /** * @param {Object} [options] -- to auto construct * @returns {MediaProperties} */ constructor() { this.webrtcMediaConnection = null; this.mediaDirection = { receiveAudio: false, receiveVideo: false, receiveShare: false, sendAudio: false, sendVideo: false, sendShare: false, }; this.videoStream = null; this.audioStream = null; this.shareVideoStream = null; this.shareAudioStream = null; this.remoteShareStream = undefined; this.remoteAudioStream = undefined; this.remoteVideoStream = undefined; this.remoteQualityLevel = QUALITY_LEVELS.HIGH; this.mediaSettings = {}; this.videoDeviceId = null; this.throttledSendMediaIssueMetric = throttle((eventPayload) => { Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED, { ...eventPayload, }); Object.keys(this.mediaIssueCounters).forEach((key) => { this.mediaIssueCounters[key] = 0; }); }, 1000 * 60 * 5); // at most once every 5 minutes } /** * Retrieves the preferred video input device * @returns {Object|null} */ getVideoDeviceId() { return this.videoDeviceId || null; } setMediaDirection(mediaDirection) { this.mediaDirection = mediaDirection; } setMediaSettings(type, values) { this.mediaSettings[type] = values; } setMediaPeerConnection(mediaPeerConnection) { this.webrtcMediaConnection = mediaPeerConnection; } setLocalVideoStream(videoStream?: LocalCameraStream) { this.videoStream = videoStream; } setLocalAudioStream(audioStream?: LocalMicrophoneStream) { this.audioStream = audioStream; } setLocalShareVideoStream(shareVideoStream?: LocalDisplayStream) { this.shareVideoStream = shareVideoStream; } setLocalShareAudioStream(shareAudioStream?: LocalSystemAudioStream) { this.shareAudioStream = shareAudioStream; } setRemoteQualityLevel(remoteQualityLevel) { this.remoteQualityLevel = remoteQualityLevel; } setRemoteShareStream(remoteShareStream: RemoteStream) { this.remoteShareStream = remoteShareStream; } /** * Sets the remote audio stream * @param {RemoteStream} remoteAudioStream RemoteStream to save * @returns {void} */ setRemoteAudioStream(remoteAudioStream: RemoteStream) { this.remoteAudioStream = remoteAudioStream; } /** * Sets the remote video stream * @param {RemoteStream} remoteVideoStream RemoteStream to save * @returns {void} */ setRemoteVideoStream(remoteVideoStream: RemoteStream) { this.remoteVideoStream = remoteVideoStream; } /** * Stores the preferred video input device * @param {string} deviceId Preferred video input device * @returns {void} */ setVideoDeviceId(deviceId: string) { this.videoDeviceId = deviceId; } /** * Clears the webrtcMediaConnection. This method should be called after * peer connection is closed and no longer needed. * @returns {void} */ unsetPeerConnection() { this.webrtcMediaConnection = null; this.throttledSendMediaIssueMetric.flush(); } /** * Removes both remote audio and video from class instance * @returns {void} */ unsetRemoteMedia() { this.remoteAudioStream = null; this.remoteVideoStream = null; } unsetRemoteShareStream() { this.remoteShareStream = null; } /** * Unsets all remote streams * @returns {void} */ unsetRemoteStreams() { this.unsetRemoteMedia(); this.unsetRemoteShareStream(); } /** * Returns if we have at least one local share stream or not. * @returns {Boolean} */ hasLocalShareStream() { return !!(this.shareAudioStream || this.shareVideoStream); } /** * Waits for the webrtc media connection to be connected. * * @param {string} correlationId * @returns {Promise} */ waitForMediaConnectionConnected(correlationId: string): Promise { const mediaConnectionAwaiter = new MediaConnectionAwaiter({ webrtcMediaConnection: this.webrtcMediaConnection, correlationId, }); return mediaConnectionAwaiter.waitForMediaConnectionConnected(); } /** * Returns ICE transport information: * - selectedCandidatePairChanges - number of times the selected candidate pair was changed, it should be at least 1 for successful connections * it will be -1 if browser doesn't supply this information * - numTransports - number of transports (should be 1 if we're using bundle) * * @param {Array} allStatsReports array of RTC stats reports * @returns {Object} */ private getTransportInfo(allStatsReports: any[]): { selectedCandidatePairChanges: number; numTransports: number; } { const transports = allStatsReports.filter((report) => report.type === 'transport'); if (transports.length > 1) { LoggerProxy.logger.warn( `Media:properties#getSelectedCandidatePairChanges --> found more than 1 transport: ${transports.length}` ); } return { selectedCandidatePairChanges: transports.length > 0 && transports[0].selectedCandidatePairChanges !== undefined ? transports[0].selectedCandidatePairChanges : -1, numTransports: transports.length, }; } /** * Checks if the given IP address is IPv6 * @param {string} ip address to check * @returns {boolean} true if the address is IPv6, false otherwise */ private isIPv6(ip: string): boolean { return ip.includes(':'); } /** Finds out if we connected using IPv4 or IPv6 * @param {RTCPeerConnection} webrtcMediaConnection * @param {Array} allStatsReports array of RTC stats reports * @returns {string} IPVersion */ private getConnectionIpVersion( webrtcMediaConnection: RTCPeerConnection, allStatsReports: any[] ): IPVersion | undefined { const transports = allStatsReports.filter((report) => report.type === 'transport'); let selectedCandidatePair; if (transports.length > 0 && transports[0].selectedCandidatePairId) { selectedCandidatePair = allStatsReports.find( (report) => report.type === 'candidate-pair' && report.id === transports[0].selectedCandidatePairId ); } else { // Firefox doesn't have selectedCandidatePairId, but has selected property on the candidate pair selectedCandidatePair = allStatsReports.find( (report) => report.type === 'candidate-pair' && report.selected ); } if (selectedCandidatePair) { const localCandidate = allStatsReports.find( (report) => report.type === 'local-candidate' && report.id === selectedCandidatePair.localCandidateId ); if (localCandidate) { if (localCandidate.address) { return this.isIPv6(localCandidate.address) ? 'IPv6' : 'IPv4'; } try { // safari doesn't have address field on the candidate, so we have to use the port to look up the candidate in the SDP const localSdp = webrtcMediaConnection.localDescription.sdp; const parsedSdp = parse(localSdp); for (const mediaLine of parsedSdp.avMedia) { const matchingCandidate = mediaLine.iceInfo.candidates.find( (candidate) => candidate.port === localCandidate.port ); if (matchingCandidate) { return this.isIPv6(matchingCandidate.connectionAddress) ? 'IPv6' : 'IPv4'; } } LoggerProxy.logger.warn( `Media:properties#getConnectionIpVersion --> failed to find local candidate in the SDP for port ${localCandidate.port}` ); } catch (error) { LoggerProxy.logger.warn( `Media:properties#getConnectionIpVersion --> error while trying to find candidate in local SDP:`, error ); return undefined; } } else { LoggerProxy.logger.warn( `Media:properties#getConnectionIpVersion --> failed to find local candidate "${selectedCandidatePair.localCandidateId}" in getStats() results` ); } } else { LoggerProxy.logger.warn( `Media:properties#getConnectionIpVersion --> failed to find selected candidate pair in getStats() results (transports.length=${transports.length}, selectedCandidatePairId=${transports[0]?.selectedCandidatePairId})` ); } return undefined; } /** * Returns the type of a connection that has been established * It should be 'UDP' | 'TCP' | 'TURN-TLS' | 'TURN-TCP' | 'TURN-UDP' | 'unknown' * * If connection was not established, it returns 'unknown' * * @param {Array} allStatsReports array of RTC stats reports * @returns {string} */ private getConnectionType(allStatsReports: any[]) { const successfulCandidatePairs = allStatsReports.filter( (report) => report.type === 'candidate-pair' && report.state?.toLowerCase() === 'succeeded' ); let foundConnectionType = 'unknown'; // all of the successful pairs should have the same connection type, so just return the type for the first one successfulCandidatePairs.some((pair) => { const localCandidate = allStatsReports.find( (report) => report.type === 'local-candidate' && report.id === pair.localCandidateId ); if (localCandidate === undefined) { LoggerProxy.logger.warn( `Media:properties#getConnectionType --> failed to find local candidate "${pair.localCandidateId}" in getStats() results` ); return false; } let connectionType; if (localCandidate.relayProtocol) { connectionType = `TURN-${localCandidate.relayProtocol.toUpperCase()}`; } else { connectionType = localCandidate.protocol?.toUpperCase(); // it will be UDP or TCP } if (connectionType) { foundConnectionType = connectionType; return true; } LoggerProxy.logger.warn( `Media:properties#getConnectionType --> missing localCandidate.protocol, candidateType=${localCandidate.candidateType}` ); return false; }); if (foundConnectionType === 'unknown') { const candidatePairStates = allStatsReports .filter((report) => report.type === 'candidate-pair') .map((report) => report.state); LoggerProxy.logger.warn( `Media:properties#getConnectionType --> all candidate pair states: ${JSON.stringify( candidatePairStates )}` ); } return foundConnectionType; } /** * Returns information about current webrtc media connection * * @returns {Promise} */ async getCurrentConnectionInfo(): Promise<{ connectionType: string; ipVersion?: IPVersion; selectedCandidatePairChanges: number; numTransports: number; }> { try { const allStatsReports = []; await new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error('timed out')); }, 1000); this.webrtcMediaConnection .getStats() .then((statsResult) => { clearTimeout(timeout); statsResult.forEach((report) => allStatsReports.push(report)); resolve(allStatsReports); }) .catch((error) => { clearTimeout(timeout); reject(error); }); }); const connectionType = this.getConnectionType(allStatsReports); const rtcPeerconnection = this.webrtcMediaConnection.multistreamConnection?.pc.pc || this.webrtcMediaConnection.mediaConnection?.pc; const ipVersion = this.getConnectionIpVersion(rtcPeerconnection, allStatsReports); const {selectedCandidatePairChanges, numTransports} = this.getTransportInfo(allStatsReports); return { connectionType, ipVersion, selectedCandidatePairChanges, numTransports, }; } catch (error) { LoggerProxy.logger.warn( `Media:properties#getCurrentConnectionInfo --> getStats() failed: ${error}` ); return { connectionType: 'unknown', ipVersion: undefined, selectedCandidatePairChanges: -1, numTransports: 0, }; } } /** * Sends a metric about a media issue. Metrics are throttled so that we don't * send too many of them, but include a count so that we know how many issues * were detected. * * @param {string} issueType * @param {string} issueSubType * @param {string} correlationId * @returns {void} */ public sendMediaIssueMetric(issueType: string, issueSubType: string, correlationId) { const key = `${issueType}_${issueSubType}`; const count = (this.mediaIssueCounters[key] || 0) + 1; this.mediaIssueCounters[key] = count; this.throttledSendMediaIssueMetric({ correlationId, ...this.mediaIssueCounters, }); } }