import { type Call, CallingState, SfuModels, hasAudio, hasPausedTrack, hasScreenShare, type StreamVideoParticipant, videoLoggerSystem, type VideoTrackType, hasVideo, isPinned, } from '@stream-io/video-client'; import { useCall, useCallStateHooks } from '@stream-io/video-react-bindings'; import type { MediaStream } from '@stream-io/react-native-webrtc'; import React, { useEffect, useCallback, useRef, useState } from 'react'; import { findNodeHandle } from 'react-native'; import { onNativeCallClosed, onNativeDimensionsUpdated, type PiPBoundsChangeEvent, type PiPChangeEvent, RTCViewPipNative, } from './RTCViewPipNative'; import { BehaviorSubject, debounceTime } from 'rxjs'; import { shouldDisableIOSLocalVideoOnBackgroundRef } from '../../../utils/internal/shouldDisableIOSLocalVideoOnBackground'; import { useTrackDimensions } from '../../../hooks/useTrackDimensions'; import { isInPiPMode$ } from '../../../utils/internal/rxSubjects'; import TrackSubscriber from '../../Participant/ParticipantView/VideoRenderer/TrackSubscriber'; // One native view per Call instance. React Native drops the events of an // unmounted view, so an event the previous call's controller queued before // the replacement cannot reach the view of its successor. const nativeViewKeys = new WeakMap(); let nextNativeViewKey = 0; const nativeViewKeyOf = (call: Call | undefined) => { if (!call) return 0; let key = nativeViewKeys.get(call); if (key === undefined) { key = ++nextNativeViewKey; nativeViewKeys.set(call, key); } return key; }; type Props = { includeLocalParticipantVideo?: boolean; /** * Optional video mirroring override. */ mirror?: boolean; /** * Callback that is called when the PiP mode state changes. * @param active - true when PiP started, false when PiP stopped */ onPiPChange?: (active: boolean) => void; }; export const RTCViewPipIOS = React.memo((props: Props) => { const { includeLocalParticipantVideo, mirror: mirrorOverride, onPiPChange, } = props; const call = useCall(); const { useCameraState, useCallCallingState } = useCallStateHooks(); const callingState = useCallCallingState(); const { direction } = useCameraState(); const [allParticipants, setAllParticipants] = useState< StreamVideoParticipant[] >(call?.state.participants ?? []); // bounds of the native window, in logical points. While the window is on // screen they are the demand of the track it renders. const [pipDimensions$] = useState( () => new BehaviorSubject(undefined), ); const [isPipActive, setIsPipActive] = useState(false); // synchronous mirror of isPipActive: native may emit the same lifecycle // state twice (e.g. failedToStart after didStop) before React re-renders. const isPipActiveRef = useRef(false); // set once the call is closed: later native events describe a disposed window. const isClosedRef = useRef(false); // the handlers below are bound to the view they were rendered for; an event // of a replaced view is rejected even if it were delivered. const nativeViewKey = nativeViewKeyOf(call); const activeNativeViewKeyRef = useRef(nativeViewKey); useEffect(() => { activeNativeViewKeyRef.current = nativeViewKey; }, [nativeViewKey]); const isStaleNativeEvent = () => isClosedRef.current || nativeViewKey !== activeNativeViewKeyRef.current; const onPiPChangeRef = useRef(onPiPChange); useEffect(() => { onPiPChangeRef.current = onPiPChange; }, [onPiPChange]); const updatePipState = useCallback((active: boolean) => { if (isPipActiveRef.current === active) return; isPipActiveRef.current = active; setIsPipActive(active); isInPiPMode$.next(active); onPiPChangeRef.current?.(active); }, []); // we debounce the participants to avoid unnecessary rerenders // that happen when participant tracks are all subscribed simultaneously useEffect(() => { if (!call) { setAllParticipants([]); return; } const subscription = call.state.participants$ .pipe(debounceTime(300)) .subscribe(setAllParticipants); return () => subscription.unsubscribe(); }, [call]); const [dominantSpeaker, dominantSpeaker2] = allParticipants.filter( (participant) => includeLocalParticipantVideo ? true : !participant.isLocalParticipant, ); // show the dominant remote speaker in PiP mode // local speaker is shown only if remote doesn't exist let participantInSpotlight: StreamVideoParticipant | undefined = dominantSpeaker; if (dominantSpeaker?.isLocalParticipant && dominantSpeaker2) { participantInSpotlight = dominantSpeaker2; } useEffect(() => { shouldDisableIOSLocalVideoOnBackgroundRef.current = !includeLocalParticipantVideo; }, [includeLocalParticipantVideo]); const nativeRef = React.useRef(null); React.useEffect(() => { const node = findNodeHandle(nativeRef.current); isClosedRef.current = false; const onCallClosed = () => { if (isClosedRef.current) { return; } isClosedRef.current = true; if (node !== null) { onNativeCallClosed(node); } shouldDisableIOSLocalVideoOnBackgroundRef.current = true; pipDimensions$.next(undefined); updatePipState(false); }; const unsubFunc = call?.on('call.ended', () => { videoLoggerSystem .getLogger('RTCViewPipIOS') .debug(`onCallClosed due to call.ended event`); onCallClosed(); }); const subscription = call?.state.callingState$.subscribe((state) => { if (state === CallingState.LEFT) { videoLoggerSystem .getLogger('RTCViewPipIOS') .debug(`onCallClosed due to callingState: ${state}`); onCallClosed(); } }); return () => { onCallClosed(); unsubFunc?.(); subscription?.unsubscribe(); }; }, [call, pipDimensions$, updatePipState]); const onDimensionsUpdated = useCallback((width: number, height: number) => { const node = findNodeHandle(nativeRef.current); if (node !== null && width > 0 && height > 0) { onNativeDimensionsUpdated(node, width, height); } }, []); const { videoStream, screenShareStream } = participantInSpotlight || {}; const isScreenSharing = participantInSpotlight ? hasScreenShare(participantInSpotlight) : false; const videoStreamToRender = (isScreenSharing ? screenShareStream : videoStream) as unknown as MediaStream | undefined; const isPublishingTrack = isScreenSharing || (participantInSpotlight && hasVideo(participantInSpotlight)); const streamURL = isPublishingTrack ? videoStreamToRender?.toURL() : undefined; const mirror = isScreenSharing ? false : mirrorOverride !== undefined ? mirrorOverride : !!participantInSpotlight?.isLocalParticipant && direction === 'front'; const handlePiPChange = (event: { nativeEvent: PiPChangeEvent }) => { if (isStaleNativeEvent()) return; updatePipState(event.nativeEvent.active); }; const handlePiPBoundsChange = (event: { nativeEvent: PiPBoundsChangeEvent; }) => { if (isStaleNativeEvent()) return; const { width, height } = event.nativeEvent; pipDimensions$.next({ width, height }); }; // Get participant info for avatar placeholder const participantName = participantInSpotlight?.name || undefined; const participantImageURL = participantInSpotlight?.image || undefined; // Determine if the call is reconnecting or offline const isReconnecting = callingState === CallingState.MIGRATING || callingState === CallingState.RECONNECTING || callingState === CallingState.RECONNECTING_FAILED || callingState === CallingState.OFFLINE; // Determine if the participant has audio enabled const participantHasAudio = participantInSpotlight ? hasAudio(participantInSpotlight) : true; // Determine if the video track is paused const trackType: VideoTrackType = isScreenSharing ? 'screenShareTrack' : 'videoTrack'; const isVideoTrackPaused = participantInSpotlight ? hasPausedTrack(participantInSpotlight, trackType) : false; // Determine if the participant is pinned const participantIsPinned = participantInSpotlight ? isPinned(participantInSpotlight) : false; // Determine if the participant is speaking const participantIsSpeaking = participantInSpotlight?.isSpeaking ?? false; // Get connection quality (convert enum to number: UNSPECIFIED=0, POOR=1, GOOD=2, EXCELLENT=3) const participantConnectionQuality = participantInSpotlight?.connectionQuality ?? SfuModels.ConnectionQuality.UNSPECIFIED; // while the native window is on screen, its bounds - and not the hidden // inline layout behind it - are the demand of the track it renders. The local // preview is never subscribed to. const pipTrackOwner = isPipActive && participantInSpotlight?.isLocalParticipant !== true ? participantInSpotlight : undefined; return ( <> {pipTrackOwner && call && ( )} {participantInSpotlight && ( )} ); }); const DimensionsUpdatedRenderless = React.memo( ({ participant, trackType, onDimensionsUpdated, }: { participant: StreamVideoParticipant; trackType: VideoTrackType; onDimensionsUpdated: (width: number, height: number) => void; }) => { const { width, height } = useTrackDimensions(participant, trackType); useEffect(() => { onDimensionsUpdated(width, height); }, [width, height, onDimensionsUpdated]); return null; }, ); DimensionsUpdatedRenderless.displayName = 'DimensionsUpdatedRenderless'; RTCViewPipIOS.displayName = 'RTCViewPipIOS';