import { isDevEnvironment, showBalloonWarning } from "../engine/debug/index.js"; import { RoomEvents } from "../engine/engine_networking.js"; import { disposeStream, NetworkedStreamEvents, NetworkedStreams, PeerHandle, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js"; import { serializable } from "../engine/engine_serialization.js"; import { delay, getParam } from "../engine/engine_utils.js"; import { AudioSource } from "./AudioSource.js"; import { Behaviour, GameObject } from "./Component.js"; import type { IPointerClickHandler, PointerEventData } from "./ui/PointerEvents.js"; import { AspectMode, VideoPlayer } from "./VideoPlayer.js"; const debug = getParam("debugscreensharing"); /** * ScreenCapture component allows you to share your screen, camera or microphone with other users in the networked room. */ export enum ScreenCaptureDevice { /** * Capture the screen of the user. */ Screen = 0, /** * Capture the camera of the user. */ Camera = 1, /** Please note that canvas streaming might not work reliably on chrome: https://bugs.chromium.org/p/chromium/issues/detail?id=1156408 */ Canvas = 2, /** When using Microphone only the voice will be send */ Microphone = 3 } /** * {@link ScreenCapture} allows you to share your screen, camera or microphone with other users in the networked room. */ declare type ScreenCaptureDeviceTypes = keyof typeof ScreenCaptureDevice; /** * The current mode of the {@link ScreenCapture} component. */ export enum ScreenCaptureMode { Idle = 0, Sending = 1, Receiving = 2 } /** * Options for the {@link ScreenCapture} component when starting to share a stream by calling the {@link ScreenCapture.share}. */ export declare type ScreenCaptureOptions = { /** * You can specify the device type to capture (e.g. Screen, Camera, Microphone) */ device?: ScreenCaptureDeviceTypes, /** * Constraints for the media stream like resolution, frame rate, etc. * @see https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamConstraints */ constraints?: MediaTrackConstraints, /** Filter video device by id. Alternatively pass in a deviceFilter callback to manually filter available devices */ deviceId?: string, /** Return false to skip the available device */ deviceFilter?: (device: MediaDeviceInfo) => boolean, } /** * The ScreenCapture component allows you to share your screen, camera or microphone with other users in the networked room. * When the stream is active the video will be displayed on the VideoPlayer component attached to the same GameObject. * * Note: For debugging append `?debugscreensharing` to the URL to see more information in the console. * * By default the component will start sharing the screen when the user clicks on the object this component is attached to. You can set {@link device} This behaviour can be disabled by setting `allowStartOnClick` to false. * It is also possible to start the stream manually from your code by calling the {@link share} method. * * @category Networking * @group Components */ export class ScreenCapture extends Behaviour implements IPointerClickHandler { /** * When enabled the stream will start when the user clicks on the object this component is attached to * It is also possible to start the stream manually from your code by calling the {@link share} method * To modify what type of device is shared you can set the {@link device} property. * @default true */ @serializable() allowStartOnClick: boolean = true; /** @internal */ onPointerEnter() { if (this.context.connection.allowEditing == false) return; if (!this.allowStartOnClick) return; this.context.input.setCursor("pointer"); } /** @internal */ onPointerExit() { if (this.context.connection.allowEditing == false) return; if (!this.allowStartOnClick) return; this.context.input.unsetCursor("pointer"); } /** @internal */ onPointerClick(evt: PointerEventData) { if (this.context.connection.allowEditing == false) return; if (!this.allowStartOnClick) return; if (evt && evt.pointerId !== 0) return; if (this.isReceiving && this.videoPlayer?.isPlaying) { if (this.videoPlayer) this.videoPlayer.screenspace = !this.videoPlayer.screenspace; return; } if (this.isSending) { this.close(); return; } this.share(); } /** When enabled the stream will start when this component becomes active (enabled in the scene) */ @serializable() autoConnect: boolean = false; /** * If a VideoPlayer component is assigned to this property the video will be displayed on the VideoPlayer component. */ @serializable(VideoPlayer) set videoPlayer(val: VideoPlayer | undefined) { if (this._videoPlayer && (this.isSending || this.isReceiving)) { this._videoPlayer.stop(); } this._videoPlayer = val; if (this._videoPlayer && this._currentStream && (this.isSending || this.isReceiving)) { this._videoPlayer.setVideo(this._currentStream); } } get videoPlayer() { return this._videoPlayer; } private _videoPlayer?: VideoPlayer; private _audioSource?: AudioSource; /** * When enabled the video will be displayed in the screenspace of the VideoPlayer component. */ get screenspace() { return this.videoPlayer?.screenspace ?? false; } set screenspace(v: boolean) { if (this.videoPlayer) this.videoPlayer.screenspace = v; } /** * Which streaming device type should be used when starting to share (if {@link share} is called without a device option). Options are Screen, Camera, Microphone. * This is e.g. used if `allowStartOnClick` is enabled and the user clicks on the object. * @default Screen */ @serializable() device: ScreenCaptureDeviceTypes = "Screen"; /** * If assigned the device the device will be selected by this id or label when starting to share. * Note: This is only supported for `Camera` devices */ @serializable() deviceName?: string; /** * Filter which device should be chosen for sharing by id or label. * Assign a method to this property to manually filter the available devices. */ deviceFilter?: (device: MediaDeviceInfo) => boolean; /** * the current stream that is being shared or received * @link https://developer.mozilla.org/en-US/docs/Web/API/MediaStream */ get currentScream(): MediaStream | null { return this._currentStream; } get currentMode(): ScreenCaptureMode { return this._currentMode; } /** * @returns true if the component is currently sending a stream */ get isSending() { return this._currentStream?.active && this._currentMode === ScreenCaptureMode.Sending; } /** * @returns true if the component is currently receiving a stream */ get isReceiving() { if (this._currentMode === ScreenCaptureMode.Receiving) { if (!this._currentStream || this._currentStream.active === false) return false; // if any track is still live consider it active const tracks = this._currentStream.getTracks(); for (const track of tracks) { if (track.readyState === "live") return true; } } return false; } private get requiresVideoPlayer() { return this.device !== "Microphone"; } private _net?: NetworkedStreams; private _requestOpen: boolean = false; private _currentStream: MediaStream | null = null; private _currentMode: ScreenCaptureMode = ScreenCaptureMode.Idle; /** @internal */ awake() { // Resolve the device type if it is a number if (typeof this.device === "number") { this.device = ScreenCaptureDevice[this.device] as ScreenCaptureDeviceTypes; } if (debug) console.log("Screensharing", this.name, this); AudioSource.registerWaitForAllowAudio(() => { if (this._videoPlayer && this._currentStream && this._currentMode === ScreenCaptureMode.Receiving) { this._videoPlayer.playInBackground = true; this._videoPlayer.setVideo(this._currentStream); } }); this._net = new NetworkedStreams(this); } /** @internal */ onEnable(): void { this._net?.enable(); //@ts-ignore this._net?.addEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream); //@ts-ignore this._net?.addEventListener(NetworkedStreamEvents.StreamEnded, this.onCallEnded); this.context.connection.beginListen(RoomEvents.JoinedRoom, this.onJoinedRoom); if (this.autoConnect) { delay(1000).then(() => { if (this.enabled && this.autoConnect && !this.isReceiving && !this.isSending && this.context.connection.isInRoom) this.share() return 0; }); } } /** @internal */ onDisable(): void { //@ts-ignore this._net?.removeEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream); //@ts-ignore this._net?.removeEventListener(NetworkedStreamEvents.StreamEnded, this.onCallEnded); this.context.connection.stopListen(RoomEvents.JoinedRoom, this.onJoinedRoom); this._net?.disable(); this.close(); } private onJoinedRoom = async () => { await delay(1000); if (this.autoConnect && !this.isSending && !this.isReceiving && this.context.connection.isInRoom) { this.share(); } } private _ensureVideoPlayer() { const vp = new VideoPlayer(); vp.aspectMode = AspectMode.AdjustWidth; GameObject.addComponent(this.gameObject, vp); this._videoPlayer = vp; } private _activeShareRequest: Promise | null = null; /** Call to begin screensharing */ async share(opts?: ScreenCaptureOptions) { if (this._activeShareRequest) return this._activeShareRequest; this._activeShareRequest = this.internalShare(opts); return this._activeShareRequest.then(() => { return this._activeShareRequest = null; }) } private async internalShare(opts?: ScreenCaptureOptions) { if (this.context.connection.isInRoom === false) { console.warn("Can not start screensharing: requires network connection"); if (isDevEnvironment()) showBalloonWarning("Can not start screensharing: requires network connection. Add a SyncedRoom component or join a room first."); return; } if (opts?.device) this.device = opts.device; if (!this.videoPlayer && this.requiresVideoPlayer) { if (!this._videoPlayer) { this._videoPlayer = GameObject.getComponent(this.gameObject, VideoPlayer) ?? undefined; } if (!this.videoPlayer) { this._ensureVideoPlayer(); } if (!this.videoPlayer) { console.warn("Can not share video without a videoPlayer assigned"); return; } } this._requestOpen = true; try { const settings: MediaTrackConstraints = opts?.constraints ?? { echoCancellation: true, autoGainControl: false, }; const displayMediaOptions: MediaStreamConstraints = { video: settings, audio: settings, }; const videoOptions = displayMediaOptions.video; if (videoOptions !== undefined && typeof videoOptions !== "boolean") { // Set default video settings if (!videoOptions.width) videoOptions.width = { max: 1920 }; if (!videoOptions.height) videoOptions.height = { max: 1920 }; if (!videoOptions.aspectRatio) videoOptions.aspectRatio = { ideal: 1.7777777778 }; if (!videoOptions.frameRate) videoOptions.frameRate = { ideal: 24 }; if (!videoOptions.facingMode) videoOptions.facingMode = { ideal: "user" }; } switch (this.device) { // Capture a connected camera case "Camera": this.tryShareUserCamera(displayMediaOptions, opts); break; // capture any screen, will show a popup case "Screen": { if (!navigator.mediaDevices.getDisplayMedia) { console.error("No getDisplayMedia support"); return; } const myVideo = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions); if (this._requestOpen) { this.setStream(myVideo, ScreenCaptureMode.Sending); } else disposeStream(myVideo); } break; // capture the canvas meaning the threejs view case "Canvas": // looks like this doesnt work reliably on chrome https://stackoverflow.com/a/66848674 // firefox updates fine // https://bugs.chromium.org/p/chromium/issues/detail?id=1156408 const fps = 0; const stream = this.context.renderer.domElement.captureStream(fps); this.setStream(stream, ScreenCaptureMode.Sending); break; case "Microphone": { if (!navigator.mediaDevices.getUserMedia) { console.error("No getDisplayMedia support"); return; } displayMediaOptions.video = false; const myStream = await navigator.mediaDevices.getUserMedia(displayMediaOptions); if (this._requestOpen) { this.setStream(myStream, ScreenCaptureMode.Sending); } else disposeStream(myStream); } break default: console.error("Can not start screen sharing: Unknown device type", this.device); } } catch (err: any) { if (err.name === "NotAllowedError") { // user cancelled stream selection console.log("Selection cancelled"); this._requestOpen = false; return; } console.error("Error opening video", err); } } close() { this._requestOpen = false; if (this._currentStream) { if (debug) console.warn("Close current stream / disposing resources, stream was active?", this._currentStream.active); this._net?.stopSendingStream(this._currentStream); disposeStream(this._currentStream); this._currentMode = ScreenCaptureMode.Idle; this._currentStream = null; } } private setStream(stream: MediaStream, mode: ScreenCaptureMode) { if (stream === this._currentStream) return; this.close(); if (!stream) return; this._currentStream = stream; this._requestOpen = true; this._currentMode = mode; const isVideoStream = this.device !== "Microphone"; const isSending = mode === ScreenCaptureMode.Sending; if (isVideoStream) { if (!this._videoPlayer) this._ensureVideoPlayer(); if (this._videoPlayer) this._videoPlayer.setVideo(stream); else console.error("No video player assigned for video stream"); } else { if (!this._audioSource) { this._audioSource = new AudioSource(); this._audioSource.spatialBlend = 0; this._audioSource.volume = 1; this.gameObject.addComponent(this._audioSource); } if (!isSending) { if (debug) console.log("PLAY", stream.getAudioTracks()) this._audioSource.volume = 1; this._audioSource?.play(stream); } } if (isSending) { this._net?.startSendingStream(stream); } // Mute audio for the video we are sending if (isSending) { if (this._videoPlayer) this._videoPlayer.muted = true; this._audioSource?.stop(); } for (const track of stream.getTracks()) { track.addEventListener("ended", () => { if (debug) console.log("Track ended", track); this.close(); }); if (debug) { if (track.kind === "video") { if (isSending) console.log("Video →", track.getSettings()); else console.log("Video ←", track.getSettings()); } } } } private onReceiveStream = (evt: StreamReceivedEvent) => { if (evt.stream?.active !== true) return; this.setStream(evt.stream, ScreenCaptureMode.Receiving); } private onCallEnded = (_evt: StreamEndedEvent) => { if (debug) console.log("CALL ENDED", this.isReceiving, this?.screenspace) if (this.isReceiving) this.screenspace = false; } private async tryShareUserCamera(constraints: MediaStreamConstraints, options?: ScreenCaptureOptions) { // let newWindow = open('', 'example', 'width=300,height=300'); // if (window) { // newWindow!.document.body.innerHTML = "Please allow access to your camera and microphone"; // } // TODO: allow user to select device const devices = (await navigator.mediaDevices.enumerateDevices()).filter(d => d.kind === "videoinput"); if (debug) console.log("Request camera. These are your kind:videoinput devices:\n", devices); let foundDevice = false; for (const dev of devices) { try { if (!this._requestOpen) { if (debug) console.log("Camera selection cancelled"); break; } if (dev.kind !== "videoinput") { if (debug) console.log("Skipping non-video device", dev); continue; } const id = dev.deviceId; // If the share method is called with filter options then those should be used const hasOptionsFilter = options?.deviceId != undefined || options?.deviceFilter != undefined; if (hasOptionsFilter) { if (options?.deviceId !== undefined) { if (id !== options.deviceId) { if (debug) console.log("Skipping device due to options.deviceId: " + dev.label + "; " + dev.deviceId); continue; } } if (options?.deviceFilter) { const useDevice = options.deviceFilter(dev); if (useDevice === false) { if (debug) console.log("Skipping device due to options.deviceFilter: " + dev.label + "; " + dev.deviceId); continue; } } } // If the share method was called without filter options then the component filter should be used else if (this.deviceFilter) { const useDevice = this.deviceFilter(dev); if (useDevice === false) { if (debug) console.log("Skipping device due to ScreenShare.deviceFilter: " + dev.label + "; " + dev.deviceId); continue; } else if(debug) console.log("Selected device by filter", dev); } else if (this.deviceName) { const lowercaseLabel = dev.label.toLowerCase(); const lowercaseName = this.deviceName.toLowerCase(); const labelMatches = lowercaseLabel.includes(lowercaseName); const idMatches = dev.deviceId === this.deviceName; if (!labelMatches && !idMatches) { if (debug) console.log("Skipping device due to ScreenShare.deviceName: " + dev.label + "; " + dev.deviceId); continue; } else if(debug) console.log("Selected device by name", dev); } if (constraints.video !== false) { if (typeof constraints.video === "undefined" || typeof constraints.video === "boolean") { constraints.video = {}; } constraints.video.deviceId = id; } foundDevice = true; const userMedia = await navigator.mediaDevices.getUserMedia(constraints).catch(err => { console.error("Failed to get user media", err); return null; }) if (userMedia === null) { continue; } else if (this._requestOpen) { this.setStream(userMedia, ScreenCaptureMode.Sending); if (debug) console.log("Selected camera", dev); } else { disposeStream(userMedia); if (debug) console.log("Camera selection cancelled"); } break; } catch (err: any) { // First message is firefox, second is chrome when the video source is already in use by another app if (err.message === "Failed to allocate videosource" || err.message === "Could not start video source") { showBalloonWarning("Failed to start video: Try another camera (Code " + err.code + ")"); console.warn(err); continue; } else { console.error("Failed to get user media", err.message, err.code, err); } } } if(!foundDevice && isDevEnvironment()){ showBalloonWarning("No camera found for sharing. Please connect a camera (see console for more information)"); console.warn("No camera found for sharing. Please connect a camera", devices, this.deviceName, "Using deviceFilter? " + this.deviceFilter != undefined, "Using options? " + options != undefined, "Using deviceName? " + this.deviceName != undefined, "Using options.deviceId? " + options?.deviceId != undefined, "Using options.deviceFilter? " + options?.deviceFilter != undefined); } } // private _cameraSelectionWindow : Window | null = null; // private openWindowToSelectCamera(){ // } }