import { Logger } from '../common/logger'; import { IImageProcessorLogger, ImageParams, ImageProcessor } from '@lenovo-software/lsa-clients-dom-renderer'; class ImageLogger implements IImageProcessorLogger { private logger: Logger; constructor(_logger: Logger) { this.logger = _logger; } logDebug(msg: string) { this.logger.logDebug(msg); } logInfo(msg: string) { this.logger.logInfo(msg); } logMessage(msg: string) { this.logger.logMessage(msg); } logWarning(msg: string) { this.logger.logWarning(msg); } logError(msg: string) { this.logger.logError(msg); } } export class FullScreenCapture { private logger: Logger; private bScreenCapEnabled = false; private initialized = false; private screen_stream: MediaStream | undefined | null; private approved_cb: Function | undefined; private denied_cb: Function | undefined; private desktop_media_id: number = -1; private streamEndedCb: Function | undefined; private video: HTMLVideoElement; private canvas: HTMLCanvasElement; private context: CanvasRenderingContext2D | null; private privacyImage: HTMLImageElement | undefined | null; private policy: any; constructor() { this.logger = Logger.getInstance(); this.video = document.createElement('video'); this.canvas = document.createElement('canvas'); // TODO: Convert to OffscreenCanvas and modify lsa-clients-dom-renderer to accept an OffscreenCanvas parameter this.context = this.canvas.getContext('2d'); if (this.context) this.context.imageSmoothingEnabled = false; if (typeof Image !== 'undefined') { this.privacyImage = new Image(); this.privacyImage.src = './images/Zoom_Out_256.png'; } else { this.privacyImage = null; } this.policy = {}; chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { switch (request.message) { case 'GetFullScreenThumbnailSync': { let width = request.params.width; let height = request.params.height; let format = request.params.format; this.generateFormattedThumbnail(width, height, format) .then((screenThumb) => { let imageData = { buffer: screenThumb, width: width, height: height, format: format, source: 'Full Screen', type: 'base64' }; if (sendResponse) { sendResponse(imageData); } }) .catch((err) => { this.logger.logError( 'FullScreenCapture.(GetFullScreenThumbnailSync): Error calling generateFormattedThumbnail: ' + err ); }); break; } case 'GetFullScreenCapSync': { this.generateScreenCap(request.params.text) .then((screenCap) => { sendResponse(screenCap); }) .catch((err) => { this.logger.logError( 'FullScreenCapture.(GetFullScreenCapSync): Error calling generateFormattedThumbnail: ' + err ); }); break; } } return true; }); } // TODO: any setPolicy(policy: any) { this.policy = policy; } onAccessApproved(id: string) { if (id === '') { // access denied this.bScreenCapEnabled = false; if (this.denied_cb) this.denied_cb(); return; } if (this.screen_stream) { this.logger.logMessage('ScreenCapture.onAccessApproved(): screen_stream exists already.'); return; } navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia; try { navigator.getUserMedia( { video: { mandatory: { chromeMediaSource: 'desktop', chromeMediaSourceId: id, maxWidth: 1280, maxHeight: 1024 } } as any // TS doesn't like the "mandatory" section so just bypass the type here. }, (s) => { this.screen_stream = s; this.video.srcObject = s; this.initialized = true; this.bScreenCapEnabled = true; this.logger.logMessage('Enabling desktop thumbnail capture'); if (this.approved_cb) { this.video .play() .then(() => { if (this.approved_cb) this.approved_cb(); }) .catch((err) => { this.logger.logMessage('Error starting playback for full screen: ' + err); if (this.approved_cb) this.approved_cb(); }); } }, (err) => { this.logger.logMessage('Error, Cannot grab screen media stream: ' + JSON.stringify(err)); if (this.denied_cb) this.denied_cb(); } ); } catch (e) { this.logger.logMessage('EXCEPT: ' + JSON.stringify(e)); if (this.denied_cb) this.denied_cb(); } } startCapture(approved_cb: Function, denied_cb: Function, streamEnded_cb: Function) { this.approved_cb = approved_cb; this.denied_cb = denied_cb; this.streamEndedCb = streamEnded_cb; if (typeof chrome.desktopCapture != 'undefined') { this.desktop_media_id = chrome.desktopCapture.chooseDesktopMedia(['screen'], (id) => this.onAccessApproved(id) ); this.logger.logMessage('using chrome.desktopCapture: ' + this.desktop_media_id); } } stopCapture() { if (this.screen_stream) { this.screen_stream.getTracks().forEach((track) => { track.stop(); }); } this.screen_stream = null; this.video.src = ''; this.initialized = false; // Goofy hack for LSA-1753 due to this change in Chrome 73: https://chromium-review.googlesource.com/c/chromium/src/+/1375098 this.video.play().then(() => { this.video.pause(); }); } streamEnded() { this.logger.logMessage('Media Stream ended'); this.stopCapture(); this.bScreenCapEnabled = false; if (this.streamEndedCb != null) this.streamEndedCb(); } getFormatString(format: string) { var str = 'image/jpeg'; if (format == 'PNG') { str = 'image/png'; } return str; } async generateFormattedThumbnail(width: number, height: number, format: string, message?: string): Promise { var imgBMP = this.policy.student_privacy ? await this.generatePrivacyThumbnail( width, height, chrome.i18n.getMessage('student_privacy_teacher_message') || 'Student Privacy' ) : await this.generateScreenCapSized(width, height, message); return imgBMP; // if (imgBMP != null) { // var imgBase64 = this.canvas.toDataURL(this.getFormatString(format)); // return imgBase64; // } // return null; } async generatePrivacyThumbnail(width: number, height: number, message: string): Promise { if (!this.privacyImage) { throw new Error('FullScreenCapture.generatePrivacyThumbnail(): No privacy image.'); } if (!this.context) { throw new Error('FullScreenCapture.generatePrivacyThumbnail(): No context.'); } // this.canvas.width = width; // this.canvas.height = height; // this.context.fillStyle = "#f1ff25"; // this.context.fillRect(0, 0, width, height); var x = width * 0.15; var y = height * 0.15; var w = width * 0.7; var h = height * 0.7; const imageParams: ImageParams = { width: w, height: h, overlayMessage: message !== null ? message : '' }; let imgBMP = await createImageBitmap(this.privacyImage); return new ImageProcessor(new ImageLogger(this.logger)).processImage(imgBMP, imageParams); // try { // this.context.drawImage(this.privacyImage, x, y, w, h); // } // catch (err) { // this.logger.logMessage("Error: Cannot grab screen media stream: " + err.stack); // } // this.context.fillStyle = "#000000"; // this.context.font = "12px"; // this.context = new TextOverlay(context).canvasWrapText(10, 10, width * 0.80, 100, 14, message); // this.imagedata = this.context.getImageData(0, 0, width, height); // return this.imagedata; } async generateScreenCapSized(width: number, height: number, message?: string): Promise { if (this.bScreenCapEnabled != true || this.screen_stream == null) return ''; // TODO: Throw error instead? if (this.screen_stream.active === false) { this.streamEnded(); return ''; // TODO: Throw error instead? } this.logger.logMessage('requested image (' + width + ' x ' + height + ')'); const imageParams: ImageParams = { width: width, height: height, overlayMessage: message !== null ? message : '' }; let imgBMP = await createImageBitmap(this.video); return new ImageProcessor(new ImageLogger(this.logger)).processImage(imgBMP, imageParams); // var xscale = width / this.video.videoWidth; // var yscale = height / this.video.videoHeight; // var scale = 1; // if (xscale < yscale) // scale = xscale; // else // scale = yscale; // //logger.logMessage( "SCALE: " + scale ); // var scale_width = Math.round(this.video.videoWidth * scale); // var scale_height = Math.round(this.video.videoHeight * scale); // //logger.logMessage("scale_width: " + scale_width + ", scale_height: " + scale_height); // this.canvas.width = scale_width; // this.canvas.height = scale_height; // // this is where we have to apply the scaling // logger.logMessage("Drawing to: (" + scale_width + ", " + scale_height + ")"); // this.context.fillStyle = "#000000"; // this.context.fillRect(0, 0, width, height); // this.context.drawImage(this.video, 0, 0, scale_width, scale_height); // if (message != null) { // this.context = new TextOverlay(this.context).canvasTextOverlay(scale_width, scale_height, message); // } // this.imagedata = this.context.getImageData(0, 0, width, height); // return this.imagedata; } _getVideoWidth() { if (this.video && this.video.videoWidth) return this.video.videoWidth; return screen.width; } _getVideoHeight() { if (this.video && this.video.videoHeight) return this.video.videoHeight; return screen.height; } async generateScreenCap(message?: string) { let width = this._getVideoWidth(); let height = this._getVideoHeight(); var imgBMP = this.policy.student_privacy ? await this.generatePrivacyThumbnail( width, height, chrome.i18n.getMessage('student_privacy_teacher_message') ) : await this.generateScreenCapSized(width, height, message); if (imgBMP != null) { return { imageBase64: imgBMP, width: width, height: height }; } else { return { imageBase64: '', width: width, height: height }; } } }