import { VIDEO_DECODE_WAIT_FRAME, VIDEO_DECODE_SEEK_TIMEOUT_FRAME, VIDEO_PLAYBACK_RATE_MAX, VIDEO_PLAYBACK_RATE_MIN, } from '../constant'; import {addListener, removeListener, removeAllListeners} from '../utils/video-listener'; import {IPHONE, WECHAT, SAFARI_OR_IOS_WEBVIEW} from '../utils/ua'; import {PAGModule} from '../pag-module'; import type {TimeRange, VideoReader as VideoReaderInterfaces} from '../interfaces'; import type {PAGPlayer} from '../pag-player'; import {isInstanceOf} from '../utils/type-utils'; import {destroyVerify} from "../utils/decorators"; const UHD_RESOLUTION = 3840; // Get video initiated token on Wechat browser. const getWechatNetwork = () => { return new Promise((resolve) => { window.WeixinJSBridge.invoke( 'getNetworkType', {}, () => { resolve(); }, () => { resolve(); }, ); }); }; const waitVideoCanPlay = (videoElement: HTMLVideoElement) => { return new Promise((resolve) => { const canplayHandle = () => { removeListener(videoElement, 'canplay', canplayHandle); clearTimeout(timer); resolve(true); }; addListener(videoElement, 'canplay', canplayHandle); const timer = setTimeout(() => { removeListener(videoElement, 'canplay', canplayHandle); resolve(false); }, 1000); }); }; @destroyVerify export class VideoReader { public static async create( source: Uint8Array | HTMLVideoElement, width: number, height: number, frameRate: number, staticTimeRanges: TimeRange[], ): Promise { return new VideoReader(source, width, height, frameRate, staticTimeRanges); } public isSought = false; public isPlaying = false; public bitmap: ImageBitmap | null = null; public isDestroyed = false; private videoEl: HTMLVideoElement | null = null; private frameRate = 0; private canplay = false; private staticTimeRanges: StaticTimeRanges | null = null; private disablePlaybackRate = false; private error: any = null; private player: PAGPlayer | null = null; private width = 0; private height = 0; private bitmapCanvas: OffscreenCanvas | null = null; private bitmapCtx: OffscreenCanvasRenderingContext2D | null = null; private currentFrame = -1; private targetFrame = -1; private visibilityHandle: (() => void) | null = null; public constructor( source: Uint8Array | HTMLVideoElement, width: number, height: number, frameRate: number, staticTimeRanges: TimeRange[], ) { if (isInstanceOf(source, globalThis.HTMLVideoElement)) { this.videoEl = source as HTMLVideoElement; this.canplay = true; } else { this.videoEl = document.createElement('video'); this.videoEl.style.display = 'none'; this.videoEl.muted = true; this.videoEl.playsInline = true; this.videoEl.preload = 'auto'; // use load() will make a bug on Chrome. this.videoEl.width = width; this.videoEl.height = height; waitVideoCanPlay(this.videoEl).then(() => { this.canplay = true; }); const buffer = (source as Uint8Array).slice(); const blob = new Blob([buffer], {type: 'video/mp4'}); this.videoEl.src = URL.createObjectURL(blob); if (IPHONE) { // use load() will make a bug on Chrome. this.videoEl.load(); } } this.frameRate = frameRate; this.width = width; this.height = height; this.staticTimeRanges = new StaticTimeRanges(staticTimeRanges); if (UHD_RESOLUTION < width || UHD_RESOLUTION < height) { this.disablePlaybackRate = true; } this.linkPlayer(PAGModule.currentPlayer); } public async prepare(targetFrame: number, playbackRate: number): Promise { if (this.isDestroyed || targetFrame === this.currentFrame) { return; } const promise = new Promise(async (resolve) => { this.setError(null); // reset error this.isSought = false; // reset seek status const {currentTime} = this.videoEl!; const targetTime = targetFrame / this.frameRate; if (currentTime === 0 && targetTime === 0) { if (!this.canplay && !SAFARI_OR_IOS_WEBVIEW) { await waitVideoCanPlay(this.videoEl!); } else { try { await waitVideoCanPlay(this.videoEl!); await this.play(); } catch (e) { this.setError(e); this.currentFrame = targetFrame; resolve(); return; } await new Promise((resolveInner) => { requestAnimationFrame(() => { if (!this.isDestroyed) { this.pause(); } resolveInner(); }); }); } } else { if (Math.round(targetTime * this.frameRate) === Math.round(currentTime * this.frameRate)) { // Current frame } else if (this.staticTimeRanges?.contains(targetFrame)) { // Static frame await this.seek(targetTime, false); this.currentFrame = targetFrame; resolve(); return; } else if ((Math.abs(currentTime - targetTime) < (1 / this.frameRate) * VIDEO_DECODE_WAIT_FRAME) && !this.videoEl!.paused) { // Within tolerable frame rate deviation } else { // Seek and play this.isSought = true; await this.seek(targetTime); this.currentFrame = targetFrame; resolve(); return; } } if (this.isDestroyed || !this.videoEl) { resolve(); return; } const targetPlaybackRate = Math.min(Math.max(playbackRate, VIDEO_PLAYBACK_RATE_MIN), VIDEO_PLAYBACK_RATE_MAX); if (!this.disablePlaybackRate && this.videoEl.playbackRate !== targetPlaybackRate) { this.videoEl.playbackRate = targetPlaybackRate; } if (this.isPlaying && this.videoEl.paused) { try { await this.play(); } catch (e) { this.setError(e); this.currentFrame = targetFrame; resolve(); return; } } this.currentFrame = targetFrame; resolve(); }); await promise; } public getCurrentFrame(): number { return this.currentFrame; } public getVideo() { return this.videoEl; } public async play() { if (!this.videoEl!.paused) return; if (WECHAT && window.WeixinJSBridge) { await getWechatNetwork(); } if (document.visibilityState !== 'visible') { // Page is hidden, defer video play until visible this.clearVisibilityListener(); this.visibilityHandle = () => { if (this.isDestroyed) { this.clearVisibilityListener(); return; } if (document.visibilityState === 'visible') { if (this.videoEl) this.videoEl.play().catch((e) => { this.setError(e); }); this.clearVisibilityListener(); } }; window.addEventListener('visibilitychange', this.visibilityHandle); throw new Error('The play() request was interrupted because the document was hidden!'); } await this.videoEl?.play(); } public pause() { this.isPlaying = false; if (this.videoEl!.paused) return; this.videoEl?.pause(); } public stop() { this.isPlaying = false; if (!this.videoEl!.paused) { this.videoEl?.pause(); } } public getError() { return this.error; } public onDestroy() { this.isDestroyed = true; if (this.player) { this.player.unlinkVideoReader(this); this.player = null; } this.clearVisibilityListener(); removeAllListeners(this.videoEl!, 'playing'); removeAllListeners(this.videoEl!, 'timeupdate'); removeAllListeners(this.videoEl!, 'seeked'); removeAllListeners(this.videoEl!, 'canplay'); this.videoEl = null; this.bitmapCanvas = null; this.bitmapCtx = null; } private seek(targetTime: number, play = true) { return new Promise((resolve, reject) => { if (!this.videoEl) { reject(new Error('Video element is not initialized.')); return; } const onSeeked = () => { if (!this.videoEl || this.isDestroyed) { clearTimeout(seekTimeout); resolve(); return; } removeListener(this.videoEl, 'seeked', onSeeked); clearTimeout(seekTimeout); if (play) { // After seeking, the video might still be in 'ended' state // Reset it by setting currentTime to itself to clear the ended flag if (this.videoEl.ended) { this.videoEl.currentTime = this.videoEl.currentTime; } this.videoEl.play().catch((e) => { this.setError(e); }); } else if (!play && !this.videoEl.paused) { this.videoEl.pause(); } resolve(); }; const onCanPlay = () => { if (!this.videoEl || this.isDestroyed) { clearTimeout(seekTimeout); resolve(); return; } removeListener(this.videoEl, 'canplay', onCanPlay); // Now that we have enough data, perform the seek. this.videoEl.currentTime = targetTime; addListener(this.videoEl, 'seeked', onSeeked); }; const seekTimeout = setTimeout(() => { if (this.videoEl) { removeListener(this.videoEl, 'canplay', onCanPlay); removeListener(this.videoEl, 'seeked', onSeeked); } this.setError('Seek operation timed out.'); resolve(); }, (1000 / this.frameRate) * VIDEO_DECODE_SEEK_TIMEOUT_FRAME); // Check if we need to wait for 'canplay' event before seeking. if (this.videoEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) { addListener(this.videoEl, 'canplay', onCanPlay); } else { // We already have enough data to perform the seek. this.videoEl!.currentTime = targetTime; addListener(this.videoEl, 'seeked', onSeeked); } }); } private setError(e: any) { this.error = e; } private clearVisibilityListener() { if (this.visibilityHandle) { window.removeEventListener('visibilitychange', this.visibilityHandle); this.visibilityHandle = null; } } private linkPlayer(player: PAGPlayer | null) { this.player = player; if (player) { player.linkVideoReader(this); } } } export class StaticTimeRanges { private timeRanges: TimeRange[]; public constructor(timeRanges: TimeRange[]) { this.timeRanges = timeRanges; } public contains(targetFrame: number) { if (this.timeRanges.length === 0) return false; for (let timeRange of this.timeRanges) { if (timeRange.start <= targetFrame && targetFrame < timeRange.end) { return true; } } return false; } }