import { Unit } from '@_unit/unit/lib/Class/Unit' import { EventEmitter_ } from '@_unit/unit/lib/EventEmitter' import { Rect } from '@_unit/unit/lib/client/util/geometry/types' import { Dict } from '@_unit/unit/lib/types/Dict' import { Unlisten } from '@_unit/unit/lib/types/Unlisten' import { rtcLiveSource2, wsLiveSource } from './cc' import icons from './icons' import { MenuOption, renderMenu } from './menu' import { defaultGetLiveMetadataUrl, defaultGetObjectMenu, defaultOnActionSelected, getCacheKey, shouldLink, } from './museme' import { MUSEME_API_ORIGIN, MUSEME_LIVE_ORIGIN, MUSEME_POLL_ORIGIN, } from './origins' import { clamp, isValidUrl } from './utils' const Hls = require('hls.js') export type MetadataColorMap = Dict export type MusemeEvents = 'connecting' | 'connected' | 'error' export type MusemeOverlayAPI = { setSetting(name: string, value: any): void getSettings(): MusemeOverlaySettings getSetting( name: K ): MusemeOverlaySettings[K] getPlayer(): any getCurrentColorMap(): MetadataColorMap addListener(event: MusemeEvents, listener: () => void): Unlisten } const SVG_NAMESPACE = 'http://www.w3.org/2000/svg' const RING_MENU_WIDTH = 200 const RING_MENU_HEIGHT = 200 const FRAME_BUFFER_SIZE = 100 const DEFAULT_RING_MENU_STYLE = { width: '100%', height: '100%', top: '0', left: '0', overflow: 'hidden', } const MARKER_INTERVAL_T = 10 export const labelToIcon = { Open: 'external-link', Close: 'circle-x', Talk: 'message-circle', Give: 'hand', Buy: 'shopping-cart', Sell: 'store', Fight: 'axe', Run: 'bike', Escape: 'door', } const labels = Object.keys(labelToIcon) const randomInArray = (array) => { return array[Math.floor(Math.random() * array.length)] } export const randomMenuOptions = () => { const n = Math.floor(2 + Math.random() * 7) return new Array(n).fill(null, 0, n).map((_, index) => { const label = labels[index] const icon = labelToIcon[label] return { value: icon, label, icon, url: 'https://museme.com', } }) } const defaultGetOrigins = (): MusemeOrigins => { return { apiHostname: MUSEME_API_ORIGIN, liveHostname: MUSEME_LIVE_ORIGIN, pollHostname: MUSEME_POLL_ORIGIN, } } export const DEFAULT_OVERLAY_SETTINGS: MusemeOverlaySettings = { enabled: true, pauseOnClick: true, showMasksOnPause: true, showMasksOnTouch: true, showMarkers: true, showOverlay: false, liveProtocol: 'rtc', liveMode: 'vote', debug: false, } const defaultGetSettings = (): MusemeOverlaySettings => { return DEFAULT_OVERLAY_SETTINGS } export type RGBA = [number, number, number, number] const hexToRgba = (hex: string) => { const bigInt = parseInt(hex.substring(1), 16) let r: number let g: number let b: number let a: number = 255 if (hex.length === 7) { r = (bigInt >> 16) & 255 g = (bigInt >> 8) & 255 b = bigInt & 255 } else if (hex.length === 9) { r = (bigInt >> 24) & 255 g = (bigInt >> 16) & 255 b = (bigInt >> 8) & 255 a = bigInt & 255 } else { throw new Error('invalid HEX color') } return [r, g, b, a] } export const rgbaToHex = (rgba: RGBA): string => { const [r, g, b, a] = rgba const h = (v: number) => v.toString(16).padStart(2, '0') return `#${h(r)}${h(g)}${h(b)}${h(a)}` } export const isBlackPixel = (r: number, g: number, b: number): boolean => { return r + g + b < 100 } export type MusemeSourceVideo = { play(): void pause(): void getCurrentTime(): number getRect(): { width: number; height: number; top: number; left: number } isPaused(): boolean addEventListener(event: string, callback: (event: Event) => void): void removeEventListener(event: string, callback: (event: Event) => void): void } export type MusemeOrigins = { apiHostname: string liveHostname: string pollHostname: string } export type LiveProtocol = 'ws' | 'rtc' export type LiveMode = 'vote' | 'link' export type MusemeOverlaySettings = { enabled: boolean pauseOnClick: boolean showMasksOnPause: boolean showMasksOnTouch: boolean showMarkers: boolean showOverlay: boolean liveProtocol: LiveProtocol liveMode: LiveMode debug: boolean } export type DetectedObject = { Class: string Color: [number, number, number] SceneId: string Id: string Rect: { x: number; y: number; w: number; h: number } Score: number Actions: string PrevSceneId?: number PrevObjectId?: number Metadata: { menu: string metadata: string name: string } } export type MusemeObjectMetadata = DetectedObject & { Hex: string ActionList?: string[] metadata?: string } export type Config = { getObjectMenu?( videoId: string, metadata: MusemeObjectMetadata, signal: AbortSignal ): Promise getOverlayVideoUrl?(): string getSocket?(): WebSocket getMetadataUrl?(): string | Promise getPixelColor?( x: number, y: number, width: number, height: number ): [number, number, number, number] onActionSelected?( metadata: MusemeObjectMetadata, icon: string, options: MenuOption[] ): boolean | void onEmptyMenuClick?(metadata: MusemeObjectMetadata): void getCurrentLatency?(ccTime: number): number getOrigins?: () => MusemeOrigins getSettings?: () => Partial } const MAX_COLOR_DISTANCE = 40 export function renderOverlay( video: MusemeSourceVideo, videoId: string, config: Config = {}, live: boolean = false ): [MusemeOverlayAPI, () => void] { const { getOrigins = defaultGetOrigins, getSettings = defaultGetSettings } = config const { apiHostname, liveHostname, pollHostname } = getOrigins() const userSettings = getSettings() const overlaySettings: MusemeOverlaySettings = { ...DEFAULT_OVERLAY_SETTINGS, ...userSettings, } const emitter = new EventEmitter_() let currentMenu = [] let animatingRingMenu = false let showingRingMenu = false let showingRingMenuForColor = null let shownObjectMetadata: MusemeObjectMetadata = null let syncedCcTimestamp: number = 0 let fetchingMenu: boolean = false const abortController = new AbortController() let cancelOldClick: boolean = false let offsetXp = 0 let offsetYp = 0 let clientXp = 0 let clientYp = 0 let menuX = 0 let menuY = 0 let menuWidth = 0 let menuHeight = 0 let zoom = 1 let videoWidth: number let videoHeight: number let videoBox: any const ROOT_ID = '__museme__root__' let musemeVideoMetadata: any let loadingMusemeVideoMetadata = true let hasMusemeVideoMetadata = undefined let colorMap: MetadataColorMap = {} let hoveredAction: string | null = null let animationFrame: number let intervalFrame: NodeJS.Timeout = undefined let overlayBboxMap = null let colorPinMap: Dict = {} let latestMetadataObjects: DetectedObject[] = [] let frameBuffer: [number, string][] = [] let metadataBuffer = [] let lastMjpg: [number, string] let lastFrameReceived: Date let lastMetadataReceived: Date let unlistenCc: Unlisten const prevRoot = document.querySelector(`#${ROOT_ID}`) if (prevRoot) { if (prevRoot.parentElement) { prevRoot.parentElement.removeChild(prevRoot) } } let root: HTMLDivElement = document.createElement('div') let overlay: HTMLDivElement = document.createElement('div') let labelGroup: HTMLDivElement = document.createElement('div') let actionLabel: HTMLDivElement = document.createElement('div') let objectLabel: HTMLDivElement = document.createElement('div') let objectLabelLinkIcon: SVGSVGElement = document.createElementNS( SVG_NAMESPACE, 'svg' ) as SVGSVGElement let objectLabelLinkIconPath: SVGPathElement = document.createElementNS( SVG_NAMESPACE, 'path' ) as SVGPathElement let extraObjectLabel: HTMLDivElement = document.createElement('div') let backgroundOverlay: HTMLDivElement = document.createElement('div') let guiOverlay: HTMLDivElement = document.createElement('div') let videoOverlay: HTMLVideoElement = document.createElement('video') let imageOverlay: HTMLImageElement = document.createElement('img') let canvasOverlay: HTMLCanvasElement = document.createElement('canvas') let foregroundOverlay: HTMLDivElement = document.createElement('div') let markerOverlay: HTMLDivElement = document.createElement('div') const defaultYoutubeOverlayVideoUrl = (videoId: string): string => { return `${apiHostname}/cvstream/${videoId}` } const defaultYoutubeVideoMetadataUrl = (videoId: string) => { const url = `${apiHostname}/ytmetadata/${videoId}` return url } const defaultGetLiveMetadataUrl_ = (videoId: string) => { return defaultGetLiveMetadataUrl(liveHostname, videoId) } const optionsCache: Dict = {} const defaultGetObjectMenu_ = ( videoId: string, metadata: MusemeObjectMetadata, signal: AbortSignal ) => { return defaultGetObjectMenu( apiHostname, videoId, metadata, live, overlaySettings.liveMode, optionsCache, signal ) } const getMenuOptions = async ( videoId: string, metatada: MusemeObjectMetadata, signal: AbortSignal ) => { const { getObjectMenu = defaultGetObjectMenu_ } = config return getObjectMenu(videoId, metatada, signal) } const getDefaultGetPixelColor = ( x: number, y: number ): [number, number, number, number] => { drawOverlayOnCanvas() const imageData = canvasCtx.getImageData(x, y, 2, 2) const r = imageData.data[0] const g = imageData.data[0 + 1] const b = imageData.data[0 + 2] const a = imageData.data[0 + 3] return [r, g, b, a] } const defaultGetLatency = (): number => { return 0 } const defaultOnActionSelected_ = ( metadata: MusemeObjectMetadata, icon: string, options: MenuOption[] ) => { return defaultOnActionSelected( pollHostname, videoId, metadata, icon, options, live, overlaySettings.liveMode, overlaySettings.debug ) } const defaultOnEmptyMenuClick = (objectMetadata: MusemeObjectMetadata) => { const { metadata: metadataUrl } = objectMetadata if (metadataUrl) { if (isValidUrl(metadataUrl)) { window.open(metadataUrl) } else { // eslint-disable-next-line console.warn('invalid metadata url') } } } const { getPixelColor = getDefaultGetPixelColor, onActionSelected = defaultOnActionSelected_, onEmptyMenuClick = defaultOnEmptyMenuClick, getOverlayVideoUrl = defaultYoutubeOverlayVideoUrl, getMetadataUrl = live ? defaultGetLiveMetadataUrl_ : defaultYoutubeVideoMetadataUrl, getCurrentLatency = defaultGetLatency, } = config let musemeOverlayUrl = getOverlayVideoUrl(videoId) root.id = ROOT_ID function handleTouch(event) { event.preventDefault() } root.addEventListener('touchmove', handleTouch, { passive: false }) root.appendChild(overlay) root.appendChild(markerOverlay) root.appendChild(foregroundOverlay) overlay.appendChild(videoOverlay) overlay.appendChild(canvasOverlay) overlay.appendChild(backgroundOverlay) backgroundOverlay.appendChild(guiOverlay) foregroundOverlay.appendChild(labelGroup) foregroundOverlay.appendChild(extraObjectLabel) foregroundOverlay.appendChild(imageOverlay) labelGroup.appendChild(actionLabel) labelGroup.appendChild(objectLabel) labelGroup.appendChild(objectLabelLinkIcon) objectLabelLinkIcon.appendChild(objectLabelLinkIconPath) objectLabelLinkIconPath.setAttribute('d', icons['external-link']) const canvasCtx = canvasOverlay.getContext('2d', { willReadFrequently: true }) let ringMenu: Unit = null let detachRingMenu: Unlisten = null root.style.position = 'fixed' root.style.zIndex = `10` root.style.pointerEvents = 'none' root.style.userSelect = 'none' overlay.style.position = 'absolute' overlay.style.opacity = '1' overlay.style.width = `100%` overlay.style.height = `100%` overlay.style.pointerEvents = 'none' overlay.style.zIndex = `1` overlay.style.transition = 'opacity 0.2s linear' backgroundOverlay.style.position = 'absolute' backgroundOverlay.style.top = `0` backgroundOverlay.style.left = `0` backgroundOverlay.style.width = `100%` backgroundOverlay.style.height = `100%` foregroundOverlay.style.position = 'absolute' foregroundOverlay.style.top = `0` foregroundOverlay.style.left = `0` foregroundOverlay.style.width = `100%` foregroundOverlay.style.height = `100%` foregroundOverlay.style.zIndex = `2` foregroundOverlay.style.pointerEvents = `none` markerOverlay.style.position = 'absolute' markerOverlay.style.top = `0` markerOverlay.style.left = `0` markerOverlay.style.width = `100%` markerOverlay.style.height = `100%` markerOverlay.style.zIndex = `3` markerOverlay.style.pointerEvents = `none` guiOverlay.style.position = 'absolute' guiOverlay.style.top = `0` guiOverlay.style.left = `0` guiOverlay.style.width = `0` guiOverlay.style.height = `0` guiOverlay.style.pointerEvents = `none` guiOverlay.style.opacity = `0` guiOverlay.style.transition = 'opacity 0.2s linear, top 0.2s linear, left 0.2s linear, width 0.2s linear, height 0.2s linear' guiOverlay.style.transform = `translate(-50%, -50%)` labelGroup.style.position = 'absolute' labelGroup.style.display = 'flex' labelGroup.style.gap = '4px' labelGroup.style.pointerEvents = 'none' labelGroup.style.backgroundColor = '#000000aa' labelGroup.style.padding = '9px' labelGroup.style.color = '#ffffff' labelGroup.style.borderRadius = '3px' labelGroup.style.transform = 'translate(-50%,-51px)' labelGroup.style.transition = 'top 0.2s linear, left 0.2s linear, transform 0.2s linear' labelGroup.style.border = '2px solid white' labelGroup.style.fontSize = '14px' labelGroup.style.fontWeight = '500' labelGroup.style.opacity = '0' extraObjectLabel.style.position = 'absolute' extraObjectLabel.style.display = 'flex' extraObjectLabel.style.gap = '4px' extraObjectLabel.style.pointerEvents = 'none' extraObjectLabel.style.backgroundColor = '#000000aa' extraObjectLabel.style.padding = '9px' extraObjectLabel.style.color = '#ffffff' extraObjectLabel.style.borderRadius = '3px' extraObjectLabel.style.transform = 'translate(-50%,-51px)' extraObjectLabel.style.transition = 'top 0.2s linear, left 0.2s linear' extraObjectLabel.style.border = '2px solid white' extraObjectLabel.style.fontSize = '14px' extraObjectLabel.style.fontWeight = '500' extraObjectLabel.style.opacity = '0' actionLabel.style.display = 'none' actionLabel.style.color = '#26bbf8' videoOverlay.crossOrigin = 'anonymous' videoOverlay.controls = false videoOverlay.autoplay = live videoOverlay.volume = 0 videoOverlay.style.position = 'absolute' videoOverlay.style.top = `0` videoOverlay.style.left = `0` videoOverlay.style.width = `100%` videoOverlay.style.height = `100%` videoOverlay.style.opacity = '0' videoOverlay.style.transition = 'opacity 0.2s linear' videoOverlay.style.pointerEvents = 'none' canvasOverlay.style.position = 'absolute' canvasOverlay.style.top = `0` canvasOverlay.style.left = `0` canvasOverlay.style.width = `100%` canvasOverlay.style.height = `100%` canvasOverlay.style.pointerEvents = 'none' canvasOverlay.style.display = 'none' imageOverlay.style.position = 'absolute' imageOverlay.style.top = `0` imageOverlay.style.left = `0` imageOverlay.style.width = `100%` imageOverlay.style.height = `100%` imageOverlay.style.opacity = '0.5' imageOverlay.style.pointerEvents = 'none' imageOverlay.style.display = 'none' objectLabel.style.gap = '9px' objectLabelLinkIconPath.style.stroke = '#44aaff' objectLabelLinkIconPath.style.strokeWidth = '2px' objectLabelLinkIcon.style.display = 'none' objectLabelLinkIcon.style.width = '16px' objectLabelLinkIcon.style.height = '16px' objectLabelLinkIcon.style.background = 'transparent' objectLabelLinkIcon.style.cursor = 'pointer' objectLabelLinkIcon.style.marginLeft = '5px' objectLabelLinkIcon.style.pointerEvents = 'all' objectLabelLinkIcon.setAttribute('viewBox', '0 0 24 24') objectLabelLinkIcon.addEventListener('click', () => { if (shownObjectMetadata) { const { Metadata } = shownObjectMetadata if (!Metadata) { return } const { metadata: url } = Metadata window.open(url) } }) objectLabelLinkIconPath.setAttribute('d', icons['external-link']) let hlsPlayer: any const drawOverlayOnCanvas = () => { videoBox = video.getRect() canvasOverlay.width = videoBox.width canvasOverlay.height = videoBox.height let overlayToDraw: CanvasImageSource if (live) { overlayToDraw = imageOverlay } else { overlayToDraw = videoOverlay } canvasCtx.drawImage(overlayToDraw, 0, 0, videoBox.width, videoBox.height) } const checkBlackPixel = (x: number, y: number) => { const [r, g, b, a] = getPixelColor(x, y, videoWidth, videoHeight) return isBlackPixel(r, g, b) } const findClosestColorCode = ( rgbaColor: [number, number, number, number] ): string => { const colorCodes = Object.keys(colorMap) let closestColorCode = null let closestColorCodeDistance = Infinity for (const colorCode of colorCodes) { const colorCodeRgba = hexToRgba(colorCode) const rDistance = Math.abs(colorCodeRgba[0] - rgbaColor[0]) const gDistance = Math.abs(colorCodeRgba[1] - rgbaColor[1]) const bDistance = Math.abs(colorCodeRgba[2] - rgbaColor[2]) const d = Math.sqrt(rDistance ** 2 + gDistance ** 2 + bDistance ** 2) if (d < closestColorCodeDistance) { closestColorCode = colorCode closestColorCodeDistance = d } } // if (closestColorCodeDistance > MAX_COLOR_DISTANCE) { // return null // } return closestColorCode } const playVideo = () => { video.play() } const playOverlayVideo = () => { videoOverlay.play() } const pauseOverlayVideo = () => { videoOverlay.pause() } const showOverlay = () => { overlay.style.pointerEvents = 'auto' // overlay.style.opacity = '1' } const hideOverlay = () => { overlay.style.pointerEvents = 'none' // overlay.style.opacity = '0' } const showOverlayVideo = () => { videoOverlay.style.opacity = '0.5' } const hideOverlayVideo = () => { videoOverlay.style.opacity = '0' } const showRingMenu = ( menu: MenuOption[], x: number, y: number, width: number, height: number, color: string, metadata: MusemeObjectMetadata ) => { if (menu.length < 1) { return } currentMenu = menu const fullMenu = [ { value: 'close', label: 'Close', icon: 'x', href: 'https://museme.com', }, ...menu, ] ringMenu.push('list', fullMenu) if (showingRingMenu) { guiOverlay.style.transition = 'opacity 0.2s linear, top 0.2s linear, left 0.2s linear, width 0.2s linear, height 0.2s linear' // disable ring menu interactivity while in transition ringMenu.push('disabled', true) setTimeout(() => { ringMenu.push('disabled', false) }, 300) } else { guiOverlay.style.transition = 'opacity 0.2s linear, width 0.2s linear, height 0.2s linear' } showingRingMenu = true showingRingMenuForColor = color shownObjectMetadata = metadata setMenuAnimating() guiOverlay.style.opacity = `1` labelGroup.style.opacity = `1` labelGroup.style.transition = `opacity 0.2s linear, top 0.2s linear, left 0.2s linear, transform 0.2s linear` // const { Metadata } = shownObjectMetadata // if (Metadata && Metadata.metadata) { // showLink() // } else { // hideLink() // } setMenuLayout(x, y, width, height) } const setMenuLayout = ( x: number, y: number, width: number, height: number ): void => { menuX = x menuY = y menuWidth = width menuHeight = height const deltaHeight = RING_MENU_HEIGHT - menuHeight zoom = menuWidth / RING_MENU_WIDTH guiOverlay.style.left = `${x}px` guiOverlay.style.top = `${y}px` guiOverlay.style.width = `${menuWidth}px` guiOverlay.style.height = `${menuHeight}px` let labelGroupTop = y + 72 - deltaHeight / 2 const paddingBottom = 100 if (y + menuHeight / 2 > videoHeight - paddingBottom) { labelGroupTop -= 36 + 6 + menuHeight } labelGroup.style.left = `${x}px` labelGroup.style.top = `${labelGroupTop}px` labelGroup.style.transform = `translate(-50%,30px) scale(${zoom})` if (live) { showOverlay() } } const removeTheseMarkers = (lastOverlayBBox: Dict) => { for (const color in lastOverlayBBox) { const pin = colorPinMap[color] if (pin) { markerOverlay.removeChild(pin) delete colorPinMap[color] } } } const setMenuAnimating = () => { animatingRingMenu = true setTimeout(() => { animatingRingMenu = false }, 200) } const hideRingMenu = () => { showingRingMenu = false setMenuAnimating() guiOverlay.style.transition = 'opacity 0.2s linear, top 0.2s linear, left 0.2s linear, width 0.2s linear, height 0.2s linear' guiOverlay.style.opacity = `0` guiOverlay.style.width = `${0}px` guiOverlay.style.height = `${0}px` labelGroup.style.opacity = '0' labelGroup.style.transform = `translate(-50%,-51px) scale(${zoom})` hideLink() } const positionMenu = (): void => { const offsetX = offsetXp * videoWidth const offsetY = offsetYp * videoHeight const clientX = clientXp * videoWidth const clientY = clientYp * videoHeight const { x, y, width, height } = getFinalMenuLayout( clientX, clientY, offsetX, offsetY ) if (!animatingRingMenu) { labelGroup.style.transition = `` guiOverlay.style.transition = `` } setMenuLayout(x, y, width, height) } const getMenuZoom = (): number => { const maxSize = Math.min(videoWidth, videoHeight) * 0.5 const menuWidth = clamp(RING_MENU_WIDTH, 0, maxSize) return menuWidth / RING_MENU_WIDTH } const positionRoot = () => { videoBox = video.getRect() videoWidth = videoBox.width videoHeight = videoBox.height videoOverlay.width = videoWidth videoOverlay.height = videoHeight root.style.top = `${videoBox.top}px` root.style.left = `${videoBox.left}px` root.style.width = `${videoBox.width}px` root.style.height = `${videoBox.height}px` } const onFrame = (mjpg: string, latestCcTimestamp: number) => { // console.log('onFrame', mjpg, latestCcTimestamp) lastFrameReceived = new Date() lastMjpg = [latestCcTimestamp, mjpg] frameBuffer.push(lastMjpg) if (frameBuffer.length > FRAME_BUFFER_SIZE) { frameBuffer.shift() } let mjpgToRender = mjpg syncedCcTimestamp = latestCcTimestamp const latency = getCurrentLatency(latestCcTimestamp) const targetTimestamp = latestCcTimestamp - latency let minD = Infinity for (let i = frameBuffer.length - 1; i >= 0; i--) { const cursor = frameBuffer[i] const [cursorTimestamp, cursorMjpg] = cursor const d = cursorTimestamp - targetTimestamp if (d < 0) { break } if (d < minD) { minD = d mjpgToRender = cursorMjpg syncedCcTimestamp = cursorTimestamp } } if (overlaySettings.debug) { // eslint-disable-next-line no-console console.log('onFrame', { latestCcTimestamp, latency, targetTimestamp, syncedCcTimestamp, }) } imageOverlay.src = mjpgToRender } let ccRetryTimeout: NodeJS.Timeout let ccRetryCount = 0 const onOpen = () => { emitter.emit('connected') hasMusemeVideoMetadata = true loadingMusemeVideoMetadata = false ccRetryCount = 0 } const onError = (err: Error) => { emitter.emit('error') if (ccRetryTimeout) { clearTimeout(ccRetryTimeout) } ccRetryCount++ ccRetryTimeout = setTimeout(() => { unlistenCc() startCc() }, 2000) } const onMetadata = (metadata: any, latestMetadataTimestamp: number) => { // console.log('onMetadata', metadata) lastMetadataReceived = new Date() colorMap = {} const { detected_objects } = metadata latestMetadataObjects = detected_objects metadataBuffer.push([latestMetadataObjects, latestMetadataTimestamp]) if (metadataBuffer.length > FRAME_BUFFER_SIZE) { metadataBuffer.shift() } let syncedDetectedObjects = latestMetadataObjects let syncedMetadataTimestamp = latestMetadataTimestamp // const transportMetadataLatency: number = // overlaySettings.liveProtocol === 'rtc' ? 2000 : 0 // const targetMetadataTimestamp = ccSyncedTimestamp + transportMetadataLatency const targetMetadataTimestamp = syncedCcTimestamp let minD = Infinity let index: number = 0 for (let i = metadataBuffer.length - 1; i >= 0; i--) { const cursor = metadataBuffer[i] const [cursorObjects, cursorTimestamp] = cursor const d = cursorTimestamp - targetMetadataTimestamp if (d < 0) { break } if (d < minD) { minD = d index = metadataBuffer.length - i syncedDetectedObjects = cursorObjects syncedMetadataTimestamp = cursorTimestamp } } if (overlaySettings.debug) { // eslint-disable-next-line no-console console.log('onMetadata', { index, latestMetadataTimestamp, syncedCcTimestamp, 'metadadataTimestamp - syncedCcTimestamp': latestMetadataTimestamp - syncedCcTimestamp, syncedMetadataTimestamp, 'metadataSyncedTimestamp - syncedCcTimestamp': syncedMetadataTimestamp - syncedCcTimestamp, syncedDetectedObjects, }) } for (const detectedObject of syncedDetectedObjects) { let { Color, Actions = '' } = detectedObject let ActionList: string[] if (typeof Actions === 'string') { ActionList = Actions.split(',') } else { ActionList = Actions } const Hex = rgbaToHex([Color[0], Color[1], Color[2], 1]) const trackedObject: MusemeObjectMetadata = { ...detectedObject, Hex, ActionList, } colorMap[trackedObject.Hex] = trackedObject } } let wsUrl: string let rtcUrl: string const startWs = async (): Promise => { emitter.emit('connecting') if (!wsUrl) { wsUrl = await getMetadataUrl(videoId) } if (overlaySettings.liveProtocol === 'ws') { unlistenCc = wsLiveSource(wsUrl, { onOpen, onFrame, onError, onMetadata }) } } const startRtc = () => { emitter.emit('connecting') rtcUrl = `${apiHostname}` unlistenCc = rtcLiveSource2( rtcUrl, videoId, video, { video: videoOverlay, canvas: canvasOverlay, }, { onOpen, onError, onFrame, onMetadata, }, overlaySettings ) } const startCc = () => { colorMap = {} imageOverlay.src = '' if (overlaySettings.liveProtocol === 'ws') { startWs() } else if (overlaySettings.liveProtocol === 'rtc') { startRtc() } else if (overlaySettings.liveProtocol === 'auto') { startRtc() } else { throw new Error(`Invalid live protocol: ${overlaySettings.liveProtocol}`) } } const startBackgroundFetch = async () => { for (const colorCode in colorMap) { const musemeObject = colorMap[colorCode] const { SceneId, Id } = musemeObject const id = getCacheKey(videoId, SceneId, Id) if (!optionsCache[id]) { getMenuOptions(videoId, musemeObject, abortController.signal) } } } const loadVideoMetadata = async () => { if (live) { startCc() } else { ;(async () => { try { const musemeVideoMetadataUrl = await getMetadataUrl(videoId) try { const response = await fetch(musemeVideoMetadataUrl) musemeVideoMetadata = await response.json() } catch (err) { return } hasMusemeVideoMetadata = true if (video.isPaused()) { refreshOverlay() } for (const SceneId in musemeVideoMetadata) { const trackedObjects = musemeVideoMetadata[SceneId] for (const trackedObject of trackedObjects) { const musemeObject: MusemeObjectMetadata = { Class: trackedObject.name, metadata: trackedObject.metadata, Id: trackedObject.object_id, Hex: trackedObject.color_code, Metadata: trackedObject.Metadata, Color: null, Rect: null, PrevObjectId: null, PrevSceneId: null, Actions: '', ActionList: [], Score: null, SceneId, } colorMap[trackedObject.color_code] = musemeObject } } startBackgroundFetch() } catch (err) { musemeVideoMetadata = undefined hasMusemeVideoMetadata = false } loadingMusemeVideoMetadata = false })() } } const loadVideoOverlay = () => { if (musemeOverlayUrl.endsWith('.mp4')) { videoOverlay.src = musemeOverlayUrl } else { if (Hls.isSupported()) { hlsPlayer = new Hls() hlsPlayer.attachMedia(videoOverlay) hlsPlayer.loadSource(musemeOverlayUrl) } else { const canPlayType = videoOverlay.canPlayType( 'application/vnd.apple.mpegurl' ) if (canPlayType) { videoOverlay.src = musemeOverlayUrl } else { // eslint-disable-next-line console.warn("Browser doesn't support HLS.") } } } } const onOverlayClick = async (event) => { const { clientX, clientY, offsetX, offsetY } = event if (hasMusemeVideoMetadata || live) { const isBlackPixel = checkBlackPixel(offsetX, offsetY) offsetXp = offsetX / videoWidth offsetYp = offsetY / videoHeight clientXp = clientX / videoWidth clientYp = clientY / videoHeight const { x: nextRingX, y: nextRingY, width: menuWidth, height: menuHeight, } = getFinalMenuLayout(clientX, clientY, offsetX, offsetY) if (isBlackPixel) { if (fetchingMenu) { cancelOldClick = true } if (showingRingMenu) { hideRingMenu() } else { playVideo() } } else { const color = getPixelColor(offsetX, offsetY, videoWidth, videoHeight) const closestColorCode = findClosestColorCode(color) const currentTime = video.getCurrentTime() /* eslint-disable-next-line no-console */ console.log({ currentDetectedObjects: latestMetadataObjects, color, closestColorCode, currentTime, }) if (closestColorCode) { const objectMetadata = colorMap[closestColorCode] if (objectMetadata) { // console.log(objectMetadata) const { SceneId, Id, Class } = objectMetadata if (!Class || Class === 'undefined') { return } if (fetchingMenu) { abortController.abort() } fetchingMenu = true let menu: MenuOption[] = [] try { menu = await getMenuOptions( videoId, objectMetadata, abortController.signal ) } catch { // } fetchingMenu = false if (cancelOldClick) { cancelOldClick = false return } if (menu.length === 0) { onEmptyMenuClick(objectMetadata) return } showRingMenu( menu, nextRingX, nextRingY, menuWidth, menuHeight, closestColorCode, objectMetadata ) objectLabel.textContent = Class // extraObjectLabel.style.transition = `` extraObjectLabel.style.opacity = '0' } else { // eslint-disable-next-line console.warn('Could not find closest color object metadata') } } else { // eslint-disable-next-line console.warn('Could not find closest color code.') } } } else { // } } const getFinalMenuLayout = ( clientX: number, clientY: number, offsetX: number, offsetY: number ): Rect => { offsetXp = offsetX / videoWidth offsetYp = offsetY / videoHeight let nextRingX = offsetX let nextRingY = offsetY const PROVIDER_CONTROL_BAR_HEIGHT = 60 const maxSize = Math.min(videoWidth, videoHeight) * 0.5 const menuWidth = clamp(RING_MENU_WIDTH, 0, maxSize) const menuHeight = clamp(RING_MENU_HEIGHT, 0, maxSize) const dx0 = offsetX - menuWidth / 2 const dy0 = clientY - menuHeight / 2 const dx1 = videoWidth - (offsetX + menuWidth / 2) const dy1 = videoHeight - (clientY + menuHeight / 2) - PROVIDER_CONTROL_BAR_HEIGHT / 2 if (dx0 < 0) { nextRingX = offsetX - dx0 } if (dy0 < 0) { nextRingY = offsetY - dy0 } if (dx1 < 0) { nextRingX = offsetX + dx1 } if (dy1 < 0) { nextRingY = offsetY + dy1 } return { x: nextRingX, y: nextRingY, width: menuWidth, height: menuHeight } } const hideExtraObjectLabel = () => { extraObjectLabel.style.transition = `opacity 0.2s linear` extraObjectLabel.style.opacity = '0' } const onPointerMove = (event) => { const { clientX, clientY } = event const offsetX = clientX - videoBox.left const offsetY = clientY - videoBox.top const isBlackPixel = checkBlackPixel(offsetX, offsetY) if (isBlackPixel) { backgroundOverlay.style.cursor = 'auto' if (showingRingMenu) { extraObjectLabel.style.opacity = '0' } else { labelGroup.style.opacity = '0' } } else { backgroundOverlay.style.cursor = 'pointer' const color = getPixelColor(offsetX, offsetY, videoWidth, videoHeight) const colorCode = findClosestColorCode(color) if (colorCode) { const { Class } = colorMap[colorCode] if (!Class || Class === 'undefined') { return } zoom = getMenuZoom() if (showingRingMenu) { if (colorCode === showingRingMenuForColor) { hideExtraObjectLabel() } else { extraObjectLabel.style.transition = `` extraObjectLabel.style.top = `${offsetY}px` extraObjectLabel.style.left = `${offsetX}px` extraObjectLabel.style.transform = `translate(-50%,${ 30 * zoom }px) scale(${zoom})` extraObjectLabel.style.opacity = '1' extraObjectLabel.textContent = Class } } else { if (!overlaySettings.pauseOnClick || video.isPaused() || live) { labelGroup.style.transition = `` labelGroup.style.top = `${offsetY}px` labelGroup.style.left = `${offsetX}px` labelGroup.style.transform = `translate(-50%,${ 30 * zoom }px) scale(${zoom})` labelGroup.style.opacity = '1' objectLabel.textContent = Class } } } } } const onPointerUp = () => { hideExtraObjectLabel() } const onPointerLeave = () => { if (showingRingMenu) { hideExtraObjectLabel() } else { labelGroup.style.opacity = '0' } } const onLoadStart = () => { // console.log('onLoadStart') } const onPause = () => { if ((!overlaySettings.pauseOnClick && hasMusemeVideoMetadata) || live) { justClicked = false video.play() return } refreshOverlay() } const onEnded = (event: Event) => { hideOverlay() } let justClicked = false const onClick = () => { justClicked = true } const syncVideoOverlay = () => { const currentTime = video.getCurrentTime() videoOverlay.currentTime = currentTime } const refreshOverlay = () => { if (live) { showOverlay() return } if (video.isPaused() && video.getCurrentTime()) { if (video.isPaused()) { if (hasMusemeVideoMetadata) { showOverlay() if (overlaySettings.showMasksOnPause) { showOverlayVideo() } syncVideoOverlay() } } else { hideOverlay() } } } const startInterval = () => { if (intervalFrame === undefined) { intervalFrame = setInterval(intervalTick, MARKER_INTERVAL_T) } } const stopInterval = () => { if (intervalFrame !== undefined) { clearInterval(intervalFrame) intervalFrame = undefined } } const onPlay = () => { if (!live) { hideOverlay() hideOverlayVideo() } else { playOverlayVideo() } if (hasMusemeVideoMetadata) { startInterval() } if (showingRingMenu && overlaySettings.pauseOnClick && !live) { hideRingMenu() } } const onOverlayVideoLoadStart = () => { // } const onOverlayVideoCanPlay = () => { // } const onOverlayVideoCanPlayThrough = () => { // } const onActionEnter = async (icon: string) => { const option = currentMenu.find((o) => o.icon === icon) if (icon === 'x') { hoveredAction = icon actionLabel.style.display = 'block' actionLabel.textContent = 'Close' objectLabel.style.display = 'none' hideLink() } else { hoveredAction = icon actionLabel.style.display = 'block' actionLabel.textContent = option.label objectLabel.style.display = 'flex' const link = shouldLink(option) if (link) { showLink() } } } const findBoundingBoxMap = ( imageData: ImageData ): Dict<{ x0: number; y0: number; x1: number; y1: number }> => { const imageDataData = imageData.data const bboxMap: Dict<{ x0: number; y0: number; x1: number; y1: number }> = {} for (let i = 0; i < imageDataData.length; i += 4) { const x = (i / 4) % imageData.width const y = Math.floor(i / 4 / imageData.width) const r = imageData.data[i] const g = imageData.data[i + 1] const b = imageData.data[i + 2] const a = imageData.data[i + 3] if (isBlackPixel(r, g, b)) { // } else { const color = findClosestColorCode([r, g, b, a]) if (color) { bboxMap[color] = bboxMap[color] ?? { x0: Infinity, y0: Infinity, x1: -Infinity, y1: -Infinity, } const { x0, y0, x1, y1 } = bboxMap[color] bboxMap[color].x0 = Math.min(x0, x) bboxMap[color].y0 = Math.min(y0, y) bboxMap[color].x1 = Math.max(x1, x) bboxMap[color].y1 = Math.max(y1, y) } } } return bboxMap } const showLink = () => { objectLabelLinkIcon.style.display = 'block' } const hideLink = () => { objectLabelLinkIcon.style.display = 'none' } const onActionLeave = async (icon: string) => { hoveredAction = null actionLabel.style.display = 'none' actionLabel.textContent = '' objectLabel.style.display = 'flex' hideLink() } const onActionClick = async (icon: string) => { if (icon === 'x') { if (showingRingMenu) { hideRingMenu() playVideo() } } else { const done = onActionSelected(shownObjectMetadata, icon, currentMenu) ?? false if (done) { hideRingMenu() } } } const intervalTick = () => { // console.log('intervalTick') // syncVideoOverlay() return if (hasMusemeVideoMetadata) { if (overlaySettings.showMarkers) { drawOverlayOnCanvas() const lastOverlayBBox = { ...overlayBboxMap } if (videoWidth === 0) { return } overlayBboxMap = findBoundingBoxMap( canvasCtx.getImageData(0, 0, videoWidth, videoHeight) ) for (const color in overlayBboxMap) { const bbox = overlayBboxMap[color] const { x0, y0, x1, y1 } = bbox const cx = (x0 + x1) / 2 const cy = (y0 + y1) / 2 let marker = colorPinMap[color] delete lastOverlayBBox[color] if (!marker) { marker = document.createElement('div') colorPinMap[color] = marker marker.style.position = 'absolute' marker.style.backgroundColor = color marker.style.width = '10px' marker.style.height = '10px' marker.style.borderRadius = '50%' marker.style.border = '2px solid white' marker.style.transition = 'top 0.2s linear, left 0.2s linear' markerOverlay.appendChild(marker) } marker.style.top = `${cy}px` marker.style.left = `${cx}px` } removeTheseMarkers(lastOverlayBBox) } else { removeAllMarkers() } } else { removeAllMarkers() } } let lastPausedTime = 0 const tick = () => { positionRoot() if (showingRingMenu && !animatingRingMenu) { positionMenu() } if (hasMusemeVideoMetadata) { if (video.isPaused()) { justClicked = false if (hasMusemeVideoMetadata && !overlaySettings.pauseOnClick) { video.play() } } } if (!detached) { if (animationFrame !== undefined) { animationFrame = undefined startTick() } } const currentTime = video.getCurrentTime() if (lastPausedTime !== currentTime) { syncVideoOverlay() lastPausedTime = currentTime } } const startTick = () => { if (animationFrame === undefined) { animationFrame = requestAnimationFrame(tick) } } const stopTick = () => { if (animationFrame !== undefined) { cancelAnimationFrame(animationFrame) animationFrame = undefined } } const removeAllMarkers = () => { removeTheseMarkers({ ...colorPinMap }) } let detached = true const detach = () => { detached = true if (unlistenCc) { unlistenCc() unlistenCc = undefined } overlay.removeEventListener('click', onOverlayClick) overlay.removeEventListener('pointermove', onPointerMove) overlay.removeEventListener('pointerdown', onPointerMove) overlay.removeEventListener('pointerup', onPointerUp) overlay.removeEventListener('pointerleave', onPointerLeave) video.removeEventListener('loadstart', onLoadStart) video.removeEventListener('pause', onPause) video.removeEventListener('play', onPlay) video.removeEventListener('ended', onEnded) video.removeEventListener('click', onClick) videoOverlay.removeEventListener('loadstart', onOverlayVideoLoadStart) videoOverlay.removeEventListener('canplay', onOverlayVideoCanPlay) videoOverlay.removeEventListener( 'canplaythrough', onOverlayVideoCanPlayThrough ) try { detachRingMenu() } catch { // } stopTick() stopInterval() removeAllMarkers() if (root.parentElement) { root.parentElement.removeChild(root) } } const attach = () => { if (!detached) { return } detached = false if (overlaySettings.enabled) { if (live) { video.addEventListener('click', onOverlayClick) video.addEventListener('pointermove', onPointerMove) video.addEventListener('pointerleave', onPointerLeave) } overlay.addEventListener('click', onOverlayClick) overlay.addEventListener('pointerdown', onPointerMove) overlay.addEventListener('pointermove', onPointerMove) overlay.addEventListener('pointerup', onPointerUp) overlay.addEventListener('pointerleave', onPointerLeave) const ifPausedOnCLick = (handler: any) => { return (event: PointerEvent) => { if (!overlaySettings.pauseOnClick) { handler(event) } } } video.addEventListener('click', ifPausedOnCLick(onOverlayClick)) video.addEventListener('pointermove', ifPausedOnCLick(onPointerMove)) video.addEventListener('pointerleave', ifPausedOnCLick(onPointerLeave)) video.addEventListener('pointerup', ifPausedOnCLick(onPointerUp)) video.addEventListener('pause', onPause.bind(null, null)) video.addEventListener('play', onPlay) video.addEventListener('ended', onEnded) video.addEventListener('click', onClick) videoOverlay.addEventListener('loadstart', onOverlayVideoLoadStart) videoOverlay.addEventListener('canplay', onOverlayVideoCanPlay) videoOverlay.addEventListener( 'canplaythrough', onOverlayVideoCanPlayThrough ) positionRoot() startInterval() startTick() const controls = document.querySelector('.ytp-chrome-bottom') if (controls) { controls.insertAdjacentElement('beforebegin', root) } else { document.body.appendChild(root) } ;[ringMenu, detachRingMenu] = renderMenu( guiOverlay, DEFAULT_RING_MENU_STYLE, currentMenu, { onActionClick, onActionEnter, onActionLeave, } ) if (video.isPaused()) { onPause() } syncVideoOverlay() } } if (live) { // } else { loadVideoOverlay() } loadVideoMetadata() attach() if (live) { showOverlay() } if (overlaySettings.showOverlay) { if (live) { imageOverlay.style.display = 'block' } else { // } } else { // } const api: MusemeOverlayAPI = { setSetting: function (key: string, value: any): void { if (overlaySettings[key] === value) { return } switch (key) { case 'enabled': overlaySettings.enabled = value if (detached && overlaySettings.enabled) { attach() } else if (!detached && !overlaySettings.enabled) { detach() } break case 'pauseOnClick': { overlaySettings.pauseOnClick = value } break case 'showMasksOnPause': { overlaySettings.showMasksOnPause = value if (hasMusemeVideoMetadata) { if (overlaySettings.showMasksOnPause) { if (video.isPaused()) { showOverlayVideo() } else { hideOverlayVideo() } } else { hideOverlayVideo() } } } break case 'showMasksOnTouch': { overlaySettings.showMasksOnTouch = value } break case 'showMarkers': { overlaySettings.showMarkers = value } break case 'showOverlay': { overlaySettings.showOverlay = value if (overlaySettings.showOverlay) { if (live) { imageOverlay.style.display = 'block' } else { // } } else { if (live) { imageOverlay.style.display = 'none' } else { // } } } break case 'liveProtocol': if (overlaySettings.liveProtocol === value) { return } overlaySettings.liveProtocol = value if (unlistenCc) { unlistenCc() } startCc() break case 'liveMode': if (overlaySettings.liveMode === value) { return } overlaySettings.liveMode = value break } }, getPlayer: function () { return hlsPlayer }, getCurrentColorMap: function (): MetadataColorMap { return colorMap }, getSettings: function (): MusemeOverlaySettings { return overlaySettings }, getSetting: function ( name: K ): MusemeOverlaySettings[K] { return overlaySettings[name] }, addListener: function ( event: MusemeEvents, listener: () => void ): Unlisten { return emitter.addListener(event, listener) }, } return [api, detach] }