/* Phaneron - Clustered, accelerated and cloud-fit video server, pre-assembled and in kit form. Copyright (C) 2020 Streampunk Media Ltd. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . https://www.streampunk.media/ mailto:furnace@streampunk.media 14 Ormiscaig, Aultbea, Achnasheen, IV22 2JJ U.K. */ import { clContext as nodenCLContext, KernelParams } from 'nodencl' import { RedioPipe, RedioEnd, isValue, Valve, nil, isEnd, RedioNil } from 'redioactive' import { OpenCLBuffer } from 'nodencl' import { AudioInputParam, filterer, Filterer, Frame } from 'beamcoder' import { VideoFormat } from './config' import { ClJobs } from './clJobQueue' import ImageProcess from './process/imageProcess' import Transition from './process/transition' import { Silence, Black } from './blackSilence' import { Producer } from './producer/producer' import { EventEmitter, once } from 'events' export type TransitionSpec = { type: string; len: number; source?: Producer; mask?: Producer } export class Transitioner { private readonly clContext: nodenCLContext private readonly layerID: string private readonly consumerFormat: VideoFormat private readonly clJobs: ClJobs private readonly endEvent: EventEmitter private silence: Silence | null private black: Black | null private audType: string private vidType: string private numFrames: number private curFrame: number private nextType: string private nextNumFrames: number private audTransition: Filterer | null = null private vidTransition: ImageProcess | null = null private audioPipe: RedioPipe | undefined private videoPipe: RedioPipe | undefined private readonly audSourcePipes: RedioPipe[] = [] private readonly vidSourcePipes: RedioPipe[] = [] private updating = true // eslint-disable-next-line @typescript-eslint/no-empty-function private layerUpdate: (ts: number[]) => void = () => {} constructor( clContext: nodenCLContext, layerID: string, consumerFormat: VideoFormat, clJobs: ClJobs, endEvent: EventEmitter, layerUpdate: (ts: number[]) => void ) { this.clContext = clContext this.layerID = `${layerID} transition` this.consumerFormat = consumerFormat this.clJobs = clJobs this.endEvent = endEvent this.silence = new Silence(this.consumerFormat) this.black = new Black(this.clContext, this.consumerFormat, this.layerID) this.layerUpdate = layerUpdate this.audType = 'cut' this.vidType = 'cut' this.numFrames = 0 this.curFrame = 0 this.nextType = 'cut' this.nextNumFrames = 0 } async initialise(): Promise { const silencePipe = (await this.silence?.initialise()) as RedioPipe const blackPipe = (await this.black?.initialise()) as RedioPipe const transitionAudValve: Valve<[Frame | RedioEnd, ...(Frame | RedioEnd)[]], Frame | RedioEnd> = async (frames) => { if (isValue(frames)) { const srcFrames = frames.slice(1, 3) if (srcFrames.length === 0) return frames[0] if (this.audType !== this.nextType && srcFrames.length === this.audSourcePipes.length) { this.audType = this.nextType await this.makeAudTransition() } let transitionResult: (Frame | RedioEnd | RedioNil)[] = [srcFrames[0]] if (srcFrames.reduce((acc, f) => acc && isValue(f), true)) { if (srcFrames.length > 1) { const srcs = srcFrames as Frame[] const pts = srcs[0].pts const filterFrames = srcs.map((f, i) => { f.pts = pts return { name: `in${i}:a`, frames: [f] } }) const ff = await this.audTransition?.filter(filterFrames) transitionResult = ff && ff[0] && ff[0].frames.length > 0 ? ff[0].frames : [nil] } } else { transitionResult = srcFrames.length > 1 && isValue(srcFrames[1]) ? [srcFrames[1]] : [srcFrames[0]] } if (isEnd(transitionResult[0])) transitionResult = [frames[0]] return transitionResult } else { return frames } } const transitionVidValve: Valve< [OpenCLBuffer | RedioEnd, ...(OpenCLBuffer | RedioEnd)[]], OpenCLBuffer | RedioEnd > = async (frames) => { if (isValue(frames)) { const srcFrames = frames.slice(1) // eslint-disable-next-line prettier/prettier this.layerUpdate(srcFrames.map((f) => (isValue(f) ? f.timestamp : this.updating ? 0 : -1))) if (srcFrames.length === 0) { if (isValue(frames[0])) frames[0].addRef() return frames[0] } if (this.vidType !== this.nextType && srcFrames.length === this.vidSourcePipes.length) { this.vidType = this.nextType this.numFrames = this.nextNumFrames this.curFrame = 0 await this.makeVidTransition() } let transitionResult = srcFrames[0] if (srcFrames.reduce((acc, f) => acc && isValue(f), true)) { this.updating = false if (srcFrames.length === 1) { if (isValue(transitionResult)) transitionResult.addRef() } else { const transitionDest = await this.clContext.createBuffer( this.consumerFormat.width * this.consumerFormat.height * 4 * 4, 'readwrite', 'coarse', { width: this.consumerFormat.width, height: this.consumerFormat.height }, 'transition' ) const timestamp = (srcFrames[1] as OpenCLBuffer).timestamp // transitionDest.loadstamp = Math.min(...srcFrames.map((f) => f.loadstamp)) transitionDest.timestamp = timestamp const params: KernelParams = { inputs: srcFrames.slice(0, 2), output: transitionDest } if (srcFrames.length === 2) { params.mix = this.numFrames > 0 ? 1.0 - this.curFrame / this.numFrames : 0.0 } else { params.mask = srcFrames[2] } this.curFrame++ await this.vidTransition?.run( params, { source: this.layerID, timestamp: timestamp }, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {} ) await this.clJobs.runQueue({ source: this.layerID, timestamp: timestamp }) transitionResult = transitionDest } } else { transitionResult = srcFrames.length > 1 && isValue(srcFrames[1]) ? srcFrames[1] : srcFrames[0] if (isValue(transitionResult)) transitionResult.addRef() } srcFrames.forEach((f) => (isValue(f) ? f.release() : {})) if (isEnd(transitionResult)) { transitionResult = frames[0] if (isValue(transitionResult)) transitionResult.addRef() } return transitionResult } else { this.layerUpdate([]) return frames } } this.audioPipe = silencePipe .zipEach(this.audSourcePipes) .valve(transitionAudValve, { oneToMany: true }) // eslint-disable-next-line prettier/prettier this.videoPipe = blackPipe .zipEach(this.vidSourcePipes) .valve(transitionVidValve) } async makeAudTransition(): Promise { if (this.audType === 'cut') this.audTransition = null else { const sampleRate = this.consumerFormat.audioSampleRate const numAudChannels = this.consumerFormat.audioChannels const audLayout = `${numAudChannels}c` const inParams: Array = [] let inStr = '' for (let s = 0; s < this.audSourcePipes.length; ++s) { inStr += `[in${s}:a]` inParams.push({ name: `in${s}:a`, timeBase: [1, sampleRate], sampleRate: sampleRate, sampleFormat: 'fltp', channelLayout: audLayout }) } this.audTransition = await filterer({ filterType: 'audio', inputParams: inParams, outputParams: [ { name: 'out0:a', sampleRate: sampleRate, sampleFormat: 'fltp', channelLayout: audLayout } ], filterSpec: `${inStr}amix=inputs=${this.audSourcePipes.length}:duration=shortest[out0:a]` }) // console.log('\nTransition audio:\n', this.audTransition.graph.dump()) } } async makeVidTransition(): Promise { if (this.vidType === 'cut') this.vidTransition = null else { this.vidTransition = new ImageProcess( this.clContext, new Transition(this.vidType, this.consumerFormat.width, this.consumerFormat.height), this.clJobs ) await this.vidTransition.init() } } update( type: string, numFrames: number, audioSrcPipes: RedioPipe[], videoSrcPipes: RedioPipe[] ): void { this.nextType = type this.nextNumFrames = numFrames > 0 ? numFrames - 1 : 0 this.updating = true this.audSourcePipes.splice(0) audioSrcPipes.forEach((p) => this.audSourcePipes.push(p)) this.vidSourcePipes.splice(0) videoSrcPipes.forEach((p) => this.vidSourcePipes.push(p)) } getAudioPipe(): RedioPipe | undefined { return this.audioPipe } getVideoPipe(): RedioPipe | undefined { return this.videoPipe } async release(): Promise { this.silence?.release() this.black?.release() await once(this.endEvent, 'end') this.silence = null this.black = null this.audTransition = null this.vidTransition?.finish() this.vidTransition = null this.audioPipe = undefined this.videoPipe = undefined } }