/* Phaneron - Clustered, accelerated and cloud-fit video server, pre-assembled and in kit form. Copyright (C) 2020 Streampunk Media Ltd. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . https://www.streampunk.media/ mailto:furnace@streampunk.media 14 Ormiscaig, Aultbea, Achnasheen, IV22 2JJ U.K. */ import { clContext as nodenCLContext, OpenCLBuffer } from 'nodencl' import { RedioPipe, RedioEnd, isValue, Valve, nil } from 'redioactive' import { Frame, Filterer, filterer /*, FilterContext*/ } from 'beamcoder' import { VideoFormat } from '../config' import { ClJobs } from '../clJobQueue' import ImageProcess from '../process/imageProcess' import Transform from '../process/transform' export interface AnchorParams { x: number y: number } export interface FillParams { xOffset: number yOffset: number xScale: number yScale: number } export type MixerParams = { anchor: AnchorParams rotation: number fill: FillParams volume: number } export const MixerDefaults = '{\ "anchor": { "x": 0, "y": 0 },\ "rotation": 0,\ "fill": { "xOffset": 0, "yOffset": 0, "xScale": 1, "yScale": 1 },\ "volume": 1\ }' // const getFilters = (filterer: Filterer): string[] => { // const filters: string[] = [] // filterer.graph.filters.forEach((f) => filters.push(f.name)) // return filters // } // type optionSpec = { // name: string // optionType: string // readonly: boolean // value: string | undefined // } // type optionSpecs = Map // const getFilterParams = (filter: FilterContext): optionSpecs => { // const specs: optionSpecs = new Map() // const priv_class = filter.filter.priv_class // if (priv_class) { // const optionsKeys = Object.keys(priv_class.options) // const options: optionSpec[] = [] // optionsKeys.forEach((k) => { // const opt = priv_class.options[k] // if (filter.priv) { // options.push({ // name: opt.name, // optionType: opt.option_type, // readonly: opt.flags.READONLY, // value: filter.priv[opt.name] // }) // } else { // console.log(`No entry found for parameter '${k}' in filter '${filter.name}'`) // const opt = priv_class.options[k] // options.push({ // name: opt.name, // optionType: opt.option_type, // readonly: opt.flags.READONLY, // value: undefined // }) // } // }) // specs.set(filter.name, options) // } // return specs // } export class Mixer { private readonly clContext: nodenCLContext private readonly consumerFormat: VideoFormat private readonly clJobs: ClJobs private transform: ImageProcess | null private mixAudio!: RedioPipe private mixVideo!: RedioPipe private audMixFilterer: Filterer | null = null private mixParams = JSON.parse(MixerDefaults) private srcLevels: number[] = [] private paused = true private running = true private audDone = false private vidDone = false constructor(clContext: nodenCLContext, consumerFormat: VideoFormat, clJobs: ClJobs) { this.clContext = clContext this.consumerFormat = consumerFormat this.clJobs = clJobs this.transform = new ImageProcess( this.clContext, new Transform(this.clContext, this.consumerFormat.width, this.consumerFormat.height), clJobs ) } async init( sourceID: string, srcAudio: RedioPipe, srcVideo: RedioPipe, srcFormat: VideoFormat ): Promise { const srcSampleRate = srcFormat.audioSampleRate const srcAudChannels = srcFormat.audioChannels const dstSampleRate = this.consumerFormat.audioSampleRate const dstAudChannels = this.consumerFormat.audioChannels const dstAudLayout = `${dstAudChannels}c` let filtSpec = '' for (let c = 0; c < srcAudChannels; ++c) { this.srcLevels.push(1.0) const panSpec = `pan=${dstAudLayout}|c${c % dstAudChannels}=${this.srcLevels[c]}*c0` filtSpec += (c === 0 ? '' : ';\n') + `[in${c}:a]highpass=mix=0, adelay=delays='', acompressor=threshold=1:mix=0, aformat=sample_fmts=fltp, ${panSpec}[c${c}:a]` } filtSpec += ';\n' for (let c = 0; c < srcAudChannels; ++c) filtSpec += `[c${c}:a]` filtSpec += `amix=inputs=${srcAudChannels}:duration=shortest:weights=` for (let c = 0; c < srcAudChannels; ++c) filtSpec += (c === 0 ? '' : ' ') + (c < dstAudChannels ? '1' : '0') filtSpec += `, volume=1.0:eval=frame:precision=float[out0:a]` // console.log(filtSpec) const inParams = [] for (let s = 0; s < srcAudChannels; ++s) inParams.push({ name: `in${s}:a`, timeBase: [1, srcSampleRate], sampleRate: srcSampleRate, sampleFormat: 'fltp', channelLayout: '1c' }) this.audMixFilterer = await filterer({ filterType: 'audio', inputParams: inParams, outputParams: [ { name: 'out0:a', sampleRate: dstSampleRate, sampleFormat: 'fltp', channelLayout: dstAudLayout } ], filterSpec: filtSpec }) // console.log('\nChannel audio:\n', this.audMixFilterer.graph.dump()) // console.log(getFilters(this.audMixFilterer)) // const filtContexts = this.audMixFilterer.graph.filters.filter( // (filt) => filt.filter.name === 'pan' // ) // filtContexts.forEach((c) => console.log(getFilterParams(c))) const audMixFilter: Valve = async (frames) => { if (isValue(frames)) { if (!this.running) return nil if (!this.audMixFilterer) return nil const inSpec: { name: string; frames: Frame[] }[] = [] frames.forEach((f, i) => inSpec.push({ name: `in${i}:a`, frames: [f] })) const ff = await this.audMixFilterer.filter(inSpec) return ff[0] && ff[0].frames.length > 0 ? ff[0].frames : nil } else { this.audMixFilterer = null this.audDone = true if (this.audDone && this.vidDone) this.running = false return frames } } await this.transform?.init() const numBytesRGBA = this.consumerFormat.width * this.consumerFormat.height * 4 * 4 const srcXscale = this.consumerFormat.squareWidth / srcFormat.squareWidth const srcYscale = this.consumerFormat.squareHeight / srcFormat.squareHeight const mixVidValve: Valve = async (frame) => { if (isValue(frame)) { if (!this.running) { frame.release() return nil } const timestamp = frame.timestamp const xfDest = await this.clContext.createBuffer( numBytesRGBA, 'readwrite', 'coarse', { width: this.consumerFormat.width, height: this.consumerFormat.height }, `mixer ${sourceID} ${timestamp}` ) // xfDest.loadstamp = frame.loadstamp xfDest.timestamp = timestamp await this.transform?.run( { input: frame, flipH: false, flipV: false, anchorX: this.mixParams.anchor.x - 0.5, anchorY: this.mixParams.anchor.y - 0.5, scaleX: srcXscale * this.mixParams.fill.xScale, scaleY: srcYscale * this.mixParams.fill.yScale, rotate: -this.mixParams.rotation / 360.0, offsetX: -this.mixParams.fill.xOffset, offsetY: -this.mixParams.fill.yOffset, output: xfDest }, { source: sourceID, timestamp: timestamp }, () => frame.release() ) await this.clJobs.runQueue({ source: sourceID, timestamp: timestamp }) return xfDest } else { this.clJobs.clearQueue(sourceID) this.transform?.finish() this.transform = null this.vidDone = true if (this.audDone && this.vidDone) this.running = false return frame } } this.mixAudio = srcAudio .pause(() => this.paused && this.running) .valve(audMixFilter, { oneToMany: true }) this.mixVideo = srcVideo .pause((frame) => { if (!this.running) { frame = nil return false } if (this.paused && isValue(frame)) (frame as OpenCLBuffer).addRef() return this.paused }) .valve(mixVidValve) } setPaused(pause: boolean): void { this.paused = pause this.setVolume(this.mixParams.volume, this.paused) } release(): void { this.running = false } setMixParams(mixParams: MixerParams): void { this.mixParams = mixParams this.setVolume(this.mixParams.volume) } setVolume(volume: number, mute?: boolean): boolean { this.mixParams.volume = volume const volFilter = this.audMixFilterer?.graph.filters.find((f) => f.filter.name === 'volume') if (volFilter && volFilter.priv) volFilter.priv = { volume: mute ? '0.0' : this.mixParams.volume.toString() } return true } getAudioPipe(): RedioPipe { return this.mixAudio } getVideoPipe(): RedioPipe { return this.mixVideo } }