/*
Phaneron - Clustered, accelerated and cloud-fit video server, pre-assembled and in kit form.
Copyright (C) 2020 Streampunk Media Ltd.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
https://www.streampunk.media/ mailto:furnace@streampunk.media
14 Ormiscaig, Aultbea, Achnasheen, IV22 2JJ U.K.
*/
import { clContext as nodenCLContext, OpenCLBuffer } from 'nodencl'
import { ConsumerFactory, Consumer } from './consumer'
import { RedioPipe, RedioEnd, nil, end, isValue, Valve, Spout } from 'redioactive'
import * as Macadam from 'macadam'
import { FromRGBA } from '../process/io'
import { Writer } from '../process/v210'
import { Frame, Filterer, filterer } from 'beamcoder'
import { ConfigParams, VideoFormat, DeviceConfig } from '../config'
import { ClJobs } from '../clJobQueue'
interface DecklinkConfig extends DeviceConfig {
keyDeviceIndex: number
embeddedAudio: boolean
latency: 'normal' | 'low' | 'default'
keyer: 'external' | 'external_separate_device' | 'internal' | 'default'
keyOnly: boolean
bufferDepth: number
}
const decklinkDefaults: DecklinkConfig = {
name: 'decklink',
deviceIndex: 1,
keyDeviceIndex: 0,
embeddedAudio: false,
latency: 'normal',
keyer: 'external',
keyOnly: false,
bufferDepth: 3
}
interface AudioBuffer {
buffer: Buffer
timestamp: number
}
const bmdSampleRates = new Map([[48000, Macadam.bmdAudioSampleRate48kHz]])
const bmdDisplayMode = new Map([
['720p5000', Macadam.bmdModeHD1080p50],
['1080i5000', Macadam.bmdModeHD1080i50],
['1080p5000', Macadam.bmdModeHD1080p50]
])
export class MacadamConsumer implements Consumer {
private readonly clContext: nodenCLContext
private readonly chanID: string
private readonly params: ConfigParams
private readonly format: VideoFormat
private readonly device: DeviceConfig
private readonly clJobs: ClJobs
private readonly logTimings = false
private playback: Macadam.PlaybackChannel | null = null
private fromRGBA: FromRGBA | null = null
private clDests: OpenCLBuffer[] = []
private vidField: number
private readonly audioChannels: number
private readonly audioTimebase: number[]
private readonly videoTimebase: number[]
private audFilterer: Filterer | null = null
constructor(
context: nodenCLContext,
chanID: string,
params: ConfigParams,
format: VideoFormat,
device: DeviceConfig,
clJobs: ClJobs
) {
this.clContext = context
this.params = params
this.format = format
this.device = device
this.chanID = `${chanID} decklink-${this.device.deviceIndex - 1}`
this.clJobs = clJobs
this.vidField = 0
this.audioChannels = this.format.audioChannels
this.audioTimebase = [1, this.format.audioSampleRate]
this.videoTimebase = [this.format.duration, this.format.timescale / this.format.fields]
if (Object.keys(this.params).length > 1)
console.log('Macadam consumer - unused params', this.params)
const defaultConfig: DeviceConfig = { name: 'decklink', deviceIndex: 0 }
this.device = Object.assign(defaultConfig, { ...decklinkDefaults }, this.device)
// Turn off single field output flag
// - have to thump it twice to get it to change!!
const macadamIndex = this.device.deviceIndex - 1
for (let x = 0; x < 2; ++x)
Macadam.setDeviceConfig({
deviceIndex: macadamIndex,
fieldFlickerRemoval: false
})
if (Macadam.getDeviceConfig(macadamIndex).fieldFlickerRemoval)
console.log(
// eslint-disable-next-line prettier/prettier
`Macadam consumer ${this.device.deviceIndex - 1} - failed to turn off single field output mode`
)
}
async initialise(): Promise {
this.playback = await Macadam.playback({
deviceIndex: this.device.deviceIndex - 1,
channels: this.audioChannels,
sampleRate: bmdSampleRates.get(this.format.audioSampleRate),
sampleType: Macadam.bmdAudioSampleType32bitInteger,
displayMode: bmdDisplayMode.get(this.format.name),
pixelFormat: Macadam.bmdFormat10BitYUV
})
const sampleRate = this.audioTimebase[1]
const audLayout = `${this.audioChannels}c`
// !!! Needs more work to handle 59.94 frame rates !!!
const samplesPerFrame =
(this.format.audioSampleRate * this.format.duration * this.format.fields) /
this.format.timescale
this.audFilterer = await filterer({
filterType: 'audio',
inputParams: [
{
name: 'in0:a',
timeBase: this.audioTimebase,
sampleRate: sampleRate,
sampleFormat: 'fltp',
channelLayout: audLayout
}
],
outputParams: [
{
name: 'out0:a',
sampleRate: sampleRate,
sampleFormat: 's32',
channelLayout: audLayout
}
],
filterSpec: `[in0:a] asetnsamples=n=${samplesPerFrame}:p=1 [out0:a]`
})
// console.log('\nMacadam consumer audio:\n', this.audFilterer.graph.dump())
this.fromRGBA = new FromRGBA(
this.clContext,
'709',
new Writer(this.playback.width, this.playback.height, this.format.fields === 2),
this.clJobs
)
await this.fromRGBA.init()
console.log(`Created Macadam consumer for Blackmagic id: ${this.device.deviceIndex - 1}`)
return Promise.resolve()
}
async waitHW(): Promise {
let delay = 0
const hwTime = this.playback?.hardwareTime()
if (hwTime) {
const nominalDelayMs = (1000 * hwTime.ticksPerFrame) / hwTime.timeScale
const targetTimeInFrame = nominalDelayMs * 0.05
const curTimeInFrame = ((nominalDelayMs * hwTime.timeInFrame) / hwTime.ticksPerFrame) >>> 0
delay = nominalDelayMs + targetTimeInFrame - curTimeInFrame
}
return new Promise((resolve) =>
setTimeout(() => {
const hwTimeNow = this.playback?.hardwareTime()
if (hwTime && hwTimeNow) {
const hwDelay = hwTimeNow.hardwareTime - hwTime.hardwareTime - hwTimeNow.ticksPerFrame
if (hwDelay > hwTimeNow.ticksPerFrame * 0.9)
console.log(
// eslint-disable-next-line prettier/prettier
`Macadam consumer ${this.device.deviceIndex - 1} - frame may be delayed (${hwDelay} ticks)`
)
}
resolve()
}, delay)
)
}
connect(
combineAudio: RedioPipe,
combineVideo: RedioPipe
): void {
this.vidField = 0
const audFilter: Valve = async (frame) => {
if (isValue(frame) && this.audFilterer) {
const ff = await this.audFilterer.filter([{ name: 'in0:a', frames: [frame] }])
const result: AudioBuffer[] = ff[0].frames.map((f) => ({
buffer: f.data[0],
timestamp: f.pts
}))
return result.length > 0 ? result : nil
} else {
return end
}
}
const vidProcess: Valve = async (frame) => {
if (isValue(frame)) {
const start = process.hrtime()
const fromRGBA = this.fromRGBA as FromRGBA
if (this.vidField === 0) {
this.clDests = await fromRGBA.createDests()
this.clDests.forEach((d) => {
// d.loadstamp = frame.loadstamp
d.timestamp = (frame.timestamp / this.format.fields) << 0
})
}
const interlace = this.format.fields ? 0x1 | (this.vidField << 1) : 0
fromRGBA.processFrame(this.chanID, frame, this.clDests, interlace)
await this.clJobs.runQueue({ source: this.chanID, timestamp: frame.timestamp })
const end = process.hrtime(start)
if (this.logTimings)
console.log(
`Macadam channel ${this.device.deviceIndex}: ${frame.timestamp} ${(
end[0] * 1000.0 +
end[1] / 1000000.0
).toFixed(2)}ms processing total`
)
if (this.format.fields === 2) this.vidField = 1 - this.vidField
else this.vidField = 0
return this.vidField === 1 ? nil : this.clDests[0]
} else {
this.clJobs.clearQueue(this.chanID)
return frame
}
}
const vidSaver: Valve = async (frame) => {
if (isValue(frame)) {
const fromRGBA = this.fromRGBA as FromRGBA
await fromRGBA.saveFrame(frame, this.clContext.queue.unload)
await this.clContext.waitFinish(this.clContext.queue.unload)
return frame
} else {
return frame
}
}
const macadamSpout: Spout<
[(OpenCLBuffer | RedioEnd | undefined)?, (AudioBuffer | RedioEnd | undefined)?] | RedioEnd
> = async (frame) => {
if (isValue(frame)) {
const vidBuf = frame[0]
const audBuf = frame[1]
if (!(audBuf && isValue(audBuf) && vidBuf && isValue(vidBuf))) {
console.log('One-legged zipper:', audBuf, vidBuf)
if (vidBuf && isValue(vidBuf)) vidBuf.release()
return Promise.resolve()
}
const atb = this.audioTimebase
const ats = (audBuf.timestamp * atb[0]) / atb[1]
const vtb = this.videoTimebase
const vts = (vidBuf.timestamp * vtb[0]) / vtb[1]
if (Math.abs(ats - vts) > 0.1)
console.log('Macadam audio and video timestamp mismatch - aud:', ats, ' vid:', vts)
await this.waitHW()
await this.playback?.displayFrame(vidBuf, audBuf.buffer)
vidBuf.release()
return Promise.resolve()
} else {
// this.clContext.logBuffers()
return Promise.resolve()
}
}
combineVideo
.valve(vidProcess)
.valve(vidSaver)
.zip(combineAudio.valve(audFilter, { oneToMany: true }))
.spout(macadamSpout)
}
}
export class MacadamConsumerFactory implements ConsumerFactory {
private readonly clContext: nodenCLContext
constructor(clContext: nodenCLContext) {
this.clContext = clContext
}
createConsumer(
chanID: string,
params: ConfigParams,
format: VideoFormat,
device: DeviceConfig,
clJobs: ClJobs
): MacadamConsumer {
const consumer = new MacadamConsumer(this.clContext, chanID, params, format, device, clJobs)
return consumer
}
}