import * as _ from 'lodash'; // app import { config } from '../config'; import { CacheManager } from './cache'; import { CompositionSequenceType, getCompositionDefinition, IComposition, IEditRate, IFile, ISequence, IUser, } from './connect-api'; import { hasOwnProperty, Nullable } from '../utils/types'; export type CustomAudioTrack = [string, number[], string[]]; type CustomChannels = Record; type TrackCustomChannels = [number[], string[], string?]; export interface IPlaylistBuildInput { fileId: string; virtualTrackId?: string; definition?: IComposition; accessToken?: string; orgSlug: string; segmentSize?: number; customAudioTrack?: CustomAudioTrack[]; cacheLocation?: string; orgId?: string; preCache?: string | boolean; user?: IUser; buildCustomSegments?: boolean; } interface IAddCplSegmentsToTrackOptions { segmentSize: number, cacheLocation: string, orgSlug: string, user?: IUser, sequences: ISequence[], trackPlaylist: ITrackPlaylist, customChannels?: CustomChannels, } export interface ITrackPlaylists { [key: string]: ITrackPlaylist } interface ITrackPlaylist { editUnits: number; type: CompositionSequenceType; segments: TranscodingSegment[]; } export interface IPlaylist { trackPlaylists?: ITrackPlaylists; segments?: TranscodingSegment[]; type: Nullable; mediaPlayList: string; sourceFile: IFile; } export enum SegmentType { Image = 'image', Audio = 'audio', TimedText = 'timed_text', SubsegmentConcat = 'subsegment_concat', ExtractAudioChannels = 'extract_audio_channels', } export interface ISegment { id?: string; url?: string; fileId?: string; entryPoint?: number; editUnits: number; duration: number; totalStartTime: number; startTime?: number; cacheLocation: string; orgId: string; type: SegmentType; cacheBuster?: number; honeycombTraceId?: string; honeycombParentId?: string; urls?: string[]; s3Path?: string; // ToDo Looks useless. Investigate. user?: Pick; size?: number; billed?: boolean; } export interface IAudioSegment extends ISegment { url: string; startTime: number; channels: number; sampleRate: number; streamIndex: number; customChannels: Nullable; layout: Nullable; mxfIndexUrl?: string; } interface ICustomAudioSegmentInput { url: string; mxfIndexUrl?: string; streams: { [key: string]: { startTime: number; duration: number; channelsCount: number; channelIndices: number[]; channelLabels: string[]; fileId: string, sampleRate: number; }, }; } export interface ICustomAudioSegment extends ISegment{ inputs: ICustomAudioSegmentInput[]; } export interface IImageSegment extends ISegment { pixFmt: string; width: number; height: number; editRate: number; streamIndex: number; codec: string; bitDepth?: string; mxfIndexUrl?: string; firstFrameIndex?: number; } export interface ITimedTextSegment extends ISegment { editRate?: number; } // ToDo: we never transcode ITimedTextSegment. Legacy? export type TranscodingSegment = IImageSegment | IAudioSegment | ICustomAudioSegment | ITimedTextSegment; export interface ITranscodingTask { segment: TranscodingSegment; sourceFile: IFile; } export interface IBuild { playlist: IPlaylist; customSegments: Nullable; } export function isImageSegment(segment: ISegment): segment is IImageSegment { return segment.type === SegmentType.Image; } export function isAudioSegment(segment: ISegment): segment is IAudioSegment { return segment.type === SegmentType.Audio; } export function isTimedTextSegment(segment: ISegment): segment is ITimedTextSegment { return segment.type === SegmentType.TimedText; } export function isCustomAudioSegment(segment: ISegment): segment is ICustomAudioSegment { return segment.type === SegmentType.Audio && hasOwnProperty(segment, 'inputs') && Array.isArray(segment.inputs) && segment.inputs.length > 0; } // Todo Transform this to an actual class and create instance fields instead of passing by reference from one static // to another export class PlaylistBuilder { public static MERGE_SMALL_SEGMENTS_THRESHOLD = 0.4; // seconds public static async build(options: IPlaylistBuildInput): Promise { const { fileId, virtualTrackId, orgSlug, user, segmentSize = 7, // seconds customAudioTrack, preCache, buildCustomSegments = false, } = options; let { definition, cacheLocation } = options; if (!definition) { const trackIds = customAudioTrack ? _.map(customAudioTrack, (tr) => tr[0]) : []; const { compositionDefinition, org } = await getCompositionDefinition(fileId, orgSlug, trackIds); definition = compositionDefinition; cacheLocation = org.cacheLocation; await CacheManager.getInstance().setMaxStorage(orgSlug, org.maxCacheSize); } const playlist = PlaylistBuilder.buildPlaylist( definition, fileId, segmentSize, cacheLocation as string, orgSlug, user, virtualTrackId, customAudioTrack, preCache, ); let customPlaylist: Nullable = null; if (buildCustomSegments) { customPlaylist = {}; PlaylistBuilder.buildSegments( definition, customPlaylist, segmentSize, cacheLocation as string, orgSlug, user, ); } return { playlist, // all IAudioSegment[] customSegments: customPlaylist && Object.fromEntries( Object.entries(customPlaylist).filter(([, value]) => value.type === 'audio'), ), }; } public static buildSegments( definition: IComposition, trackPlaylists: ITrackPlaylists, segmentSize: number, cacheLocation: string, orgSlug: string, user?: IUser, customAudioTrack?: CustomAudioTrack[], ): void { for (const segment of definition.segments) { for (const sequence of segment.sequences) { if ( [CompositionSequenceType.Image, CompositionSequenceType.Audio, CompositionSequenceType.TimedText] .includes(sequence.type) ) { if (!trackPlaylists[sequence.virtualTrackId]) { trackPlaylists[sequence.virtualTrackId] = { editUnits: 0, type: sequence.type, segments: [], }; } PlaylistBuilder.addCplSegmentToTrackPlaylist({ segmentSize, cacheLocation, orgSlug, user, sequences: [sequence], trackPlaylist: trackPlaylists[sequence.virtualTrackId], }); } } if (customAudioTrack) { // build custom audio track const customTrackId = 'customAudio'; const virtualTrackIds = new Set(customAudioTrack.map((item) => item[0])); if (!trackPlaylists[customTrackId]) { trackPlaylists[customTrackId] = { editUnits: 0, type: CompositionSequenceType.Audio, segments: [], }; } const sequences = segment.sequences.filter((sequence) => virtualTrackIds.has(sequence.virtualTrackId)); const customChannels: CustomChannels = {}; for (const item of customAudioTrack) { customChannels[item[0]] = [item[1], item[2]]; } const customChannelsTrackIds = Object.keys(customChannels); for (const customChTrackId of customChannelsTrackIds) { const res = _.find(sequences, (s) => s.virtualTrackId === customChTrackId); if (res) { const resource = _.find(res.resources, (r) => r.trackId === customChTrackId); if (resource) { const chFileId = resource.track.fileId; customChannels[customChTrackId].push(chFileId); } } } PlaylistBuilder.addCplSegmentToTrackPlaylist({ orgSlug, customChannels, segmentSize, cacheLocation, user, sequences, trackPlaylist: trackPlaylists[customTrackId], }); } } } public static buildPlaylist( definition: IComposition, fileId: string, segmentSize: number, cacheLocation: string, orgSlug: string, user?: IUser, virtualTrackId?: string, customAudioTrack?: CustomAudioTrack[], preCache?: string | boolean, ): IPlaylist { const trackPlaylists: ITrackPlaylists = {}; PlaylistBuilder.buildSegments(definition, trackPlaylists, segmentSize, cacheLocation, orgSlug, user, customAudioTrack); if (preCache) { let allSegments = _.chain(trackPlaylists).values().map((x) => x.segments).flatten() .value(); if (preCache === 'just_one_audio') { // when using _.get(,, [default_value]) ts compiler gets confused const imageSegments = _.get(_.find(_.values(trackPlaylists), ['type', 'image']), 'segments') ?? []; const audioSegments = _.get(_.find(_.values(trackPlaylists), ['type', 'audio']), 'segments') ?? []; allSegments = imageSegments.concat(audioSegments); } return { trackPlaylists, segments: allSegments, type: null, mediaPlayList: '', sourceFile: definition.file, }; } if (!virtualTrackId) { return PlaylistBuilder.buildMasterPlaylist(definition, fileId, trackPlaylists, customAudioTrack); } const playlist = []; const trackPlaylist = trackPlaylists[virtualTrackId]; playlist.push('#EXTM3U'); playlist.push('#EXT-X-VERSION:3'); playlist.push(`#EXT-X-TARGETDURATION:${segmentSize + 1}`); playlist.push('#EXT-X-MEDIA-SEQUENCE:0'); trackPlaylist.segments.forEach((segment, index) => { const duration = segment.duration; playlist.push(`#EXTINF:${duration}`); if (virtualTrackId !== 'customAudio') { playlist.push(`${config.apiHost}/segment?fileId=${fileId}&virtualTrackId=${virtualTrackId}&index=${index}`); } else { playlist.push(`${config.apiHost}/segment?fileId=${fileId}&virtualTrackId=${virtualTrackId}&customAudioTrack=${encodeURIComponent(JSON.stringify(customAudioTrack))}&index=${index}`); } }); playlist.push('#EXT-X-ENDLIST'); return { segments: trackPlaylist.segments, type: trackPlaylist.type, mediaPlayList: playlist.join('\n'), sourceFile: definition.file, }; } public static addCplSegmentToTrackPlaylist(options: IAddCplSegmentsToTrackOptions): void { const { sequences, trackPlaylist, cacheLocation, orgSlug, user, customChannels = null, } = options; let { segmentSize } = options; if (trackPlaylist.type === 'timed_text') { segmentSize = 10000000; } if (trackPlaylist.type === 'audio') { segmentSize *= (47 * 1024) / 48000; } // entryPoint: virtualTrackId -> entryPoint // resourceIndex: virtualTrackId - > resourceIndex const resourceIndex: Record = {}; const resourceProcessedEditUnits: Record = {}; for (const sequence of sequences) { resourceIndex[sequence.virtualTrackId] = 0; resourceProcessedEditUnits[sequence.virtualTrackId] = 0; // maybe rename to allResourcesProcessedEditUnits } const editRate = sequences[0].resources[0].editRate as IEditRate; const editUnitDuration = editRate.denominator / editRate.numerator; let cplSegmentDuration = 0; for (const resource of sequences[0].resources) { cplSegmentDuration += resource.sourceDuration; } let processedEditUnits = 0; while (processedEditUnits < cplSegmentDuration) { // next resource end let nextResourceEndDuration = Math.round(segmentSize * (editRate.numerator / editRate.denominator)); // move out of while for (const sequence of sequences) { const id = sequence.virtualTrackId; const resource = sequence.resources[resourceIndex[id]]; nextResourceEndDuration = Math.min( nextResourceEndDuration, resource.sourceDuration - resourceProcessedEditUnits[id], ); } const hlsSegments: ISegment[] = []; let isFirstSegmentOfResource = false; // true if at least one segment starts at the beginning of a resource for (const sequence of sequences) { const id = sequence.virtualTrackId; const resource = sequence.resources[resourceIndex[id]]; const entryPoint = resource.entryPoint + resourceProcessedEditUnits[id]; /** * Cache Signatures * baseLength should be the segment length expressed in frames rounded for the case where the fps is * non integer value * the calculation below normalizes the first segment and (ergo) the segments after to always start and end * at fixed intervals therefore the cache should not invalidate if we change the segmentEntryPoint */ if (resourceProcessedEditUnits[id] === 0) { const baseLength = nextResourceEndDuration; // hls segment size const normalizedSegmentEnd = resource.entryPoint + (baseLength - (resource.entryPoint % baseLength)); nextResourceEndDuration = baseLength - ((resource.entryPoint + baseLength - normalizedSegmentEnd)); // can be simplified, represents duration until we hit the next hls segment } isFirstSegmentOfResource = isFirstSegmentOfResource || (resourceProcessedEditUnits[id] === 0); // Some formats' start time is not 0, for example mpegts // Seems that for audio this is not necessary let startTimeOffset = Number.parseFloat(resource.track.properties.probeResult.startTime) || 0; if (trackPlaylist.type !== CompositionSequenceType.Image) { startTimeOffset = 0; } const segment: ISegment = { user, cacheLocation, orgId: orgSlug, fileId: resource.track.fileId, url: resource.trackFileLocator.url, entryPoint, editUnits: nextResourceEndDuration, duration: nextResourceEndDuration * editUnitDuration, totalStartTime: trackPlaylist.editUnits * editUnitDuration, // maybe rename to totalTimeSinceStart startTime: startTimeOffset + entryPoint * editUnitDuration, type: trackPlaylist.type as unknown as SegmentType, }; if (isImageSegment(segment)) { segment.pixFmt = resource.track.properties.probeResult.pixFmt; segment.width = resource.track.properties.probeResult.width; segment.height = resource.track.properties.probeResult.fieldHeight ? resource.track.properties.probeResult.fieldHeight : resource.track.properties.probeResult.height; segment.editRate = (resource.editRate as IEditRate).numerator / (resource.editRate as IEditRate).denominator; segment.streamIndex = resource.track?.properties?.probeResult?.index ?? 0; segment.codec = resource.track?.properties?.probeResult?.codecName ?? undefined; segment.bitDepth = resource.track?.properties?.probeResult?.bitsPerRawSample ?? undefined; segment.mxfIndexUrl = (resource.track?.file?.mxfIndexLocator?.url || null) as string; segment.firstFrameIndex = sequence.firstFrameIndex != null ? sequence.firstFrameIndex : undefined; } if (isAudioSegment(segment)) { segment.channels = resource.track.properties.probeResult.channels; segment.sampleRate = Number.parseInt((resource.editRate?.numerator as unknown as string), 10); segment.streamIndex = resource.track?.properties?.probeResult?.index ?? 0; segment.customChannels = customChannels ? customChannels[id] : null; segment.layout = resource.track.properties.probeResult.channelLayout || null; segment.mxfIndexUrl = (resource.track?.file?.mxfIndexLocator?.url || null) as string; } if (isTimedTextSegment(segment)) { segment.editRate = (resource.editRate as IEditRate).numerator / (resource.editRate as IEditRate).denominator; } hlsSegments.push(segment); resourceProcessedEditUnits[id] += nextResourceEndDuration; if (resourceProcessedEditUnits[id] === resource.sourceDuration) { resourceIndex[sequence.virtualTrackId] += 1; resourceProcessedEditUnits[sequence.virtualTrackId] = 0; } } let hlsSegment: TranscodingSegment; if (hlsSegments.length === 1 && !(hlsSegments[0] as IAudioSegment).customChannels) { hlsSegment = hlsSegments[0]; } else { // create a custom audio segment const inputs: ICustomAudioSegmentInput[] = []; for (const segment of (hlsSegments as IAudioSegment[])) { const streams: Record = {}; streams[segment.streamIndex] = { startTime: segment.startTime, duration: segment.duration, channelsCount: segment.channels, sampleRate: segment.sampleRate, channelIndices: (segment.customChannels as TrackCustomChannels)[0], channelLabels: (segment.customChannels as TrackCustomChannels)[1], fileId: (segment.customChannels as TrackCustomChannels)[2] || null, layout: segment.layout, }; inputs.push({ streams, // ToDo This is just one stream. For every segment, we push one stream to the inputs array! url: segment.url, mxfIndexUrl: segment.mxfIndexUrl, }); } // ToDo Why wouldn't we just do hlsSegment = { ...hlsSegments[0], inputs }. This way, we have all info for // custom audio channels segments, and we wouldn't need to send playlist.customSegments, since we have all info here... hlsSegment = { user, inputs, cacheLocation, orgId: orgSlug, totalStartTime: hlsSegments[0].totalStartTime, duration: hlsSegments[0].duration, editUnits: hlsSegments[0].editUnits, type: hlsSegments[0].type, }; } // merge `hlsSegment` with the previous segment if its duration is < MERGE_SMALL_SEGMENTS_THRESHOLD and // they use the same resources if (hlsSegment.duration < PlaylistBuilder.MERGE_SMALL_SEGMENTS_THRESHOLD && !isFirstSegmentOfResource) { const prevSegment = trackPlaylist.segments[trackPlaylist.segments.length - 1]; prevSegment.duration += hlsSegment.duration; prevSegment.editUnits += hlsSegment.editUnits; } else { trackPlaylist.segments.push(hlsSegment); } processedEditUnits += nextResourceEndDuration; trackPlaylist.editUnits += nextResourceEndDuration; } } private static buildMasterPlaylist( definition: IComposition, fileId: string, trackPlaylists: ITrackPlaylists, customAudioTrack?: CustomAudioTrack[], ) { const playlist = []; playlist.push('#EXTM3U'); const audioTracks = []; const timedTextTracks = []; const imageTracks = []; for (const playlistVirtualTrackId of Object.keys(trackPlaylists)) { const virtualTrackPlaylist = trackPlaylists[playlistVirtualTrackId]; // If this track is audio with unknown channel layout, we don't add it to the playlist if (virtualTrackPlaylist.type === CompositionSequenceType.Audio && !virtualTrackPlaylist.segments.some( (segment) => isAudioSegment(segment) && segment.channels != null && ![6, 2, 1].includes(segment.channels), )) { audioTracks.push(playlistVirtualTrackId); } if (virtualTrackPlaylist.type === CompositionSequenceType.TimedText) { timedTextTracks.push(playlistVirtualTrackId); } if (virtualTrackPlaylist.type === CompositionSequenceType.Image) { imageTracks.push(playlistVirtualTrackId); } } const alternateMediaRefs = `${audioTracks.length ? ',AUDIO="audio"' : ''}${timedTextTracks.length ? ',SUBTITLES="imsc"' : ''}`; for (let i = 0; i < audioTracks.length; i += 1) { const virtualAudioTrackId = audioTracks[i]; // ToDo Investigate. As far as I've seen customAudio virtualTrackId gets created only when we have // customAudioTrack track set. if (!customAudioTrack || virtualAudioTrackId === 'customAudio') { let uri = `${config.apiHost}/media-playlist?fileId=${fileId}&virtualTrackId=${virtualAudioTrackId}`; if (virtualAudioTrackId === 'customAudio') { uri += `&customAudioTrack=${encodeURIComponent(JSON.stringify(customAudioTrack))}`; } // If it's the last one we specify it as the stream it needs to play. playlist.push(`#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",LANGUAGE="eng",NAME="English",AUTOSELECT=YES, DEFAULT=YES,URI="${uri}"`); if (i === audioTracks.length - 1 && imageTracks.length === 0) { playlist.push(`#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=195023,CODECS="mp4a.40.2,stpp.TTML.im1t"${alternateMediaRefs}`); playlist.push(uri); } } } for (const virtualTimeTextTrackId of timedTextTracks) { const uri = `${config.apiHost}/media-playlist?fileId=${fileId}&virtualTrackId=${virtualTimeTextTrackId}`; playlist.push(`#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="imsc",LANGUAGE="eng",NAME="English", URI="${uri}"`); } for (const virtualImageTrackId of imageTracks) { const uri = `${config.apiHost}/media-playlist?fileId=${fileId}&virtualTrackId=${virtualImageTrackId}`; playlist.push(`#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=195023,CODECS="avc1.4d0028,mp4a.40.2,stpp.TTML.im1t"${alternateMediaRefs}`); playlist.push(uri); } return { type: 'master', mediaPlayList: playlist.join('\n'), sourceFile: definition.file, }; } }