/*! * Copyright (c) 2026-present, Vanilagy and contributors * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ import { AudioCodec, MediaCodec, PCM_AUDIO_CODECS, VideoCodec } from './codec'; import { determineVideoPacketType } from './codec-data'; import { customAudioDecoders, customVideoDecoders } from './custom-coder'; import { Input } from './input'; import { Logging } from './logging'; import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink'; import { assert, extractRotationFromMatrix, isNumber, isThenable, matrixIsFlipped, MaybePromise, Rational, roundToDivisor, simplifyRational, TransformationMatrix, } from './misc'; import { TrackType } from './output'; import { EncodedPacket, PacketType } from './packet'; import { TrackDisposition } from './metadata'; import { DurationMetadataRequestOptions } from './demuxer'; /** * Contains aggregate statistics about the encoded packets of a track. * @group Input files & tracks * @public */ export type PacketStats = { /** The total number of packets. */ packetCount: number; /** The average number of packets per second. For video tracks, this will equal the average frame rate (FPS). */ averagePacketRate: number; /** The average number of bits per second. */ averageBitrate: number; }; /** * Computed metrics about the frame rate of a video track. * @group Input files & tracks * @public */ export type FrameRateMetrics = { /** * The true, underlying frame rate of the video that produced the frame timestamps. This value is heuristically * determined by examining the time differences between frames, excluding outliers, and then fitting them to * fractions. This algorithm is stable even when frames are dropped, as long as all frames lie roughly on one * uniform lattice. * * If this field is not `null`, Mediabunny is extremely confident in having found the actual frame rate of * the video. * * This field is `null` for videos where no consistent underlying frame rate was detected, indicating a variable * frame-rate (VFR) video. */ underlyingFrameRate: number | null; /** * Mediabunny's best guess for the video's actual intended frame rate. * * This value is determined heuristically. If `underlyingFrameRate` exists, this field will be equal to it. If it * doesn't, Mediabunny will check if `medianFrameRate` is close to a "common" frame rate and if so, snap to it. * Otherwise, this field simply falls back to `medianFrameRate`. */ bestGuessFrameRate: number; /** * The minimum frame rate of the video at any given point, based on the largest distance between two * consecutive frames. */ minFrameRate: number; /** * The maximum frame rate of the video at any given point, based on the smallest distance between two * consecutive frames. */ maxFrameRate: number; /** * The average frame rate across the duration of the probed packets. This value represents the average frames per * second; it is not the average of frame rates across the video. */ averageFrameRate: number; /** * The median frame rate, as determined by the distance between any two consecutive frames. For variable frame-rate * (VFR) videos, this field gives you a good idea of the _de facto_ frame rate of the video. */ medianFrameRate: number; /** Is `true` only when the underlying video is CFR (constant frame-rate) and no frames are skipped. */ frameRateIsConstant: boolean; /** * The number of packets that were probed/inspected to estimate the frame rate. Probe at least 256 packets for a * reliable estimation of frame rate. */ probedPacketCount: number; }; /** * Options for controlling how a video track's {@link FrameRateMetrics} are computed. * @group Input files & tracks * @public */ export type FrameRateMetricsOptions = { /** * The target number of packets to inspect to compute the frame rate. Defaults to `256`. Pass `Infinity` to scan * the entire file. */ targetPacketCount?: number; }; export interface InputTrackBacking { getType(): TrackType; getId(): number; getNumber(): number; getCodec(): MaybePromise; getInternalCodecId(): MaybePromise; getName(): MaybePromise; getLanguageCode(): MaybePromise; getTimeResolution(): MaybePromise; isRelativeToUnixEpoch(): MaybePromise; getUnixTimeForTimestamp(timestamp: number): MaybePromise; getDisposition(): MaybePromise; getPairingMask(): bigint; getBitrate(): MaybePromise; getAverageBitrate(): MaybePromise; getDurationFromMetadata(options: DurationMetadataRequestOptions): Promise; getLiveRefreshInterval(): Promise; getHasOnlyKeyPackets?(): MaybePromise; getDecoderConfig(): Promise; getMetadataCodecParameterString?(): MaybePromise; getFirstPacket(options: PacketRetrievalOptions): Promise; getPacket(timestamp: number, options: PacketRetrievalOptions): Promise; getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise; getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise; getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise; } /** * Represents a media track in an input file. * @group Input files & tracks * @public */ export abstract class InputTrack { /** The input file this track belongs to. */ readonly input: Input; /** @internal */ _backing: InputTrackBacking; /** @internal */ constructor(input: Input, backing: InputTrackBacking) { this.input = input; this._backing = backing; } /** The type of the track. */ abstract get type(): TrackType; /** Returns the codec of the track's packets. */ abstract getCodec(): Promise; /** * The codec of the track's packets. * @deprecated Use {@link InputTrack.getCodec} instead. */ // eslint-disable-next-line @typescript-eslint/no-deprecated abstract get codec(): MediaCodec | null; /** Returns the full codec parameter string for this track. */ abstract getCodecParameterString(): Promise; /** Checks if this track's packets can be decoded by the browser. */ abstract canDecode(): Promise; /** * For a given packet of this track, this method determines the actual type of this packet (key/delta) by looking * into its bitstream. Returns null if the type couldn't be determined. */ abstract determinePacketType(packet: EncodedPacket): Promise; /** * Returns whether the track metadata says that this track only contains key packets. The actual packets may * differ. */ abstract hasOnlyKeyPackets(): Promise; /** Returns true if and only if this track is a video track. */ isVideoTrack(): this is InputVideoTrack { return this instanceof InputVideoTrack; } /** Returns true if and only if this track is an audio track. */ isAudioTrack(): this is InputAudioTrack { return this instanceof InputAudioTrack; } /** The unique ID of this track in the input file. */ get id() { return this._backing.getId(); } /** * The 1-based index of this track among all tracks of the same type in the input file. For example, the first * video track has number 1, the second video track has number 2, and so on. The index refers to the order in * which the tracks are returned by {@link Input.getTracks}. */ get number() { return this._backing.getNumber(); } /** * Returns the identifier of the codec used internally by the container. It is not homogenized by Mediabunny * and depends entirely on the container format. * * This method can be used to determine the codec of a track in case Mediabunny doesn't know that codec. * * - For ISOBMFF files, this resolves to the name of the Sample Description Box (e.g. `'avc1'`). * - For Matroska files, this resolves to the value of the `CodecID` element. * - For WAVE files, this resolves to the value of the format tag in the `'fmt '` chunk. * - For ADTS files, this resolves to the `MPEG-4 Audio Object Type`. * - For MPEG-TS files, this resolves to the `streamType` value from the Program Map Table. * - In all other cases, this resolves to `null`. */ async getInternalCodecId() { return this._backing.getInternalCodecId(); } /** * See {@link InputTrack.getInternalCodecId}. * @deprecated Use {@link InputTrack.getInternalCodecId} instead. */ get internalCodecId() { return requireSync(this._backing.getInternalCodecId(), 'internalCodecId', 'getInternalCodecId'); } /** * Returns the ISO 639-2/T language code for this track. If the language is unknown, this resolves to `'und'` * (undetermined). */ async getLanguageCode() { return this._backing.getLanguageCode(); } /** * The ISO 639-2/T language code for this track. If the language is unknown, this field is `'und'` (undetermined). * @deprecated Use {@link InputTrack.getLanguageCode} instead. */ get languageCode() { return requireSync(this._backing.getLanguageCode(), 'languageCode', 'getLanguageCode'); } /** Returns the user-defined name for this track. */ async getName() { return this._backing.getName(); } /** * A user-defined name for this track. * @deprecated Use {@link InputTrack.getName} instead. */ get name() { return requireSync(this._backing.getName(), 'name', 'getName'); } /** * Returns a positive number x such that all timestamps and durations of all packets of this track are * integer multiples of 1/x. */ async getTimeResolution() { return this._backing.getTimeResolution(); } /** * A positive number x such that all timestamps and durations of all packets of this track are * integer multiples of 1/x. * @deprecated Use {@link InputTrack.getTimeResolution} instead. */ get timeResolution() { return requireSync(this._backing.getTimeResolution(), 'timeResolution', 'getTimeResolution'); } /** * Returns whether the timestamps of this track are relative to the Unix epoch (January 1, 1970 00:00:00 UTC). * When `true`, each timestamp maps to a definitive point in time. */ async isRelativeToUnixEpoch() { return this._backing.isRelativeToUnixEpoch(); } /** * Returns the Unix time (in seconds since January 1, 1970 00:00:00 UTC) that the given track timestamp (in seconds) * maps to, or `null` if there is no such mapping. This provides a piecewise-continuous mapping from this track's * timestamp space into wall-clock time. Such mapping exists, for example, for HLS playlists with * `#EXT-X-PROGRAM-DATE-TIME` tags present. * * This mapping can be available even when {@link InputTrack.isRelativeToUnixEpoch} is `false`, for example for HLS * streams with program date time information but with {@link HlsInputFormatOptions.offsetTimestampsByDateTime} * set to `false`. */ async getUnixTimeForTimestamp(timestamp: number): Promise { return this._backing.getUnixTimeForTimestamp(timestamp); } /** * Whether the track's timestamps can be mapped to Unix wall clock time via * {@link InputTrack.getUnixTimeForTimestamp}. */ async hasUnixTimeMapping(): Promise { return (await this._backing.getUnixTimeForTimestamp(await this.getFirstTimestamp())) !== null; } /** Returns the track's disposition, i.e. information about its intended usage. */ async getDisposition() { return this._backing.getDisposition(); } /** * The track's disposition, i.e. information about its intended usage. * @deprecated Use {@link InputTrack.getDisposition} instead. */ get disposition() { return requireSync(this._backing.getDisposition(), 'disposition', 'getDisposition'); } /** * Returns the peak bitrate of the track in bits per second, as specified in the track's metadata. This might not * match the actual media data's bitrate. */ async getBitrate() { return this._backing.getBitrate(); } /** * Returns the average bitrate of the track in bits per second, as specified in the track's metadata. This might * not match the actual media data's bitrate. */ async getAverageBitrate() { return this._backing.getAverageBitrate(); } /** * Returns the start timestamp of the first packet of this track, in seconds. While often near zero, this value * may be positive or even negative. A negative starting timestamp means the track's timing has been offset. Samples * with a negative timestamp should not be presented. */ async getFirstTimestamp() { const firstPacket = await this._backing.getFirstPacket({ metadataOnly: true }); return firstPacket?.timestamp ?? 0; } /** * Returns the end timestamp of the last packet of this track, in seconds. * * By default, when the underlying media is live, this method will only resolve once the live stream ends. If you * want to query the current end timestamp of the stream, set {@link PacketRetrievalOptions.skipLiveWait} to `true` * in the options. */ async computeDuration(options?: PacketRetrievalOptions) { const lastPacket = await this._backing.getPacket(Infinity, { metadataOnly: true, ...options }); const result = (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0); return roundToDivisor(result, await this.getTimeResolution()); } /** * Gets the duration (end timestamp) in seconds of this track from metadata stored in the file. This value may be * approximate or diverge from the actual, precise duration returned by `.computeDuration()`, but compared to that * method, this method is cheaper. When the duration cannot be determined from the file metadata, `null` * is returned. * * By default, when the underlying media is live, this method will only resolve once the live stream * ends. If you want to query the current duration of the media, set * {@link DurationMetadataRequestOptions.skipLiveWait} to `true` in the options. */ async getDurationFromMetadata(options: DurationMetadataRequestOptions = {}) { return this._backing.getDurationFromMetadata(options); } /** * Computes aggregate packet statistics for this track, such as average packet rate or bitrate. * * @param targetPacketCount - This optional parameter sets a target for how many packets this method must have * looked at before it can return early; this means, you can use it to aggregate only a subset (prefix) of all * packets. This is very useful for getting a great estimate of video frame rate without having to scan through the * entire file. * * By default, when the underlying media is live and `targetPacketCount` is not set, this method will only resolve * once the live stream ends. If you want to query the current packet statistics of the stream, set * {@link PacketRetrievalOptions.skipLiveWait} to `true` in the options. */ async computePacketStats(targetPacketCount = Infinity, options?: PacketRetrievalOptions): Promise { const sink = new EncodedPacketSink(this); let startTimestamp = Infinity; let endTimestamp = -Infinity; let packetCount = 0; let totalPacketBytes = 0; for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true, ...options })) { if ( packetCount >= targetPacketCount // This additional condition is needed to produce correct results with out-of-presentation-order packets && packet.timestamp >= endTimestamp ) { break; } startTimestamp = Math.min(startTimestamp, packet.timestamp); endTimestamp = Math.max(endTimestamp, packet.timestamp + packet.duration); packetCount++; totalPacketBytes += packet.byteLength; } return { packetCount, averagePacketRate: packetCount ? Number((packetCount / (endTimestamp - startTimestamp)).toPrecision(16)) : 0, averageBitrate: packetCount ? Number((8 * totalPacketBytes / (endTimestamp - startTimestamp)).toPrecision(16)) : 0, }; } /** * Whether or not this track is currently live, meaning the media's end is still unknown. * * The value returned by this method may change over time as the track stops being live. To keep track of the * track's live status, poll this method at the track's refresh interval * via {@link InputTrack.getLiveRefreshInterval}. */ async isLive() { return (await this._backing.getLiveRefreshInterval()) !== null; } /** * Returns the track's live refresh interval in seconds, or `null` if the track is not live. This interval describes * the time it takes, on average, for new live media data to become available. */ async getLiveRefreshInterval() { return this._backing.getLiveRefreshInterval(); } /** * Returns `true` if this track can be paired with the given track. Two tracks being pairable means they can be * presented (displayed) together. * * Returns `false` if `other` equals `this`. */ canBePairedWith(other: InputTrack) { if (!(other instanceof InputTrack)) { throw new TypeError('other must be an InputTrack.'); } if (this.input !== other.input || this === other) { return false; } return (this._backing.getPairingMask() & other._backing.getPairingMask()) !== 0n; } /** * Gets the list of other tracks that can be paired with this track. An optional query can be provided to narrow * down the results. */ async getPairableTracks(query?: InputTrackQuery) { return this.input.getTracks(mergeInputTrackQueries({ filter: t => t.canBePairedWith(this), }, query)); } /** * Gets the list of other video tracks that can be paired with this track. An optional query can be provided to * narrow down the results. */ async getPairableVideoTracks(query?: InputTrackQuery) { return this.input.getVideoTracks(mergeInputTrackQueries({ filter: t => t.canBePairedWith(this), }, query)); } /** * Gets the list of other audio tracks that can be paired with this track. An optional query can be provided to * narrow down the results. */ async getPairableAudioTracks(query?: InputTrackQuery) { return this.input.getAudioTracks(mergeInputTrackQueries({ filter: t => t.canBePairedWith(this), }, query)); } /** Returns the primary track that can be paired with this track, optionally steered by the provided query. */ async getPrimaryPairableVideoTrack(query?: InputTrackQuery) { return this.input.getPrimaryVideoTrack(mergeInputTrackQueries({ filter: t => t.canBePairedWith(this), }, query)); } /** Returns the primary track that can be paired with this track, optionally steered by the provided query. */ async getPrimaryPairableAudioTrack(query?: InputTrackQuery) { return this.input.getPrimaryAudioTrack(mergeInputTrackQueries({ filter: t => t.canBePairedWith(this), }, query)); } /** Returns `true` if there is another track that can be paired with this track. */ async hasPairableTrack(predicate?: (track: InputTrack) => MaybePromise): Promise { predicate &&= toValidatedPredicate(predicate); const tracks = await this.input.getTracks(); for (const track of tracks) { if (!this.canBePairedWith(track)) { continue; } if (!predicate || await predicate(track)) { return true; } } return false; } /** Returns `true` if there is a video track that can be paired with this track. */ hasPairableVideoTrack(predicate?: (track: InputVideoTrack) => MaybePromise): Promise { predicate &&= toValidatedPredicate(predicate); return this.hasPairableTrack(async x => x.isVideoTrack() && (!predicate || await predicate(x)), ); } /** Returns `true` if there is an audio track that can be paired with this track. */ hasPairableAudioTrack(predicate?: (track: InputAudioTrack) => MaybePromise): Promise { predicate &&= toValidatedPredicate(predicate); return this.hasPairableTrack(async x => x.isAudioTrack() && (!predicate || await predicate(x)), ); } } const requireSync = (value: MaybePromise, getterName: string, asyncName: string): T => { if (isThenable(value)) { throw new Error( `'${getterName}' is deprecated and not available synchronously for this track. Use the preferred` + ` '${asyncName}()' instead.`, ); } return value; }; const toValidatedPredicate = ( predicate?: (track: T) => MaybePromise, ) => { if (predicate !== undefined && typeof predicate !== 'function') { throw new TypeError('predicate, when provided, must be a function.'); } return predicate ? (track: T) => { const handle = (result: boolean) => { if (typeof result !== 'boolean') { throw new TypeError('predicate must return or resolve to a boolean value.'); } return result; }; const result = predicate(track); if (isThenable(result)) { return result.then(handle); } return handle(result); } : undefined; }; export interface InputVideoTrackBacking extends InputTrackBacking { getType(): 'video'; getCodec(): MaybePromise; getCodedWidth(): MaybePromise; getCodedHeight(): MaybePromise; getSquarePixelWidth(): MaybePromise; getSquarePixelHeight(): MaybePromise; getMetadataDisplayWidth?(): MaybePromise; getMetadataDisplayHeight?(): MaybePromise; getTransformationMatrix(): MaybePromise; getColorSpace(): Promise; canBeTransparent(): Promise; getDecoderConfig(): Promise; } /** * Represents a video track in an input file. * @group Input files & tracks * @public */ export class InputVideoTrack extends InputTrack { /** @internal */ declare _backing: InputVideoTrackBacking; /** @internal */ _pixelAspectRatioCache: Rational | null = null; /** @internal */ constructor(input: Input, backing: InputVideoTrackBacking) { super(input, backing); } get type(): TrackType { return 'video'; } /** The codec of the track's packets. */ async getCodec(): Promise { return this._backing.getCodec(); } /** * The codec of the track's packets. * @deprecated Use {@link InputVideoTrack.getCodec} instead. */ get codec(): VideoCodec | null { return requireSync(this._backing.getCodec(), 'codec', 'getCodec'); } async hasOnlyKeyPackets() { return (await this._backing.getHasOnlyKeyPackets?.()) ?? (await this._backing.getCodec() === 'prores'); // Only ProRes is fully intra-frame } /** Returns the width in pixels of the track's coded samples, before any transformations or rotations. */ async getCodedWidth() { return this._backing.getCodedWidth(); } /** * The width in pixels of the track's coded samples, before any transformations or rotations. * @deprecated Use {@link InputVideoTrack.getCodedWidth} instead. */ get codedWidth() { return requireSync(this._backing.getCodedWidth(), 'codedWidth', 'getCodedWidth'); } /** Returns the height in pixels of the track's coded samples, before any transformations or rotations. */ async getCodedHeight() { return this._backing.getCodedHeight(); } /** * The height in pixels of the track's coded samples, before any transformations or rotations. * @deprecated Use {@link InputVideoTrack.getCodedHeight} instead. */ get codedHeight() { return requireSync(this._backing.getCodedHeight(), 'codedHeight', 'getCodedHeight'); } /** * Returns the row-major 3x3 affine transformation matrix that is used to transform the raw video frames before they * are displayed. The raw frames are assumed to be placed with their top-left corner at the origin (0,0). * * Usually, this matrix models a rotation and a flip. Depending on the file, it may also contain a translation * component. When displaying video, you should typically ignore the translation component and display the center of * the transformed frame in the center of the viewport. */ async getTransformationMatrix() { return this._backing.getTransformationMatrix(); } /** * Returns the angle in degrees by which the track's frames should be rotated (clockwise). The rotation is * applied before flipping. */ async getRotation() { return extractRotationFromMatrix(await this._backing.getTransformationMatrix()); } /** * The angle in degrees by which the track's frames should be rotated (clockwise). * @deprecated Use {@link InputVideoTrack.getRotation} instead. */ get rotation() { return extractRotationFromMatrix( requireSync(this._backing.getTransformationMatrix(), 'rotation', 'getRotation'), ); } /** * Returns whether the track's frames should be flipped horizontally (about the vertical axis), after rotation. */ async getFlip() { return matrixIsFlipped(await this._backing.getTransformationMatrix()); } /** * Returns the width of the track's frames in square pixels, adjusted for pixel aspect ratio but before rotation. */ async getSquarePixelWidth() { return this._backing.getSquarePixelWidth(); } /** * The width of the track's frames in square pixels, adjusted for pixel aspect ratio but before rotation. * @deprecated Use {@link InputVideoTrack.getSquarePixelWidth} instead. */ get squarePixelWidth() { return requireSync(this._backing.getSquarePixelWidth(), 'squarePixelWidth', 'getSquarePixelWidth'); } /** * Returns the height of the track's frames in square pixels, adjusted for pixel aspect ratio but before rotation. */ async getSquarePixelHeight() { return this._backing.getSquarePixelHeight(); } /** * The height of the track's frames in square pixels, adjusted for pixel aspect ratio but before rotation. * @deprecated Use {@link InputVideoTrack.getSquarePixelHeight} instead. */ get squarePixelHeight() { return requireSync(this._backing.getSquarePixelHeight(), 'squarePixelHeight', 'getSquarePixelHeight'); } /** * Returns the pixel aspect ratio of the track's frames as a rational number in its reduced form. Most videos use * square pixels (1:1). */ async getPixelAspectRatio() { // Potential minor async race condition here if called twice, but doesn't matter since the computation is // so cheap return this._pixelAspectRatioCache ??= simplifyRational({ num: (await this.getSquarePixelWidth()) * (await this.getCodedHeight()), den: (await this.getSquarePixelHeight()) * (await this.getCodedWidth()), }); } /** * The pixel aspect ratio of the track's frames, as a rational number in its reduced form. Most videos use * square pixels (1:1). * @deprecated Use {@link InputVideoTrack.getPixelAspectRatio} instead. */ get pixelAspectRatio() { return this._pixelAspectRatioCache ??= simplifyRational({ num: requireSync(this._backing.getSquarePixelWidth(), 'pixelAspectRatio', 'getPixelAspectRatio') * requireSync(this._backing.getCodedHeight(), 'pixelAspectRatio', 'getPixelAspectRatio'), den: requireSync(this._backing.getSquarePixelHeight(), 'pixelAspectRatio', 'getPixelAspectRatio') * requireSync(this._backing.getCodedWidth(), 'pixelAspectRatio', 'getPixelAspectRatio'), }); } /** Returns the display width of the track's frames in pixels, after aspect ratio adjustment and rotation. */ async getDisplayWidth() { const metadata = await this._backing.getMetadataDisplayWidth?.(); if (metadata != null) { return metadata; } const rotation = await this.getRotation(); return rotation % 180 === 0 ? this.getSquarePixelWidth() : this.getSquarePixelHeight(); } /** * The display width of the track's frames in pixels, after aspect ratio adjustment and rotation. * @deprecated Use {@link InputVideoTrack.getDisplayWidth} instead. */ get displayWidth() { const metadataRaw = this._backing.getMetadataDisplayWidth?.(); if (metadataRaw !== undefined) { const metadata = requireSync(metadataRaw, 'displayWidth', 'getDisplayWidth'); if (metadata !== null) { return metadata; } } const rotation = extractRotationFromMatrix( requireSync(this._backing.getTransformationMatrix(), 'displayWidth', 'getDisplayWidth'), ); const value = rotation % 180 === 0 ? this._backing.getSquarePixelWidth() : this._backing.getSquarePixelHeight(); return requireSync(value, 'displayWidth', 'getDisplayWidth'); } /** Returns the display height of the track's frames in pixels, after aspect ratio adjustment and rotation. */ async getDisplayHeight() { const metadata = await this._backing.getMetadataDisplayHeight?.(); if (metadata != null) { return metadata; } const rotation = await this.getRotation(); return rotation % 180 === 0 ? this.getSquarePixelHeight() : this.getSquarePixelWidth(); } /** * The display height of the track's frames in pixels, after aspect ratio adjustment and rotation. * @deprecated Use {@link InputVideoTrack.getDisplayHeight} instead. */ get displayHeight() { const metadataRaw = this._backing.getMetadataDisplayHeight?.(); if (metadataRaw !== undefined) { const metadata = requireSync(metadataRaw, 'displayHeight', 'getDisplayHeight'); if (metadata !== null) { return metadata; } } const rotation = extractRotationFromMatrix( requireSync(this._backing.getTransformationMatrix(), 'displayHeight', 'getDisplayHeight'), ); const value = rotation % 180 === 0 ? this._backing.getSquarePixelHeight() : this._backing.getSquarePixelWidth(); return requireSync(value, 'displayHeight', 'getDisplayHeight'); } /** Returns the color space of the track's samples. */ async getColorSpace() { return this._backing.getColorSpace(); } /** If this method returns true, the track's samples use a high dynamic range (HDR). */ async hasHighDynamicRange() { const colorSpace = await this._backing.getColorSpace(); return (colorSpace.primaries as string) === 'bt2020' || (colorSpace.primaries as string) === 'smpte432' || (colorSpace.transfer as string) === 'pq' || (colorSpace.transfer as string) === 'hlg' || (colorSpace.matrix as string) === 'bt2020-ncl'; } /** Checks if this track may contain transparent samples with alpha data. */ async canBeTransparent() { return this._backing.canBeTransparent(); } /** * Returns the [decoder configuration](https://www.w3.org/TR/webcodecs/#video-decoder-config) for decoding the * track's packets using a [`VideoDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder). Returns * null if the track's codec is unknown. */ async getDecoderConfig() { return this._backing.getDecoderConfig(); } async getCodecParameterString() { const fromMetadata = await this._backing.getMetadataCodecParameterString?.(); if (fromMetadata != null) { return fromMetadata; } const decoderConfig = await this._backing.getDecoderConfig(); return decoderConfig?.codec ?? null; } async canDecode() { try { const decoderConfig = await this._backing.getDecoderConfig(); if (!decoderConfig) { return false; } const codec = await this._backing.getCodec(); assert(codec !== null); if (customVideoDecoders.some(x => x.supports(codec, decoderConfig))) { return true; } if (typeof VideoDecoder === 'undefined') { return false; } const support = await VideoDecoder.isConfigSupported(decoderConfig); return support.supported === true; } catch (error) { Logging._error('Error during decodability check:', error); return false; } } async determinePacketType(packet: EncodedPacket): Promise { if (!(packet instanceof EncodedPacket)) { throw new TypeError('packet must be an EncodedPacket.'); } if (packet.isMetadataOnly) { throw new TypeError('packet must not be metadata-only to determine its type.'); } const codec = await this.getCodec(); if (codec === null) { return null; } const decoderConfig = await this.getDecoderConfig(); assert(decoderConfig); return determineVideoPacketType(codec, decoderConfig, packet.data); } /** * Computes frame rate metrics for this video track, i.e. estimates the video's frame rate. Frame rate is never * determined from file metadata (which is unreliable) but is always deduced directly from the actual frame * timestamps. */ async computeFrameRateMetrics(options: FrameRateMetricsOptions = {}): Promise { if (!options || typeof options !== 'object') { throw new TypeError('options must be an object.'); } if ( options.targetPacketCount !== undefined && (!isNumber(options.targetPacketCount) || options.targetPacketCount < 0) ) { throw new TypeError('options.targetPacketCount must be a non-negative number.'); } const timeResolution = await this.getTimeResolution(); const targetPacketCount = options.targetPacketCount ?? 256; const sink = new EncodedPacketSink(this); const timestamps: number[] = []; let maxTimestamp = -Infinity; let probedPacketCount = 0; for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true })) { if ( timestamps.length >= targetPacketCount // Needed for out-of-presentation-order packets. && packet.timestamp >= maxTimestamp ) { break; } timestamps.push(packet.timestamp); maxTimestamp = Math.max(maxTimestamp, packet.timestamp); probedPacketCount++; } const ticks = new Float64Array(timestamps.length); for (let i = 0; i < timestamps.length; i++) { ticks[i] = Math.round(timestamps[i]! * timeResolution); } ticks.sort(); // Deduplicate in-place; only ticks[0..n) remains active. let n = 1; for (let i = 1; i < ticks.length; i++) { if (ticks[i] !== ticks[n - 1]) { ticks[n++] = ticks[i]!; } } if (n < 2) { return { underlyingFrameRate: null, bestGuessFrameRate: timeResolution, minFrameRate: timeResolution, maxFrameRate: timeResolution, averageFrameRate: timeResolution, medianFrameRate: timeResolution, frameRateIsConstant: true, probedPacketCount, }; } const activeTicks = ticks.subarray(0, n); const underlyingFrameRate = findUnderlyingFrameRate(activeTicks, timeResolution); // If an underlying frame rate exists, differences are expressed in inferred frame intervals. Otherwise they // remain raw timestamp ticks. const unitRate = underlyingFrameRate ?? timeResolution; const ticksPerFrame = underlyingFrameRate !== null ? timeResolution / underlyingFrameRate : null; const histogram = new Map(); let minDifference = Infinity; let maxDifference = -Infinity; let totalDifference = 0; for (let i = 1; i < n; i++) { const tickDifference = activeTicks[i]! - activeTicks[i - 1]!; const difference = ticksPerFrame !== null ? Math.max(1, Math.round(tickDifference / ticksPerFrame)) : tickDifference; histogram.set(difference, (histogram.get(difference) ?? 0) + 1); minDifference = Math.min(minDifference, difference); maxDifference = Math.max(maxDifference, difference); totalDifference += difference; } const differenceCount = n - 1; // Get the two middle differences so the even-sized case has a true median. const sortedDifferences = [...histogram.keys()].sort((a, b) => a - b); const middleA = (differenceCount - 1) >> 1; const middleB = differenceCount >> 1; let medianDifferenceA = 0; let medianDifferenceB = 0; let cumulativeCount = 0; for (const difference of sortedDifferences) { cumulativeCount += histogram.get(difference)!; if (medianDifferenceA === 0 && cumulativeCount > middleA) { medianDifferenceA = difference; } if (cumulativeCount > middleB) { medianDifferenceB = difference; break; } } // Median is defined over instantaneous frame rates, not durations. const medianFrameRate = ( unitRate / medianDifferenceA + unitRate / medianDifferenceB ) / 2; return { underlyingFrameRate, bestGuessFrameRate: underlyingFrameRate !== null ? underlyingFrameRate : getBestGuessFrameRate(medianFrameRate), minFrameRate: unitRate / maxDifference, maxFrameRate: unitRate / minDifference, averageFrameRate: unitRate * differenceCount / totalDifference, medianFrameRate, frameRateIsConstant: underlyingFrameRate !== null && minDifference === 1 && maxDifference === 1, probedPacketCount, }; } } export interface InputAudioTrackBacking extends InputTrackBacking { getType(): 'audio'; getCodec(): MaybePromise; getNumberOfChannels(): MaybePromise; getSampleRate(): MaybePromise; getDecoderConfig(): Promise; } /** * Represents an audio track in an input file. * @group Input files & tracks * @public */ export class InputAudioTrack extends InputTrack { /** @internal */ declare _backing: InputAudioTrackBacking; /** @internal */ constructor(input: Input, backing: InputAudioTrackBacking) { super(input, backing); } get type(): TrackType { return 'audio'; } /** The codec of the track's packets. */ async getCodec(): Promise { return this._backing.getCodec(); } /** * The codec of the track's packets. * @deprecated Use {@link InputAudioTrack.getCodec} instead. */ get codec(): AudioCodec | null { return requireSync(this._backing.getCodec(), 'codec', 'getCodec'); } async hasOnlyKeyPackets() { return (await this._backing.getHasOnlyKeyPackets?.()) ?? true; } /** Returns the number of audio channels in the track. */ async getNumberOfChannels() { return this._backing.getNumberOfChannels(); } /** * The number of audio channels in the track. * @deprecated Use {@link InputAudioTrack.getNumberOfChannels} instead. */ get numberOfChannels() { return requireSync(this._backing.getNumberOfChannels(), 'numberOfChannels', 'getNumberOfChannels'); } /** Returns the track's audio sample rate in hertz. */ async getSampleRate() { return this._backing.getSampleRate(); } /** * The track's audio sample rate in hertz. * @deprecated Use {@link InputAudioTrack.getSampleRate} instead. */ get sampleRate() { return requireSync(this._backing.getSampleRate(), 'sampleRate', 'getSampleRate'); } /** * Returns the [decoder configuration](https://www.w3.org/TR/webcodecs/#audio-decoder-config) for decoding the * track's packets using an [`AudioDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder). Returns * null if the track's codec is unknown. */ async getDecoderConfig() { return this._backing.getDecoderConfig(); } async getCodecParameterString() { const fromMetadata = await this._backing.getMetadataCodecParameterString?.(); if (fromMetadata != null) { return fromMetadata; } const decoderConfig = await this._backing.getDecoderConfig(); return decoderConfig?.codec ?? null; } async canDecode() { try { const decoderConfig = await this._backing.getDecoderConfig(); if (!decoderConfig) { return false; } const codec = await this._backing.getCodec(); assert(codec !== null); if (customAudioDecoders.some(x => x.supports(codec, decoderConfig))) { return true; } if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) { return true; // Since we decode it ourselves } else { if (typeof AudioDecoder === 'undefined') { return false; } const support = await AudioDecoder.isConfigSupported(decoderConfig); return support.supported === true; } } catch (error) { Logging._error('Error during decodability check:', error); return false; } } async determinePacketType(packet: EncodedPacket): Promise { if (!(packet instanceof EncodedPacket)) { throw new TypeError('packet must be an EncodedPacket.'); } if ((await this.getCodec()) === null) { return null; } return 'key'; // No audio codec with delta packets } } /** * Defines a query for input tracks. Can be used to query tracks tersely and expressively, which is especially useful * for media inputs with many tracks, such as HLS manifests. * * @group Input files & tracks * @public */ export type InputTrackQuery = { /** * A filter predicate function called for every track. Returning or resolving to `false` excludes the track from * the result. */ filter?: (track: T) => MaybePromise; /** * A function called for every track, used to define a track ordering. Tracks are ordered in ascending order using * the value returned by this function. When the function returns an array of numbers `arr`, tracks will be sorted * by `arr[0]` unless they have the same value, in which case they will be sorted by `arr[1]`, and so on. This * allows you to construct a list of ordering criteria, sorted by importance. * * To help construct complex ordering criteria, the {@link asc}, {@link desc}, and {@link prefer} helper functions * can be used. */ sortBy?: (track: T) => MaybePromise; }; /** * Helper function for use in {@link InputTrackQuery.sortBy}, used to describe sorting tracks by a numeric property in * ascending order. `null` and `undefined` are accepted too and are last in the order (sorted to the end). * * @group Input files & tracks * @public */ export const asc = (value: number | null | undefined) => { return value ?? Infinity; // nulls and undefined last }; /** * Helper function for use in {@link InputTrackQuery.sortBy}, used to describe sorting tracks by a numeric property in * descending order. `null` and `undefined` are accepted too and are last in the order (sorted to the end). * * @group Input files & tracks * @public */ export const desc = (value: number | null | undefined) => { return -(value ?? -Infinity); // nulls and undefined last }; /** * Helper function for use in {@link InputTrackQuery.sortBy}, used to sort tracks by boolean properties. `true` is * sorted to the start, `false` to the end. Useful for expressing soft preferences (e.g., "I'd prefer 1080p, but other * resolutions are fine too") as opposed to {@link InputTrackQuery.filter} which expresses hard requirements for * tracks. * * @group Input files & tracks * @public */ export const prefer = (value: boolean) => { return -value; }; export const toValidatedInputTrackQuery = ( query: InputTrackQuery, ): InputTrackQuery => { if (typeof query !== 'object' || !query) { throw new TypeError('query must be an object.'); } if (query.filter !== undefined && typeof query.filter !== 'function') { throw new TypeError('query.filter, when provided, must be a function.'); } if (query.sortBy !== undefined && typeof query.sortBy !== 'function') { throw new TypeError('query.sortBy, when provided, must be a function.'); } // Instead of validating the return types of the functions everywhere the query is used, simply return a new query // which wraps the old one while validating it. return { filter: query.filter ? (track) => { const handle = (bool: boolean) => { if (typeof bool !== 'boolean') { throw new TypeError('query.filter must return or resolve to a boolean.'); } return bool; }; const result = query.filter!(track); if (isThenable(result)) { return result.then(handle); } else { return handle(result); } } : undefined, sortBy: query.sortBy ? (track) => { const handle = (value: number | number[]) => { if ( typeof value !== 'number' && (!Array.isArray(value) || !value.every(x => typeof x === 'number')) ) { throw new TypeError( 'query.sortBy must return or resolve to a number or an array of numbers.', ); } return value; }; const result = query.sortBy!(track); if (isThenable(result)) { return result.then(handle); } else { return handle(result); } } : undefined, }; }; export const mergeInputTrackQueries = ( queryA: InputTrackQuery | undefined, queryB: InputTrackQuery | undefined, ): InputTrackQuery => { return { filter: queryA?.filter || queryB?.filter ? (track) => { const resultA = queryA?.filter?.(track) ?? true; const handleResultA = (resultA: boolean) => { if (resultA === false) { return false; } return queryB?.filter?.(track) ?? true; }; if (isThenable(resultA)) { return resultA.then(handleResultA); } else { return handleResultA(resultA); } } : undefined, sortBy: queryA?.sortBy || queryB?.sortBy ? (track) => { const resultA = queryA?.sortBy?.(track) ?? []; const resultB = queryB?.sortBy?.(track) ?? []; type Result = Awaited; const join = (resultA: Result, resultB: Result) => { return [ ...(Array.isArray(resultA) ? resultA : [resultA]), ...(Array.isArray(resultB) ? resultB : [resultB]), ]; }; if (isThenable(resultA) || isThenable(resultB)) { return Promise.all([resultA, resultB]).then(([resultA, resultB]) => { return join(resultA, resultB); }); } else { return join(resultA, resultB); } } : undefined, }; }; export const queryInputTracks = async ( tracks: T[], query?: InputTrackQuery, ): Promise => { let matched = tracks; if (query?.filter) { const filterMatches = tracks.map(t => query.filter!(t)); const hasAsyncFilter = filterMatches.some(x => isThenable(x)); if (hasAsyncFilter) { // eslint-disable-next-line @typescript-eslint/await-thenable const resolvedFilterMatches = await Promise.all(filterMatches); matched = tracks.filter((_, i) => resolvedFilterMatches[i]); } else { matched = tracks.filter((_, i) => filterMatches[i] as boolean); } } if (!query?.sortBy) { return matched; } const sortValues = matched.map(t => query.sortBy!(t)); const hasAsyncSort = sortValues.some(x => isThenable(x)); const resolvedSortValues = hasAsyncSort // eslint-disable-next-line @typescript-eslint/await-thenable ? await Promise.all(sortValues) : sortValues as (number | number[])[]; return matched .map((track, i) => ({ track, sortValue: resolvedSortValues[i] })) .sort((a, b) => { const aValues = Array.isArray(a.sortValue) ? a.sortValue : [a.sortValue]; const bValues = Array.isArray(b.sortValue) ? b.sortValue : [b.sortValue]; const maxLength = Math.max(aValues.length, bValues.length); for (let i = 0; i < maxLength; i++) { const aValue = aValues[i] ?? 0; const bValue = bValues[i] ?? 0; if (aValue === bValue) { continue; } return aValue - bValue; } return 0; }) .map(x => x.track); }; /** * Estimates the underlying CFR-like frame rate from timestamp deltas. * * Each delta is modeled as an integer multiple of one frame period, allowing dropped frames and a small number of * malformed timestamps. */ export const findUnderlyingFrameRate = ( ticks: Float64Array, resolution: number, ): number | null => { const MAX_DENOMINATOR = 1_000_000; const MIN_INLIER_RATIO = 0.98; const DELTA_TOLERANCE = 1 + 1e-9; const MAX_EFFECTIVE_FRAME_SPAN = 1000; const KNOWN_FRAME_RATES = [ 12, 15, 20, 24000 / 1001, 24, 25, 30000 / 1001, 30, 48, 50, 60000 / 1001, 60, 100, 120000 / 1001, 120, 144, 240, ]; if (ticks.length < 2) { return null; } const gaps = new Float64Array(ticks.length - 1); for (let i = 1; i < ticks.length; i++) { const gap = ticks[i]! - ticks[i - 1]!; if (!(gap > 0)) { return null; } gaps[i - 1] = gap; } // Start near the low end so dropped frames don't inflate the estimate, without letting one anomalously short gap // determine it. const sortedGaps = gaps.slice(); sortedGaps.sort(); let period = sortedGaps[Math.floor(sortedGaps.length * 0.05)]!; // Repeatedly infer how many frame periods each gap spans and refine the underlying period from the locally // consistent gaps. for (let iteration = 0; iteration < 6; iteration++) { let totalTicks = 0; let totalFrames = 0; for (const gap of gaps) { const multiple = Math.max(1, Math.round(gap / period)); if (Math.abs(gap - multiple * period) >= DELTA_TOLERANCE) { continue; } totalTicks += gap; totalFrames += multiple; } if (totalFrames === 0) { return null; } const refinedPeriod = totalTicks / totalFrames; if ( Math.abs(refinedPeriod - period) <= 1e-12 * Math.max(1, period) ) { period = refinedPeriod; break; } period = refinedPeriod; } let inlierCount = 0; let totalTicks = 0; let totalFrames = 0; for (const gap of gaps) { const multiple = Math.max(1, Math.round(gap / period)); if (Math.abs(gap - multiple * period) >= DELTA_TOLERANCE) { continue; } inlierCount++; totalTicks += gap; totalFrames += multiple; } if (inlierCount / gaps.length < MIN_INLIER_RATIO) { return null; } period = totalTicks / totalFrames; // Don't let arbitrarily long files force arbitrarily precise rational reconstruction from microscopic clock drift. const uncertainty = 1 / Math.min( totalFrames, MAX_EFFECTIVE_FRAME_SPAN, ); const periodLo = Math.max(Number.EPSILON, period - uncertainty); const periodHi = period + uncertainty; const fpsLo = resolution / periodHi; const fpsHi = resolution / periodLo; const fittedFps = resolution / period; // Prefer established frame rates when they're supported by the measured interval. If several fit, use the one // closest to the measured cadence. let fps: number | null = null; let bestKnownError = Infinity; for (const candidate of KNOWN_FRAME_RATES) { if (candidate < fpsLo || candidate > fpsHi) { continue; } const error = Math.abs(candidate / fittedFps - 1); if (error < bestKnownError) { fps = candidate; bestKnownError = error; } } if (fps === null) { // Otherwise, the natural fraction may be simpler either as FPS or as ticks/frame. const periodFraction = simplestFractionBetween( periodLo, periodHi, MAX_DENOMINATOR, ); const fpsFraction = simplestFractionBetween( fpsLo, fpsHi, MAX_DENOMINATOR, ); if ( fpsFraction && ( !periodFraction || fpsFraction.den < periodFraction.den || ( fpsFraction.den === periodFraction.den && fpsFraction.num <= periodFraction.num ) ) ) { fps = fpsFraction.num / fpsFraction.den; } else if (periodFraction) { fps = resolution * periodFraction.den / periodFraction.num; } else { return null; } } // Make sure the chosen rate still explains the deltas. const finalPeriod = resolution / fps; let finalInlierCount = 0; for (const gap of gaps) { const multiple = Math.max(1, Math.round(gap / finalPeriod)); if ( Math.abs(gap - multiple * finalPeriod) < DELTA_TOLERANCE ) { finalInlierCount++; } } if (finalInlierCount / gaps.length < MIN_INLIER_RATIO) { return null; } return fps; }; const simplestFractionBetween = ( lo: number, hi: number, maxDenominator: number, ): Rational | null => { for (let den = 1; den <= maxDenominator; den++) { const num = Math.floor(lo * den) + 1; if (num / den < hi) { return simplifyRational({ num, den }); } } return null; }; const getBestGuessFrameRate = (frameRate: number) => { const SPECIAL_FRAME_RATES = [ 24 / 1.001, 30 / 1.001, 60 / 1.001, 120 / 1.001, ]; const COMMON_FRAME_RATES = [ 12, 15, 20, 24, 25, 30, 48, 50, 60, 100, 120, 144, 240, ]; const SPECIAL_TOLERANCE = 0.0005; const COMMON_TOLERANCE = 0.025; for (const candidate of SPECIAL_FRAME_RATES) { if (Math.abs(candidate / frameRate - 1) <= SPECIAL_TOLERANCE) { return candidate; } } let best = frameRate; let bestError = Infinity; for (const candidate of COMMON_FRAME_RATES) { const error = Math.abs(candidate / frameRate - 1); if (error <= COMMON_TOLERANCE && error < bestError) { best = candidate; bestError = error; } } return best; };