/* * libmedia isobmff encoder * * 版权所有 (C) 2024 赵高兴 * Copyright (C) 2024 Gaoxing Zhao * * 此文件是 libmedia 的一部分 * This file is part of libmedia. * * libmedia 是自由软件;您可以根据 GNU Lesser General Public License(GNU LGPL)3.1 * 或任何其更新的版本条款重新分发或修改它 * libmedia is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 3.1 of the License, or (at your option) any later version. * * libmedia 希望能够为您提供帮助,但不提供任何明示或暗示的担保,包括但不限于适销性或特定用途的保证 * 您应自行承担使用 libmedia 的风险,并且需要遵守 GNU Lesser General Public License 中的条款和条件。 * libmedia is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * */ import OFormat from './OFormat' import type { AVOFormatContext } from '../AVFormatContext' import type { IsobmffContext, IsobmffStreamContext } from './isobmff/type' import createIsobmffContext from './isobmff/function/createIsobmffContext' import mktag from '../function/mktag' import * as oisobmff from './isobmff/oisobmff' import { BoxType, SampleFlags, TKHDFlags } from './isobmff/boxType' import createIsobmffStreamContext from './isobmff/function/createIsobmffStreamContext' import createFragmentTrack from './isobmff/function/createFragmentTrack' import updatePositionSize from './isobmff/function/updatePositionSize' import { Mp4FragmentMode, Mp4Mode } from './isobmff/isobmff' import rewriteIO from '../function/rewriteIO' import arrayItemSame from '../function/arrayItemSame' import Annexb2AvccFilter from '../bsf/h2645/Annexb2AvccFilter' import getSampleDuration from './isobmff/function/getSampleDuration' import { mapUint8Array } from '@libmedia/cheap' import { AVFormat, AVMediaType, AVCodecID, type AVPacket, type AVStream, type AVStreamMetadataEncryption, AVDisposition, AVStreamMetadataKey, createAVPacket, destroyAVPacket, getAVPacketData, getAVPacketSideData, getSideData, avQ2D, avRescaleQ2, AVPacketFlags, NOPTS_VALUE_BIGINT, AVPacketSideDataType, avRescaleQ, encryptionSideData2Info } from '@libmedia/avutil' import { UINT32_MAX, AV_MILLI_TIME_BASE_Q, AV_TIME_BASE, AV_TIME_BASE_Q, ac3 } from '@libmedia/avutil/internal' import { array, concatTypeArray, logger, is, object, bigint } from '@libmedia/common' import { IOWriterSync as IOWriter } from '@libmedia/common/io' export { Mp4FragmentMode, Mp4Mode } export interface OIsobmffFormatOptions { /** * fragment 按 gop 分段还是按帧分段 */ fragmentMode?: Mp4FragmentMode /** * mp4 还是 mov */ mp4Mode?: Mp4Mode /** * fragment 模式 */ fragment?: boolean /** * moov 放到文件开头 */ fastOpen?: boolean /** * data offset 基于 moof box(mse 使用) */ defaultBaseIsMoof?: boolean /** * 忽略 editlist box 的约束 */ ignoreEditlist?: boolean /** * drm 加密信息 */ encryption?: AVStreamMetadataEncryption /** * 保留 avcc 码流中的 sps,用于封装 sps 中途更改的流 */ reverseSpsInAvcc?: boolean /** * 忽略 drm 数据写入 */ ignoreEncryption?: boolean /** * fragment 最短时长(只有音频时使用,默认 5 秒) */ minFragmentLength?: number /** * fragment index 最短时长(只有音频时使用,默认 5 秒) */ minFragmentIndexLength?: number /** * fragment 结束时是否追加 tfra 用于 seek */ hasTfra?: boolean /** * Use mdta atom for metadata */ useMetadataTags?: boolean /** * 新的 fragment 开始 * * @returns */ onFragmentStart?: () => void } const defaultOptions: OIsobmffFormatOptions = { fragmentMode: Mp4FragmentMode.GOP, mp4Mode: Mp4Mode.MP4, fragment: false, fastOpen: false, defaultBaseIsMoof: false, reverseSpsInAvcc: false, ignoreEncryption: false, minFragmentLength: 5000, minFragmentIndexLength: 5000, hasTfra: true, useMetadataTags: false } export default class OIsobmffFormat extends OFormat { public type: AVFormat = AVFormat.ISOBMFF private context: IsobmffContext public options: OIsobmffFormatOptions private annexb2AvccFilter: Annexb2AvccFilter private avpacket: pointer constructor(options: OIsobmffFormatOptions = {}) { super() this.options = object.extend({}, defaultOptions, options) this.context = createIsobmffContext() } public init(formatContext: AVOFormatContext): number { formatContext.ioWriter.setEndian(true) formatContext.streams.forEach((stream) => { if (!this.annexb2AvccFilter && (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H264 || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_HEVC || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_VVC ) ) { this.annexb2AvccFilter = new Annexb2AvccFilter(this.options.reverseSpsInAvcc) this.annexb2AvccFilter.init(addressof(stream.codecpar), addressof(stream.timeBase)) } }) this.avpacket = createAVPacket() this.context.metadata = formatContext.metadata this.context.chapters = formatContext.chapters return 0 } public async destroy(formatContext: AVOFormatContext) { if (this.annexb2AvccFilter) { this.annexb2AvccFilter.destroy() this.annexb2AvccFilter = null } if (this.avpacket) { destroyAVPacket(this.avpacket) this.avpacket = nullptr } } /* * stream.disposition controls the "enabled" flag in the tkhd tag. * QuickTime will not play a track if it is not enabled. So make sure * that one track of each type (audio, video, subtitle) is enabled. * * Subtitles are special. For audio and video, setting "enabled" also * makes the track "default" (i.e. it is rendered when played). For * subtitles, an "enabled" subtitle is not rendered by default, but * if no subtitle is enabled, the subtitle menu in QuickTime will be * empty! */ private enableStreams(formatContext: AVOFormatContext) { const enabled = [] const first = [] for (let i = 0; i < AVMediaType.AVMEDIA_TYPE_NB; i++) { enabled[i] = 0 first[i] = -1 } array.each(formatContext.streams, (stream, index) => { if (stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_UNKNOWN || stream.codecpar.codecType >= AVMediaType.AVMEDIA_TYPE_NB || (stream.disposition & AVDisposition.ATTACHED_PIC) ) { return true } if (first[stream.codecpar.codecType] < 0) { first[stream.codecpar.codecType] = index } if (stream.disposition & AVDisposition.DEFAULT) { enabled[stream.codecpar.codecType]++ (stream.privData as IsobmffStreamContext).flags |= TKHDFlags.ENABLED } }) for (let i = 0; i < AVMediaType.AVMEDIA_TYPE_NB; i++) { switch (i) { case AVMediaType.AVMEDIA_TYPE_VIDEO: case AVMediaType.AVMEDIA_TYPE_AUDIO: case AVMediaType.AVMEDIA_TYPE_SUBTITLE: if (enabled[i] > 1) { (formatContext.streams[enabled[i]].privData as IsobmffStreamContext).perStreamGrouping = true } if (!enabled[i] && first[i] >= 0) { (formatContext.streams[first[i]].privData as IsobmffStreamContext).flags |= TKHDFlags.ENABLED } break } } } public writeHeader(formatContext: AVOFormatContext): number { this.context.majorBrand = mktag('isom') this.context.minorVersion = 512 this.context.compatibleBrand = [mktag('isom')] this.context.timescale = 1000 this.context.ignoreEncryption = this.options.ignoreEncryption this.context.useMetadataTags = this.options.useMetadataTags const cover = formatContext.streams.find((stream) => { return (stream.disposition & AVDisposition.ATTACHED_PIC) && (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_MJPEG || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_PNG || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_BMP ) }) if (cover && cover.attachedPic) { this.context.covr = { type: 0x1b, data: getAVPacketData(cover.attachedPic) } if (cover.codecpar.codecId === AVCodecID.AV_CODEC_ID_PNG) { this.context.covr.type = 0xe } else if (cover.codecpar.codecId === AVCodecID.AV_CODEC_ID_MJPEG) { this.context.covr.type = 0xd } } if (this.options.fragment) { this.context.compatibleBrand.push(mktag('iso6')) this.context.fragment = true } if (this.options.mp4Mode === Mp4Mode.MOV) { this.context.isom = true this.context.majorBrand = mktag('qt ') this.context.compatibleBrand = [this.context.majorBrand] } if (this.options.mp4Mode !== Mp4Mode.MOV) { this.context.compatibleBrand.push(mktag('iso2')) const videoStream = formatContext.getStreamByMediaType(AVMediaType.AVMEDIA_TYPE_VIDEO) if (videoStream && videoStream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H264) { this.context.compatibleBrand.push(mktag('avc1')) } this.context.compatibleBrand.push(mktag('mp41')) } oisobmff.writeFtyp(formatContext.ioWriter, this.context) this.context.holdMoovPos = formatContext.ioWriter.getPos() if (this.options.fragment) { this.context.currentFragment = { pos: 0n, currentTrack: null, sequence: 1, tracks: [], size: 0, firstWrote: false } array.each(formatContext.streams, (stream, index) => { const streamContext = createIsobmffStreamContext() stream.privData = streamContext streamContext.chunkOffsets = [] streamContext.cttsSampleCounts = [] streamContext.cttsSampleOffsets = [] streamContext.stscFirstChunk = [] streamContext.stscSamplesPerChunk = [] streamContext.stscSampleDescriptionIndex = [] streamContext.stssSampleNumbers = [] streamContext.sampleSizes = [] streamContext.sttsSampleCounts = [] streamContext.sttsSampleDeltas = [] streamContext.alternateGroup = index if (!(stream.disposition & AVDisposition.ATTACHED_PIC)) { const track = createFragmentTrack() track.baseIsMoof = this.options.defaultBaseIsMoof track.streamIndex = stream.index track.trackId = this.context.nextTrackId++ streamContext.trackId = track.trackId const encryption = this.options.encryption || stream.metadata[AVStreamMetadataKey.ENCRYPTION] as AVStreamMetadataEncryption if (encryption) { const cenc = this.context.cencs || {} cenc[track.trackId] = { schemeType: encryption.schemeType, schemeVersion: encryption.schemeVersion, cryptByteBlock: encryption.cryptByteBlock, skipByteBlock: encryption.skipByteBlock, defaultConstantIV: encryption.constantIV, isProtected: 1, defaultPerSampleIVSize: encryption.perSampleIVSize, defaultKeyId: encryption.kid, pattern: !!encryption.pattern } this.context.cencs = cenc } track.ioWriter = new IOWriter() track.ioWriter.onFlush = (data) => { track.buffers.push(data.slice()) return 0 } this.context.currentFragment.tracks.push(track) } }) this.enableStreams(formatContext) oisobmff.writeMoov(formatContext.ioWriter, formatContext, this.context) formatContext.ioWriter.flush() } else { array.each(formatContext.streams, (stream, index) => { const streamContext = createIsobmffStreamContext() stream.privData = streamContext streamContext.trackId = this.context.nextTrackId++ streamContext.chunkOffsets = [] streamContext.cttsSampleCounts = [] streamContext.cttsSampleOffsets = [] streamContext.stscFirstChunk = [] streamContext.stscSamplesPerChunk = [] streamContext.stscSampleDescriptionIndex = [] streamContext.stssSampleNumbers = [] streamContext.sampleSizes = [] streamContext.sttsSampleCounts = [] streamContext.sttsSampleDeltas = [] streamContext.alternateGroup = index }) this.enableStreams(formatContext) const pos = formatContext.ioWriter.getPos() formatContext.ioWriter.writeUint32(0) formatContext.ioWriter.writeUint32(mktag(BoxType.MDAT)) this.context.boxsPositionInfo.push({ pos, type: BoxType.MDAT, size: 0 }) } if (!formatContext.getStreamByMediaType(AVMediaType.AVMEDIA_TYPE_VIDEO)) { this.context.audioOnly = true } return 0 } private updateCurrentChunk(formatContext: AVOFormatContext) { let currentChunk = this.context.currentChunk if (!currentChunk || !currentChunk.sampleCount) { return } const prevStream = formatContext.streams.find((stream) => { return stream.index === currentChunk.streamIndex }) const prevIsobmffStreamContext = prevStream.privData as IsobmffStreamContext prevIsobmffStreamContext.chunkCount++ prevIsobmffStreamContext.chunkOffsets.push(currentChunk.pos) if (!prevIsobmffStreamContext.stscFirstChunk.length) { prevIsobmffStreamContext.stscFirstChunk.push(prevIsobmffStreamContext.chunkCount) prevIsobmffStreamContext.stscSamplesPerChunk.push(currentChunk.sampleCount) prevIsobmffStreamContext.stscSampleDescriptionIndex.push(1) prevIsobmffStreamContext.lastStscCount = currentChunk.sampleCount } else { if (prevIsobmffStreamContext.lastStscCount !== currentChunk.sampleCount) { prevIsobmffStreamContext.stscFirstChunk.push(prevIsobmffStreamContext.chunkCount) prevIsobmffStreamContext.stscSamplesPerChunk.push(currentChunk.sampleCount) prevIsobmffStreamContext.stscSampleDescriptionIndex.push(1) prevIsobmffStreamContext.lastStscCount = currentChunk.sampleCount } } } private updateCurrentFragment(formatContext: AVOFormatContext, currentDts?: int64) { if (this.context.currentFragment.firstWrote) { array.each(this.context.currentFragment.tracks, (track) => { const stream = formatContext.streams.find((stream) => { return stream.index === track.streamIndex }) if (!track.sampleCount || !stream) { return true } const streamContext = stream.privData as IsobmffStreamContext track.baseDataOffset = formatContext.ioWriter.getPos() this.context.currentFragment.pos = formatContext.ioWriter.getPos() if (!track.sampleDurations.length) { if (stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_AUDIO) { if (currentDts) { track.sampleDurations.push(static_cast(currentDts - streamContext.lastDts)) } else if (stream.codecpar.frameSize > 0) { track.sampleDurations.push(static_cast(avRescaleQ( static_cast((stream.codecpar.frameSize / stream.codecpar.sampleRate * AV_TIME_BASE) as double), AV_TIME_BASE_Q, stream.timeBase ))) } else if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_AAC) { track.sampleDurations.push(static_cast(avRescaleQ( static_cast((1024 / stream.codecpar.sampleRate * AV_TIME_BASE) as double), AV_TIME_BASE_Q, stream.timeBase ))) } else { // 随便猜一个?每帧一个 fragment 没有 sampleDuration QuickTime 无法播放 track.sampleDurations.push(static_cast(avRescaleQ( static_cast((1024 / stream.codecpar.sampleRate * AV_TIME_BASE) as double), AV_TIME_BASE_Q, stream.timeBase ))) } } else if (stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_VIDEO) { if (currentDts) { track.sampleDurations.push(static_cast(currentDts - streamContext.lastDts)) } else if (avQ2D(stream.codecpar.framerate) > 0) { track.sampleDurations.push(static_cast(avRescaleQ( static_cast(1 / avQ2D(stream.codecpar.framerate) * AV_TIME_BASE), AV_TIME_BASE_Q, stream.timeBase ))) } else { // 随便猜一个?每帧一个 fragment 没有 sampleDuration QuickTime 无法播放 // 取帧率 30 track.sampleDurations.push((stream.timeBase.den / (30 * stream.timeBase.num)) >>> 0) } } else { track.sampleDurations.push(0) } } else if (currentDts && track.sampleDurations.length === track.sampleSizes.length - 1) { track.sampleDurations.push(static_cast(currentDts - streamContext.lastDts)) } streamContext.lastDuration = track.sampleDurations[track.sampleSizes.length - 1] if (track.sampleFlags.length === 1 || arrayItemSame(track.sampleFlags, 1)) { track.firstSampleFlags = track.sampleFlags[0] track.defaultSampleFlags = track.sampleFlags[1] ?? track.firstSampleFlags track.sampleFlags = [] } if (track.sampleSizes.length === 1 || arrayItemSame(track.sampleSizes)) { track.defaultSampleSize = track.sampleSizes[0] track.sampleSizes = [] } if (track.sampleDurations.length === 1 || arrayItemSame(track.sampleDurations)) { track.defaultSampleDuration = track.sampleDurations[0] track.sampleDurations = [] } if (track.cenc) { if (track.cenc.sampleSizes.length === 1 || arrayItemSame(track.cenc.sampleSizes)) { track.cenc.defaultSampleInfoSize = track.cenc.sampleSizes[0] track.cenc.sampleSizes = [] } } if (stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_AUDIO) { track.defaultSampleFlags = SampleFlags.DEPENDS_NO } else if (track.sampleFlags.length) { track.defaultSampleFlags = track.sampleFlags[0] } if (track.sampleSizes.length) { track.defaultSampleSize = track.sampleSizes[0] } if (track.sampleDurations.length) { track.defaultSampleDuration = track.sampleDurations[0] } if (this.options.hasTfra) { if (!streamContext.fragIndexes.length || this.options.fragmentMode === Mp4FragmentMode.GOP || avRescaleQ(track.baseDataOffset - track.lastFragIndexDts, stream.timeBase, AV_MILLI_TIME_BASE_Q) >= this.options.minFragmentIndexLength ) { streamContext.fragIndexes.push({ pos: track.baseDataOffset, time: track.baseMediaDecodeTime + static_cast(track.sampleCompositionTimeOffset[0] ?? 0) }) track.lastFragIndexDts = streamContext.lastDts } } }) formatContext.ioWriter.flush() if (this.options.onFragmentStart) { this.options.onFragmentStart() } oisobmff.writeMoof(formatContext.ioWriter, formatContext, this.context) let dataOffset = this.context.currentFragment.size + 8 const buffers = [] let mdatSize = 8 array.each(this.context.currentFragment.tracks, (track) => { if (!track.sampleCount) { return true } track.ioWriter.flush() const buffer = concatTypeArray(Uint8Array, track.buffers) track.dataOffset = dataOffset dataOffset += buffer.length mdatSize += buffer.length buffers.push(buffer) rewriteIO(formatContext.ioWriter, track.dataOffsetPos, track.dataOffset, 'int32') if (track.cenc) { if (track.cenc.offsetPos) { rewriteIO(formatContext.ioWriter, track.cenc.offsetPos, track.cenc.offset, 'int32') } } track.buffers = [] track.sampleFlags = [] track.sampleSizes = [] track.sampleDurations = [] track.sampleCompositionTimeOffset = [] track.sampleCount = 0 track.firstSampleFlags = 0 track.cenc = null }) formatContext.ioWriter.writeUint32(mdatSize) formatContext.ioWriter.writeUint32(mktag(BoxType.MDAT)) array.each(buffers, (buffer) => { formatContext.ioWriter.writeBuffer(buffer) }) updatePositionSize(formatContext.ioWriter, this.context) formatContext.ioWriter.flush() this.context.currentFragment.firstWrote = false this.context.currentFragment.sequence++ } } private handleEAC3(avpacket: pointer, stream: AVStream) { if (!this.context.ac3Info) { this.context.ac3Info = { done: false, numBlocks: 0, dataRate: 0, ac3BitrateCode: -1, numIndSub: 0, substream: [] } } const ac3Info = this.context.ac3Info const info = ac3.parseHeader(mapUint8Array(avpacket.data, reinterpret_cast(avpacket.size))) if (is.number(info)) { ac3Info.done = true return } ac3Info.dataRate = Math.max(ac3Info.dataRate, info.bitrate / 1000) ac3Info.ac3BitrateCode = Math.max(ac3Info.ac3BitrateCode, info.ac3BitrateCode) if (!ac3Info.done) { if (info.bitstreamId <= 10 && info.substreamId != 0) { return } if (info.frameType === ac3.EAC3FrameType.EAC3_FRAME_TYPE_INDEPENDENT || info.frameType == ac3.EAC3FrameType.EAC3_FRAME_TYPE_AC3_CONVERT) { /* substream ids must be incremental */ if (info.substreamId > ac3Info.numIndSub + 1) { return } if (info.substreamId == ac3Info.numIndSub + 1) { return } else if (info.substreamId < ac3Info.numIndSub || info.substreamId == 0 && ac3Info.substream[0]?.bsid) { ac3Info.done = true return } } else { if (info.substreamId != 0) { return } } if (!ac3Info.substream[info.substreamId]) { ac3Info.substream[info.substreamId] = { fscod: 0, bsid: 0, bsmod: 0, acmod: 0, lfeon: 0, numDepSub: 0, chanLoc: 0 } } ac3Info.substream[info.substreamId].fscod = info.srCode ac3Info.substream[info.substreamId].bsid = info.bitstreamId ac3Info.substream[info.substreamId].bsmod = info.bitstreamMode ac3Info.substream[info.substreamId].acmod = info.channelMode ac3Info.substream[info.substreamId].lfeon = info.lfeOn if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_AC3) { ac3Info.done = true return } } } private writeTrackData(ioWriter: IOWriter, avpacket: pointer, dts: int64, stream: AVStream) { if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_WEBVTT) { const streamContext = stream.privData as IsobmffStreamContext if (streamContext.lastDuration > 0) { const lastCueEndTimestamp = streamContext.lastDts + static_cast(streamContext.lastDuration) // add empty sample vtte if (lastCueEndTimestamp < dts && avRescaleQ(dts - lastCueEndTimestamp, stream.timeBase, AV_MILLI_TIME_BASE_Q) > 200) { ioWriter.writeUint32(8) ioWriter.writeString(BoxType.VTTE) const duration = static_cast((dts - lastCueEndTimestamp) as int64) if (this.context.fragment) { const track = this.context.currentFragment.tracks.find((track) => { return track.streamIndex === avpacket.streamIndex }) track.sampleCount++ if (!track.sampleSizes.length) { track.baseMediaDecodeTime = lastCueEndTimestamp } track.sampleDurations.push(duration) track.sampleSizes.push(8) } else { const deltas = static_cast((lastCueEndTimestamp - streamContext.lastDts) as int64) this.context.currentChunk.sampleCount++ streamContext.sampleSizes.push(8) if (streamContext.sttsSampleDeltas[streamContext.sttsSampleDeltas.length - 1] === deltas) { streamContext.sttsSampleCounts[streamContext.sttsSampleCounts.length - 1]++ } else { streamContext.sttsSampleCounts.push(1) streamContext.sttsSampleDeltas.push(deltas) } } streamContext.lastPts = lastCueEndTimestamp streamContext.lastDts = lastCueEndTimestamp streamContext.lastDuration = duration } } const identifier = getAVPacketSideData(avpacket, AVPacketSideDataType.AV_PKT_DATA_WEBVTT_IDENTIFIER) const settings = getAVPacketSideData(avpacket, AVPacketSideDataType.AV_PKT_DATA_WEBVTT_SETTINGS) let size = avpacket.size + 8 if (identifier) { size += 8 + static_cast(identifier.size) } if (settings) { size += 8 + static_cast(settings.size) } ioWriter.writeUint32(8 + size) ioWriter.writeString(BoxType.VTTC) if (identifier) { ioWriter.writeUint32(8 + identifier.size) ioWriter.writeString('iden') ioWriter.writeBuffer(mapUint8Array(identifier.data, reinterpret_cast(identifier.size))) } if (settings) { ioWriter.writeUint32(8 + settings.size) ioWriter.writeString('sttg') ioWriter.writeBuffer(mapUint8Array(settings.data, reinterpret_cast(settings.size))) } ioWriter.writeUint32(8 + avpacket.size) ioWriter.writeString(BoxType.PAYL) ioWriter.writeBuffer(getAVPacketData(avpacket)) return 8 + size } else { ioWriter.writeBuffer(getAVPacketData(avpacket)) return avpacket.size } } public writeAVPacket(formatContext: AVOFormatContext, avpacket: pointer): number { if (!avpacket.size) { logger.warn(`packet\'s size is 0: ${avpacket.streamIndex}, ignore it`) return 0 } const stream = formatContext.getStreamByIndex(avpacket.streamIndex) if (!stream || (stream.disposition & AVDisposition.ATTACHED_PIC)) { logger.warn(`can not found the stream width the avpacket\'s streamIndex: ${avpacket.streamIndex}, ignore it`) return } const streamContext = stream.privData as IsobmffStreamContext let dts: int64 = avRescaleQ2(avpacket.dts, addressof(avpacket.timeBase), stream.timeBase) let pts: int64 = avRescaleQ2(avpacket.pts !== NOPTS_VALUE_BIGINT ? avpacket.pts : avpacket.dts, addressof(avpacket.timeBase), stream.timeBase) const duration: int64 = avpacket.duration !== NOPTS_VALUE_BIGINT ? avRescaleQ2(avpacket.duration, addressof(avpacket.timeBase), stream.timeBase) : NOPTS_VALUE_BIGINT if ((stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H264 || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_HEVC || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_VVC ) && (avpacket.flags & AVPacketFlags.AV_PKT_FLAG_H26X_ANNEXB) ) { this.annexb2AvccFilter.sendAVPacket(avpacket) this.annexb2AvccFilter.receiveAVPacket(this.avpacket) avpacket = this.avpacket } else if ((stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_AC3 || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_EAC3 ) && (!this.context.ac3Info || !this.context.ac3Info.done) ) { this.handleEAC3(avpacket, stream) } if (this.context.fragment) { const track = this.context.currentFragment.tracks.find((track) => { return track.streamIndex === avpacket.streamIndex }) if (track) { if (track.tfdtDelay === NOPTS_VALUE_BIGINT) { if (dts < 0) { track.tfdtDelay = -dts } else { track.tfdtDelay = 0n } if (pts < 0) { track.trunPtsDelay = track.tfdtDelay } } dts += track.tfdtDelay pts += track.trunPtsDelay if (this.options.fragmentMode === Mp4FragmentMode.GOP && (stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_VIDEO && avpacket.flags & AVPacketFlags.AV_PKT_FLAG_KEY || this.context.audioOnly && avRescaleQ(dts - track.baseMediaDecodeTime, stream.timeBase, AV_MILLI_TIME_BASE_Q) >= this.options.minFragmentLength ) || this.options.fragmentMode === Mp4FragmentMode.FRAME ) { if (this.context.currentFragment.tracks.length === 1) { this.updateCurrentFragment(formatContext, dts) } else { this.updateCurrentFragment(formatContext) } } const dataSize = this.writeTrackData(track.ioWriter, avpacket, dts, stream) if (!track.sampleSizes.length) { track.baseMediaDecodeTime = dts } if (track.sampleSizes.length && (!track.sampleDurations[track.sampleSizes.length - 1] || track.sampleDurations[track.sampleSizes.length - 1] <= 0 ) ) { track.sampleDurations[track.sampleSizes.length - 1] = static_cast(dts - streamContext.lastDts) } if (duration > 0) { track.sampleDurations.push(static_cast(duration)) } track.sampleSizes.push(dataSize) if (stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_VIDEO) { let flag = 0 if (avpacket.flags & AVPacketFlags.AV_PKT_FLAG_KEY) { flag |= SampleFlags.DEPENDS_NO } else { flag |= (SampleFlags.DEPENDS_YES | SampleFlags.IS_NON_SYN) } track.sampleCompositionTimeOffset.push(static_cast((pts !== NOPTS_VALUE_BIGINT ? pts : dts) - dts)) track.sampleFlags.push(flag) } const sideData = getSideData( addressof(avpacket.sideData), addressof(avpacket.sideDataElems), AVPacketSideDataType.AV_PKT_DATA_ENCRYPTION_INFO ) const cenc = this.context.cencs && this.context.cencs[streamContext.trackId] if (sideData && cenc) { const info = encryptionSideData2Info(mapUint8Array(sideData.data, sideData.size)) if (!track.cenc) { track.cenc = { sampleCount: 0, defaultSampleInfoSize: 0, sampleSizes: [], offset: 0, useSubsamples: false, sampleInfoOffset: [], sampleEncryption: [] } } if (info.subsamples.length) { if (!track.cenc.useSubsamples && track.cenc.sampleSizes.length) { track.cenc.sampleSizes = track.cenc.sampleSizes.map((value) => { return value + 2 }) } track.cenc.useSubsamples = true } track.cenc.sampleCount++ track.cenc.sampleSizes.push(cenc.defaultPerSampleIVSize + (track.cenc.useSubsamples ? (2 + info.subsamples.length * 6) : 0)) track.cenc.sampleEncryption.push({ iv: info.iv, subsamples: info.subsamples }) } track.sampleCount++ streamContext.lastPts = bigint.max(streamContext.lastPts, pts + (duration !== NOPTS_VALUE_BIGINT ? duration : 0n)) streamContext.lastDts = dts if (duration > 0) { streamContext.lastDuration = static_cast(duration) } this.context.currentFragment.firstWrote = true } else { logger.warn(`can not found track width streamIndex ${avpacket.streamIndex}, ignore it`) } } else { const pos = formatContext.ioWriter.getPos() let currentChunk = this.context.currentChunk if (!currentChunk) { currentChunk = this.context.currentChunk = { pos, streamIndex: avpacket.streamIndex, sampleCount: 1 } } else if (currentChunk.streamIndex !== avpacket.streamIndex) { this.updateCurrentChunk(formatContext) currentChunk.streamIndex = avpacket.streamIndex currentChunk.sampleCount = 1 currentChunk.pos = pos } else { currentChunk.sampleCount++ } streamContext.sampleSizes.push(this.writeTrackData(formatContext.ioWriter, avpacket, dts, stream)) if (stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_VIDEO && avpacket.flags & AVPacketFlags.AV_PKT_FLAG_KEY ) { streamContext.stssSampleNumbers.push(streamContext.sampleSizes.length) } if (!streamContext.firstWrote) { streamContext.startDts = dts streamContext.startCT = static_cast((pts !== NOPTS_VALUE_BIGINT ? pts : dts) - dts) streamContext.firstWrote = true } else { const deltas = static_cast(dts - streamContext.lastDts) if (streamContext.sttsSampleDeltas[streamContext.sttsSampleDeltas.length - 1] === deltas) { streamContext.sttsSampleCounts[streamContext.sttsSampleCounts.length - 1]++ } else { streamContext.sttsSampleCounts.push(1) streamContext.sttsSampleDeltas.push(deltas) } } if (pts >= 0) { if (streamContext.startPts === NOPTS_VALUE_BIGINT) { streamContext.startPts = pts } else { streamContext.startPts = bigint.min(streamContext.startPts, pts) } } if (stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_VIDEO) { const ctts = static_cast((pts !== NOPTS_VALUE_BIGINT ? pts : dts) - dts) if (!streamContext.cttsSampleCounts.length) { streamContext.cttsSampleCounts.push(1) streamContext.cttsSampleOffsets.push(ctts) } else { if (streamContext.cttsSampleOffsets[streamContext.cttsSampleOffsets.length - 1] === ctts) { streamContext.cttsSampleCounts[streamContext.cttsSampleCounts.length - 1]++ } else { streamContext.cttsSampleCounts.push(1) streamContext.cttsSampleOffsets.push(ctts) } } } streamContext.lastPts = bigint.max(streamContext.lastPts, pts + (duration !== NOPTS_VALUE_BIGINT ? duration : 0n)) streamContext.lastDts = dts streamContext.lastDuration = static_cast(duration as int64) } return 0 } public writeTrailer(formatContext: AVOFormatContext): number { if (!this.context.fragment) { this.updateCurrentChunk(formatContext) let duration = 0n array.each(formatContext.streams, (stream) => { if (stream.disposition & AVDisposition.ATTACHED_PIC) { return true } const streamContext = stream.privData as IsobmffStreamContext if (streamContext.sampleSizes.length) { if (streamContext.sttsSampleDeltas.length) { if (streamContext.lastDuration > 0 && streamContext.lastDuration !== streamContext.sttsSampleDeltas[streamContext.sttsSampleDeltas.length - 1] ) { streamContext.sttsSampleCounts.push(1) streamContext.sttsSampleDeltas.push(streamContext.lastDuration) } else { streamContext.sttsSampleCounts[streamContext.sttsSampleCounts.length - 1]++ } } else { streamContext.sttsSampleCounts = [1] streamContext.sttsSampleDeltas = [0] } } const streamDuration = avRescaleQ( getSampleDuration(streamContext), stream.timeBase, { den: 1000, num: 1 } ) if (streamDuration > duration) { duration = streamDuration } }) this.context.duration = duration const mdat = this.context.boxsPositionInfo[this.context.boxsPositionInfo.length - 1] if (mdat.type !== BoxType.MDAT) { logger.error('last box is not mdat') } mdat.size = static_cast(formatContext.ioWriter.getPos() - mdat.pos) if (mdat.size > UINT32_MAX) { const now = formatContext.ioWriter.getPos() formatContext.ioWriter.seek(mdat.pos - 8n) // overwrite 'free' placeholder atom formatContext.ioWriter.writeUint32(1) formatContext.ioWriter.writeUint32(mktag(BoxType.MDAT)) formatContext.ioWriter.writeUint64(static_cast(mdat.size as uint32) + 8n) formatContext.ioWriter.seek(now) this.context.boxsPositionInfo.pop() this.context.use64Mdat = true } updatePositionSize(formatContext.ioWriter, this.context) if (this.options.fastOpen) { formatContext.ioWriter.flush() let buffers = [] const rawFlush = formatContext.ioWriter.onFlush formatContext.ioWriter.onFlush = (buffer) => { buffers.push(buffer.slice()) return 0 } oisobmff.writeMoov(formatContext.ioWriter, formatContext, this.context) formatContext.ioWriter.flush() let data = concatTypeArray(Uint8Array, buffers) array.each(formatContext.streams, (stream) => { const streamContext = stream.privData as IsobmffStreamContext if (streamContext.chunkOffsets.length) { for (let i = 0; i < streamContext.chunkOffsets.length; i++) { streamContext.chunkOffsets[i] += static_cast(data.length) } } }) buffers = [] oisobmff.writeMoov(formatContext.ioWriter, formatContext, this.context) formatContext.ioWriter.flush() data = concatTypeArray(Uint8Array, buffers) if (rawFlush) { rawFlush(data, this.context.holdMoovPos) } formatContext.ioWriter.onFlush = rawFlush } else { oisobmff.writeMoov(formatContext.ioWriter, formatContext, this.context) formatContext.ioWriter.flush() } } else { array.each(this.context.currentFragment.tracks, (track) => { const stream = formatContext.streams.find((stream) => { return stream.index === track.streamIndex }) const streamContext = stream.privData as IsobmffStreamContext if (track.sampleCount) { if (track.sampleDurations.length) { track.sampleDurations.push(track.sampleDurations[track.sampleDurations.length - 1]) } else { track.sampleDurations = [streamContext.lastDuration] } } }) this.updateCurrentFragment(formatContext) if (this.options.hasTfra) { oisobmff.writeMfra(formatContext.ioWriter, formatContext, this.context) } else { formatContext.ioWriter.writeUint32(8) formatContext.ioWriter.writeString(BoxType.MFRA) } formatContext.ioWriter.flush() } return 0 } public flush(formatContext: AVOFormatContext): number { if (this.options.fragment) { array.each(this.context.currentFragment.tracks, (track) => { const stream = formatContext.streams.find((stream) => { return stream.index === track.streamIndex }) const streamContext = stream.privData as IsobmffStreamContext if (track.sampleCount) { if (track.sampleDurations.length) { track.sampleDurations.push(track.sampleDurations[track.sampleDurations.length - 1]) } else { track.sampleDurations = [streamContext.lastDuration] } } }) this.updateCurrentFragment(formatContext) } formatContext.ioWriter.flush() return 0 } public getCapabilities() { return OIsobmffFormat.Capabilities } static Capabilities: AVCodecID[] = [ AVCodecID.AV_CODEC_ID_VORBIS, AVCodecID.AV_CODEC_ID_OPUS, AVCodecID.AV_CODEC_ID_AAC, AVCodecID.AV_CODEC_ID_MP3, AVCodecID.AV_CODEC_ID_FLAC, AVCodecID.AV_CODEC_ID_SPEEX, AVCodecID.AV_CODEC_ID_AC3, AVCodecID.AV_CODEC_ID_EAC3, AVCodecID.AV_CODEC_ID_VP9, AVCodecID.AV_CODEC_ID_AV1, AVCodecID.AV_CODEC_ID_H264, AVCodecID.AV_CODEC_ID_HEVC, AVCodecID.AV_CODEC_ID_VVC, AVCodecID.AV_CODEC_ID_MPEG4, AVCodecID.AV_CODEC_ID_WEBVTT, AVCodecID.AV_CODEC_ID_MOV_TEXT ] }