/* * libmedia mpegts decoder * * 版权所有 (C) 2024 赵高兴 * Copyright (C) 2024 Gaoxing Zhao * * 此文件是 libmedia 的一部分 * This file is part of libmedia. * * libmedia 是自由软件;您可以根据 GNU Lesser General Public License(GNU LGPL)3.1 * 或任何其更新的版本条款重新分发或修改它 * libmedia is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 3.1 of the License, or (at your option) any later version. * * libmedia 希望能够为您提供帮助,但不提供任何明示或暗示的担保,包括但不限于适销性或特定用途的保证 * 您应自行承担使用 libmedia 的风险,并且需要遵守 GNU Lesser General Public License 中的条款和条件。 * libmedia is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * */ import type { AVIFormatContext } from '../AVFormatContext' import type { MpegtsContext, MpegtsStreamContext } from './mpegts/type' import createMpegtsContext from './mpegts/function/createMpegtsContext' import * as impegts from './mpegts/impegts' import * as mpegts from './mpegts/mpegts' import handleSectionSlice from './mpegts/function/handleSectionSlice' import parsePES from './mpegts/function/parsePES' import parsePESSlice from './mpegts/function/parsePESSlice' import clearTSSliceQueue from './mpegts/function/clearTSSliceQueue' import type { PES } from './mpegts/struct' import { TSSliceQueue } from './mpegts/struct' import IFormat from './IFormat' import initStream from './mpegts/function/initStream' import seekInBytes from '../function/seekInBytes' import { memcpyFromUint8Array, mapSafeUint8Array, memcpy } from '@libmedia/cheap' import { AVFormat, AVSeekFlags, AVMediaType, AVCodecID, type AVPacket, type AVStream, avMalloc, addAVPacketData, createAVPacket, AVPacketFlags, destroyAVPacket, getAVPacketData, getAVPacketSideData, NOPTS_VALUE, NOPTS_VALUE_BIGINT, AVPacketSideDataType, deleteAVPacketSideData, avRescaleQ, errorType, nalu as nalusUtil } from '@libmedia/avutil' import { AV_MILLI_TIME_BASE_Q, aac, mp3, h264, hevc, vvc, opus, ac3, dts } from '@libmedia/avutil/internal' import { array, concatTypeArray, logger, is } from '@libmedia/common' import { IOError } from '@libmedia/common/io' export default class IMpegtsFormat extends IFormat { public type: AVFormat = AVFormat.MPEGTS private context: MpegtsContext private firstTSPacketPos: int64 private cacheAVPacket: pointer constructor() { super() this.context = createMpegtsContext() } public init(formatContext: AVIFormatContext): void { if (formatContext.ioReader) { formatContext.ioReader.setEndian(true) } this.cacheAVPacket = nullptr } public async destroy(formatContext: AVIFormatContext) { if (this.cacheAVPacket) { destroyAVPacket(this.cacheAVPacket) this.cacheAVPacket = nullptr } array.each(formatContext.streams, (stream) => { const streamContext = stream.privData as MpegtsStreamContext if (streamContext.filter) { streamContext.filter.destroy() streamContext.filter = null } }) } public async readHeader(formatContext: AVIFormatContext): Promise { try { let ret = 0 let packetSize = await impegts.getPacketSize(formatContext.ioReader) if (!packetSize) { packetSize = mpegts.TS_PACKET_SIZE } this.context.tsPacketSize = packetSize // 码流可能存在一些非 ts packet 数据,跳过 if (this.context.tsPacketSize !== mpegts.TS_DVHS_PACKET_SIZE && (await formatContext.ioReader.peekUint8() !== 0x47) ) { await this.syncTSPacket(formatContext, false) } while ((!this.context.hasPAT || !this.context.hasPMT)) { const tsPacket = await impegts.parseTSPacket(formatContext.ioReader, this.context) if (!tsPacket.payload) { continue } if (tsPacket.pid === 0 || tsPacket.pid === this.context.currentPmtPid || this.context.pmt.pid2StreamType.get(tsPacket.pid) === mpegts.TSStreamType.kSCTE35 ) { handleSectionSlice(tsPacket, this.context) } } if (!this.context.hasPAT || !this.context.hasPMT) { return errorType.DATA_INVALID } else { this.firstTSPacketPos = formatContext.ioReader.getPos() } return ret } catch (error) { logger.error(error.message) return formatContext.ioReader.error } } private checkExtradata(avpacket: pointer, stream: AVStream) { if (!stream.codecpar.extradata) { let element = getAVPacketSideData(avpacket, AVPacketSideDataType.AV_PKT_DATA_NEW_EXTRADATA) if (!element) { return } stream.codecpar.extradata = avMalloc(element.size) memcpy(stream.codecpar.extradata, element.data, element.size) stream.codecpar.extradataSize = static_cast(element.size) deleteAVPacketSideData(avpacket, AVPacketSideDataType.AV_PKT_DATA_NEW_EXTRADATA) if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_AAC) { aac.parseAVCodecParameters(stream, mapSafeUint8Array(stream.codecpar.extradata, reinterpret_cast(stream.codecpar.extradataSize))) } else if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_OPUS) { opus.parseAVCodecParameters(stream, mapSafeUint8Array(stream.codecpar.extradata, reinterpret_cast(stream.codecpar.extradataSize))) } } } private handlePES(formatContext: AVIFormatContext, avpacket: pointer, pes: PES, stream: AVStream) { if (pes.randomAccessIndicator || stream.codecpar.codecType === AVMediaType.AVMEDIA_TYPE_AUDIO) { avpacket.flags |= AVPacketFlags.AV_PKT_FLAG_KEY } const streamContext = stream.privData as MpegtsStreamContext if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H264 || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H265 || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_VVC ) { avpacket.flags |= AVPacketFlags.AV_PKT_FLAG_H26X_ANNEXB } avpacket.streamIndex = stream.index avpacket.dts = pes.dts avpacket.pts = pes.pts avpacket.pos = pes.pos avpacket.timeBase.den = 90000 avpacket.timeBase.num = 1 avpacket.flags |= pes.flags if (stream.startTime === NOPTS_VALUE_BIGINT) { stream.startTime = avpacket.pts || avpacket.dts } const payload: pointer = avMalloc(pes.payload.length) memcpyFromUint8Array(payload, pes.payload.length, pes.payload) addAVPacketData(avpacket, payload, pes.payload.length) if (streamContext.filter) { let ret = 0 ret = streamContext.filter.sendAVPacket(avpacket) if (ret < 0) { logger.error('send avpacket to bsf failed') return errorType.DATA_INVALID } ret = streamContext.filter.receiveAVPacket(avpacket) if (ret < 0) { if (!this.context.ioEnd) { return IOError.AGAIN } logger.error('receive avpacket from bsf failed') return errorType.DATA_INVALID } avpacket.timeBase.den = 90000 avpacket.timeBase.num = 1 avpacket.streamIndex = stream.index this.checkExtradata(avpacket, stream) while (true) { const avpacket = this.cacheAVPacket || createAVPacket() ret = streamContext.filter.receiveAVPacket(avpacket) if (ret === 0) { avpacket.timeBase.den = 90000 avpacket.timeBase.num = 1 avpacket.streamIndex = stream.index this.checkExtradata(avpacket, stream) formatContext.interval.packetBuffer.push(avpacket) this.cacheAVPacket = nullptr } else { this.cacheAVPacket = avpacket break } } } else { const streamType = this.context.pmt.pid2StreamType.get(streamContext.pid) if (streamType === mpegts.TSStreamType.AUDIO_MPEG1 || streamType === mpegts.TSStreamType.AUDIO_MPEG2 ) { avpacket.flags |= AVPacketFlags.AV_PKT_FLAG_KEY const buffer = getAVPacketData(avpacket) const ver = (buffer[1] >>> 3) & 0x03 const layer = (buffer[1] & 0x06) >> 1 // const bitrateIndex = (buffer[2] & 0xF0) >>> 4 const samplingFreqIndex = (buffer[2] & 0x0C) >>> 2 const channelMode = (buffer[3] >>> 6) & 0x03 const channelCount = channelMode !== 3 ? 2 : 1 const profile = mp3.getProfileByLayer(layer) const sampleRate = mp3.getSampleRateByVersionIndex(ver, samplingFreqIndex) const hasNewExtraData = stream.codecpar.profile !== profile || stream.codecpar.sampleRate !== sampleRate || stream.codecpar.chLayout.nbChannels !== channelCount if (hasNewExtraData) { stream.codecpar.profile = profile stream.codecpar.sampleRate = sampleRate stream.codecpar.chLayout.nbChannels = channelCount } } else if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H264) { if (!stream.codecpar.extradata) { const extradata = h264.generateAnnexbExtradata(getAVPacketData(avpacket)) if (extradata) { stream.codecpar.extradata = avMalloc(extradata.length) memcpyFromUint8Array(stream.codecpar.extradata, extradata.length, extradata) stream.codecpar.extradataSize = extradata.length h264.parseAVCodecParameters(stream, extradata) avpacket.flags |= AVPacketFlags.AV_PKT_FLAG_KEY } } } else if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_HEVC) { if (!stream.codecpar.extradata) { const extradata = hevc.generateAnnexbExtradata(getAVPacketData(avpacket)) if (extradata) { stream.codecpar.extradata = avMalloc(extradata.length) memcpyFromUint8Array(stream.codecpar.extradata, extradata.length, extradata) stream.codecpar.extradataSize = extradata.length hevc.parseAVCodecParameters(stream, extradata) avpacket.flags |= AVPacketFlags.AV_PKT_FLAG_KEY } } } else if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_VVC) { if (!stream.codecpar.extradata) { const extradata = vvc.generateAnnexbExtradata(getAVPacketData(avpacket)) if (extradata) { stream.codecpar.extradata = avMalloc(extradata.length) memcpyFromUint8Array(stream.codecpar.extradata, extradata.length, extradata) stream.codecpar.extradataSize = extradata.length vvc.parseAVCodecParameters(stream, extradata) avpacket.flags |= AVPacketFlags.AV_PKT_FLAG_KEY } } } else if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_AC3 || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_EAC3 ) { if (stream.codecpar.sampleRate === NOPTS_VALUE) { const info = ac3.parseHeader(getAVPacketData(avpacket)) if (!is.number(info)) { stream.codecpar.sampleRate = reinterpret_cast(info.sampleRate) stream.codecpar.chLayout.nbChannels = reinterpret_cast(info.channels) } } } else if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_DTS) { if (stream.codecpar.sampleRate === NOPTS_VALUE) { const info = dts.parseHeader(getAVPacketData(avpacket)) if (!is.number(info)) { stream.codecpar.sampleRate = reinterpret_cast(info.sampleRate) stream.codecpar.chLayout.nbChannels = reinterpret_cast(info.channels) } } } } return 0 } private parsePESSlice(formatContext: AVIFormatContext, avpacket: pointer, queue: TSSliceQueue, stream: AVStream) { const streamContext = stream.privData as MpegtsStreamContext let pes = parsePESSlice(queue) let ret = parsePES(pes) if (ret) { return ret } clearTSSliceQueue(queue) if (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H264 || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H265 || stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_VVC ) { if (!streamContext.pendingPES && !this.context.ioEnd) { streamContext.pendingPES = pes return IOError.AGAIN } if (streamContext.pendingPES) { let offset = 0 while (true) { const next = nalusUtil.getNextNaluStart(pes.payload, offset) if (next.offset >= 0) { offset = next.offset if (next.startCode === 4 || next.startCode === 3 && ((stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H265 && (next.offset + 3 < pes.payload.length) && (((pes.payload[next.offset + 3] >>> 1) & 0x3f) === hevc.HEVCNaluType.kSliceSEI_PREFIX || ((pes.payload[next.offset + 3] >>> 1) & 0x3f) === hevc.HEVCNaluType.kSliceAUD ) ) || (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_VVC && (next.offset + 4 < pes.payload.length) && (((pes.payload[next.offset + 4] >>> 3) & 0x1f) === vvc.VVCNaluType.kPREFIX_SEI_NUT || ((pes.payload[next.offset + 4] >>> 3) & 0x1f) === vvc.VVCNaluType.kAUD_NUT ) ) || (stream.codecpar.codecId === AVCodecID.AV_CODEC_ID_H264 && (next.offset + 3 < pes.payload.length) && (pes.payload[next.offset + 3] & 0x1f) === h264.H264NaluType.kSliceAUD ) ) ) { break } offset += 3 } else { offset = -1 break } } if (offset >= 0) { if (offset > 0) { streamContext.pendingPES.payload = concatTypeArray(Uint8Array, [streamContext.pendingPES.payload, pes.payload.subarray(0, offset)]) pes.payload = pes.payload.subarray(offset) } let pendingPES = streamContext.pendingPES streamContext.pendingPES = pes pes = pendingPES } else { streamContext.pendingPES.payload = concatTypeArray(Uint8Array, [streamContext.pendingPES.payload, pes.payload]) if (this.context.ioEnd) { pes = streamContext.pendingPES streamContext.pendingPES = null } else { return IOError.AGAIN } } } } return this.handlePES(formatContext, avpacket, pes, stream) } private async readAVPacket_(formatContext: AVIFormatContext, avpacket: pointer): Promise { if (this.context.ioEnd) { const end = () => { const stream = formatContext.streams.find((stream) => { return !!(stream.privData as MpegtsStreamContext).pendingPES }) if (stream) { const context = stream.privData as MpegtsStreamContext const pes = context.pendingPES context.pendingPES = null return this.handlePES(formatContext, avpacket, pes, stream) } return IOError.END } if (!this.context.tsSliceQueueMap.size) { return end() } const it = this.context.tsSliceQueueMap.values() let queue: TSSliceQueue while (true) { const next = it.next() if (next.value && next.value.slices.length) { queue = next.value break } if (next.done) { break } } if (!queue) { return end() } const stream = formatContext.streams.find((stream) => { return (stream.privData as MpegtsStreamContext).pid === queue.pid }) if (stream) { return this.parsePESSlice(formatContext, avpacket, queue, stream) } else { clearTSSliceQueue(queue) return this.readAVPacket_(formatContext, avpacket) } } else { try { while (true) { // 码流可能存在一些非 ts packet 数据,跳过 if (this.context.tsPacketSize !== mpegts.TS_DVHS_PACKET_SIZE && (await formatContext.ioReader.peekUint8() !== 0x47) ) { // 将剩余缓冲区移动到头部,方便 syncTSPacket 往回 seek // m3u8 切片是不支持通过字节位置 seek 的 try { await formatContext.ioReader.flush() } catch (e) {} await this.syncTSPacket(formatContext, false) } const tsPacket = await impegts.parseTSPacket(formatContext.ioReader, this.context) if (!tsPacket.payload) { continue } if (tsPacket.pid === 0 || tsPacket.pid === this.context.currentPmtPid || this.context.pmt.pid2StreamType.get(tsPacket.pid) === mpegts.TSStreamType.kSCTE35 ) { handleSectionSlice(tsPacket, this.context) continue } const streamType = this.context.pmt.pid2StreamType.get(tsPacket.pid) if (!streamType) { continue } let stream = formatContext.streams.find((stream) => { return (stream.privData as MpegtsStreamContext).pid === tsPacket.pid }) if (!stream) { stream = formatContext.createStream() initStream(tsPacket.pid, stream, this.context) } let pesPacketLength = (tsPacket.payload[4] << 8) | tsPacket.payload[5] let pesSliceQueue = this.context.tsSliceQueueMap.get(tsPacket.pid) let packetGot = false if (pesSliceQueue) { if (pesSliceQueue.totalLength > 0 && tsPacket.payloadUnitStartIndicator) { const ret = this.parsePESSlice(formatContext, avpacket, pesSliceQueue, stream) if (ret < 0) { if (ret === IOError.AGAIN) { pesSliceQueue.randomAccessIndicator = tsPacket.adaptationFieldInfo?.randomAccessIndicator ?? 0 pesSliceQueue.pos = tsPacket.pos pesSliceQueue.pid = tsPacket.pid pesSliceQueue.streamType = streamType pesSliceQueue.expectedLength = pesPacketLength === 0 ? 0 : pesPacketLength + 6 pesSliceQueue.slices.push(tsPacket.payload) pesSliceQueue.totalLength += tsPacket.payload.length continue } return ret } packetGot = true } } else { if (!tsPacket.payloadUnitStartIndicator) { if (defined(ENABLE_LOG_TRACE)) { logger.trace('got ts packet before payload unit start indicator, ignore it') } continue } pesSliceQueue = new TSSliceQueue() this.context.tsSliceQueueMap.set(tsPacket.pid, pesSliceQueue) } if (tsPacket.payloadUnitStartIndicator) { pesSliceQueue.randomAccessIndicator = tsPacket.adaptationFieldInfo?.randomAccessIndicator ?? 0 pesSliceQueue.pos = tsPacket.pos pesSliceQueue.pid = tsPacket.pid pesSliceQueue.streamType = streamType pesSliceQueue.expectedLength = pesPacketLength === 0 ? 0 : pesPacketLength + 6 } pesSliceQueue.slices.push(tsPacket.payload) pesSliceQueue.totalLength += tsPacket.payload.length if (pesSliceQueue.expectedLength > 0 && pesSliceQueue.expectedLength === pesSliceQueue.totalLength) { const ret = this.parsePESSlice(formatContext, avpacket, pesSliceQueue, stream) if (ret < 0) { if (ret === IOError.AGAIN) { continue } return ret } packetGot = true } if (packetGot) { return 0 } } } catch (error) { if (formatContext.ioReader.error === IOError.END && !this.context.ioEnd) { this.context.ioEnd = true return this.readAVPacket_(formatContext, avpacket) } else if (formatContext.ioReader.error === IOError.END || formatContext.ioReader.error === IOError.ABORT ) { return formatContext.ioReader.error } else { logger.error(`read packet error, ${error}`) return errorType.DATA_INVALID } } } } public async readAVPacket(formatContext: AVIFormatContext, avpacket: pointer): Promise { try { return this.readAVPacket_(formatContext, avpacket) } catch (error) { if (formatContext.ioReader.error !== IOError.END && formatContext.ioReader.error !== IOError.ABORT ) { logger.error(error.message) } return formatContext.ioReader.error } } private async syncTSPacket(formatContext: AVIFormatContext, syncPES: boolean = true) { let pos: int64 = NOPTS_VALUE_BIGINT const analyzeCount = 10 while (true) { try { const byte = await formatContext.ioReader.readUint8() if (byte === 0x47) { if (this.context.tsPacketSize === mpegts.TS_DVHS_PACKET_SIZE) { pos = formatContext.ioReader.getPos() - 5n } else { pos = formatContext.ioReader.getPos() - 1n } let count = 0 let now = formatContext.ioReader.getPos() while (count <= analyzeCount) { await formatContext.ioReader.skip(this.context.tsPacketSize - 1) const byte = await formatContext.ioReader.readUint8() if (byte === 0x47) { count++ } else { break } } if (count < analyzeCount) { pos = NOPTS_VALUE_BIGINT await formatContext.ioReader.seek(now) continue } else { break } } } catch (error) { pos = NOPTS_VALUE_BIGINT break } } if (pos !== NOPTS_VALUE_BIGINT) { // 移动到 ts packet 的开始 await formatContext.ioReader.seek(pos) if (syncPES) { while (true) { const tsPacket = await impegts.parseTSPacket(formatContext.ioReader, this.context) // 移动到下一个 pes 的开始 if (tsPacket.payloadUnitStartIndicator) { // 返回到上一个 ts packet 的开始 await formatContext.ioReader.seek(pos) formatContext.streams.forEach((stream) => { let pesSliceQueue = this.context.tsSliceQueueMap.get((stream.privData as MpegtsStreamContext).pid) if (pesSliceQueue) { clearTSSliceQueue(pesSliceQueue) } const streamContext = stream.privData as MpegtsStreamContext streamContext.pendingPES = null if (streamContext.filter) { streamContext.filter.reset() } }) break } pos = formatContext.ioReader.getPos() } } } } public async seek( formatContext: AVIFormatContext, stream: AVStream, timestamp: int64, flags: int32 ): Promise { let now = formatContext.ioReader.getPos() this.context.tsSliceQueueMap.forEach((queue) => { if (queue.slices.length && queue.pos < now) { now = queue.pos } }) formatContext.streams.forEach((stream) => { const streamContext = stream.privData as MpegtsStreamContext if (streamContext.pendingPES?.pos < now) { now = streamContext.pendingPES.pos } }) const end = () => { this.context.tsSliceQueueMap.forEach((queue) => { clearTSSliceQueue(queue) }) formatContext.streams.forEach((stream) => { const streamContext = stream.privData as MpegtsStreamContext streamContext.pendingPES = null if (streamContext.filter) { streamContext.filter.reset() } }) } this.context.pmt.pid2StreamType.forEach((streamType, pid) => { this.context.tsSliceQueueMap.delete(pid) }) // m3u8 使用时间戳去 seek if (flags & AVSeekFlags.TIMESTAMP) { const seekTime = avRescaleQ(timestamp, stream.timeBase, AV_MILLI_TIME_BASE_Q) await formatContext.ioReader.seek(seekTime, true) this.context.ioEnd = false end() return 0n } if (flags & AVSeekFlags.BYTE) { const size = await formatContext.ioReader.fileSize() if (size <= 0n) { return static_cast(errorType.FORMAT_NOT_SUPPORT) } if (timestamp < 0n) { timestamp = 0n } else if (timestamp > size) { timestamp = size } await formatContext.ioReader.seek(timestamp) if (!(flags & AVSeekFlags.ANY)) { await this.syncTSPacket(formatContext) } this.context.ioEnd = false end() return now } else { if (stream && stream.sampleIndexes.length) { let index = array.binarySearch(stream.sampleIndexes, (item) => { if (item.pts > timestamp) { return -1 } return 1 }) if (index > 0 && avRescaleQ(timestamp - stream.sampleIndexes[index - 1].pts, stream.timeBase, AV_MILLI_TIME_BASE_Q) < 10000n) { logger.debug(`seek in sampleIndexes, found index: ${index}, pts: ${stream.sampleIndexes[index - 1].pts}, pos: ${stream.sampleIndexes[index - 1].pos}`) await formatContext.ioReader.seek(stream.sampleIndexes[index - 1].pos) this.context.ioEnd = false end() return now } } logger.debug('not found any keyframe index, try to seek in bytes') let ret = await seekInBytes( formatContext, stream, timestamp, this.firstTSPacketPos, this.readAVPacket.bind(this), this.syncTSPacket.bind(this) ) if (ret >= 0) { this.context.ioEnd = false end() } return ret } } public getAnalyzeStreamsCount(): number { return this.context.pmt?.pid2StreamType.size ?? 1 } }