feat(muxer): add support for HEVC for MP4

core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4MuxerHelper.kt

@@ -27,7 +27,7 @@ class MP4MuxerHelper : IMuxerHelper {
 
 class AudioMP4MuxerHelper : IAudioMuxerHelper {
     /**
-     * Get TS Muxer supported audio encoders list
+     * Get MP4 Muxer supported audio encoders list
      */
     override val supportedEncoders =
         listOf(MediaFormat.MIMETYPE_AUDIO_AAC)
@@ -39,10 +39,11 @@ class AudioMP4MuxerHelper : IAudioMuxerHelper {
 
 class VideoMP4MuxerHelper : IVideoMuxerHelper {
     /**
-     * Get TS Muxer supported video encoders list
+     * Get MP4 Muxer supported video encoders list
      */
     override val supportedEncoders =
         listOf(
-            MediaFormat.MIMETYPE_VIDEO_AVC
+            MediaFormat.MIMETYPE_VIDEO_AVC,
+            MediaFormat.MIMETYPE_VIDEO_HEVC
         )
 }

core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/boxes/HEVCConfigurationBox.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2023 Thibault B.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.github.thibaultbee.streampack.internal.muxers.mp4.boxes
+
+import io.github.thibaultbee.streampack.internal.utils.av.video.hevc.HEVCDecoderConfigurationRecord
+import java.nio.ByteBuffer
+
+class HEVCConfigurationBox(private val config: HEVCDecoderConfigurationRecord) : Box("hvcC") {
+    override val size: Int = super.size + config.size
+
+    override fun write(output: ByteBuffer) {
+        super.write(output)
+        config.write(output)
+    }
+}

core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/boxes/SampleEntry.kt

@@ -104,6 +104,35 @@ class AVCSampleEntry(
     pasp
 )
 
+class HEVCSampleEntry(
+    resolution: Size,
+    horizontalResolution: Int = 72,
+    verticalResolution: Int = 72,
+    frameCount: Short = 1,
+    compressorName: String? = "HEVC Coding",
+    depth: Short = 0x0018,
+    hvcC: HEVCConfigurationBox,
+    btrt: BitRateBox? = null,
+    extensionDescriptorsBox: List<Box> = emptyList(),
+    clap: CleanApertureBox? = null,
+    pasp: PixelAspectRatioBox? = null
+) : VisualSampleEntry(
+    "hvc1",
+    resolution,
+    horizontalResolution,
+    verticalResolution,
+    frameCount,
+    compressorName,
+    depth,
+    mutableListOf<Box>(hvcC).apply {
+        btrt?.let { add(it) }
+        addAll(extensionDescriptorsBox)
+    },
+    clap,
+    pasp
+)
+
+
 class MP4AudioSampleEntry(
     channelCount: Short,
     sampleSize: Short,

core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/models/TrackChunks.kt

@@ -21,14 +21,39 @@ import io.github.thibaultbee.streampack.data.AudioConfig
 import io.github.thibaultbee.streampack.data.Config
 import io.github.thibaultbee.streampack.data.VideoConfig
 import io.github.thibaultbee.streampack.internal.data.Frame
-import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.*
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AVCConfigurationBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AVCSampleEntry
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.ChunkLargeOffsetBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.DataEntryUrlBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.DataInformationBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.DataReferenceBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.ESDSBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.HEVCConfigurationBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.HEVCSampleEntry
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.MP4AudioSampleEntry
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.MediaBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.MediaHeaderBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.MediaInformationBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.SampleDescriptionBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.SampleSizeBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.SampleTableBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.SampleToChunkBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.SyncSampleBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.TimeToSampleBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.TrackBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.TrackExtendsBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.TrackFragmentBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.TrackFragmentHeaderBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.TrackHeaderBox
+import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.TrackRunBox
 import io.github.thibaultbee.streampack.internal.muxers.mp4.utils.createHandlerBox
 import io.github.thibaultbee.streampack.internal.muxers.mp4.utils.createTypeMediaHeaderBox
 import io.github.thibaultbee.streampack.internal.utils.TimeUtils
 import io.github.thibaultbee.streampack.internal.utils.av.descriptors.AudioSpecificConfigDescriptor
 import io.github.thibaultbee.streampack.internal.utils.av.descriptors.ESDescriptor
 import io.github.thibaultbee.streampack.internal.utils.av.descriptors.SLConfigDescriptor
 import io.github.thibaultbee.streampack.internal.utils.av.video.avc.AVCDecoderConfigurationRecord
+import io.github.thibaultbee.streampack.internal.utils.av.video.hevc.HEVCDecoderConfigurationRecord
 import io.github.thibaultbee.streampack.internal.utils.extensions.clone
 import java.nio.ByteBuffer
 
@@ -51,6 +76,10 @@ class TrackChunks(
                 this.extra.size == 2
             }
 
+            MediaFormat.MIMETYPE_VIDEO_HEVC -> {
+                this.extra.size == 3
+            }
+
             MediaFormat.MIMETYPE_AUDIO_AAC -> {
                 this.extra.size == 1
             }
@@ -162,7 +191,7 @@ class TrackChunks(
                             it.id,
                             it.numOfSamples,
                             1
-                            )
+                        )
                     )
                 }
             } catch (e: NoSuchElementException) {
@@ -172,7 +201,7 @@ class TrackChunks(
                         it.id,
                         it.numOfSamples,
                         1
-                        )
+                    )
                 )
             }
         }
@@ -238,6 +267,23 @@ class TrackChunks(
                     ),
                 )
             }
+
+            MediaFormat.MIMETYPE_VIDEO_HEVC -> {
+                val extra = this.extra
+                require(extra.size == 3) { "For HEVC, extra must contains 3 parameter sets" }
+                (track.config as VideoConfig)
+                HEVCSampleEntry(
+                    track.config.resolution,
+                    hvcC = HEVCConfigurationBox(
+                        HEVCDecoderConfigurationRecord.fromParameterSets(
+                            extra[0],
+                            extra[1],
+                            extra[2]
+                        )
+                    ),
+                )
+            }
+
             MediaFormat.MIMETYPE_AUDIO_AAC -> {
                 (track.config as AudioConfig)
                 MP4AudioSampleEntry(
@@ -260,6 +306,7 @@ class TrackChunks(
                     )
                 )
             }
+
             else -> throw IllegalArgumentException("Unsupported mimeType")
         }
         return SampleDescriptionBox(sampleEntry)

core/src/main/java/io/github/thibaultbee/streampack/internal/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt

@@ -42,13 +42,9 @@ data class HEVCDecoderConfigurationRecord(
     private val numTemporalLayers: Byte = 0, // 0 = Unknown
     private val temporalIdNested: Boolean = false,
     private val lengthSizeMinusOne: Byte = 3,
-    private val sps: List<ByteBuffer>,
-    private val pps: List<ByteBuffer>,
-    private val vps: List<ByteBuffer>
+    private val parameterSets: List<NalUnit>,
 ) {
-    private val spsNoStartCode = sps.map { it.removeStartCode() }
-    private val ppsNoStartCode = pps.map { it.removeStartCode() }
-    private val vpsNoStartCode = vps.map { it.removeStartCode() }
+    val size: Int = getSize(parameterSets)
 
     fun write(buffer: ByteBuffer) {
         buffer.put(configurationVersion) // configurationVersion
@@ -77,17 +73,10 @@ data class HEVCDecoderConfigurationRecord(
                     or lengthSizeMinusOne.toInt()
         ) // constantFrameRate 2 bits = 1 for stable / numTemporalLayers 3 bits /  temporalIdNested 1 bit / lengthSizeMinusOne 2 bits
 
-        buffer.put(spsNoStartCode.size + ppsNoStartCode.size + vpsNoStartCode.size) // numOfArrays
-        spsNoStartCode.forEach { writeArray(buffer, it, NalUnitType.SPS) }
-        ppsNoStartCode.forEach { writeArray(buffer, it, NalUnitType.PPS) }
-        vpsNoStartCode.forEach { writeArray(buffer, it, NalUnitType.VPS) }
-    }
-
-    private fun writeArray(buffer: ByteBuffer, nalUnit: ByteBuffer, nalUnitType: NalUnitType) {
-        buffer.put((1 shl 7) or nalUnitType.value.toInt()) // array_completeness + reserved 1bit + naluType 6 bytes
-        buffer.putShort(1) // numNalus
-        buffer.putShort(nalUnit.remaining().toShort()) // nalUnitLength
-        buffer.put(nalUnit)
+        buffer.put(parameterSets.size) // numOfArrays
+        parameterSets.forEach {
+            it.write(buffer)
+        }
     }
 
     companion object {
@@ -109,12 +98,21 @@ data class HEVCDecoderConfigurationRecord(
             pps: List<ByteBuffer>,
             vps: List<ByteBuffer>
         ): HEVCDecoderConfigurationRecord {
-            val spsNoStartCode = sps.map { it.removeStartCode() }
-            val ppsNoStartCode = pps.map { it.removeStartCode() }
-            val vpsNoStartCode = vps.map { it.removeStartCode() }
+
+            val parameterSets = mutableListOf<NalUnit>()
+            sps.forEach {
+                parameterSets.add(NalUnit(NalUnit.Type.SPS, it))
+            }
+            pps.forEach {
+                parameterSets.add(NalUnit(NalUnit.Type.PPS, it))
+            }
+            vps.forEach {
+                parameterSets.add(NalUnit(NalUnit.Type.VPS, it))
+            }
 
             // profile_tier_level
-            val parsedSps = SequenceParameterSets.parse(spsNoStartCode.first())
+            val parsedSps =
+                SequenceParameterSets.parse(parameterSets.first { it.type == NalUnit.Type.SPS }.noStartCodeData)
             return HEVCDecoderConfigurationRecord(
                 generalProfileSpace = parsedSps.profileTierLevel.generalProfileSpace,
                 generalTierFlag = parsedSps.profileTierLevel.generalTierFlag,
@@ -126,9 +124,7 @@ data class HEVCDecoderConfigurationRecord(
                 bitDepthLumaMinus8 = parsedSps.bitDepthLumaMinus8,
                 bitDepthChromaMinus8 = parsedSps.bitDepthChromaMinus8,
                 // TODO get minSpatialSegmentationIdc from VUI
-                sps = spsNoStartCode,
-                pps = ppsNoStartCode,
-                vps = vpsNoStartCode
+                parameterSets = parameterSets
             )
         }
 
@@ -147,25 +143,45 @@ data class HEVCDecoderConfigurationRecord(
             pps: List<ByteBuffer>,
             vps: List<ByteBuffer>
         ): Int {
-            var size =
-                HEVC_DECODER_CONFIGURATION_RECORD_SIZE
+            val parameterSets = mutableListOf<NalUnit>()
+
             sps.forEach {
-                size += it.remaining() - it.getStartCodeSize() + HEVC_PARAMETER_SET_HEADER_SIZE
+                parameterSets.add(NalUnit(NalUnit.Type.SPS, it))
             }
             pps.forEach {
-                size += it.remaining() - it.getStartCodeSize() + HEVC_PARAMETER_SET_HEADER_SIZE
+                parameterSets.add(NalUnit(NalUnit.Type.PPS, it))
             }
             vps.forEach {
-                size += it.remaining() - it.getStartCodeSize() + HEVC_PARAMETER_SET_HEADER_SIZE
+                parameterSets.add(NalUnit(NalUnit.Type.VPS, it))
             }
 
+            return getSize(parameterSets)
+        }
+
+        fun getSize(parameterSets: List<NalUnit>): Int {
+            var size =
+                HEVC_DECODER_CONFIGURATION_RECORD_SIZE
+            parameterSets.forEach {
+                size += it.data.remaining() - it.data.getStartCodeSize() + HEVC_PARAMETER_SET_HEADER_SIZE
+            }
             return size
         }
     }
 
-    enum class NalUnitType(val value: Byte) {
-        VPS(32),
-        SPS(33),
-        PPS(34)
+    data class NalUnit(val type: Type, val data: ByteBuffer, val completeness: Boolean = true) {
+        val noStartCodeData: ByteBuffer = data.removeStartCode()
+
+        fun write(buffer: ByteBuffer) {
+            buffer.put((completeness shl 7) or type.value.toInt()) // array_completeness + reserved 1bit + naluType 6 bytes
+            buffer.putShort(1) // numNalus
+            buffer.putShort(noStartCodeData.remaining().toShort()) // nalUnitLength
+            buffer.put(noStartCodeData)
+        }
+
+        enum class Type(val value: Byte) {
+            VPS(32),
+            SPS(33),
+            PPS(34)
+        }
     }
 }