From 9d7dae1af05be195ac0e20a64d5a0533a5ff03cc Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Thu, 27 Jun 2024 14:56:56 -0700 Subject: [PATCH 01/13] Implement IAMF audio decoder --- starboard/android/shared/BUILD.gn | 4 + .../shared/media_is_audio_supported.cc | 3 +- .../shared/platform_configuration/BUILD.gn | 5 + .../shared/player_components_factory.h | 8 + starboard/linux/shared/BUILD.gn | 4 + .../linux/shared/media_is_audio_supported.cc | 4 + .../linux/shared/player_components_factory.cc | 6 + .../shared/platform_configuration/BUILD.gn | 4 + .../shared/libiamf/iamf_audio_decoder.cc | 332 ++++++++++++++++++ starboard/shared/libiamf/iamf_audio_decoder.h | 85 +++++ .../shared/libiamf/iamf_config_reader.cc | 184 ++++++++++ starboard/shared/libiamf/iamf_config_reader.h | 75 ++++ 12 files changed, 713 insertions(+), 1 deletion(-) create mode 100644 starboard/shared/libiamf/iamf_audio_decoder.cc create mode 100644 starboard/shared/libiamf/iamf_audio_decoder.h create mode 100644 starboard/shared/libiamf/iamf_config_reader.cc create mode 100644 starboard/shared/libiamf/iamf_config_reader.h diff --git a/starboard/android/shared/BUILD.gn b/starboard/android/shared/BUILD.gn index 9ad0f528ea76..f3ff9e59cf92 100644 --- a/starboard/android/shared/BUILD.gn +++ b/starboard/android/shared/BUILD.gn @@ -75,6 +75,10 @@ static_library("starboard_platform") { "//starboard/shared/libevent/socket_waiter_wait.cc", "//starboard/shared/libevent/socket_waiter_wait_timed.cc", "//starboard/shared/libevent/socket_waiter_wake_up.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.h", + "//starboard/shared/libiamf/iamf_config_reader.cc", + "//starboard/shared/libiamf/iamf_config_reader.h", "//starboard/shared/linux/byte_swap.cc", "//starboard/shared/linux/cpu_features_get.cc", "//starboard/shared/linux/memory_get_stack_bounds.cc", diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index 023cbfc5f8fb..bb60f6124ab4 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -56,7 +56,8 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, // Android uses a libopus based opus decoder for clear content, or a platform // opus decoder for encrypted content, if available. - if (audio_codec == kSbMediaAudioCodecOpus) { + if (audio_codec == kSbMediaAudioCodecOpus || + audio_codec == kSbMediaAudioCodecIamf) { return true; } diff --git a/starboard/android/shared/platform_configuration/BUILD.gn b/starboard/android/shared/platform_configuration/BUILD.gn index f638914483e8..785a7ad48f78 100644 --- a/starboard/android/shared/platform_configuration/BUILD.gn +++ b/starboard/android/shared/platform_configuration/BUILD.gn @@ -175,6 +175,7 @@ config("platform_configuration") { "-Wl,--wrap=readdir_r", ] } + configs += [ ":libraries" ] } config("size") { @@ -217,3 +218,7 @@ config("pedantic_warnings") { "-Wno-unused-parameter", ] } + +config("libraries") { + libs = [ "//third_party/libiamf/code/android/libiamf.a" ] +} diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index d6060d8406e4..b3b766d7be1a 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -34,6 +34,7 @@ #include "starboard/common/media.h" #include "starboard/common/ref_counted.h" #include "starboard/media.h" +#include "starboard/shared/libiamf/iamf_audio_decoder.h" #include "starboard/shared/opus/opus_audio_decoder.h" #include "starboard/shared/starboard/media/media_util.h" #include "starboard/shared/starboard/media/mime_type.h" @@ -179,6 +180,7 @@ class PlayerComponentsPassthrough class PlayerComponentsFactory : public starboard::shared::starboard::player:: filter::PlayerComponents::Factory { typedef starboard::shared::starboard::media::MimeType MimeType; + typedef starboard::shared::libiamf::IamfAudioDecoder IamfAudioDecoder; typedef starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; typedef starboard::shared::starboard::player::filter::AdaptiveAudioDecoder AdaptiveAudioDecoder; @@ -444,6 +446,12 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: return std::unique_ptr( std::move(audio_decoder_impl)); } + } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { + scoped_ptr audio_decoder_impl( + new IamfAudioDecoder(audio_stream_info)); + if (audio_decoder_impl->is_valid()) { + return audio_decoder_impl.PassAs(); + } } else { SB_LOG(ERROR) << "Unsupported audio codec " << audio_stream_info.codec; diff --git a/starboard/linux/shared/BUILD.gn b/starboard/linux/shared/BUILD.gn index 23bfa1e68e7f..1b6339f9e243 100644 --- a/starboard/linux/shared/BUILD.gn +++ b/starboard/linux/shared/BUILD.gn @@ -125,6 +125,10 @@ static_library("starboard_platform_sources") { "//starboard/shared/libfdkaac/fdk_aac_audio_decoder.h", "//starboard/shared/libfdkaac/libfdkaac_library_loader.cc", "//starboard/shared/libfdkaac/libfdkaac_library_loader.h", + "//starboard/shared/libiamf/iamf_audio_decoder.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.h", + "//starboard/shared/libiamf/iamf_config_reader.cc", + "//starboard/shared/libiamf/iamf_config_reader.h", "//starboard/shared/libvpx/vpx_video_decoder.cc", "//starboard/shared/libvpx/vpx_video_decoder.h", "//starboard/shared/linux/byte_swap.cc", diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index c1b03d18d851..e0d17062e521 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -32,6 +32,10 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } + if (audio_codec == kSbMediaAudioCodecIamf) { + return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; + } + if (audio_codec == kSbMediaAudioCodecAc3 || audio_codec == kSbMediaAudioCodecEac3) { #if SB_API_VERSION < 15 diff --git a/starboard/linux/shared/player_components_factory.cc b/starboard/linux/shared/player_components_factory.cc index 67d98405b87d..4b82253d958b 100644 --- a/starboard/linux/shared/player_components_factory.cc +++ b/starboard/linux/shared/player_components_factory.cc @@ -25,6 +25,7 @@ #include "starboard/shared/libde265/de265_video_decoder.h" #include "starboard/shared/libfdkaac/fdk_aac_audio_decoder.h" #include "starboard/shared/libfdkaac/libfdkaac_library_loader.h" +#include "starboard/shared/libiamf/iamf_audio_decoder.h" #include "starboard/shared/libvpx/vpx_video_decoder.h" #include "starboard/shared/openh264/openh264_library_loader.h" #include "starboard/shared/openh264/openh264_video_decoder.h" @@ -68,6 +69,7 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { SB_DCHECK(audio_renderer_sink); typedef ::starboard::shared::ffmpeg::AudioDecoder FfmpegAudioDecoder; + typedef ::starboard::shared::libiamf::IamfAudioDecoder IamfAudioDecoder; typedef ::starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; typedef ::starboard::shared::libfdkaac::FdkAacAudioDecoder FdkAacAudioDecoder; @@ -86,6 +88,10 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { libfdkaac::LibfdkaacHandle::GetHandle()->IsLoaded()) { SB_LOG(INFO) << "Playing audio using FdkAacAudioDecoder."; return std::unique_ptr(new FdkAacAudioDecoder()); + } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { + SB_LOG(INFO) << "Playing audio using IamfAudioDecoder."; + return std::unique_ptr( + new IamfAudioDecoder(audio_stream_info)); } else { std::unique_ptr audio_decoder_impl( FfmpegAudioDecoder::Create(audio_stream_info)); diff --git a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn index 66eedfc23e3f..4874cf39e506 100644 --- a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn +++ b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn @@ -30,6 +30,10 @@ config("platform_configuration") { config("libraries") { configs = [ "//starboard/linux/shared/platform_configuration:libraries" ] + libs = [ + "//third_party/libiamf/code/libiamf.a", + "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", + ] } config("linker_flags") { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc new file mode 100644 index 000000000000..8b537f9d51cc --- /dev/null +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -0,0 +1,332 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "starboard/shared/libiamf/iamf_audio_decoder.h" + +#include + +#include "third_party/libiamf/source/code/include/IAMF_defines.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +namespace { +using shared::starboard::player::DecodedAudio; +} // namespace + +IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) + : audio_stream_info_(audio_stream_info) {} + +IamfAudioDecoder::~IamfAudioDecoder() { + TeardownCodec(); +} + +bool IamfAudioDecoder::is_valid() const { + return decoder_ != NULL; +} + +void IamfAudioDecoder::Initialize(const OutputCB& output_cb, + const ErrorCB& error_cb) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(output_cb); + SB_DCHECK(!output_cb_); + SB_DCHECK(error_cb); + SB_DCHECK(!error_cb_); + + output_cb_ = output_cb; + error_cb_ = error_cb; +} + +void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, + const ConsumedCB& consumed_cb) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(!input_buffers.empty()); + SB_DCHECK(pending_audio_buffers_.empty()); + SB_DCHECK(output_cb_); + + if (stream_ended_) { + SB_LOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; + return; + } + if (input_buffers.size() > kMinimumBuffersToDecode) { + std::copy(std::begin(input_buffers), std::end(input_buffers), + std::back_inserter(pending_audio_buffers_)); + consumed_cb_ = consumed_cb; + DecodePendingBuffers(); + } else { + for (const auto& input_buffer : input_buffers) { + if (!DecodeInternal(input_buffer)) { + return; + } + } + Schedule(consumed_cb); + } +} + +void IamfAudioDecoder::DecodePendingBuffers() { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(!pending_audio_buffers_.empty()); + SB_DCHECK(consumed_cb_); + + for (int i = 0; i < kMinimumBuffersToDecode; ++i) { + if (!DecodeInternal(pending_audio_buffers_.front())) { + return; + } + pending_audio_buffers_.pop_front(); + if (pending_audio_buffers_.empty()) { + Schedule(consumed_cb_); + consumed_cb_ = nullptr; + if (stream_ended_) { + Schedule(std::bind(&IamfAudioDecoder::WriteEndOfStream, this)); + stream_ended_ = false; + } + return; + } + } + + SB_DCHECK(!pending_audio_buffers_.empty()); + Schedule(std::bind(&IamfAudioDecoder::DecodePendingBuffers, this)); +} + +bool IamfAudioDecoder::DecodeInternal( + const scoped_refptr& input_buffer) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(input_buffer); + SB_DCHECK(input_buffer->size() > 0); + SB_DCHECK(output_cb_); + SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); + + SB_LOG(INFO) << "Start reading"; + reader_.Read(input_buffer); + SB_LOG(INFO) << "Back in iamf decoder"; + if (!decoder_) { + bool init = InitializeCodec(); + SB_LOG(INFO) << init; + if (!init) { + SB_LOG(INFO) << "Decoder init failure"; + error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); + return false; + } + } + + scoped_refptr decoded_audio = new DecodedAudio( + audio_stream_info_.number_of_channels, GetSampleType(), + kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), + audio_stream_info_.number_of_channels * frames_per_au_ * + starboard::media::GetBytesPerSample(GetSampleType())); + uint32_t rsize = 0; + SB_LOG(INFO) << "Start decode"; + int ret = IAMF_decoder_decode(decoder_, reader_.data().data(), + reader_.data().size(), &rsize, + reinterpret_cast(decoded_audio->data())); + if (ret < 1) { + SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex << ret; + error_cb_(kSbPlayerErrorDecode, "Failed to decode sample"); + return false; + } + + frames_per_au_ = ret; + decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * + frames_per_au_ * + starboard::media::GetBytesPerSample(GetSampleType())); + const auto& sample_info = input_buffer->audio_sample_info(); + decoded_audio->AdjustForDiscardedDurations( + audio_stream_info_.samples_per_second, + sample_info.discarded_duration_from_front, + sample_info.discarded_duration_from_back); + decoded_audios_.push(decoded_audio); + output_cb_(); + SB_LOG(INFO) << "Returning decoded audio"; + + return true; +} + +void IamfAudioDecoder::WriteEndOfStream() { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(output_cb_); + + // Opus has no dependent frames so we needn't flush the decoder. Set the + // flag to ensure that Decode() is not called when the stream is ended. + stream_ended_ = true; + if (!pending_audio_buffers_.empty()) { + return; + } + + // Put EOS into the queue. + decoded_audios_.push(new DecodedAudio); + + Schedule(output_cb_); +} + +bool IamfAudioDecoder::InitializeCodec() { + int channels = audio_stream_info_.number_of_channels; + if (channels > 8 || channels < 1) { + SB_LOG(ERROR) << "Can't create decoder with " << channels << " channels"; + return false; + } + SB_LOG(INFO) << "1"; + + decoder_ = IAMF_decoder_open(); + if (!decoder_) { + SB_LOG(ERROR) << "Error creating libiamf decoder"; + return false; + } + SB_LOG(INFO) << "2"; + + int error = IAMF_decoder_set_bit_depth(decoder_, 32); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; + return false; + } + SB_LOG(INFO) << "3"; + + error = IAMF_decoder_set_sampling_rate(decoder_, 48000); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "4"; + + IAMF_SoundSystem sound_system = SOUND_SYSTEM_INVALID; + switch (channels) { + case 1: + sound_system = SOUND_SYSTEM_MONO; + break; + case 2: + sound_system = SOUND_SYSTEM_A; + break; + case 6: + sound_system = SOUND_SYSTEM_B; + break; + case 8: + sound_system = SOUND_SYSTEM_C; + break; + default: + SB_NOTREACHED(); + } + error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); + if (error != IAMF_OK) { + SB_LOG(ERROR) + << "IAMF_decoder_output_layout_set_sound_system() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "5"; + + // TODO: Accurately set pts upon resume, if needed. + error = IAMF_decoder_set_pts(decoder_, 0, 90000); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_set_pts() fails with error " << error; + return false; + } + + SB_LOG(INFO) << "6"; + + if (reader_.has_mix_presentation_id()) { + error = IAMF_decoder_set_mix_presentation_id(decoder_, + reader_.mix_presentation_id()); + if (error != IAMF_OK) { + SB_LOG(ERROR) + << "IAMF_decoder_set_mix_presentation_id() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "7"; + } + + error = IAMF_decoder_peak_limiter_enable(decoder_, 0); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_peak_limiter_enable() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "8"; + + error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); + if (error != IAMF_OK) { + SB_LOG(ERROR) + << "IAMF_decoder_set_normalization_loudness() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "9"; + + uint32_t rsize = 0; + error = IAMF_decoder_configure(decoder_, reader_.config_obus().data(), + reader_.config_size(), &rsize); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_configure() fails with error " << error + << ", rsize " << rsize; + return false; + } + SB_LOG(INFO) << "rsize is " << rsize; + return true; +} + +void IamfAudioDecoder::TeardownCodec() { + if (is_valid()) { + IAMF_decoder_close(decoder_); + decoder_ = NULL; + } +} + +scoped_refptr IamfAudioDecoder::Read( + int* samples_per_second) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(output_cb_); + SB_DCHECK(!decoded_audios_.empty()); + + scoped_refptr result; + if (!decoded_audios_.empty()) { + result = decoded_audios_.front(); + decoded_audios_.pop(); + } + *samples_per_second = 48000; + return result; +} + +void IamfAudioDecoder::Reset() { + SB_DCHECK(BelongsToCurrentThread()); + + if (is_valid()) { + // If fail to reset opus decoder, re-create it. + TeardownCodec(); + InitializeCodec(); + } + + frames_per_au_ = kMaxOpusFramesPerAU; + stream_ended_ = false; + while (!decoded_audios_.empty()) { + decoded_audios_.pop(); + } + pending_audio_buffers_.clear(); + consumed_cb_ = nullptr; + + CancelPendingJobs(); +} + +SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { + SB_DCHECK(BelongsToCurrentThread()); +#if SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) + return kSbMediaAudioSampleTypeInt16; +#else // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) + return kSbMediaAudioSampleTypeFloat32; +#endif // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) +} + +} // namespace libiamf +} // namespace shared +} // namespace starboard diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h new file mode 100644 index 000000000000..47279a0cf986 --- /dev/null +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -0,0 +1,85 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_AUDIO_DECODER_H_ +#define STARBOARD_SHARED_LIBIAMF_IAMF_AUDIO_DECODER_H_ + +#include +#include +#include + +#include "starboard/common/ref_counted.h" +#include "starboard/media.h" +#include "starboard/shared/internal_only.h" +#include "starboard/shared/libiamf/iamf_config_reader.h" +#include "starboard/shared/starboard/media/media_util.h" +#include "starboard/shared/starboard/player/decoded_audio_internal.h" +#include "starboard/shared/starboard/player/filter/audio_decoder_internal.h" +#include "starboard/shared/starboard/player/job_queue.h" +#include "third_party/libiamf/source/code/include/IAMF_decoder.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +class IamfAudioDecoder + : public ::starboard::shared::starboard::player::filter::AudioDecoder, + private starboard::player::JobQueue::JobOwner { + public: + typedef starboard::media::AudioStreamInfo AudioStreamInfo; + + explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); + ~IamfAudioDecoder() override; + + bool is_valid() const; + + // AudioDecoder functions + void Initialize(const OutputCB& output_cb, const ErrorCB& error_cb) override; + void Decode(const InputBuffers& input_buffers, + const ConsumedCB& consumed_cb) override; + void WriteEndOfStream() override; + scoped_refptr Read(int* samples_per_second) override; + void Reset() override; + + private: + static constexpr int kMinimumBuffersToDecode = 2; + + bool InitializeCodec(); + void TeardownCodec(); + void DecodePendingBuffers(); + bool DecodeInternal(const scoped_refptr& input_buffer); + static const int kMaxOpusFramesPerAU = 9600; + + SbMediaAudioSampleType GetSampleType() const; + + OutputCB output_cb_; + ErrorCB error_cb_; + + IAMF_Decoder* decoder_ = NULL; + bool stream_ended_ = false; + std::queue> decoded_audios_; + AudioStreamInfo audio_stream_info_; + int frames_per_au_ = kMaxOpusFramesPerAU; + + std::deque> pending_audio_buffers_; + ConsumedCB consumed_cb_; + + IamfConfigReader reader_; +}; + +} // namespace libiamf +} // namespace shared +} // namespace starboard + +#endif // STARBOARD_SHARED_LIBIAMF_IAMF_AUDIO_DECODER_H_ diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc new file mode 100644 index 000000000000..bc797937d0d3 --- /dev/null +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -0,0 +1,184 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "starboard/shared/libiamf/iamf_config_reader.h" + +#include "starboard/common/string.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +namespace { +// From +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax. +constexpr int kObuTypeCodecConfig = 0; +constexpr int kObuTypeAudioElement = 1; +constexpr int kObuTypeMixPresentation = 2; +constexpr int kObuTypeSequenceHeader = 31; + +// Decodes an Leb128 value and stores it in |value|. Returns the number of bytes +// read. Returns -1 on error. +int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { + SB_DCHECK(buf); + SB_DCHECK(value); + *value = 0; + bool error = true; + size_t i = 0; + for (; i < sizeof(uint32_t); ++i) { + uint8_t byte = buf[i]; + *value |= ((byte & 0x7f) << (i * 7)); + if (!(byte & 0x80)) { + error = false; + break; + } + } + + if (error) { + return -1; + } + return i + 1; +} +} // namespace + +bool IamfConfigReader::Read(scoped_refptr input_buffer) { + buffer_head_ = 0; + has_mix_presentation_id_ = false; + SB_LOG(INFO) << "Input buffer size is " << input_buffer->size(); + const uint8_t* buf = input_buffer->data(); + SB_DCHECK(buf); + + bool completed_parsing = false; + while (!completed_parsing && buffer_head_ < input_buffer->size()) { + if (!ReadOBU(buf, completed_parsing)) { + SB_LOG(INFO) << "Error parsing config OBUs"; + return false; + } + } + + SB_CHECK(completed_parsing); + + SB_LOG(INFO) << "End OBU loop"; + + data_size_ = input_buffer->size() - config_size_; + config_obus_.assign(buf, buf + config_size_); + data_.assign(buf + config_size_, buf + input_buffer->size()); + SB_LOG(INFO) << ::starboard::HexEncode(config_obus_.data(), + config_obus_.size()); + SB_LOG(INFO) << "Return read"; + + return true; +} + +bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { + uint8_t obu_type = 0; + uint32_t obu_size = 0; + uint32_t header_size = 0; + if (!ReadOBUHeader(buf, &obu_type, &obu_size, &header_size)) { + SB_LOG(ERROR) << "Error reading OBU header"; + return false; + } + SB_LOG(INFO) << "OBU size is " << obu_size << " with current buffer head " + << buffer_head_; + + // const uint8_t* last_byte = buf + buffer_head_ + obu_size; + int next_obu_pos = buffer_head_ + obu_size; + int bytes_read = 0; + + switch (static_cast(obu_type)) { + case kObuTypeCodecConfig: + SB_LOG(INFO) << "Reading codec config OBU"; + break; + case kObuTypeAudioElement: + SB_LOG(INFO) << "Reading audio element OBU"; + break; + case kObuTypeSequenceHeader: + SB_LOG(INFO) << "Reading sequence header OBU"; + break; + case kObuTypeMixPresentation: + SB_LOG(INFO) << "Reading mix presentation OBU"; + has_mix_presentation_id_ = true; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &mix_presentation_id_); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + break; + default: + SB_LOG(INFO) << "OBU type " << static_cast(obu_type); + completed_parsing = true; + break; + } + + if (!completed_parsing) { + // Skip to next OBU + buffer_head_ = next_obu_pos; + config_size_ += obu_size + header_size; + } + return true; +} + +bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, + uint8_t* obu_type, + uint32_t* obu_size, + uint32_t* header_size) { + SB_LOG(INFO) << "buffer head is " << buffer_head_; + uint8_t header_flags = buf[buffer_head_]; + *obu_type = (header_flags >> 3) & 0x1f; + buffer_head_++; + + const bool obu_redundant_copy = (header_flags >> 2) & 1; + const bool obu_trimming_status_flag = (header_flags >> 1) & 1; + const bool obu_extension_flag = header_flags & 1; + + SB_LOG(INFO) << static_cast(*obu_type); + + *header_size = 1; + + // redundant_copy |= obu_redundant_copy; + + *obu_size = 0; + int bytes_read = ReadLeb128Value(&buf[buffer_head_], obu_size); + if (bytes_read < 0) { + SB_LOG(INFO) << "Error reading OBU size"; + return false; + } + buffer_head_ += bytes_read; + *header_size += bytes_read; + + if (obu_trimming_status_flag) { + uint32_t value; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); + buffer_head_ += bytes_read; + *header_size += bytes_read; + *obu_size -= bytes_read; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); + buffer_head_ += bytes_read; + *header_size += bytes_read; + *obu_size -= bytes_read; + } + + if (obu_extension_flag) { + uint32_t extension_header_size; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &extension_header_size); + buffer_head_ += extension_header_size; + *obu_size -= extension_header_size; + *header_size += bytes_read; + } + return true; +} + +} // namespace libiamf +} // namespace shared +} // namespace starboard diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h new file mode 100644 index 000000000000..d1be99c50218 --- /dev/null +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -0,0 +1,75 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ +#define STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ + +#include + +#include "starboard/common/ref_counted.h" +#include "starboard/shared/internal_only.h" +#include "starboard/shared/starboard/player/input_buffer_internal.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +class IamfConfigReader { + public: + typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; + + IamfConfigReader() = default; + + bool Read(scoped_refptr input_buffer); + + int sample_rate() { return sample_rate_; } + bool config_changed() { return false; } + uint32_t config_size() { + SB_LOG(INFO) << "Config size is " << config_size_; + SB_LOG(INFO) << "Buffer size is " << config_obus_.size(); + return config_size_; + } + bool has_mix_presentation_id() { return has_mix_presentation_id_; } + uint32_t mix_presentation_id() { return mix_presentation_id_; } + uint32_t data_size() { return data_size_; } + + std::vector config_obus() { return config_obus_; } + std::vector data() { return data_; } + + private: + bool ReadOBU(const uint8_t* buf, bool& completed_parsing); + bool ReadOBUHeader(const uint8_t* buf, + uint8_t* obu_type, + uint32_t* obu_size, + uint32_t* header_size); + + int buffer_head_ = 0; + int sample_rate_ = 0; + int samples_per_buffer_ = 0; + uint32_t config_size_ = 0; + uint32_t data_size_ = 0; + + bool has_mix_presentation_id_ = false; + uint32_t mix_presentation_id_ = 0; + + bool has_valid_config_ = false; + std::vector config_obus_; + std::vector data_; +}; + +} // namespace libiamf +} // namespace shared +} // namespace starboard + +#endif // STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ From a0cc666bc960c7be8fa151ee0bbeb9f8d25a2cb4 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 9 Jul 2024 14:43:08 -0700 Subject: [PATCH 02/13] Update config parsing --- .../shared/libiamf/iamf_audio_decoder.cc | 56 ++++++++----------- starboard/shared/libiamf/iamf_audio_decoder.h | 4 +- .../shared/libiamf/iamf_config_reader.cc | 54 +++++++++++++++++- 3 files changed, 78 insertions(+), 36 deletions(-) diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 8b537f9d51cc..aa80f3d02e9d 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -124,28 +124,30 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * frames_per_au_ * + audio_stream_info_.number_of_channels * kMaxOpusFramesPerAU * starboard::media::GetBytesPerSample(GetSampleType())); - uint32_t rsize = 0; - SB_LOG(INFO) << "Start decode"; + SB_LOG(INFO) << "Start decode with AU size " << reader_.data().size(); int ret = IAMF_decoder_decode(decoder_, reader_.data().data(), - reader_.data().size(), &rsize, + reader_.data().size(), nullptr, reinterpret_cast(decoded_audio->data())); if (ret < 1) { SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex << ret; error_cb_(kSbPlayerErrorDecode, "Failed to decode sample"); return false; + } else { + SB_LOG(INFO) << "Decoded " << ret << " samples"; + SB_DCHECK(ret <= kMaxOpusFramesPerAU); } frames_per_au_ = ret; decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * frames_per_au_ * starboard::media::GetBytesPerSample(GetSampleType())); - const auto& sample_info = input_buffer->audio_sample_info(); - decoded_audio->AdjustForDiscardedDurations( - audio_stream_info_.samples_per_second, - sample_info.discarded_duration_from_front, - sample_info.discarded_duration_from_back); + // const auto& sample_info = input_buffer->audio_sample_info(); + // decoded_audio->AdjustForDiscardedDurations( + // audio_stream_info_.samples_per_second, + // sample_info.discarded_duration_from_front, + // sample_info.discarded_duration_from_back); decoded_audios_.push(decoded_audio); output_cb_(); SB_LOG(INFO) << "Returning decoded audio"; @@ -176,47 +178,48 @@ bool IamfAudioDecoder::InitializeCodec() { SB_LOG(ERROR) << "Can't create decoder with " << channels << " channels"; return false; } - SB_LOG(INFO) << "1"; decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_LOG(ERROR) << "Error creating libiamf decoder"; return false; } - SB_LOG(INFO) << "2"; - int error = IAMF_decoder_set_bit_depth(decoder_, 32); + int error = IAMF_decoder_set_bit_depth(decoder_, kOutputBitDepth); if (error != IAMF_OK) { SB_LOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; return false; } - SB_LOG(INFO) << "3"; - error = IAMF_decoder_set_sampling_rate(decoder_, 48000); + error = IAMF_decoder_set_sampling_rate(decoder_, kOutputSamplesPerSecond); if (error != IAMF_OK) { SB_LOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " << error; return false; } - SB_LOG(INFO) << "4"; IAMF_SoundSystem sound_system = SOUND_SYSTEM_INVALID; switch (channels) { case 1: sound_system = SOUND_SYSTEM_MONO; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for mono output"; break; case 2: sound_system = SOUND_SYSTEM_A; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; break; case 6: sound_system = SOUND_SYSTEM_B; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; break; case 8: sound_system = SOUND_SYSTEM_C; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; break; default: SB_NOTREACHED(); } + error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); if (error != IAMF_OK) { SB_LOG(ERROR) @@ -224,7 +227,8 @@ bool IamfAudioDecoder::InitializeCodec() { << error; return false; } - SB_LOG(INFO) << "5"; + SB_LOG(INFO) << "num channels: " + << IAMF_layout_sound_system_channels_count(sound_system); // TODO: Accurately set pts upon resume, if needed. error = IAMF_decoder_set_pts(decoder_, 0, 90000); @@ -233,8 +237,6 @@ bool IamfAudioDecoder::InitializeCodec() { return false; } - SB_LOG(INFO) << "6"; - if (reader_.has_mix_presentation_id()) { error = IAMF_decoder_set_mix_presentation_id(decoder_, reader_.mix_presentation_id()); @@ -244,7 +246,6 @@ bool IamfAudioDecoder::InitializeCodec() { << error; return false; } - SB_LOG(INFO) << "7"; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); @@ -253,7 +254,6 @@ bool IamfAudioDecoder::InitializeCodec() { << error; return false; } - SB_LOG(INFO) << "8"; error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); if (error != IAMF_OK) { @@ -262,17 +262,14 @@ bool IamfAudioDecoder::InitializeCodec() { << error; return false; } - SB_LOG(INFO) << "9"; - uint32_t rsize = 0; error = IAMF_decoder_configure(decoder_, reader_.config_obus().data(), - reader_.config_size(), &rsize); + reader_.config_size(), nullptr); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_configure() fails with error " << error - << ", rsize " << rsize; + SB_LOG(ERROR) << "IAMF_decoder_configure() fails with error " << error; return false; } - SB_LOG(INFO) << "rsize is " << rsize; + return true; } @@ -294,7 +291,7 @@ scoped_refptr IamfAudioDecoder::Read( result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = 48000; + *samples_per_second = kOutputSamplesPerSecond; return result; } @@ -302,7 +299,6 @@ void IamfAudioDecoder::Reset() { SB_DCHECK(BelongsToCurrentThread()); if (is_valid()) { - // If fail to reset opus decoder, re-create it. TeardownCodec(); InitializeCodec(); } @@ -320,11 +316,7 @@ void IamfAudioDecoder::Reset() { SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { SB_DCHECK(BelongsToCurrentThread()); -#if SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) - return kSbMediaAudioSampleTypeInt16; -#else // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) return kSbMediaAudioSampleTypeFloat32; -#endif // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) } } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 47279a0cf986..3dd29023e65b 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -59,7 +59,9 @@ class IamfAudioDecoder void TeardownCodec(); void DecodePendingBuffers(); bool DecodeInternal(const scoped_refptr& input_buffer); - static const int kMaxOpusFramesPerAU = 9600; + static const int kMaxOpusFramesPerAU = 960; + static const int kOutputSamplesPerSecond = 48000; + static const int kOutputBitDepth = 32; SbMediaAudioSampleType GetSampleType() const; diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index bc797937d0d3..1391ae7132ff 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -14,6 +14,8 @@ #include "starboard/shared/libiamf/iamf_config_reader.h" +#include + #include "starboard/common/string.h" namespace starboard { @@ -50,11 +52,26 @@ int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { } return i + 1; } + +int ReadString(const uint8_t* buf, std::string& value) { + SB_DCHECK(buf); + value = ""; + value.reserve(128); + + int bytes_read = 0; + while (bytes_read < 128 && buf[bytes_read] != '\0') { + value.push_back(static_cast(buf[bytes_read])); + bytes_read++; + } + + return bytes_read; +} } // namespace bool IamfConfigReader::Read(scoped_refptr input_buffer) { buffer_head_ = 0; has_mix_presentation_id_ = false; + config_size_ = 0; SB_LOG(INFO) << "Input buffer size is " << input_buffer->size(); const uint8_t* buf = input_buffer->data(); SB_DCHECK(buf); @@ -72,10 +89,11 @@ bool IamfConfigReader::Read(scoped_refptr input_buffer) { SB_LOG(INFO) << "End OBU loop"; data_size_ = input_buffer->size() - config_size_; + SB_LOG(INFO) << "Input size: " << input_buffer->size() + << ", Data size: " << input_buffer->size() - config_size_ + << ", config size: " << config_size_; config_obus_.assign(buf, buf + config_size_); data_.assign(buf + config_size_, buf + input_buffer->size()); - SB_LOG(INFO) << ::starboard::HexEncode(config_obus_.data(), - config_obus_.size()); SB_LOG(INFO) << "Return read"; return true; @@ -96,6 +114,10 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { int next_obu_pos = buffer_head_ + obu_size; int bytes_read = 0; + uint32_t count_label = 0; + std::string str; + uint32_t num_sub_mixes = 0; + switch (static_cast(obu_type)) { case kObuTypeCodecConfig: SB_LOG(INFO) << "Reading codec config OBU"; @@ -106,7 +128,7 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { case kObuTypeSequenceHeader: SB_LOG(INFO) << "Reading sequence header OBU"; break; - case kObuTypeMixPresentation: + case kObuTypeMixPresentation: { SB_LOG(INFO) << "Reading mix presentation OBU"; has_mix_presentation_id_ = true; bytes_read = ReadLeb128Value(&buf[buffer_head_], &mix_presentation_id_); @@ -114,7 +136,33 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { return false; } buffer_head_ += bytes_read; + + // count_label + bytes_read = ReadLeb128Value(&buf[buffer_head_], &count_label); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + + // language_label + for (int i = 0; i < count_label; ++i) { + buffer_head_ += ReadString(&buf[buffer_head_], str); + } + + // MixPresentationAnnotations + for (int i = 0; i < count_label; ++i) { + buffer_head_ += ReadString(&buf[buffer_head_], str); + } + + // num_sub_mixes + bytes_read = ReadLeb128Value(&buf[buffer_head_], &num_sub_mixes); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + break; + } default: SB_LOG(INFO) << "OBU type " << static_cast(obu_type); completed_parsing = true; From fc99f42519da7287e08c56cab656fa2303644dec Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 10 Jul 2024 12:57:46 -0700 Subject: [PATCH 03/13] Decode samples to int16 --- .../shared/media_is_audio_supported.cc | 9 +- .../linux/shared/media_is_audio_supported.cc | 2 + .../shared/libiamf/iamf_audio_decoder.cc | 79 ++++++------ starboard/shared/libiamf/iamf_audio_decoder.h | 8 +- .../shared/libiamf/iamf_config_reader.cc | 116 ++++++++++++++---- starboard/shared/libiamf/iamf_config_reader.h | 18 ++- 6 files changed, 156 insertions(+), 76 deletions(-) diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index bb60f6124ab4..84b288614b51 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -56,11 +56,16 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, // Android uses a libopus based opus decoder for clear content, or a platform // opus decoder for encrypted content, if available. - if (audio_codec == kSbMediaAudioCodecOpus || - audio_codec == kSbMediaAudioCodecIamf) { + if (audio_codec == kSbMediaAudioCodecOpus) { return true; } +#if SB_API_VERSION >= 15 + if (audio_codec == kSbMediaAudioCodecIamf) { + return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; + } +#endif // SB_API_VERSION >= 15 + bool media_codec_supported = MediaCapabilitiesCache::GetInstance()->HasAudioDecoderFor(mime, bitrate); diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index e0d17062e521..6f686eb7737f 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -32,9 +32,11 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } +#if SB_API_VERSION >= 15 if (audio_codec == kSbMediaAudioCodecIamf) { return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } +#endif // SB_API_VERSION >= 15 if (audio_codec == kSbMediaAudioCodecAc3 || audio_codec == kSbMediaAudioCodecEac3) { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index aa80f3d02e9d..e380cb0af537 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -108,14 +108,10 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(output_cb_); SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); - SB_LOG(INFO) << "Start reading"; reader_.Read(input_buffer); - SB_LOG(INFO) << "Back in iamf decoder"; if (!decoder_) { - bool init = InitializeCodec(); - SB_LOG(INFO) << init; - if (!init) { - SB_LOG(INFO) << "Decoder init failure"; + bool decoder_initialized = InitializeCodec(); + if (!decoder_initialized) { error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); return false; } @@ -124,33 +120,36 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * kMaxOpusFramesPerAU * + audio_stream_info_.number_of_channels * kMaxIamfFramesPerAU * starboard::media::GetBytesPerSample(GetSampleType())); - SB_LOG(INFO) << "Start decode with AU size " << reader_.data().size(); - int ret = IAMF_decoder_decode(decoder_, reader_.data().data(), - reader_.data().size(), nullptr, - reinterpret_cast(decoded_audio->data())); - if (ret < 1) { - SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex << ret; - error_cb_(kSbPlayerErrorDecode, "Failed to decode sample"); + int samples_decoded = IAMF_decoder_decode( + decoder_, reader_.data().data(), reader_.data().size(), nullptr, + reinterpret_cast(decoded_audio->data())); + if (samples_decoded < 1) { + SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex + << samples_decoded; + error_cb_(kSbPlayerErrorDecode, "Failed to decode IAMF sample"); return false; - } else { - SB_LOG(INFO) << "Decoded " << ret << " samples"; - SB_DCHECK(ret <= kMaxOpusFramesPerAU); } - frames_per_au_ = ret; + SB_DCHECK(samples_decoded <= kMaxIamfFramesPerAU); + SB_LOG(INFO) << "Decoded " << samples_decoded << " samples"; + + frames_per_au_ = samples_decoded; decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * frames_per_au_ * starboard::media::GetBytesPerSample(GetSampleType())); + + // TODO: Enable once float32 pcm output is fixed. // const auto& sample_info = input_buffer->audio_sample_info(); // decoded_audio->AdjustForDiscardedDurations( // audio_stream_info_.samples_per_second, // sample_info.discarded_duration_from_front, // sample_info.discarded_duration_from_back); + decoded_audios_.push(decoded_audio); + output_cb_(); - SB_LOG(INFO) << "Returning decoded audio"; return true; } @@ -159,8 +158,6 @@ void IamfAudioDecoder::WriteEndOfStream() { SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(output_cb_); - // Opus has no dependent frames so we needn't flush the decoder. Set the - // flag to ensure that Decode() is not called when the stream is ended. stream_ended_ = true; if (!pending_audio_buffers_.empty()) { return; @@ -175,23 +172,25 @@ void IamfAudioDecoder::WriteEndOfStream() { bool IamfAudioDecoder::InitializeCodec() { int channels = audio_stream_info_.number_of_channels; if (channels > 8 || channels < 1) { - SB_LOG(ERROR) << "Can't create decoder with " << channels << " channels"; + SB_DLOG(ERROR) << "Can't create decoder with " << channels << " channels"; return false; } decoder_ = IAMF_decoder_open(); if (!decoder_) { - SB_LOG(ERROR) << "Error creating libiamf decoder"; + SB_DLOG(ERROR) << "Error creating libiamf decoder"; return false; } - int error = IAMF_decoder_set_bit_depth(decoder_, kOutputBitDepth); + // TODO: libiamf has an issue outputting 32 bit float samples, set to 16 bit + // for now. + int error = IAMF_decoder_set_bit_depth(decoder_, 16); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; return false; } - error = IAMF_decoder_set_sampling_rate(decoder_, kOutputSamplesPerSecond); + error = IAMF_decoder_set_sampling_rate(decoder_, reader_.sample_rate()); if (error != IAMF_OK) { SB_LOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " << error; @@ -202,17 +201,19 @@ bool IamfAudioDecoder::InitializeCodec() { switch (channels) { case 1: sound_system = SOUND_SYSTEM_MONO; - SB_LOG(INFO) << "Configuring IamfAudioDecoder for mono output"; break; case 2: + // Stereo output. sound_system = SOUND_SYSTEM_A; SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; break; case 6: + // 5.1 output. sound_system = SOUND_SYSTEM_B; SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; break; case 8: + // 7.1 output. sound_system = SOUND_SYSTEM_C; SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; break; @@ -237,15 +238,12 @@ bool IamfAudioDecoder::InitializeCodec() { return false; } - if (reader_.has_mix_presentation_id()) { - error = IAMF_decoder_set_mix_presentation_id(decoder_, - reader_.mix_presentation_id()); - if (error != IAMF_OK) { - SB_LOG(ERROR) - << "IAMF_decoder_set_mix_presentation_id() fails with error " - << error; - return false; - } + error = IAMF_decoder_set_mix_presentation_id(decoder_, + reader_.mix_presentation_id()); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_set_mix_presentation_id() fails with error " + << error; + return false; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); @@ -291,7 +289,7 @@ scoped_refptr IamfAudioDecoder::Read( result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = kOutputSamplesPerSecond; + *samples_per_second = reader_.sample_rate(); return result; } @@ -303,7 +301,7 @@ void IamfAudioDecoder::Reset() { InitializeCodec(); } - frames_per_au_ = kMaxOpusFramesPerAU; + frames_per_au_ = kMaxIamfFramesPerAU; stream_ended_ = false; while (!decoded_audios_.empty()) { decoded_audios_.pop(); @@ -316,7 +314,10 @@ void IamfAudioDecoder::Reset() { SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { SB_DCHECK(BelongsToCurrentThread()); - return kSbMediaAudioSampleTypeFloat32; + // if (audio_stream_info_.bits_per_sample == 32) { + // return kSbMediaAudioSampleTypeFloat32; + // } + return kSbMediaAudioSampleTypeInt16Deprecated; } } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 3dd29023e65b..362dc4b27c72 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -17,7 +17,6 @@ #include #include -#include #include "starboard/common/ref_counted.h" #include "starboard/media.h" @@ -54,14 +53,13 @@ class IamfAudioDecoder private: static constexpr int kMinimumBuffersToDecode = 2; + static constexpr int kMaxIamfFramesPerAU = 2048; + static constexpr int kMaxOpusFramesPerAU = 960; bool InitializeCodec(); void TeardownCodec(); void DecodePendingBuffers(); bool DecodeInternal(const scoped_refptr& input_buffer); - static const int kMaxOpusFramesPerAU = 960; - static const int kOutputSamplesPerSecond = 48000; - static const int kOutputBitDepth = 32; SbMediaAudioSampleType GetSampleType() const; @@ -72,7 +70,7 @@ class IamfAudioDecoder bool stream_ended_ = false; std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; - int frames_per_au_ = kMaxOpusFramesPerAU; + int frames_per_au_ = kMaxIamfFramesPerAU; std::deque> pending_audio_buffers_; ConsumedCB consumed_cb_; diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index 1391ae7132ff..e3ec533a1478 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -30,6 +30,21 @@ constexpr int kObuTypeAudioElement = 1; constexpr int kObuTypeMixPresentation = 2; constexpr int kObuTypeSequenceHeader = 31; +// From +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig. +constexpr int kFourccOpus = 0x4f707573; +constexpr int kFourccMp4a = 0x6d703461; +constexpr int kFourccFlac = 0x664c6143; +constexpr int kFourccIpcm = 0x6970636d; + +inline uint32_t ByteSwap(uint32_t x) { +#if defined(COMPILER_MSVC) + return _byteswap_ulong(x); +#else + return __builtin_bswap32(x); +#endif +} + // Decodes an Leb128 value and stores it in |value|. Returns the number of bytes // read. Returns -1 on error. int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { @@ -69,10 +84,8 @@ int ReadString(const uint8_t* buf, std::string& value) { } // namespace bool IamfConfigReader::Read(scoped_refptr input_buffer) { - buffer_head_ = 0; - has_mix_presentation_id_ = false; - config_size_ = 0; - SB_LOG(INFO) << "Input buffer size is " << input_buffer->size(); + Reset(); + const uint8_t* buf = input_buffer->data(); SB_DCHECK(buf); @@ -86,19 +99,24 @@ bool IamfConfigReader::Read(scoped_refptr input_buffer) { SB_CHECK(completed_parsing); - SB_LOG(INFO) << "End OBU loop"; - data_size_ = input_buffer->size() - config_size_; - SB_LOG(INFO) << "Input size: " << input_buffer->size() - << ", Data size: " << input_buffer->size() - config_size_ - << ", config size: " << config_size_; config_obus_.assign(buf, buf + config_size_); data_.assign(buf + config_size_, buf + input_buffer->size()); - SB_LOG(INFO) << "Return read"; return true; } +void IamfConfigReader::Reset() { + buffer_head_ = 0; + has_mix_presentation_id_ = false; + mix_presentation_id_ = 0; + config_size_ = 0; + data_size_ = 0; + sample_rate_ = 0; + samples_per_buffer_ = 0; + sample_size_ = 0; +} + bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { uint8_t obu_type = 0; uint32_t obu_size = 0; @@ -107,10 +125,7 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { SB_LOG(ERROR) << "Error reading OBU header"; return false; } - SB_LOG(INFO) << "OBU size is " << obu_size << " with current buffer head " - << buffer_head_; - // const uint8_t* last_byte = buf + buffer_head_ + obu_size; int next_obu_pos = buffer_head_ + obu_size; int bytes_read = 0; @@ -119,17 +134,73 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { uint32_t num_sub_mixes = 0; switch (static_cast(obu_type)) { - case kObuTypeCodecConfig: - SB_LOG(INFO) << "Reading codec config OBU"; + case kObuTypeCodecConfig: { + sample_rate_ = 0; + uint32_t codec_config_id; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &codec_config_id); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + + uint32_t codec_id = 0; + std::memcpy(&codec_id, &buf[buffer_head_], sizeof(uint32_t)); + // Mp4 is in big-endian + codec_id = ByteSwap(codec_id); + buffer_head_ += 4; + SB_LOG(INFO) << std::hex << codec_id; + + bytes_read = ReadLeb128Value(&buf[buffer_head_], &samples_per_buffer_); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + + // audio_roll_distance + buffer_head_ += 2; + + switch (codec_id) { + case kFourccOpus: + sample_rate_ = 48000; + break; + case kFourccMp4a: + // TODO: Properly parse for the sample rate. 48000 is the assumed + // sample rate. + sample_rate_ = 48000; + break; + case kFourccFlac: { + // Skip METADATA_BLOCK_HEADER. + buffer_head_ += 4; + // Skip first 10 bytes of METADATA_BLOCK_STREAMINFO. + buffer_head_ += 10; + + std::memcpy(&sample_rate_, &buf[buffer_head_], sizeof(uint32_t)); + sample_rate_ = ByteSwap(sample_rate_); + sample_rate_ = sample_rate_ >> 12; + break; + } + case kFourccIpcm: { + // sample_format_flags + buffer_head_++; + uint8_t sample_size_byte = buf[buffer_head_]; + sample_size_ = static_cast(sample_size_byte); + buffer_head_++; + + std::memcpy(&sample_rate_, &buf[buffer_head_], sizeof(uint32_t)); + sample_rate_ = ByteSwap(sample_rate_); + break; + } + default: + SB_NOTREACHED(); + } + break; + } case kObuTypeAudioElement: - SB_LOG(INFO) << "Reading audio element OBU"; - break; case kObuTypeSequenceHeader: - SB_LOG(INFO) << "Reading sequence header OBU"; break; case kObuTypeMixPresentation: { - SB_LOG(INFO) << "Reading mix presentation OBU"; + // TODO: Complete Mix Presentation OBU parsing has_mix_presentation_id_ = true; bytes_read = ReadLeb128Value(&buf[buffer_head_], &mix_presentation_id_); if (bytes_read < 0) { @@ -164,7 +235,8 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { break; } default: - SB_LOG(INFO) << "OBU type " << static_cast(obu_type); + // Once an OBU is read that is not a descriptor, descriptor parsing is + // assumed to be complete. completed_parsing = true; break; } @@ -181,7 +253,6 @@ bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, uint8_t* obu_type, uint32_t* obu_size, uint32_t* header_size) { - SB_LOG(INFO) << "buffer head is " << buffer_head_; uint8_t header_flags = buf[buffer_head_]; *obu_type = (header_flags >> 3) & 0x1f; buffer_head_++; @@ -194,12 +265,9 @@ bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, *header_size = 1; - // redundant_copy |= obu_redundant_copy; - *obu_size = 0; int bytes_read = ReadLeb128Value(&buf[buffer_head_], obu_size); if (bytes_read < 0) { - SB_LOG(INFO) << "Error reading OBU size"; return false; } buffer_head_ += bytes_read; diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index d1be99c50218..04a6d76d98ae 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -25,6 +25,7 @@ namespace starboard { namespace shared { namespace libiamf { +// TODO: Add handling for non-redundant OBUS class IamfConfigReader { public: typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; @@ -33,13 +34,17 @@ class IamfConfigReader { bool Read(scoped_refptr input_buffer); + void Reset(); + + bool is_valid() { + return data_size_ > 0 && config_size_ > 0 && has_mix_presentation_id_ && + sample_rate_ > 0; + } int sample_rate() { return sample_rate_; } + uint32_t samples_per_buffer() { return samples_per_buffer_; } bool config_changed() { return false; } - uint32_t config_size() { - SB_LOG(INFO) << "Config size is " << config_size_; - SB_LOG(INFO) << "Buffer size is " << config_obus_.size(); - return config_size_; - } + uint32_t config_size() { return config_size_; } + // TODO: Allow for selection of multiple mix presentation IDs. bool has_mix_presentation_id() { return has_mix_presentation_id_; } uint32_t mix_presentation_id() { return mix_presentation_id_; } uint32_t data_size() { return data_size_; } @@ -56,7 +61,8 @@ class IamfConfigReader { int buffer_head_ = 0; int sample_rate_ = 0; - int samples_per_buffer_ = 0; + int sample_size_ = 0; + uint32_t samples_per_buffer_ = 0; uint32_t config_size_ = 0; uint32_t data_size_ = 0; From 0c7578508ac34fc57ccad14e15f0120a2d1714c7 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 10 Jul 2024 17:43:47 -0700 Subject: [PATCH 04/13] Enable Android IAMF decode support --- starboard/android/shared/media_common.h | 5 ++++ .../shared/media_is_audio_supported.cc | 2 +- .../shared/player_components_factory.h | 6 +++-- starboard/android/shared/player_create.cc | 6 ++++- .../shared/libiamf/iamf_audio_decoder.cc | 24 ++++++++++++------- starboard/shared/libiamf/iamf_audio_decoder.h | 2 ++ .../shared/libiamf/iamf_config_reader.cc | 3 --- 7 files changed, 32 insertions(+), 16 deletions(-) diff --git a/starboard/android/shared/media_common.h b/starboard/android/shared/media_common.h index dcbd0b36d108..4a2b6b86731c 100644 --- a/starboard/android/shared/media_common.h +++ b/starboard/android/shared/media_common.h @@ -61,6 +61,11 @@ inline const char* SupportedAudioCodecToMimeType( if (audio_codec == kSbMediaAudioCodecOpus) { return "audio/opus"; } +#if SB_API_VERSION >= 15 + if (audio_codec == kSbMediaAudioCodecIamf) { + return "audio/iamf"; + } +#endif // SB_API_VERSION >= 15 return nullptr; } diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index 84b288614b51..6074dce486f0 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -62,7 +62,7 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, #if SB_API_VERSION >= 15 if (audio_codec == kSbMediaAudioCodecIamf) { - return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; + return true; } #endif // SB_API_VERSION >= 15 diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index b3b766d7be1a..98a4d8700075 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -447,10 +447,12 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: std::move(audio_decoder_impl)); } } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { - scoped_ptr audio_decoder_impl( + SB_LOG(INFO) << "Creating IAMF audio decoder"; + std::unique_ptr audio_decoder_impl( new IamfAudioDecoder(audio_stream_info)); if (audio_decoder_impl->is_valid()) { - return audio_decoder_impl.PassAs(); + return std::unique_ptr( + std::move(audio_decoder_impl)); } } else { SB_LOG(ERROR) << "Unsupported audio codec " diff --git a/starboard/android/shared/player_create.cc b/starboard/android/shared/player_create.cc index 78b4ba46f252..2b9a88903b09 100644 --- a/starboard/android/shared/player_create.cc +++ b/starboard/android/shared/player_create.cc @@ -127,7 +127,11 @@ SbPlayer SbPlayerCreate(SbWindow window, audio_codec != kSbMediaAudioCodecAac && audio_codec != kSbMediaAudioCodecAc3 && audio_codec != kSbMediaAudioCodecEac3 && - audio_codec != kSbMediaAudioCodecOpus) { + audio_codec != kSbMediaAudioCodecOpus +#if SB_API_VERSION >= 15 + && audio_codec != kSbMediaAudioCodecIamf +#endif // SB_API_VERSION >= 15 + ) { SB_LOG(ERROR) << "Unsupported audio codec: " << starboard::GetMediaAudioCodecName(audio_codec) << "."; player_error_func( diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index e380cb0af537..075aa44a2d07 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -27,7 +27,12 @@ using shared::starboard::player::DecodedAudio; } // namespace IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) - : audio_stream_info_(audio_stream_info) {} + : audio_stream_info_(audio_stream_info) { + decoder_ = IAMF_decoder_open(); + if (!decoder_) { + SB_DLOG(ERROR) << "Error creating libiamf decoder"; + } +} IamfAudioDecoder::~IamfAudioDecoder() { TeardownCodec(); @@ -107,11 +112,13 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(input_buffer->size() > 0); SB_DCHECK(output_cb_); SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); + SB_DCHECK(is_valid()); reader_.Read(input_buffer); - if (!decoder_) { + if (!decoder_is_configured_) { bool decoder_initialized = InitializeCodec(); if (!decoder_initialized) { + SB_LOG(INFO) << "Failed to initialize IAMF decoder"; error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); return false; } @@ -170,18 +177,13 @@ void IamfAudioDecoder::WriteEndOfStream() { } bool IamfAudioDecoder::InitializeCodec() { + SB_DCHECK(is_valid()); int channels = audio_stream_info_.number_of_channels; if (channels > 8 || channels < 1) { SB_DLOG(ERROR) << "Can't create decoder with " << channels << " channels"; return false; } - decoder_ = IAMF_decoder_open(); - if (!decoder_) { - SB_DLOG(ERROR) << "Error creating libiamf decoder"; - return false; - } - // TODO: libiamf has an issue outputting 32 bit float samples, set to 16 bit // for now. int error = IAMF_decoder_set_bit_depth(decoder_, 16); @@ -268,6 +270,8 @@ bool IamfAudioDecoder::InitializeCodec() { return false; } + decoder_is_configured_ = true; + return true; } @@ -298,9 +302,11 @@ void IamfAudioDecoder::Reset() { if (is_valid()) { TeardownCodec(); - InitializeCodec(); + decoder_ = IAMF_decoder_open(); } + decoder_is_configured_ = false; + frames_per_au_ = kMaxIamfFramesPerAU; stream_ended_ = false; while (!decoded_audios_.empty()) { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 362dc4b27c72..a166f60b5a86 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -72,6 +72,8 @@ class IamfAudioDecoder AudioStreamInfo audio_stream_info_; int frames_per_au_ = kMaxIamfFramesPerAU; + bool decoder_is_configured_ = false; + std::deque> pending_audio_buffers_; ConsumedCB consumed_cb_; diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index e3ec533a1478..a9fd377d0f97 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -148,7 +148,6 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { // Mp4 is in big-endian codec_id = ByteSwap(codec_id); buffer_head_ += 4; - SB_LOG(INFO) << std::hex << codec_id; bytes_read = ReadLeb128Value(&buf[buffer_head_], &samples_per_buffer_); if (bytes_read < 0) { @@ -261,8 +260,6 @@ bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, const bool obu_trimming_status_flag = (header_flags >> 1) & 1; const bool obu_extension_flag = header_flags & 1; - SB_LOG(INFO) << static_cast(*obu_type); - *header_size = 1; *obu_size = 0; From 01d087d9e8ad57aaba0e74f92af849ce9e0002aa Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 17 Jul 2024 16:10:32 -0700 Subject: [PATCH 05/13] Add build args Also clean up logs and reformat decoder. --- starboard/android/shared/BUILD.gn | 16 +- .../shared/media_is_audio_supported.cc | 2 +- .../shared/platform_configuration/BUILD.gn | 10 +- .../shared/player_components_factory.h | 14 +- starboard/build/config/BUILDCONFIG.gn | 2 + starboard/linux/shared/BUILD.gn | 15 +- .../linux/shared/media_is_audio_supported.cc | 2 +- .../linux/shared/player_components_factory.cc | 11 +- starboard/linux/x64x11/BUILD.gn | 1 + .../shared/platform_configuration/BUILD.gn | 10 +- .../shared/libiamf/iamf_audio_decoder.cc | 197 +++++++++++------- starboard/shared/libiamf/iamf_audio_decoder.h | 8 +- .../shared/libiamf/iamf_config_reader.cc | 70 +++---- starboard/shared/libiamf/iamf_config_reader.h | 26 ++- 14 files changed, 233 insertions(+), 151 deletions(-) diff --git a/starboard/android/shared/BUILD.gn b/starboard/android/shared/BUILD.gn index f3ff9e59cf92..243b49a0db7c 100644 --- a/starboard/android/shared/BUILD.gn +++ b/starboard/android/shared/BUILD.gn @@ -75,10 +75,6 @@ static_library("starboard_platform") { "//starboard/shared/libevent/socket_waiter_wait.cc", "//starboard/shared/libevent/socket_waiter_wait_timed.cc", "//starboard/shared/libevent/socket_waiter_wake_up.cc", - "//starboard/shared/libiamf/iamf_audio_decoder.cc", - "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_config_reader.cc", - "//starboard/shared/libiamf/iamf_config_reader.h", "//starboard/shared/linux/byte_swap.cc", "//starboard/shared/linux/cpu_features_get.cc", "//starboard/shared/linux/memory_get_stack_bounds.cc", @@ -491,6 +487,18 @@ static_library("starboard_platform") { if (sb_evergreen_compatible_use_libunwind) { deps += [ "//third_party/llvm-project/libunwind:unwind_starboard" ] } + + defines = [] + if (enable_iamf_decode) { + sources += [ + "//starboard/shared/libiamf/iamf_audio_decoder.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.h", + "//starboard/shared/libiamf/iamf_config_reader.cc", + "//starboard/shared/libiamf/iamf_config_reader.h", + ] + + defines += [ "ENABLE_IAMF_DECODE" ] + } } static_library("starboard_base_symbolize") { diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index 6074dce486f0..b2c9d0b9c62f 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -60,7 +60,7 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return true; } -#if SB_API_VERSION >= 15 +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE if (audio_codec == kSbMediaAudioCodecIamf) { return true; } diff --git a/starboard/android/shared/platform_configuration/BUILD.gn b/starboard/android/shared/platform_configuration/BUILD.gn index 785a7ad48f78..dfcdc08ab6d9 100644 --- a/starboard/android/shared/platform_configuration/BUILD.gn +++ b/starboard/android/shared/platform_configuration/BUILD.gn @@ -175,7 +175,9 @@ config("platform_configuration") { "-Wl,--wrap=readdir_r", ] } - configs += [ ":libraries" ] + if (enable_iamf_decode) { + configs += [ ":libiamf_config" ] + } } config("size") { @@ -219,6 +221,8 @@ config("pedantic_warnings") { ] } -config("libraries") { - libs = [ "//third_party/libiamf/code/android/libiamf.a" ] +if (enable_iamf_decode) { + config("libiamf_config") { + libs = [ "//third_party/libiamf/code/android/libiamf.a" ] + } } diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index 98a4d8700075..83cb08c37651 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -34,7 +34,6 @@ #include "starboard/common/media.h" #include "starboard/common/ref_counted.h" #include "starboard/media.h" -#include "starboard/shared/libiamf/iamf_audio_decoder.h" #include "starboard/shared/opus/opus_audio_decoder.h" #include "starboard/shared/starboard/media/media_util.h" #include "starboard/shared/starboard/media/mime_type.h" @@ -49,6 +48,10 @@ #include "starboard/shared/starboard/player/filter/video_renderer_internal_impl.h" #include "starboard/shared/starboard/player/filter/video_renderer_sink.h" +#if ENABLE_IAMF_DECODE +#include "starboard/shared/libiamf/iamf_audio_decoder.h" +#endif // ENABLE_IAMF_DECODE + namespace starboard { namespace android { namespace shared { @@ -180,7 +183,6 @@ class PlayerComponentsPassthrough class PlayerComponentsFactory : public starboard::shared::starboard::player:: filter::PlayerComponents::Factory { typedef starboard::shared::starboard::media::MimeType MimeType; - typedef starboard::shared::libiamf::IamfAudioDecoder IamfAudioDecoder; typedef starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; typedef starboard::shared::starboard::player::filter::AdaptiveAudioDecoder AdaptiveAudioDecoder; @@ -446,14 +448,18 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: return std::unique_ptr( std::move(audio_decoder_impl)); } +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { SB_LOG(INFO) << "Creating IAMF audio decoder"; - std::unique_ptr audio_decoder_impl( - new IamfAudioDecoder(audio_stream_info)); + std::unique_ptr + audio_decoder_impl( + new starboard::shared::libiamf::IamfAudioDecoder( + audio_stream_info, /* prefer_binarual_audio */ false)); if (audio_decoder_impl->is_valid()) { return std::unique_ptr( std::move(audio_decoder_impl)); } +#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else { SB_LOG(ERROR) << "Unsupported audio codec " << audio_stream_info.codec; diff --git a/starboard/build/config/BUILDCONFIG.gn b/starboard/build/config/BUILDCONFIG.gn index 77d7034b160e..ad514f23c8de 100644 --- a/starboard/build/config/BUILDCONFIG.gn +++ b/starboard/build/config/BUILDCONFIG.gn @@ -40,6 +40,8 @@ declare_args() { build_with_separate_cobalt_toolchain = false use_xcode_clang = false + + enable_iamf_decode = false } _is_on_pythonpath = exec_script("//starboard/build/is_on_path.py", [], "json") diff --git a/starboard/linux/shared/BUILD.gn b/starboard/linux/shared/BUILD.gn index 1b6339f9e243..7ec64f380174 100644 --- a/starboard/linux/shared/BUILD.gn +++ b/starboard/linux/shared/BUILD.gn @@ -125,10 +125,6 @@ static_library("starboard_platform_sources") { "//starboard/shared/libfdkaac/fdk_aac_audio_decoder.h", "//starboard/shared/libfdkaac/libfdkaac_library_loader.cc", "//starboard/shared/libfdkaac/libfdkaac_library_loader.h", - "//starboard/shared/libiamf/iamf_audio_decoder.cc", - "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_config_reader.cc", - "//starboard/shared/libiamf/iamf_config_reader.h", "//starboard/shared/libvpx/vpx_video_decoder.cc", "//starboard/shared/libvpx/vpx_video_decoder.h", "//starboard/shared/linux/byte_swap.cc", @@ -445,6 +441,17 @@ static_library("starboard_platform_sources") { if (is_debug || is_devel) { defines += [ "SB_PLAYER_ENABLE_VIDEO_DUMPER" ] } + + if (enable_iamf_decode) { + sources += [ + "//starboard/shared/libiamf/iamf_audio_decoder.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.h", + "//starboard/shared/libiamf/iamf_config_reader.cc", + "//starboard/shared/libiamf/iamf_config_reader.h", + ] + + defines += [ "ENABLE_IAMF_DECODE" ] + } } if (current_toolchain == starboard_toolchain) { diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index 6f686eb7737f..8541cd76c55b 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -32,7 +32,7 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } -#if SB_API_VERSION >= 15 +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE if (audio_codec == kSbMediaAudioCodecIamf) { return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } diff --git a/starboard/linux/shared/player_components_factory.cc b/starboard/linux/shared/player_components_factory.cc index 4b82253d958b..a49a33f934a6 100644 --- a/starboard/linux/shared/player_components_factory.cc +++ b/starboard/linux/shared/player_components_factory.cc @@ -25,7 +25,6 @@ #include "starboard/shared/libde265/de265_video_decoder.h" #include "starboard/shared/libfdkaac/fdk_aac_audio_decoder.h" #include "starboard/shared/libfdkaac/libfdkaac_library_loader.h" -#include "starboard/shared/libiamf/iamf_audio_decoder.h" #include "starboard/shared/libvpx/vpx_video_decoder.h" #include "starboard/shared/openh264/openh264_library_loader.h" #include "starboard/shared/openh264/openh264_video_decoder.h" @@ -42,6 +41,10 @@ #include "starboard/shared/starboard/player/filter/video_render_algorithm_impl.h" #include "starboard/shared/starboard/player/filter/video_renderer_sink.h" +#if ENABLE_IAMF_DECODE +#include "starboard/shared/libiamf/iamf_audio_decoder.h" +#endif // ENABLE_IAMF_DECODE + namespace starboard { namespace shared { namespace starboard { @@ -69,7 +72,6 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { SB_DCHECK(audio_renderer_sink); typedef ::starboard::shared::ffmpeg::AudioDecoder FfmpegAudioDecoder; - typedef ::starboard::shared::libiamf::IamfAudioDecoder IamfAudioDecoder; typedef ::starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; typedef ::starboard::shared::libfdkaac::FdkAacAudioDecoder FdkAacAudioDecoder; @@ -88,10 +90,13 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { libfdkaac::LibfdkaacHandle::GetHandle()->IsLoaded()) { SB_LOG(INFO) << "Playing audio using FdkAacAudioDecoder."; return std::unique_ptr(new FdkAacAudioDecoder()); +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { SB_LOG(INFO) << "Playing audio using IamfAudioDecoder."; return std::unique_ptr( - new IamfAudioDecoder(audio_stream_info)); + new ::starboard::shared::libiamf::IamfAudioDecoder( + audio_stream_info, /* prefer_binarual_audio */ false)); +#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else { std::unique_ptr audio_decoder_impl( FfmpegAudioDecoder::Create(audio_stream_info)); diff --git a/starboard/linux/x64x11/BUILD.gn b/starboard/linux/x64x11/BUILD.gn index 3666f36cb196..fa50bb0cff0c 100644 --- a/starboard/linux/x64x11/BUILD.gn +++ b/starboard/linux/x64x11/BUILD.gn @@ -34,5 +34,6 @@ if (sb_is_modular && !sb_is_evergreen) { sources = [ "//starboard/linux/x64x11/main.cc" ] configs += [ "//starboard/build/config:starboard_implementation" ] public_deps = [ ":starboard_platform" ] + deps = [ "//third_party/flac" ] } } diff --git a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn index 4874cf39e506..5ebd592449ed 100644 --- a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn +++ b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn @@ -30,10 +30,12 @@ config("platform_configuration") { config("libraries") { configs = [ "//starboard/linux/shared/platform_configuration:libraries" ] - libs = [ - "//third_party/libiamf/code/libiamf.a", - "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", - ] + if (enable_iamf_decode) { + libs = [ + "//third_party/libiamf/code/libiamf.a", + "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", + ] + } } config("linker_flags") { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 075aa44a2d07..acc2f395cfb0 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -15,6 +15,7 @@ #include "starboard/shared/libiamf/iamf_audio_decoder.h" #include +#include #include "third_party/libiamf/source/code/include/IAMF_defines.h" @@ -24,10 +25,39 @@ namespace libiamf { namespace { using shared::starboard::player::DecodedAudio; + +constexpr int kForceBinauralAudio = false; +// Keep disabled as surround audio may require changes further up the SbPlayer. +constexpr int kEnableSurroundAudio = false; + +std::string ErrorCodeToString(int code) { + switch (code) { + case IAMF_OK: + return "IAMF_OK"; + case IAMF_ERR_BAD_ARG: + return "IAMF_ERR_BAD_ARG"; + case IAMF_ERR_BUFFER_TOO_SMALL: + return "IAMF_ERR_BUFFER_TOO_SMALL"; + case IAMF_ERR_INTERNAL: + return "IAMF_ERR_INTERNAL"; + case IAMF_ERR_INVALID_PACKET: + return "IAMF_ERR_INVALID_PACKET"; + case IAMF_ERR_INVALID_STATE: + return "IAMF_ERR_INVALID_STATE"; + case IAMF_ERR_UNIMPLEMENTED: + return "IAMF_ERR_UNIMPLEMENTED"; + case IAMF_ERR_ALLOC_FAIL: + return "IAMF_ERR_ALLOC_FAIL"; + default: + return "Unknown IAMF error code " + std::to_string(code); + } +} } // namespace -IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) - : audio_stream_info_(audio_stream_info) { +IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, + bool prefer_binaural_audio) + : audio_stream_info_(audio_stream_info), + prefer_binarual_audio_(prefer_binaural_audio) { decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_DLOG(ERROR) << "Error creating libiamf decoder"; @@ -62,9 +92,10 @@ void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, SB_DCHECK(output_cb_); if (stream_ended_) { - SB_LOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; + SB_DLOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; return; } + if (input_buffers.size() > kMinimumBuffersToDecode) { std::copy(std::begin(input_buffers), std::end(input_buffers), std::back_inserter(pending_audio_buffers_)); @@ -114,11 +145,15 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); SB_DCHECK(is_valid()); - reader_.Read(input_buffer); + reader_.ResetAndRead(input_buffer); + if (!reader_.is_valid()) { + SB_DLOG(INFO) << "Failed to parse IA Descriptors"; + error_cb_(kSbPlayerErrorDecode, "Failed to parse IA Descriptors"); + return false; + } if (!decoder_is_configured_) { - bool decoder_initialized = InitializeCodec(); - if (!decoder_initialized) { - SB_LOG(INFO) << "Failed to initialize IAMF decoder"; + if (!InitializeCodec()) { + SB_DLOG(INFO) << "Failed to initialize IAMF decoder"; error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); return false; } @@ -127,32 +162,31 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * kMaxIamfFramesPerAU * + audio_stream_info_.number_of_channels * reader_.samples_per_buffer() * starboard::media::GetBytesPerSample(GetSampleType())); int samples_decoded = IAMF_decoder_decode( decoder_, reader_.data().data(), reader_.data().size(), nullptr, reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { - SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex - << samples_decoded; - error_cb_(kSbPlayerErrorDecode, "Failed to decode IAMF sample"); + SB_DLOG(INFO) << "IAMF_decoder_decode() error " + << ErrorCodeToString(samples_decoded); + error_cb_(kSbPlayerErrorDecode, "Failed to decode IAMF sample, error " + + ErrorCodeToString(samples_decoded)); return false; } - SB_DCHECK(samples_decoded <= kMaxIamfFramesPerAU); - SB_LOG(INFO) << "Decoded " << samples_decoded << " samples"; + SB_DCHECK(samples_decoded <= reader_.samples_per_buffer()); - frames_per_au_ = samples_decoded; decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * - frames_per_au_ * + reader_.samples_per_buffer() * starboard::media::GetBytesPerSample(GetSampleType())); - // TODO: Enable once float32 pcm output is fixed. - // const auto& sample_info = input_buffer->audio_sample_info(); - // decoded_audio->AdjustForDiscardedDurations( - // audio_stream_info_.samples_per_second, - // sample_info.discarded_duration_from_front, - // sample_info.discarded_duration_from_back); + // TODO: Enable partial audio once float32 pcm output is available. + const auto& sample_info = input_buffer->audio_sample_info(); + decoded_audio->AdjustForDiscardedDurations( + audio_stream_info_.samples_per_second, + sample_info.discarded_duration_from_front, + sample_info.discarded_duration_from_back); decoded_audios_.push(decoded_audio); @@ -178,95 +212,116 @@ void IamfAudioDecoder::WriteEndOfStream() { bool IamfAudioDecoder::InitializeCodec() { SB_DCHECK(is_valid()); - int channels = audio_stream_info_.number_of_channels; - if (channels > 8 || channels < 1) { - SB_DLOG(ERROR) << "Can't create decoder with " << channels << " channels"; - return false; - } + SB_DCHECK(!decoder_is_configured_); // TODO: libiamf has an issue outputting 32 bit float samples, set to 16 bit // for now. int error = IAMF_decoder_set_bit_depth(decoder_, 16); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " + << ErrorCodeToString(error); return false; } error = IAMF_decoder_set_sampling_rate(decoder_, reader_.sample_rate()); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " - << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " + << ErrorCodeToString(error); return false; } - IAMF_SoundSystem sound_system = SOUND_SYSTEM_INVALID; - switch (channels) { - case 1: - sound_system = SOUND_SYSTEM_MONO; - break; - case 2: - // Stereo output. - sound_system = SOUND_SYSTEM_A; - SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; - break; - case 6: - // 5.1 output. - sound_system = SOUND_SYSTEM_B; - SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; - break; - case 8: - // 7.1 output. - sound_system = SOUND_SYSTEM_C; - SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; - break; - default: - SB_NOTREACHED(); - } + if (kForceBinauralAudio || prefer_binarual_audio_) { + SB_DLOG(INFO) << "Configuring IamfAudioDecoder for binaural output"; + error = IAMF_decoder_output_layout_set_binaural(decoder_); + if (error != IAMF_OK) { + SB_DLOG(ERROR) + << "IAMF_decoder_output_layout_set_binaural() fails with error " + << ErrorCodeToString(error); + return false; + } + } else { + // Default to stereo output. If kEnableSurroundAudio is true, set to a sound + // system matching, the platform's audio configuration, if available. + IAMF_SoundSystem sound_system = SOUND_SYSTEM_A; + if (kEnableSurroundAudio) { + SbMediaAudioConfiguration out_config; + SbMediaGetAudioConfiguration(0, &out_config); + int channels = std::max(out_config.number_of_channels, 2); + if (channels > 8 || channels < 1) { + SB_DLOG(ERROR) << "Can't create decoder with " << channels + << " channels"; + return false; + } + switch (channels) { + case 1: + sound_system = SOUND_SYSTEM_MONO; + break; + case 2: + // Stereo output. + sound_system = SOUND_SYSTEM_A; + SB_DLOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; + break; + case 6: + // 5.1 output. + sound_system = SOUND_SYSTEM_B; + SB_DLOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; + break; + case 8: + // 7.1 output. + sound_system = SOUND_SYSTEM_C; + SB_DLOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; + break; + default: + SB_NOTREACHED(); + } + } else { + SB_DLOG(INFO) << "Defaulting to stereo output."; + } - error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); - if (error != IAMF_OK) { - SB_LOG(ERROR) - << "IAMF_decoder_output_layout_set_sound_system() fails with error " - << error; - return false; + error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); + if (error != IAMF_OK) { + SB_DLOG(ERROR) + << "IAMF_decoder_output_layout_set_sound_system() fails with error " + << ErrorCodeToString(error); + return false; + } } - SB_LOG(INFO) << "num channels: " - << IAMF_layout_sound_system_channels_count(sound_system); - // TODO: Accurately set pts upon resume, if needed. error = IAMF_decoder_set_pts(decoder_, 0, 90000); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_set_pts() fails with error " << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_pts() fails with error " + << ErrorCodeToString(error); return false; } error = IAMF_decoder_set_mix_presentation_id(decoder_, reader_.mix_presentation_id()); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_set_mix_presentation_id() fails with error " - << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_mix_presentation_id() fails with error " + << ErrorCodeToString(error); return false; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_peak_limiter_enable() fails with error " - << error; + SB_DLOG(ERROR) << "IAMF_decoder_peak_limiter_enable() fails with error " + << ErrorCodeToString(error); return false; } error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); if (error != IAMF_OK) { - SB_LOG(ERROR) + SB_DLOG(ERROR) << "IAMF_decoder_set_normalization_loudness() fails with error " - << error; + << ErrorCodeToString(error); return false; } error = IAMF_decoder_configure(decoder_, reader_.config_obus().data(), reader_.config_size(), nullptr); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_configure() fails with error " << error; + SB_DLOG(ERROR) << "IAMF_decoder_configure() fails with error " + << ErrorCodeToString(error); return false; } @@ -307,7 +362,6 @@ void IamfAudioDecoder::Reset() { decoder_is_configured_ = false; - frames_per_au_ = kMaxIamfFramesPerAU; stream_ended_ = false; while (!decoded_audios_.empty()) { decoded_audios_.pop(); @@ -320,9 +374,6 @@ void IamfAudioDecoder::Reset() { SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { SB_DCHECK(BelongsToCurrentThread()); - // if (audio_stream_info_.bits_per_sample == 32) { - // return kSbMediaAudioSampleTypeFloat32; - // } return kSbMediaAudioSampleTypeInt16Deprecated; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index a166f60b5a86..55568dc8de8d 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -38,7 +38,8 @@ class IamfAudioDecoder public: typedef starboard::media::AudioStreamInfo AudioStreamInfo; - explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); + explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, + bool prefer_binaural_audio); ~IamfAudioDecoder() override; bool is_valid() const; @@ -53,8 +54,6 @@ class IamfAudioDecoder private: static constexpr int kMinimumBuffersToDecode = 2; - static constexpr int kMaxIamfFramesPerAU = 2048; - static constexpr int kMaxOpusFramesPerAU = 960; bool InitializeCodec(); void TeardownCodec(); @@ -70,7 +69,6 @@ class IamfAudioDecoder bool stream_ended_ = false; std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; - int frames_per_au_ = kMaxIamfFramesPerAU; bool decoder_is_configured_ = false; @@ -78,6 +76,8 @@ class IamfAudioDecoder ConsumedCB consumed_cb_; IamfConfigReader reader_; + + const bool prefer_binarual_audio_; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index a9fd377d0f97..2643e336a92f 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -37,6 +37,10 @@ constexpr int kFourccMp4a = 0x6d703461; constexpr int kFourccFlac = 0x664c6143; constexpr int kFourccIpcm = 0x6970636d; +constexpr int kLayoutTypeReserved = 1; +constexpr int kLayoutTypeSpeaker = 2; +constexpr int kLayoutTypeBinaural = 3; + inline uint32_t ByteSwap(uint32_t x) { #if defined(COMPILER_MSVC) return _byteswap_ulong(x); @@ -74,25 +78,44 @@ int ReadString(const uint8_t* buf, std::string& value) { value.reserve(128); int bytes_read = 0; + bool error = true; while (bytes_read < 128 && buf[bytes_read] != '\0') { value.push_back(static_cast(buf[bytes_read])); bytes_read++; + error = false; + } + + if (error) { + return -1; } return bytes_read; } } // namespace -bool IamfConfigReader::Read(scoped_refptr input_buffer) { +bool IamfConfigReader::ResetAndRead(scoped_refptr input_buffer) { Reset(); + return Read(input_buffer); +} +void IamfConfigReader::Reset() { + buffer_head_ = 0; + mix_presentation_id_ = std::optional(); + config_size_ = 0; + data_size_ = 0; + sample_rate_ = 0; + samples_per_buffer_ = 0; + sample_size_ = 0; +} + +bool IamfConfigReader::Read(scoped_refptr input_buffer) { const uint8_t* buf = input_buffer->data(); SB_DCHECK(buf); bool completed_parsing = false; while (!completed_parsing && buffer_head_ < input_buffer->size()) { if (!ReadOBU(buf, completed_parsing)) { - SB_LOG(INFO) << "Error parsing config OBUs"; + SB_DLOG(INFO) << "Error parsing config OBUs"; return false; } } @@ -106,23 +129,12 @@ bool IamfConfigReader::Read(scoped_refptr input_buffer) { return true; } -void IamfConfigReader::Reset() { - buffer_head_ = 0; - has_mix_presentation_id_ = false; - mix_presentation_id_ = 0; - config_size_ = 0; - data_size_ = 0; - sample_rate_ = 0; - samples_per_buffer_ = 0; - sample_size_ = 0; -} - bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { uint8_t obu_type = 0; uint32_t obu_size = 0; uint32_t header_size = 0; if (!ReadOBUHeader(buf, &obu_type, &obu_size, &header_size)) { - SB_LOG(ERROR) << "Error reading OBU header"; + SB_DLOG(ERROR) << "Error reading OBU header"; return false; } @@ -200,36 +212,14 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { break; case kObuTypeMixPresentation: { // TODO: Complete Mix Presentation OBU parsing - has_mix_presentation_id_ = true; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &mix_presentation_id_); - if (bytes_read < 0) { - return false; - } - buffer_head_ += bytes_read; - - // count_label - bytes_read = ReadLeb128Value(&buf[buffer_head_], &count_label); + uint32_t value; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); if (bytes_read < 0) { return false; } - buffer_head_ += bytes_read; - - // language_label - for (int i = 0; i < count_label; ++i) { - buffer_head_ += ReadString(&buf[buffer_head_], str); - } - - // MixPresentationAnnotations - for (int i = 0; i < count_label; ++i) { - buffer_head_ += ReadString(&buf[buffer_head_], str); - } - - // num_sub_mixes - bytes_read = ReadLeb128Value(&buf[buffer_head_], &num_sub_mixes); - if (bytes_read < 0) { - return false; + if (!mix_presentation_id_.has_value()) { + mix_presentation_id_ = value; } - buffer_head_ += bytes_read; break; } diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index 04a6d76d98ae..aa2e7bc529ab 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -15,8 +15,10 @@ #ifndef STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ #define STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ +#include #include +#include "starboard/common/log.h" #include "starboard/common/ref_counted.h" #include "starboard/shared/internal_only.h" #include "starboard/shared/starboard/player/input_buffer_internal.h" @@ -26,33 +28,37 @@ namespace shared { namespace libiamf { // TODO: Add handling for non-redundant OBUS +// TODO: Implement or depend on a buffer reader to simplify the implementation. class IamfConfigReader { public: typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; IamfConfigReader() = default; - bool Read(scoped_refptr input_buffer); + bool ResetAndRead(scoped_refptr input_buffer); void Reset(); bool is_valid() { - return data_size_ > 0 && config_size_ > 0 && has_mix_presentation_id_ && - sample_rate_ > 0; + return data_size_ > 0 && config_size_ > 0 && has_mix_presentation_id() && + sample_rate_ > 0 && samples_per_buffer_ > 0; } int sample_rate() { return sample_rate_; } uint32_t samples_per_buffer() { return samples_per_buffer_; } - bool config_changed() { return false; } uint32_t config_size() { return config_size_; } - // TODO: Allow for selection of multiple mix presentation IDs. - bool has_mix_presentation_id() { return has_mix_presentation_id_; } - uint32_t mix_presentation_id() { return mix_presentation_id_; } - uint32_t data_size() { return data_size_; } + // TODO: Allow for selection of multiple mix presentation IDs. Currently, + // only the first mix presentation parsed is selected. + bool has_mix_presentation_id() { return mix_presentation_id_.has_value(); } + uint32_t mix_presentation_id() { + SB_DCHECK(mix_presentation_id_.has_value()); + return mix_presentation_id_.value(); + } std::vector config_obus() { return config_obus_; } std::vector data() { return data_; } private: + bool Read(scoped_refptr input_buffer); bool ReadOBU(const uint8_t* buf, bool& completed_parsing); bool ReadOBUHeader(const uint8_t* buf, uint8_t* obu_type, @@ -66,10 +72,10 @@ class IamfConfigReader { uint32_t config_size_ = 0; uint32_t data_size_ = 0; - bool has_mix_presentation_id_ = false; - uint32_t mix_presentation_id_ = 0; + std::optional mix_presentation_id_; bool has_valid_config_ = false; + bool binaural_mix_presentation_id_ = -1; std::vector config_obus_; std::vector data_; }; From a63fcd9a66b23384acfad10d5ddfee133dc859d5 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Thu, 18 Jul 2024 15:53:53 -0700 Subject: [PATCH 06/13] Remove flac dep --- starboard/linux/x64x11/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/starboard/linux/x64x11/BUILD.gn b/starboard/linux/x64x11/BUILD.gn index fa50bb0cff0c..3666f36cb196 100644 --- a/starboard/linux/x64x11/BUILD.gn +++ b/starboard/linux/x64x11/BUILD.gn @@ -34,6 +34,5 @@ if (sb_is_modular && !sb_is_evergreen) { sources = [ "//starboard/linux/x64x11/main.cc" ] configs += [ "//starboard/build/config:starboard_implementation" ] public_deps = [ ":starboard_platform" ] - deps = [ "//third_party/flac" ] } } From cd4e3265e461e6fb894f22eda3484beebe5e53b3 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Mon, 5 Aug 2024 12:20:22 -0700 Subject: [PATCH 07/13] Move location of libiamf binaries --- starboard/android/shared/platform_configuration/BUILD.gn | 2 +- starboard/linux/x64x11/shared/platform_configuration/BUILD.gn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/starboard/android/shared/platform_configuration/BUILD.gn b/starboard/android/shared/platform_configuration/BUILD.gn index dfcdc08ab6d9..2885fccd10f5 100644 --- a/starboard/android/shared/platform_configuration/BUILD.gn +++ b/starboard/android/shared/platform_configuration/BUILD.gn @@ -223,6 +223,6 @@ config("pedantic_warnings") { if (enable_iamf_decode) { config("libiamf_config") { - libs = [ "//third_party/libiamf/code/android/libiamf.a" ] + libs = [ "//third_party/libiamf/platforms/android/libiamf.a" ] } } diff --git a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn index 5ebd592449ed..21c6d95015f0 100644 --- a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn +++ b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn @@ -32,7 +32,7 @@ config("libraries") { configs = [ "//starboard/linux/shared/platform_configuration:libraries" ] if (enable_iamf_decode) { libs = [ - "//third_party/libiamf/code/libiamf.a", + "//third_party/libiamf/platforms/linux/libiamf.a", "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", ] } From f820fde10bcf228fe0a7c3dd718e8e77c88b40c6 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Sun, 1 Sep 2024 03:12:51 -0700 Subject: [PATCH 08/13] Max IamfConfigReader actually servicable --- .../shared/platform_configuration/BUILD.gn | 5 +- .../shared/libiamf/iamf_audio_decoder.cc | 7 +- starboard/shared/libiamf/iamf_audio_decoder.h | 2 +- .../shared/libiamf/iamf_config_reader.cc | 514 +++++++++++++----- starboard/shared/libiamf/iamf_config_reader.h | 100 +++- 5 files changed, 467 insertions(+), 161 deletions(-) diff --git a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn index 21c6d95015f0..39fed6b2bce2 100644 --- a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn +++ b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn @@ -31,10 +31,7 @@ config("platform_configuration") { config("libraries") { configs = [ "//starboard/linux/shared/platform_configuration:libraries" ] if (enable_iamf_decode) { - libs = [ - "//third_party/libiamf/platforms/linux/libiamf.a", - "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", - ] + libs = [ "//third_party/libiamf/platforms/linux/libiamf.a" ] } } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index acc2f395cfb0..9f88f5a83e3c 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -57,7 +57,8 @@ std::string ErrorCodeToString(int code) { IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, bool prefer_binaural_audio) : audio_stream_info_(audio_stream_info), - prefer_binarual_audio_(prefer_binaural_audio) { + prefer_binarual_audio_(prefer_binaural_audio), + reader_(false, false) { decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_DLOG(ERROR) << "Error creating libiamf decoder"; @@ -223,7 +224,7 @@ bool IamfAudioDecoder::InitializeCodec() { return false; } - error = IAMF_decoder_set_sampling_rate(decoder_, reader_.sample_rate()); + error = IAMF_decoder_set_sampling_rate(decoder_, 48000); if (error != IAMF_OK) { SB_DLOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " << ErrorCodeToString(error); @@ -348,7 +349,7 @@ scoped_refptr IamfAudioDecoder::Read( result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = reader_.sample_rate(); + *samples_per_second = 48000; return result; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 55568dc8de8d..d89958c1fdde 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -65,7 +65,7 @@ class IamfAudioDecoder OutputCB output_cb_; ErrorCB error_cb_; - IAMF_Decoder* decoder_ = NULL; + IAMF_Decoder* decoder_ = nullptr; bool stream_ended_ = false; std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index 2643e336a92f..d1a4e904d1b6 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -14,15 +14,27 @@ #include "starboard/shared/libiamf/iamf_config_reader.h" +#include #include #include "starboard/common/string.h" +#include "third_party/libiamf/source/code/include/IAMF_defines.h" namespace starboard { namespace shared { namespace libiamf { namespace { + +// From //media/formats/mp4/rcheck.h. +#define RCHECK(condition) \ + do { \ + if (!(condition)) { \ + SB_DLOG(ERROR) << "Failure while parsing IAMF config: " #condition; \ + return false; \ + } \ + } while (0) + // From // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax. constexpr int kObuTypeCodecConfig = 0; @@ -33,25 +45,78 @@ constexpr int kObuTypeSequenceHeader = 31; // From // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig. constexpr int kFourccOpus = 0x4f707573; -constexpr int kFourccMp4a = 0x6d703461; -constexpr int kFourccFlac = 0x664c6143; constexpr int kFourccIpcm = 0x6970636d; -constexpr int kLayoutTypeReserved = 1; -constexpr int kLayoutTypeSpeaker = 2; -constexpr int kLayoutTypeBinaural = 3; +} // namespace + +IamfConfigReader::BufferReader::BufferReader(const uint8_t* buf, size_t size) + : buf_(buf), pos_(0), size_(size), error_(false) {} + +bool IamfConfigReader::BufferReader::Read1(uint8_t* ptr) { + if (!HasBytes(sizeof(uint8_t)) || !ptr) { + error_ = true; + return false; + } + *ptr = buf_[pos_++]; + return true; +} + +bool IamfConfigReader::BufferReader::Read4(uint32_t* ptr) { + if (!HasBytes(sizeof(uint32_t)) || !ptr) { + error_ = true; + return false; + } + std::memcpy(ptr, &buf_[pos_], sizeof(uint32_t)); + *ptr = ByteSwap(*ptr); + pos_ += sizeof(uint32_t); + return true; +} + +bool IamfConfigReader::BufferReader::ReadLeb128(uint32_t* ptr) { + if (!HasBytes(sizeof(uint32_t)) || !ptr) { + error_ = true; + return false; + } + int bytes_read = ReadLeb128Internal(buf_ + pos_, ptr); + if (bytes_read < 0) { + error_ = true; + return false; + } + pos_ += bytes_read; + return true; +} + +bool IamfConfigReader::BufferReader::ReadString(std::string& str) { + int bytes_read = ReadStringInternal(buf_ + pos_, str); + if (bytes_read < 0) { + error_ = true; + return false; + } + pos_ += bytes_read; + return true; +} + +bool IamfConfigReader::BufferReader::SkipBytes(size_t size) { + if (!HasBytes(size)) { + error_ = true; + return false; + } + pos_ += size; + return true; +} + +bool IamfConfigReader::BufferReader::SkipLeb128() { + uint32_t val; + return ReadLeb128(&val); +} -inline uint32_t ByteSwap(uint32_t x) { -#if defined(COMPILER_MSVC) - return _byteswap_ulong(x); -#else - return __builtin_bswap32(x); -#endif +bool IamfConfigReader::BufferReader::SkipString() { + std::string str; + return ReadString(str); } -// Decodes an Leb128 value and stores it in |value|. Returns the number of bytes -// read. Returns -1 on error. -int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { +int IamfConfigReader::BufferReader::ReadLeb128Internal(const uint8_t* buf, + uint32_t* value) { SB_DCHECK(buf); SB_DCHECK(value); *value = 0; @@ -72,26 +137,46 @@ int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { return i + 1; } -int ReadString(const uint8_t* buf, std::string& value) { +int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, + std::string& str) { SB_DCHECK(buf); - value = ""; - value.reserve(128); - int bytes_read = 0; - bool error = true; - while (bytes_read < 128 && buf[bytes_read] != '\0') { - value.push_back(static_cast(buf[bytes_read])); - bytes_read++; - error = false; + int remaining_size = static_cast(size_) - pos_; + if (remaining_size <= 0) { + return -1; } - if (error) { + // The size of the string is capped to 128 bytes. + int max_str_size = std::min(remaining_size, 128); + str.clear(); + + size_t size = 0; + while (buf[size] != '\0' && size < max_str_size) { + size++; + } + + if (buf[size] != '\0') { return -1; } - return bytes_read; + if (size > 0) { + str.resize(size); + std::memcpy(str.data(), reinterpret_cast(buf), size); + } + + // Account for null terminator byte. + return ++size; +} + +IamfConfigReader::IamfConfigReader(bool prefer_binaural_audio, + bool prefer_surround_audio) + : prefer_binaural_audio_(prefer_binaural_audio), + prefer_surround_audio_(prefer_surround_audio) { +#if SB_IS_BIG_ENDIAN +#error IamfConfigReader assumes little-endianness. +#endif // SB_IS_BIG_ENDIAN + SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); } -} // namespace bool IamfConfigReader::ResetAndRead(scoped_refptr input_buffer) { Reset(); @@ -99,8 +184,8 @@ bool IamfConfigReader::ResetAndRead(scoped_refptr input_buffer) { } void IamfConfigReader::Reset() { - buffer_head_ = 0; mix_presentation_id_ = std::optional(); + binaural_audio_element_ids_ = std::unordered_set(); config_size_ = 0; data_size_ = 0; sample_rate_ = 0; @@ -109,116 +194,295 @@ void IamfConfigReader::Reset() { } bool IamfConfigReader::Read(scoped_refptr input_buffer) { - const uint8_t* buf = input_buffer->data(); - SB_DCHECK(buf); + SB_DCHECK(input_buffer->data()); + BufferReader reader(input_buffer->data(), input_buffer->size()); - bool completed_parsing = false; - while (!completed_parsing && buffer_head_ < input_buffer->size()) { - if (!ReadOBU(buf, completed_parsing)) { - SB_DLOG(INFO) << "Error parsing config OBUs"; - return false; - } + while (!is_valid() && reader.pos() < reader.size()) { + RCHECK(ReadOBU(&reader)); } - SB_CHECK(completed_parsing); + if (reader.error()) { + return false; + } - data_size_ = input_buffer->size() - config_size_; - config_obus_.assign(buf, buf + config_size_); - data_.assign(buf + config_size_, buf + input_buffer->size()); + data_size_ = reader.size() - config_size_; + config_obus_.assign(reader.buf(), reader.buf() + config_size_); + data_.assign(reader.buf() + config_size_, reader.buf() + reader.size()); return true; } -bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { +bool IamfConfigReader::ReadOBU(BufferReader* reader) { + SB_DCHECK(reader); uint8_t obu_type = 0; uint32_t obu_size = 0; - uint32_t header_size = 0; - if (!ReadOBUHeader(buf, &obu_type, &obu_size, &header_size)) { + if (!ReadOBUHeader(reader, &obu_type, &obu_size)) { SB_DLOG(ERROR) << "Error reading OBU header"; return false; } - int next_obu_pos = buffer_head_ + obu_size; - int bytes_read = 0; - - uint32_t count_label = 0; - std::string str; - uint32_t num_sub_mixes = 0; + int next_obu_pos = reader->pos() + obu_size; + + auto skip_param_definition = [&]() { + // parameter_id + RCHECK(reader->SkipLeb128()); + // parameter_rate + RCHECK(reader->SkipLeb128()); + uint8_t param_definition_mode; + RCHECK(reader->Read1(¶m_definition_mode)); + param_definition_mode = param_definition_mode >> 7; + if (param_definition_mode == static_cast(0)) { + // duration + RCHECK(reader->SkipLeb128()); + uint32_t constant_subblock_duration; + RCHECK(reader->ReadLeb128(&constant_subblock_duration)); + if (constant_subblock_duration == 0) { + uint32_t num_subblocks; + RCHECK(reader->ReadLeb128(&num_subblocks)); + for (int i = 0; i < num_subblocks; ++i) { + // subblock_duration + RCHECK(reader->SkipLeb128()); + } + } + } + return true; + }; switch (static_cast(obu_type)) { case kObuTypeCodecConfig: { - sample_rate_ = 0; - uint32_t codec_config_id; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &codec_config_id); - if (bytes_read < 0) { - return false; - } - buffer_head_ += bytes_read; + RCHECK(reader->SkipLeb128()); uint32_t codec_id = 0; - std::memcpy(&codec_id, &buf[buffer_head_], sizeof(uint32_t)); - // Mp4 is in big-endian - codec_id = ByteSwap(codec_id); - buffer_head_ += 4; - - bytes_read = ReadLeb128Value(&buf[buffer_head_], &samples_per_buffer_); - if (bytes_read < 0) { - return false; - } - buffer_head_ += bytes_read; + RCHECK(reader->Read4(&codec_id)); + + RCHECK(reader->ReadLeb128(&samples_per_buffer_)); // audio_roll_distance - buffer_head_ += 2; + RCHECK(reader->SkipBytes(2)); switch (codec_id) { case kFourccOpus: sample_rate_ = 48000; break; - case kFourccMp4a: - // TODO: Properly parse for the sample rate. 48000 is the assumed - // sample rate. - sample_rate_ = 48000; - break; - case kFourccFlac: { - // Skip METADATA_BLOCK_HEADER. - buffer_head_ += 4; - // Skip first 10 bytes of METADATA_BLOCK_STREAMINFO. - buffer_head_ += 10; - - std::memcpy(&sample_rate_, &buf[buffer_head_], sizeof(uint32_t)); - sample_rate_ = ByteSwap(sample_rate_); - sample_rate_ = sample_rate_ >> 12; - break; - } case kFourccIpcm: { // sample_format_flags - buffer_head_++; - uint8_t sample_size_byte = buf[buffer_head_]; - sample_size_ = static_cast(sample_size_byte); - buffer_head_++; + RCHECK(reader->SkipBytes(1)); - std::memcpy(&sample_rate_, &buf[buffer_head_], sizeof(uint32_t)); - sample_rate_ = ByteSwap(sample_rate_); + uint8_t sample_size_unsigned; + RCHECK(reader->Read1(&sample_size_unsigned)); + sample_size_ = static_cast(sample_size_unsigned); + + uint32_t sample_rate_unsigned; + RCHECK(reader->Read4(&sample_rate_unsigned)); + sample_rate_ = static_cast(sample_rate_unsigned); break; } default: SB_NOTREACHED(); + return false; } break; } - case kObuTypeAudioElement: + case kObuTypeAudioElement: { + uint32_t audio_element_id; + RCHECK(reader->ReadLeb128(&audio_element_id)); + + uint8_t audio_element_type; + RCHECK(reader->Read1(&audio_element_type)); + audio_element_type = audio_element_type >> 5; + + // codec_config_id + RCHECK(reader->SkipLeb128()); + + uint32_t num_substreams; + RCHECK(reader->ReadLeb128(&num_substreams)); + + for (int i = 0; i < num_substreams; ++i) { + // audio_substream_id + RCHECK(reader->SkipLeb128()); + } + + uint32_t num_parameters; + RCHECK(reader->ReadLeb128(&num_parameters)); + + for (int i = 0; i < num_parameters; ++i) { + uint32_t param_definition_type; + RCHECK(reader->ReadLeb128(¶m_definition_type)); + + if (param_definition_type == IAMF_PARAMETER_TYPE_DEMIXING) { + skip_param_definition(); + // DemixingParamDefintion + RCHECK(reader->SkipBytes(1)); + } else if (param_definition_type == IAMF_PARAMETER_TYPE_RECON_GAIN) { + skip_param_definition(); + } else if (param_definition_type > 2) { + uint32_t param_definition_size; + RCHECK(reader->ReadLeb128(¶m_definition_size)); + RCHECK(reader->SkipBytes(param_definition_size)); + } + } + + if (static_cast(audio_element_type) == + AUDIO_ELEMENT_CHANNEL_BASED && + (prefer_binaural_audio_ || prefer_surround_audio_)) { + // Parse ScalableChannelLayoutConfig for binaural and surround + // loudspeaker layouts + uint8_t num_layers; + RCHECK(reader->Read1(&num_layers)); + num_layers = num_layers >> 5; + // Read ChannelAudioLayerConfigs + for (int i = 0; i < static_cast(num_layers); ++i) { + uint8_t loudspeaker_layout; + bool output_gain_is_present_flag; + RCHECK(reader->Read1(&loudspeaker_layout)); + output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; + loudspeaker_layout = loudspeaker_layout >> 4; + if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL) { + binaural_audio_element_ids_.insert(audio_element_id); + } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && + loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT) { + surround_audio_element_ids_.insert(audio_element_id); + } + + // substream_count and coupled_substream_count + RCHECK(reader->SkipBytes(2)); + + if (output_gain_is_present_flag) { + // output_gain_flags and output_gain + RCHECK(reader->SkipBytes(3)); + } + + if (i == 1 && loudspeaker_layout == static_cast(15)) { + // expanded_loudspeaker_layout + RCHECK(reader->SkipBytes(1)); + } + } + } + break; + } case kObuTypeSequenceHeader: break; case kObuTypeMixPresentation: { - // TODO: Complete Mix Presentation OBU parsing - uint32_t value; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); - if (bytes_read < 0) { - return false; + uint32_t mix_presentation_id; + RCHECK(reader->ReadLeb128(&mix_presentation_id)); + + uint32_t count_label; + RCHECK(reader->ReadLeb128(&count_label)); + for (int i = 0; i < count_label; ++i) { + // language_label; + RCHECK(reader->SkipString()); + } + + for (int i = 0; i < count_label; ++i) { + // MixPresentationAnnotations; + RCHECK(reader->SkipString()); + } + + uint32_t num_sub_mixes; + RCHECK(reader->ReadLeb128(&num_sub_mixes)); + for (int i = 0; i < num_sub_mixes; ++i) { + uint32_t num_audio_elements; + RCHECK(reader->ReadLeb128(&num_audio_elements)); + for (int j = 0; j < num_audio_elements; ++j) { + uint32_t audio_element_id; + RCHECK(reader->ReadLeb128(&audio_element_id)); + + // Set a mix presentation for binaural or surround streams. The mix + // presentation is chosen if there exists an audio element that has + // the qualities it requires - such as an audio element with a + // binaural loudspeaker layout. + if (!mix_presentation_id_.has_value() || + (prefer_binaural_audio_ && + binaural_mix_selection_ > + kBinauralMixSelectionLoudspeakerLayout)) { + if (prefer_binaural_audio_ && + binaural_audio_element_ids_.find(audio_element_id) != + binaural_audio_element_ids_.end()) { + mix_presentation_id_ = mix_presentation_id; + binaural_mix_selection_ = kBinauralMixSelectionLoudspeakerLayout; + } else if (prefer_surround_audio_ && + surround_audio_element_ids_.find(audio_element_id) != + surround_audio_element_ids_.end()) { + mix_presentation_id_ = mix_presentation_id; + } + } + + for (int k = 0; k < count_label; ++k) { + // MixPresentationElementAnnotatoions + RCHECK(reader->SkipString()); + } + + // The following fields are for the RenderingConfig + // headphones_rendering_mode + RCHECK(reader->SkipBytes(1)); + uint32_t rendering_config_extension_size; + RCHECK(reader->ReadLeb128(&rendering_config_extension_size)); + // rendering_config_extension_bytes + RCHECK(reader->SkipBytes(rendering_config_extension_size)); + + // The following fields are for the ElementMixConfig + RCHECK(skip_param_definition()); + // default_mix_gain + RCHECK(reader->SkipBytes(2)); + } + // The following fields are for the OutputMixConfig + RCHECK(skip_param_definition()); + // default_mix_gain + RCHECK(reader->SkipBytes(2)); + + uint32_t num_layouts; + RCHECK(reader->ReadLeb128(&num_layouts)); + for (int j = 0; j < num_layouts; ++j) { + uint8_t layout_type; + RCHECK(reader->Read1(&layout_type)); + layout_type = layout_type >> 6; + // If a binaural mix presentation is preferred and the mix + // presentation id has not yet been set, set the mix presentation id + // if the current mix presentation has a binaural loudness layout. The + // mix presentation id will change if a different mix presentation is + // found that uses an audio element with a binaural loudspeaker + // layout, as that is higher priority. + if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && + prefer_binaural_audio_ && + (!mix_presentation_id_.has_value() || + binaural_mix_selection_ > kBinauralMixSelectionLoudnessLayout)) { + mix_presentation_id_ = mix_presentation_id; + binaural_mix_selection_ = kBinauralMixSelectionLoudnessLayout; + } + + // The following fields are for the LoudnessInfo + uint8_t info_type; + RCHECK(reader->Read1(&info_type)); + // integrated_loudness and digital_loudness + RCHECK(reader->SkipBytes(4)); + if (info_type & 1) { + // true_peak + RCHECK(reader->SkipBytes(2)); + } + if (info_type & 2) { + uint8_t num_anchored_loudness; + RCHECK(reader->Read1(&num_anchored_loudness)); + for (uint8_t k = 0; k < num_anchored_loudness; ++k) { + // anchor_element and anchored_loudness + RCHECK(reader->SkipBytes(3)); + } + } + if ((info_type & 0b11111100) > 0) { + uint32_t info_type_size; + RCHECK(reader->ReadLeb128(&info_type_size)); + // info_type_bytes + RCHECK(reader->SkipBytes(info_type_size)); + } + } } + + // If the mix presentation id is unassigned at this point, the stream is + // stereo, or a proper mix presentation for binaural or surround preferred + // streams hasn't yet been parsed. Default to the first read mix + // presentation in case a preferred mix does not exist. if (!mix_presentation_id_.has_value()) { - mix_presentation_id_ = value; + mix_presentation_id_ = mix_presentation_id; } break; @@ -226,59 +490,51 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { default: // Once an OBU is read that is not a descriptor, descriptor parsing is // assumed to be complete. - completed_parsing = true; - break; + SB_DCHECK(mix_presentation_id_.has_value()); + return true; } - if (!completed_parsing) { - // Skip to next OBU - buffer_head_ = next_obu_pos; - config_size_ += obu_size + header_size; - } + // Skip to the next OBU. + const size_t remaining_size = next_obu_pos - reader->pos(); + RCHECK(reader->SkipBytes(remaining_size)); + config_size_ = reader->pos(); return true; } -bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, +bool IamfConfigReader::ReadOBUHeader(BufferReader* reader, uint8_t* obu_type, - uint32_t* obu_size, - uint32_t* header_size) { - uint8_t header_flags = buf[buffer_head_]; + uint32_t* obu_size) { + uint8_t header_flags; + RCHECK(reader->Read1(&header_flags)); *obu_type = (header_flags >> 3) & 0x1f; - buffer_head_++; const bool obu_redundant_copy = (header_flags >> 2) & 1; const bool obu_trimming_status_flag = (header_flags >> 1) & 1; const bool obu_extension_flag = header_flags & 1; - *header_size = 1; - *obu_size = 0; - int bytes_read = ReadLeb128Value(&buf[buffer_head_], obu_size); - if (bytes_read < 0) { - return false; - } - buffer_head_ += bytes_read; - *header_size += bytes_read; + + RCHECK(reader->ReadLeb128(obu_size)); + + // |obu_size| contains the size of the OBU after its own field. + // If either of the flags are set, subtract the number of bytes read + // from the flags from |obu_size| before returning to ReadOBU(). + size_t reader_pos_before_flags = reader->pos(); if (obu_trimming_status_flag) { - uint32_t value; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); - buffer_head_ += bytes_read; - *header_size += bytes_read; - *obu_size -= bytes_read; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); - buffer_head_ += bytes_read; - *header_size += bytes_read; - *obu_size -= bytes_read; + RCHECK(reader->SkipLeb128()); + RCHECK(reader->SkipLeb128()); } if (obu_extension_flag) { - uint32_t extension_header_size; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &extension_header_size); - buffer_head_ += extension_header_size; - *obu_size -= extension_header_size; - *header_size += bytes_read; + RCHECK(reader->SkipLeb128()); + } + + size_t flag_bytes_read = reader->pos() - reader_pos_before_flags; + if (flag_bytes_read >= *obu_size) { + return false; } + *obu_size -= flag_bytes_read; return true; } diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index aa2e7bc529ab..0012a1d67829 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -16,6 +16,8 @@ #define STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ #include +#include +#include #include #include "starboard/common/log.h" @@ -27,45 +29,91 @@ namespace starboard { namespace shared { namespace libiamf { -// TODO: Add handling for non-redundant OBUS -// TODO: Implement or depend on a buffer reader to simplify the implementation. +// TODO: Add handling for non-redundant OBUs class IamfConfigReader { public: typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; - IamfConfigReader() = default; + IamfConfigReader(bool prefer_binaural_audio, bool prefer_surround_audio); bool ResetAndRead(scoped_refptr input_buffer); - void Reset(); - - bool is_valid() { - return data_size_ > 0 && config_size_ > 0 && has_mix_presentation_id() && - sample_rate_ > 0 && samples_per_buffer_ > 0; + bool is_valid() const { + return mix_presentation_id_.has_value() && sample_rate_ > 0 && + samples_per_buffer_ > 0; } - int sample_rate() { return sample_rate_; } - uint32_t samples_per_buffer() { return samples_per_buffer_; } - uint32_t config_size() { return config_size_; } - // TODO: Allow for selection of multiple mix presentation IDs. Currently, - // only the first mix presentation parsed is selected. - bool has_mix_presentation_id() { return mix_presentation_id_.has_value(); } - uint32_t mix_presentation_id() { + int sample_rate() const { return sample_rate_; } + uint32_t samples_per_buffer() const { return samples_per_buffer_; } + uint32_t config_size() const { return config_size_; } + uint32_t mix_presentation_id() const { SB_DCHECK(mix_presentation_id_.has_value()); return mix_presentation_id_.value(); } - std::vector config_obus() { return config_obus_; } - std::vector data() { return data_; } + const std::vector& config_obus() const { return config_obus_; } + const std::vector& data() const { return data_; } private: + class BufferReader { + public: + BufferReader(const uint8_t* buf, size_t size); + + bool Read1(uint8_t* ptr); + bool Read4(uint32_t* ptr); + bool ReadLeb128(uint32_t* ptr); + bool ReadString(std::string& str); + + bool SkipBytes(size_t size); + bool SkipLeb128(); + bool SkipString(); + + size_t size() const { return size_; } + int pos() const { return pos_; } + const uint8_t* buf() const { return buf_; } + bool error() const { return error_; } + + private: + bool HasBytes(size_t size) const { return size + pos_ <= size_; } + inline uint32_t ByteSwap(uint32_t x) { +#if defined(COMPILER_MSVC) + return _byteswap_ulong(x); +#else + return __builtin_bswap32(x); +#endif + } + // Decodes an Leb128 value and stores it in |value|. Returns the number of + // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, + // or -1 on error. + int ReadLeb128Internal(const uint8_t* buf, uint32_t* value); + // Reads a c-string into |str|. Returns the number of bytes read, or -1 on + // error. + int ReadStringInternal(const uint8_t* buf, std::string& str); + + int pos_ = 0; + const uint8_t* buf_; + const size_t size_ = 0; + bool error_ = false; + }; + + // Used in the selection of a binaural mix presentation, using the strategy + // defined in + // https://aomediacodec.github.io/iamf/#processing-mixpresentation-selection. + // The preferred methods of choosing a binaural mix presentation are listed + // from high to low. + enum BinauralMixSelection { + kBinauralMixSelectionLoudspeakerLayout, + kBinauralMixSelectionLoudnessLayout, + kBinauralMixSelectionNotFound + }; + + void Reset(); bool Read(scoped_refptr input_buffer); - bool ReadOBU(const uint8_t* buf, bool& completed_parsing); - bool ReadOBUHeader(const uint8_t* buf, + // Reads a single Descriptor OBU. Returns false on error. + bool ReadOBU(BufferReader* reader); + bool ReadOBUHeader(BufferReader* reader, uint8_t* obu_type, - uint32_t* obu_size, - uint32_t* header_size); + uint32_t* obu_size); - int buffer_head_ = 0; int sample_rate_ = 0; int sample_size_ = 0; uint32_t samples_per_buffer_ = 0; @@ -73,9 +121,13 @@ class IamfConfigReader { uint32_t data_size_ = 0; std::optional mix_presentation_id_; + std::unordered_set binaural_audio_element_ids_; + std::unordered_set surround_audio_element_ids_; + const bool prefer_binaural_audio_; + const bool prefer_surround_audio_; + + BinauralMixSelection binaural_mix_selection_ = kBinauralMixSelectionNotFound; - bool has_valid_config_ = false; - bool binaural_mix_presentation_id_ = -1; std::vector config_obus_; std::vector data_; }; From 75cade95883dce85bfc073633a91c47e506b9ec3 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 3 Sep 2024 09:28:27 -0700 Subject: [PATCH 09/13] Max IamfAudioDecoder actually servicable --- .../shared/media_is_audio_supported.cc | 2 +- .../shared/player_components_factory.h | 2 +- .../linux/shared/player_components_factory.cc | 2 +- .../shared/libiamf/iamf_audio_decoder.cc | 130 ++++++++++-------- starboard/shared/libiamf/iamf_audio_decoder.h | 14 +- .../shared/libiamf/iamf_config_reader.cc | 24 +--- starboard/shared/libiamf/iamf_config_reader.h | 17 ++- 7 files changed, 94 insertions(+), 97 deletions(-) diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index b2c9d0b9c62f..364a093210da 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -64,7 +64,7 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, if (audio_codec == kSbMediaAudioCodecIamf) { return true; } -#endif // SB_API_VERSION >= 15 +#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE bool media_codec_supported = MediaCapabilitiesCache::GetInstance()->HasAudioDecoderFor(mime, bitrate); diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index 83cb08c37651..d5225d71cc4a 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -454,7 +454,7 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: std::unique_ptr audio_decoder_impl( new starboard::shared::libiamf::IamfAudioDecoder( - audio_stream_info, /* prefer_binarual_audio */ false)); + audio_stream_info)); if (audio_decoder_impl->is_valid()) { return std::unique_ptr( std::move(audio_decoder_impl)); diff --git a/starboard/linux/shared/player_components_factory.cc b/starboard/linux/shared/player_components_factory.cc index a49a33f934a6..3b9d0b46087e 100644 --- a/starboard/linux/shared/player_components_factory.cc +++ b/starboard/linux/shared/player_components_factory.cc @@ -95,7 +95,7 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { SB_LOG(INFO) << "Playing audio using IamfAudioDecoder."; return std::unique_ptr( new ::starboard::shared::libiamf::IamfAudioDecoder( - audio_stream_info, /* prefer_binarual_audio */ false)); + audio_stream_info)); #endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else { std::unique_ptr audio_decoder_impl( diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 9f88f5a83e3c..0c3d5375d710 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -15,8 +15,8 @@ #include "starboard/shared/libiamf/iamf_audio_decoder.h" #include -#include +#include "starboard/common/string.h" #include "third_party/libiamf/source/code/include/IAMF_defines.h" namespace starboard { @@ -27,8 +27,7 @@ namespace { using shared::starboard::player::DecodedAudio; constexpr int kForceBinauralAudio = false; -// Keep disabled as surround audio may require changes further up the SbPlayer. -constexpr int kEnableSurroundAudio = false; +constexpr int kForceSurroundAudio = false; std::string ErrorCodeToString(int code) { switch (code) { @@ -54,19 +53,16 @@ std::string ErrorCodeToString(int code) { } } // namespace -IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, - bool prefer_binaural_audio) - : audio_stream_info_(audio_stream_info), - prefer_binarual_audio_(prefer_binaural_audio), - reader_(false, false) { +IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) + : audio_stream_info_(audio_stream_info) { decoder_ = IAMF_decoder_open(); if (!decoder_) { - SB_DLOG(ERROR) << "Error creating libiamf decoder"; + SB_LOG(ERROR) << "Error creating libiamf decoder"; } } IamfAudioDecoder::~IamfAudioDecoder() { - TeardownCodec(); + TeardownDecoder(); } bool IamfAudioDecoder::is_valid() const { @@ -93,7 +89,7 @@ void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, SB_DCHECK(output_cb_); if (stream_ended_) { - SB_DLOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; + SB_LOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; return; } @@ -141,21 +137,23 @@ bool IamfAudioDecoder::DecodeInternal( const scoped_refptr& input_buffer) { SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(input_buffer); - SB_DCHECK(input_buffer->size() > 0); SB_DCHECK(output_cb_); SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); SB_DCHECK(is_valid()); - reader_.ResetAndRead(input_buffer); - if (!reader_.is_valid()) { - SB_DLOG(INFO) << "Failed to parse IA Descriptors"; - error_cb_(kSbPlayerErrorDecode, "Failed to parse IA Descriptors"); + if (input_buffer->size() == 0) { + SB_LOG(ERROR) << "Empty input buffer written to IamfAudioDecoder"; + return false; + } + + IamfConfigReader reader(input_buffer, kForceBinauralAudio, + kForceSurroundAudio); + if (!reader.is_valid()) { + ReportError("Failed to parse IA Descriptors"); return false; } if (!decoder_is_configured_) { - if (!InitializeCodec()) { - SB_DLOG(INFO) << "Failed to initialize IAMF decoder"; - error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); + if (!ConfigureDecoder(&reader, input_buffer->timestamp())) { return false; } } @@ -163,23 +161,21 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * reader_.samples_per_buffer() * + audio_stream_info_.number_of_channels * reader.samples_per_buffer() * starboard::media::GetBytesPerSample(GetSampleType())); int samples_decoded = IAMF_decoder_decode( - decoder_, reader_.data().data(), reader_.data().size(), nullptr, + decoder_, reader.data().data(), reader.data().size(), nullptr, reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { - SB_DLOG(INFO) << "IAMF_decoder_decode() error " - << ErrorCodeToString(samples_decoded); - error_cb_(kSbPlayerErrorDecode, "Failed to decode IAMF sample, error " + - ErrorCodeToString(samples_decoded)); + ReportError("Failed to decode IAMF sample, error " + + ErrorCodeToString(samples_decoded)); return false; } - SB_DCHECK(samples_decoded <= reader_.samples_per_buffer()); + SB_DCHECK(samples_decoded <= reader.samples_per_buffer()); decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * - reader_.samples_per_buffer() * + reader.samples_per_buffer() * starboard::media::GetBytesPerSample(GetSampleType())); // TODO: Enable partial audio once float32 pcm output is available. @@ -211,7 +207,8 @@ void IamfAudioDecoder::WriteEndOfStream() { Schedule(output_cb_); } -bool IamfAudioDecoder::InitializeCodec() { +bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, + int64_t timestamp) { SB_DCHECK(is_valid()); SB_DCHECK(!decoder_is_configured_); @@ -219,38 +216,38 @@ bool IamfAudioDecoder::InitializeCodec() { // for now. int error = IAMF_decoder_set_bit_depth(decoder_, 16); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_bit_depth() fails with error " + + ErrorCodeToString(error)); return false; } error = IAMF_decoder_set_sampling_rate(decoder_, 48000); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_sampling_rate() fails with error " + + ErrorCodeToString(error)); return false; } - if (kForceBinauralAudio || prefer_binarual_audio_) { - SB_DLOG(INFO) << "Configuring IamfAudioDecoder for binaural output"; + if (kForceBinauralAudio) { + SB_LOG(INFO) << "Configuring IamfAudioDecoder for binaural output"; error = IAMF_decoder_output_layout_set_binaural(decoder_); if (error != IAMF_OK) { - SB_DLOG(ERROR) - << "IAMF_decoder_output_layout_set_binaural() fails with error " - << ErrorCodeToString(error); + ReportError( + "IAMF_decoder_output_layout_set_binaural() fails with error " + + ErrorCodeToString(error)); return false; } } else { - // Default to stereo output. If kEnableSurroundAudio is true, set to a sound + // Default to stereo output. If kForceSurroundAudio is true, set to a sound // system matching, the platform's audio configuration, if available. IAMF_SoundSystem sound_system = SOUND_SYSTEM_A; - if (kEnableSurroundAudio) { + if (kForceSurroundAudio) { SbMediaAudioConfiguration out_config; SbMediaGetAudioConfiguration(0, &out_config); int channels = std::max(out_config.number_of_channels, 2); if (channels > 8 || channels < 1) { - SB_DLOG(ERROR) << "Can't create decoder with " << channels - << " channels"; + ReportError(::starboard::FormatString( + "Can't create decoder with %i channels", channels)); return false; } switch (channels) { @@ -274,6 +271,7 @@ bool IamfAudioDecoder::InitializeCodec() { break; default: SB_NOTREACHED(); + return false; } } else { SB_DLOG(INFO) << "Defaulting to stereo output."; @@ -281,48 +279,49 @@ bool IamfAudioDecoder::InitializeCodec() { error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); if (error != IAMF_OK) { - SB_DLOG(ERROR) - << "IAMF_decoder_output_layout_set_sound_system() fails with error " - << ErrorCodeToString(error); + ReportError( + "IAMF_decoder_output_layout_set_sound_system() fails with error " + + ErrorCodeToString(error)); return false; } } - error = IAMF_decoder_set_pts(decoder_, 0, 90000); + // Time base is set to 9000, as it is in the iamfplayer example + // https://github.com/AOMediaCodec/libiamf/blob/v1.0.0-errata/code/test/tools/iamfplayer/player/iamfplayer.c#L450 + error = IAMF_decoder_set_pts(decoder_, timestamp, 90000); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_pts() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_pts() fails with error " + + ErrorCodeToString(error)); return false; } error = IAMF_decoder_set_mix_presentation_id(decoder_, - reader_.mix_presentation_id()); + reader->mix_presentation_id()); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_mix_presentation_id() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_mix_presentation_id() fails with error " + + ErrorCodeToString(error)); return false; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_peak_limiter_enable() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_peak_limiter_enable() fails with error " + + ErrorCodeToString(error)); return false; } error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); if (error != IAMF_OK) { - SB_DLOG(ERROR) - << "IAMF_decoder_set_normalization_loudness() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_normalization_loudness() fails with error " + + ErrorCodeToString(error)); return false; } - error = IAMF_decoder_configure(decoder_, reader_.config_obus().data(), - reader_.config_size(), nullptr); + error = IAMF_decoder_configure(decoder_, reader->config_obus().data(), + reader->config_size(), nullptr); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_configure() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_configure() fails with error " + + ErrorCodeToString(error)); return false; } @@ -331,7 +330,7 @@ bool IamfAudioDecoder::InitializeCodec() { return true; } -void IamfAudioDecoder::TeardownCodec() { +void IamfAudioDecoder::TeardownDecoder() { if (is_valid()) { IAMF_decoder_close(decoder_); decoder_ = NULL; @@ -357,7 +356,7 @@ void IamfAudioDecoder::Reset() { SB_DCHECK(BelongsToCurrentThread()); if (is_valid()) { - TeardownCodec(); + TeardownDecoder(); decoder_ = IAMF_decoder_open(); } @@ -375,9 +374,18 @@ void IamfAudioDecoder::Reset() { SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { SB_DCHECK(BelongsToCurrentThread()); +#if SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) + return kSbMediaAudioSampleTypeInt16; +#endif // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) return kSbMediaAudioSampleTypeInt16Deprecated; } +void IamfAudioDecoder::ReportError(const std::string& message) const { + SB_DCHECK(error_cb_); + SB_LOG(ERROR) << "IamfAudioDecoder error: " << message; + error_cb_(kSbPlayerErrorDecode, message); +} + } // namespace libiamf } // namespace shared } // namespace starboard diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index d89958c1fdde..5aee80f5aaf5 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -17,6 +17,7 @@ #include #include +#include #include "starboard/common/ref_counted.h" #include "starboard/media.h" @@ -38,8 +39,7 @@ class IamfAudioDecoder public: typedef starboard::media::AudioStreamInfo AudioStreamInfo; - explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, - bool prefer_binaural_audio); + explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); ~IamfAudioDecoder() override; bool is_valid() const; @@ -55,13 +55,15 @@ class IamfAudioDecoder private: static constexpr int kMinimumBuffersToDecode = 2; - bool InitializeCodec(); - void TeardownCodec(); + bool ConfigureDecoder(IamfConfigReader* reader, int64_t timestamp); + void TeardownDecoder(); void DecodePendingBuffers(); bool DecodeInternal(const scoped_refptr& input_buffer); SbMediaAudioSampleType GetSampleType() const; + void ReportError(const std::string& message) const; + OutputCB output_cb_; ErrorCB error_cb_; @@ -74,10 +76,6 @@ class IamfAudioDecoder std::deque> pending_audio_buffers_; ConsumedCB consumed_cb_; - - IamfConfigReader reader_; - - const bool prefer_binarual_audio_; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index d1a4e904d1b6..0a257c6d7136 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -168,32 +168,20 @@ int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, return ++size; } -IamfConfigReader::IamfConfigReader(bool prefer_binaural_audio, - bool prefer_surround_audio) +IamfConfigReader::IamfConfigReader( + const scoped_refptr& input_buffer, + bool prefer_binaural_audio, + bool prefer_surround_audio) : prefer_binaural_audio_(prefer_binaural_audio), prefer_surround_audio_(prefer_surround_audio) { #if SB_IS_BIG_ENDIAN #error IamfConfigReader assumes little-endianness. #endif // SB_IS_BIG_ENDIAN SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); + Read(input_buffer); } -bool IamfConfigReader::ResetAndRead(scoped_refptr input_buffer) { - Reset(); - return Read(input_buffer); -} - -void IamfConfigReader::Reset() { - mix_presentation_id_ = std::optional(); - binaural_audio_element_ids_ = std::unordered_set(); - config_size_ = 0; - data_size_ = 0; - sample_rate_ = 0; - samples_per_buffer_ = 0; - sample_size_ = 0; -} - -bool IamfConfigReader::Read(scoped_refptr input_buffer) { +bool IamfConfigReader::Read(const scoped_refptr& input_buffer) { SB_DCHECK(input_buffer->data()); BufferReader reader(input_buffer->data(), input_buffer->size()); diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index 0012a1d67829..72653d8f3e51 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -34,9 +34,9 @@ class IamfConfigReader { public: typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; - IamfConfigReader(bool prefer_binaural_audio, bool prefer_surround_audio); - - bool ResetAndRead(scoped_refptr input_buffer); + IamfConfigReader(const scoped_refptr& input_buffer, + bool prefer_binaural_audio, + bool prefer_surround_audio); bool is_valid() const { return mix_presentation_id_.has_value() && sample_rate_ > 0 && @@ -47,6 +47,10 @@ class IamfConfigReader { uint32_t config_size() const { return config_size_; } uint32_t mix_presentation_id() const { SB_DCHECK(mix_presentation_id_.has_value()); + if (prefer_binaural_audio_ && + binaural_mix_selection_ == kBinauralMixSelectionNotFound) { + SB_LOG(INFO) << "Could not find binaural mix presentation."; + } return mix_presentation_id_.value(); } @@ -85,8 +89,8 @@ class IamfConfigReader { // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, // or -1 on error. int ReadLeb128Internal(const uint8_t* buf, uint32_t* value); - // Reads a c-string into |str|. Returns the number of bytes read, or -1 on - // error. + // Reads a c-string into |str|. Returns the number of bytes read, capped to + // 128 bytes, or -1 on error. int ReadStringInternal(const uint8_t* buf, std::string& str); int pos_ = 0; @@ -106,8 +110,7 @@ class IamfConfigReader { kBinauralMixSelectionNotFound }; - void Reset(); - bool Read(scoped_refptr input_buffer); + bool Read(const scoped_refptr& input_buffer); // Reads a single Descriptor OBU. Returns false on error. bool ReadOBU(BufferReader* reader); bool ReadOBUHeader(BufferReader* reader, From 3d57408f81e6a7c6593a01e3ba1d1c4266374101 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 3 Sep 2024 09:43:39 -0700 Subject: [PATCH 10/13] Minor formatting --- starboard/shared/libiamf/iamf_audio_decoder.cc | 4 ++-- starboard/shared/libiamf/iamf_audio_decoder.h | 5 ++--- starboard/shared/libiamf/iamf_config_reader.cc | 12 ++++++------ starboard/shared/libiamf/iamf_config_reader.h | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 0c3d5375d710..ba315b88f6eb 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -221,7 +221,7 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, return false; } - error = IAMF_decoder_set_sampling_rate(decoder_, 48000); + error = IAMF_decoder_set_sampling_rate(decoder_, kDefaultSampleRate); if (error != IAMF_OK) { ReportError("IAMF_decoder_set_sampling_rate() fails with error " + ErrorCodeToString(error)); @@ -348,7 +348,7 @@ scoped_refptr IamfAudioDecoder::Read( result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = 48000; + *samples_per_second = kDefaultSampleRate; return result; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 5aee80f5aaf5..a922921ff14b 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -54,6 +54,7 @@ class IamfAudioDecoder private: static constexpr int kMinimumBuffersToDecode = 2; + static constexpr int kDefaultSampleRate = 48000; bool ConfigureDecoder(IamfConfigReader* reader, int64_t timestamp); void TeardownDecoder(); @@ -66,16 +67,14 @@ class IamfAudioDecoder OutputCB output_cb_; ErrorCB error_cb_; + ConsumedCB consumed_cb_; IAMF_Decoder* decoder_ = nullptr; bool stream_ended_ = false; std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; - bool decoder_is_configured_ = false; - std::deque> pending_audio_buffers_; - ConsumedCB consumed_cb_; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index 0a257c6d7136..342b62706967 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -86,7 +86,7 @@ bool IamfConfigReader::BufferReader::ReadLeb128(uint32_t* ptr) { return true; } -bool IamfConfigReader::BufferReader::ReadString(std::string& str) { +bool IamfConfigReader::BufferReader::ReadString(std::string* str) { int bytes_read = ReadStringInternal(buf_ + pos_, str); if (bytes_read < 0) { error_ = true; @@ -112,7 +112,7 @@ bool IamfConfigReader::BufferReader::SkipLeb128() { bool IamfConfigReader::BufferReader::SkipString() { std::string str; - return ReadString(str); + return ReadString(&str); } int IamfConfigReader::BufferReader::ReadLeb128Internal(const uint8_t* buf, @@ -138,7 +138,7 @@ int IamfConfigReader::BufferReader::ReadLeb128Internal(const uint8_t* buf, } int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, - std::string& str) { + std::string* str) { SB_DCHECK(buf); int remaining_size = static_cast(size_) - pos_; @@ -148,7 +148,7 @@ int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, // The size of the string is capped to 128 bytes. int max_str_size = std::min(remaining_size, 128); - str.clear(); + str->clear(); size_t size = 0; while (buf[size] != '\0' && size < max_str_size) { @@ -160,8 +160,8 @@ int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, } if (size > 0) { - str.resize(size); - std::memcpy(str.data(), reinterpret_cast(buf), size); + str->resize(size); + std::memcpy(str->data(), reinterpret_cast(buf), size); } // Account for null terminator byte. diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index 72653d8f3e51..d38d0fd9781d 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -65,14 +65,14 @@ class IamfConfigReader { bool Read1(uint8_t* ptr); bool Read4(uint32_t* ptr); bool ReadLeb128(uint32_t* ptr); - bool ReadString(std::string& str); + bool ReadString(std::string* str); bool SkipBytes(size_t size); bool SkipLeb128(); bool SkipString(); size_t size() const { return size_; } - int pos() const { return pos_; } + size_t pos() const { return pos_; } const uint8_t* buf() const { return buf_; } bool error() const { return error_; } @@ -91,7 +91,7 @@ class IamfConfigReader { int ReadLeb128Internal(const uint8_t* buf, uint32_t* value); // Reads a c-string into |str|. Returns the number of bytes read, capped to // 128 bytes, or -1 on error. - int ReadStringInternal(const uint8_t* buf, std::string& str); + int ReadStringInternal(const uint8_t* buf, std::string* str); int pos_ = 0; const uint8_t* buf_; From 03221af1c617b6e763abe9f7103f3d5da98d6b93 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 3 Sep 2024 12:43:12 -0700 Subject: [PATCH 11/13] Update DCHECK in mix_presentation_id() --- starboard/shared/libiamf/iamf_config_reader.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index d38d0fd9781d..0a7db26bba9a 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -46,7 +46,8 @@ class IamfConfigReader { uint32_t samples_per_buffer() const { return samples_per_buffer_; } uint32_t config_size() const { return config_size_; } uint32_t mix_presentation_id() const { - SB_DCHECK(mix_presentation_id_.has_value()); + SB_DCHECK(is_valid()) + << "Called mix_presentation_id() on invalid IamfConfigReader."; if (prefer_binaural_audio_ && binaural_mix_selection_ == kBinauralMixSelectionNotFound) { SB_LOG(INFO) << "Could not find binaural mix presentation."; From 7678d987b065999b429e9d339c019438b1ec1e57 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 4 Sep 2024 17:28:32 -0700 Subject: [PATCH 12/13] IamfConfigReader becomes IamfBufferParser --- starboard/android/shared/BUILD.gn | 4 +- starboard/linux/shared/BUILD.gn | 4 +- .../shared/libiamf/iamf_audio_decoder.cc | 86 +-- starboard/shared/libiamf/iamf_audio_decoder.h | 7 +- .../shared/libiamf/iamf_buffer_parser.cc | 588 ++++++++++++++++++ starboard/shared/libiamf/iamf_buffer_parser.h | 109 ++++ .../shared/libiamf/iamf_config_reader.cc | 531 ---------------- starboard/shared/libiamf/iamf_config_reader.h | 143 ----- 8 files changed, 734 insertions(+), 738 deletions(-) create mode 100644 starboard/shared/libiamf/iamf_buffer_parser.cc create mode 100644 starboard/shared/libiamf/iamf_buffer_parser.h delete mode 100644 starboard/shared/libiamf/iamf_config_reader.cc delete mode 100644 starboard/shared/libiamf/iamf_config_reader.h diff --git a/starboard/android/shared/BUILD.gn b/starboard/android/shared/BUILD.gn index 173a78f7b192..a53baa3089f3 100644 --- a/starboard/android/shared/BUILD.gn +++ b/starboard/android/shared/BUILD.gn @@ -491,8 +491,8 @@ static_library("starboard_platform") { sources += [ "//starboard/shared/libiamf/iamf_audio_decoder.cc", "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_config_reader.cc", - "//starboard/shared/libiamf/iamf_config_reader.h", + "//starboard/shared/libiamf/iamf_buffer_parser.cc", + "//starboard/shared/libiamf/iamf_buffer_parser.h", ] defines += [ "ENABLE_IAMF_DECODE" ] diff --git a/starboard/linux/shared/BUILD.gn b/starboard/linux/shared/BUILD.gn index dbb40f6bf238..d4575a12f507 100644 --- a/starboard/linux/shared/BUILD.gn +++ b/starboard/linux/shared/BUILD.gn @@ -444,8 +444,8 @@ static_library("starboard_platform_sources") { sources += [ "//starboard/shared/libiamf/iamf_audio_decoder.cc", "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_config_reader.cc", - "//starboard/shared/libiamf/iamf_config_reader.h", + "//starboard/shared/libiamf/iamf_buffer_parser.cc", + "//starboard/shared/libiamf/iamf_buffer_parser.h", ] defines += [ "ENABLE_IAMF_DECODE" ] diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index ba315b88f6eb..58db3987d272 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -93,44 +93,12 @@ void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, return; } - if (input_buffers.size() > kMinimumBuffersToDecode) { - std::copy(std::begin(input_buffers), std::end(input_buffers), - std::back_inserter(pending_audio_buffers_)); - consumed_cb_ = consumed_cb; - DecodePendingBuffers(); - } else { - for (const auto& input_buffer : input_buffers) { - if (!DecodeInternal(input_buffer)) { - return; - } - } - Schedule(consumed_cb); - } -} - -void IamfAudioDecoder::DecodePendingBuffers() { - SB_DCHECK(BelongsToCurrentThread()); - SB_DCHECK(!pending_audio_buffers_.empty()); - SB_DCHECK(consumed_cb_); - - for (int i = 0; i < kMinimumBuffersToDecode; ++i) { - if (!DecodeInternal(pending_audio_buffers_.front())) { - return; - } - pending_audio_buffers_.pop_front(); - if (pending_audio_buffers_.empty()) { - Schedule(consumed_cb_); - consumed_cb_ = nullptr; - if (stream_ended_) { - Schedule(std::bind(&IamfAudioDecoder::WriteEndOfStream, this)); - stream_ended_ = false; - } + for (const auto& input_buffer : input_buffers) { + if (!DecodeInternal(input_buffer)) { return; } } - - SB_DCHECK(!pending_audio_buffers_.empty()); - Schedule(std::bind(&IamfAudioDecoder::DecodePendingBuffers, this)); + Schedule(consumed_cb); } bool IamfAudioDecoder::DecodeInternal( @@ -146,14 +114,15 @@ bool IamfAudioDecoder::DecodeInternal( return false; } - IamfConfigReader reader(input_buffer, kForceBinauralAudio, - kForceSurroundAudio); - if (!reader.is_valid()) { + IamfBufferParser::IamfBufferInfo info; + IamfBufferParser().ParseInputBuffer(input_buffer, &info, kForceBinauralAudio, + kForceSurroundAudio); + if (!info.is_valid()) { ReportError("Failed to parse IA Descriptors"); return false; } if (!decoder_is_configured_) { - if (!ConfigureDecoder(&reader, input_buffer->timestamp())) { + if (!ConfigureDecoder(&info, input_buffer->timestamp())) { return false; } } @@ -161,28 +130,30 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * reader.samples_per_buffer() * + audio_stream_info_.number_of_channels * info.num_samples * starboard::media::GetBytesPerSample(GetSampleType())); - int samples_decoded = IAMF_decoder_decode( - decoder_, reader.data().data(), reader.data().size(), nullptr, - reinterpret_cast(decoded_audio->data())); + int samples_decoded = + IAMF_decoder_decode(decoder_, info.data.data(), info.data_size, nullptr, + reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { ReportError("Failed to decode IAMF sample, error " + ErrorCodeToString(samples_decoded)); return false; } - SB_DCHECK(samples_decoded <= reader.samples_per_buffer()); + SB_DCHECK(samples_decoded <= info.num_samples) + << "Samples decoded (" << samples_decoded + << ") is greater than the number of samples indicated by the stream (" + << info.num_samples << ")"; - decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * - reader.samples_per_buffer() * - starboard::media::GetBytesPerSample(GetSampleType())); + if (samples_per_second_ == 0) { + samples_per_second_ = info.sample_rate; + } // TODO: Enable partial audio once float32 pcm output is available. const auto& sample_info = input_buffer->audio_sample_info(); decoded_audio->AdjustForDiscardedDurations( - audio_stream_info_.samples_per_second, - sample_info.discarded_duration_from_front, + samples_per_second_, sample_info.discarded_duration_from_front, sample_info.discarded_duration_from_back); decoded_audios_.push(decoded_audio); @@ -207,7 +178,7 @@ void IamfAudioDecoder::WriteEndOfStream() { Schedule(output_cb_); } -bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, +bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, int64_t timestamp) { SB_DCHECK(is_valid()); SB_DCHECK(!decoder_is_configured_); @@ -221,7 +192,7 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, return false; } - error = IAMF_decoder_set_sampling_rate(decoder_, kDefaultSampleRate); + error = IAMF_decoder_set_sampling_rate(decoder_, info->sample_rate); if (error != IAMF_OK) { ReportError("IAMF_decoder_set_sampling_rate() fails with error " + ErrorCodeToString(error)); @@ -286,7 +257,7 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, } } - // Time base is set to 9000, as it is in the iamfplayer example + // Time base is set to 90000, as it is in the iamfplayer example // https://github.com/AOMediaCodec/libiamf/blob/v1.0.0-errata/code/test/tools/iamfplayer/player/iamfplayer.c#L450 error = IAMF_decoder_set_pts(decoder_, timestamp, 90000); if (error != IAMF_OK) { @@ -295,8 +266,8 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, return false; } - error = IAMF_decoder_set_mix_presentation_id(decoder_, - reader->mix_presentation_id()); + error = IAMF_decoder_set_mix_presentation_id( + decoder_, info->mix_presentation_id.value()); if (error != IAMF_OK) { ReportError("IAMF_decoder_set_mix_presentation_id() fails with error " + ErrorCodeToString(error)); @@ -317,8 +288,8 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, return false; } - error = IAMF_decoder_configure(decoder_, reader->config_obus().data(), - reader->config_size(), nullptr); + error = IAMF_decoder_configure(decoder_, info->config_obus.data(), + info->config_obus_size, nullptr); if (error != IAMF_OK) { ReportError("IAMF_decoder_configure() fails with error " + ErrorCodeToString(error)); @@ -342,13 +313,14 @@ scoped_refptr IamfAudioDecoder::Read( SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(output_cb_); SB_DCHECK(!decoded_audios_.empty()); + SB_DCHECK(samples_per_second_ > 0); scoped_refptr result; if (!decoded_audios_.empty()) { result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = kDefaultSampleRate; + *samples_per_second = samples_per_second_; return result; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index a922921ff14b..6f457d548b90 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -22,7 +22,7 @@ #include "starboard/common/ref_counted.h" #include "starboard/media.h" #include "starboard/shared/internal_only.h" -#include "starboard/shared/libiamf/iamf_config_reader.h" +#include "starboard/shared/libiamf/iamf_buffer_parser.h" #include "starboard/shared/starboard/media/media_util.h" #include "starboard/shared/starboard/player/decoded_audio_internal.h" #include "starboard/shared/starboard/player/filter/audio_decoder_internal.h" @@ -38,6 +38,7 @@ class IamfAudioDecoder private starboard::player::JobQueue::JobOwner { public: typedef starboard::media::AudioStreamInfo AudioStreamInfo; + typedef shared::libiamf::IamfBufferParser::IamfBufferInfo IamfBufferInfo; explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); ~IamfAudioDecoder() override; @@ -56,9 +57,8 @@ class IamfAudioDecoder static constexpr int kMinimumBuffersToDecode = 2; static constexpr int kDefaultSampleRate = 48000; - bool ConfigureDecoder(IamfConfigReader* reader, int64_t timestamp); + bool ConfigureDecoder(IamfBufferInfo* info, int64_t timestamp); void TeardownDecoder(); - void DecodePendingBuffers(); bool DecodeInternal(const scoped_refptr& input_buffer); SbMediaAudioSampleType GetSampleType() const; @@ -75,6 +75,7 @@ class IamfAudioDecoder AudioStreamInfo audio_stream_info_; bool decoder_is_configured_ = false; std::deque> pending_audio_buffers_; + int samples_per_second_ = 0; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_buffer_parser.cc b/starboard/shared/libiamf/iamf_buffer_parser.cc new file mode 100644 index 000000000000..02cd04bef7a6 --- /dev/null +++ b/starboard/shared/libiamf/iamf_buffer_parser.cc @@ -0,0 +1,588 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "starboard/shared/libiamf/iamf_buffer_parser.h" + +#include +#include +#include + +#include "third_party/libiamf/source/code/include/IAMF_defines.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +namespace { + +// From //media/formats/mp4/rcheck.h. +#define RCHECK(condition) \ + do { \ + if (!(condition)) { \ + SB_DLOG(ERROR) << "Failure while parsing IAMF config: " #condition; \ + return false; \ + } \ + } while (0) + +// From +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax. +constexpr int kObuTypeCodecConfig = 0; +constexpr int kObuTypeAudioElement = 1; +constexpr int kObuTypeMixPresentation = 2; +constexpr int kObuTypeSequenceHeader = 31; + +// From +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig. +constexpr int kFourccOpus = 0x4f707573; +constexpr int kFourccIpcm = 0x6970636d; + +} // namespace + +class BufferReader { + public: + BufferReader(const uint8_t* buf, size_t size) + : buf_(buf), pos_(0), size_(size) { +#if SB_IS_BIG_ENDIAN +#error BufferReader assumes little-endianness. +#endif // SB_IS_BIG_ENDIAN + } + + bool Read1(uint8_t* ptr) { + if (!HasBytes(sizeof(uint8_t)) || !ptr) { + return false; + } + *ptr = buf_[pos_++]; + return true; + } + + bool Read4(uint32_t* ptr) { + if (!HasBytes(sizeof(uint32_t)) || !ptr) { + return false; + } + std::memcpy(ptr, &buf_[pos_], sizeof(uint32_t)); + *ptr = ByteSwap(*ptr); + pos_ += sizeof(uint32_t); + return true; + } + + bool ReadLeb128(uint32_t* ptr) { + if (!HasBytes(sizeof(uint32_t)) || !ptr) { + return false; + } + int bytes_read = ReadLeb128Internal(buf_ + pos_, ptr); + if (bytes_read < 0) { + return false; + } + pos_ += bytes_read; + return true; + } + + bool ReadString(std::string* str) { + int bytes_read = ReadStringInternal(buf_ + pos_, str); + if (bytes_read < 0) { + return false; + } + pos_ += bytes_read; + return true; + } + + bool SkipBytes(size_t size) { + if (!HasBytes(size)) { + return false; + } + pos_ += size; + return true; + } + + bool SkipLeb128() { + uint32_t val; + return ReadLeb128(&val); + } + + bool SkipString() { + std::string str; + return ReadString(&str); + } + + size_t size() const { return size_; } + size_t pos() const { return pos_; } + const uint8_t* buf() const { return buf_; } + + private: + bool HasBytes(size_t size) const { return size + pos_ <= size_; } + inline uint32_t ByteSwap(uint32_t x) { +#if defined(COMPILER_MSVC) + return _byteswap_ulong(x); +#else + return __builtin_bswap32(x); +#endif + } + + // Decodes an Leb128 value and stores it in |value|. Returns the number of + // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, + // or -1 on error. + int ReadLeb128Internal(const uint8_t* buf, uint32_t* value) { + SB_DCHECK(buf); + SB_DCHECK(value); + *value = 0; + bool error = true; + size_t i = 0; + for (; i < sizeof(uint32_t); ++i) { + uint8_t byte = buf[i]; + *value |= ((byte & 0x7f) << (i * 7)); + if (!(byte & 0x80)) { + error = false; + break; + } + } + + if (error) { + return -1; + } + return i + 1; + } + + // Reads a c-string into |str|. Returns the number of bytes read, capped to + // 128 bytes, or -1 on error. + int ReadStringInternal(const uint8_t* buf, std::string* str) { + SB_DCHECK(buf); + + int remaining_size = static_cast(size_) - pos_; + if (remaining_size <= 0) { + return -1; + } + + // The size of the string is capped to 128 bytes. + const int kMaxStringSize = 128; + int str_size = std::min(remaining_size, kMaxStringSize); + str->clear(); + + size_t bytes_read = 0; + while (bytes_read < str_size && buf[bytes_read] != '\0') { + bytes_read++; + } + + if (bytes_read == str_size) { + if (buf[bytes_read - 1] != '\0') { + return -1; + } + } else { + if (buf[bytes_read] != '\0') { + return -1; + } + } + + if (bytes_read > 0) { + str->resize(bytes_read); + std::memcpy(str->data(), reinterpret_cast(buf), bytes_read); + } + + // Account for null terminator byte. + return ++bytes_read; + } + + int pos_ = 0; + const uint8_t* buf_; + const size_t size_ = 0; +}; + +IamfBufferParser::IamfBufferParser() {} + +bool IamfBufferParser::ParseInputBuffer( + const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + SB_DCHECK(info); + SB_DCHECK(input_buffer->data()); + SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); + RCHECK(ParseInputBufferInternal(input_buffer, info, prefer_binaural_audio, + prefer_surround_audio)); + return true; +} + +bool IamfBufferParser::ParseInputBufferInternal( + const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + BufferReader reader(input_buffer->data(), input_buffer->size()); + + while (!info->is_valid() && reader.pos() < reader.size()) { + RCHECK(ParseDescriptorOBU(&reader, info, prefer_binaural_audio, + prefer_surround_audio)); + } + + info->data_size = reader.size() - info->config_obus_size; + info->config_obus.assign(reader.buf(), reader.buf() + info->config_obus_size); + info->data.assign(reader.buf() + info->config_obus_size, + reader.buf() + reader.size()); + + return true; +} + +bool IamfBufferParser::ParseDescriptorOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + SB_DCHECK(reader); + uint8_t obu_type = 0; + uint32_t obu_size = 0; + if (!ParseOBUHeader(reader, &obu_type, &obu_size)) { + SB_DLOG(ERROR) << "Error parsing OBU header"; + return false; + } + + int next_obu_pos = reader->pos() + obu_size; + + switch (static_cast(obu_type)) { + case kObuTypeCodecConfig: + RCHECK(ParseCodecConfigOBU(reader, info)); + break; + case kObuTypeAudioElement: + RCHECK(ParseAudioElementOBU(reader, info, prefer_binaural_audio, + prefer_surround_audio)); + break; + case kObuTypeSequenceHeader: + break; + case kObuTypeMixPresentation: + RCHECK(ParseMixPresentationOBU(reader, info, prefer_binaural_audio, + prefer_surround_audio)); + break; + default: + // Once an OBU is read that is not a descriptor, descriptor parsing is + // assumed to be complete. + SB_DCHECK(info->is_valid()); + return true; + } + + // Skip to the next OBU. + const size_t remaining_size = next_obu_pos - reader->pos(); + RCHECK(reader->SkipBytes(remaining_size)); + info->config_obus_size = reader->pos(); + return true; +} + +bool IamfBufferParser::ParseOBUHeader(BufferReader* reader, + uint8_t* obu_type, + uint32_t* obu_size) const { + uint8_t header_flags; + RCHECK(reader->Read1(&header_flags)); + *obu_type = (header_flags >> 3) & 0x1f; + + const bool obu_redundant_copy = (header_flags >> 2) & 1; + const bool obu_trimming_status_flag = (header_flags >> 1) & 1; + const bool obu_extension_flag = header_flags & 1; + + *obu_size = 0; + + RCHECK(reader->ReadLeb128(obu_size)); + + // |obu_size| contains the size of the OBU after its own field. + // If either of the flags are set, subtract the number of bytes read + // from the flags from |obu_size| before returning to ParseDescriptorOBU(). + size_t reader_pos_before_flags = reader->pos(); + + if (obu_trimming_status_flag) { + RCHECK(reader->SkipLeb128()); + RCHECK(reader->SkipLeb128()); + } + + if (obu_extension_flag) { + RCHECK(reader->SkipLeb128()); + } + + size_t flag_bytes_read = reader->pos() - reader_pos_before_flags; + if (flag_bytes_read >= *obu_size) { + return false; + } + *obu_size -= flag_bytes_read; + return true; +} + +bool IamfBufferParser::ParseCodecConfigOBU(BufferReader* reader, + IamfBufferInfo* info) { + RCHECK(reader->SkipLeb128()); + + uint32_t codec_id = 0; + RCHECK(reader->Read4(&codec_id)); + + RCHECK(reader->ReadLeb128(&info->num_samples)); + + // audio_roll_distance + RCHECK(reader->SkipBytes(2)); + + const int kOpusSampleRate = 48000; + switch (codec_id) { + case kFourccOpus: + info->sample_rate = kOpusSampleRate; + break; + case kFourccIpcm: { + // sample_format_flags + RCHECK(reader->SkipBytes(1)); + + // sample_size + RCHECK(reader->SkipBytes(1)); + + uint32_t sample_rate_unsigned; + RCHECK(reader->Read4(&sample_rate_unsigned)); + info->sample_rate = static_cast(sample_rate_unsigned); + break; + } + default: + SB_NOTREACHED(); + return false; + } + + return true; +} + +bool IamfBufferParser::ParseAudioElementOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + uint32_t audio_element_id; + RCHECK(reader->ReadLeb128(&audio_element_id)); + + uint8_t audio_element_type; + RCHECK(reader->Read1(&audio_element_type)); + audio_element_type = audio_element_type >> 5; + + // codec_config_id + RCHECK(reader->SkipLeb128()); + + uint32_t num_substreams; + RCHECK(reader->ReadLeb128(&num_substreams)); + + for (int i = 0; i < num_substreams; ++i) { + // audio_substream_id + RCHECK(reader->SkipLeb128()); + } + + uint32_t num_parameters; + RCHECK(reader->ReadLeb128(&num_parameters)); + + for (int i = 0; i < num_parameters; ++i) { + uint32_t param_definition_type; + RCHECK(reader->ReadLeb128(¶m_definition_type)); + + if (param_definition_type == IAMF_PARAMETER_TYPE_DEMIXING) { + SkipParamDefinition(reader); + // DemixingParamDefintion + RCHECK(reader->SkipBytes(1)); + } else if (param_definition_type == IAMF_PARAMETER_TYPE_RECON_GAIN) { + SkipParamDefinition(reader); + } else if (param_definition_type > 2) { + uint32_t param_definition_size; + RCHECK(reader->ReadLeb128(¶m_definition_size)); + RCHECK(reader->SkipBytes(param_definition_size)); + } + } + + if (static_cast(audio_element_type) == + AUDIO_ELEMENT_CHANNEL_BASED && + (prefer_binaural_audio || prefer_surround_audio)) { + // Parse ScalableChannelLayoutConfig for binaural and surround + // loudspeaker layouts + uint8_t num_layers; + RCHECK(reader->Read1(&num_layers)); + num_layers = num_layers >> 5; + // Read ChannelAudioLayerConfigs + for (int i = 0; i < static_cast(num_layers); ++i) { + uint8_t loudspeaker_layout; + bool output_gain_is_present_flag; + RCHECK(reader->Read1(&loudspeaker_layout)); + output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; + loudspeaker_layout = loudspeaker_layout >> 4; + if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL) { + binaural_audio_element_ids_.insert(audio_element_id); + } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && + loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT) { + surround_audio_element_ids_.insert(audio_element_id); + } + + // substream_count and coupled_substream_count + RCHECK(reader->SkipBytes(2)); + + if (output_gain_is_present_flag) { + // output_gain_flags and output_gain + RCHECK(reader->SkipBytes(3)); + } + + if (i == 1 && loudspeaker_layout == static_cast(15)) { + // expanded_loudspeaker_layout + RCHECK(reader->SkipBytes(1)); + } + } + } + return true; +} + +bool IamfBufferParser::ParseMixPresentationOBU( + BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + uint32_t mix_presentation_id; + RCHECK(reader->ReadLeb128(&mix_presentation_id)); + + uint32_t count_label; + RCHECK(reader->ReadLeb128(&count_label)); + for (int i = 0; i < count_label; ++i) { + // language_label; + RCHECK(reader->SkipString()); + } + + for (int i = 0; i < count_label; ++i) { + // MixPresentationAnnotations; + RCHECK(reader->SkipString()); + } + + uint32_t num_sub_mixes; + RCHECK(reader->ReadLeb128(&num_sub_mixes)); + for (int i = 0; i < num_sub_mixes; ++i) { + uint32_t num_audio_elements; + RCHECK(reader->ReadLeb128(&num_audio_elements)); + for (int j = 0; j < num_audio_elements; ++j) { + uint32_t audio_element_id; + RCHECK(reader->ReadLeb128(&audio_element_id)); + + // Set a mix presentation for binaural or surround streams. The mix + // presentation is chosen if there exists an audio element that has + // the qualities it requires - such as an audio element with a + // binaural loudspeaker layout. + if (!info->mix_presentation_id.has_value() || + (prefer_binaural_audio && + binaural_mix_selection_ > kBinauralMixSelectionLoudspeakerLayout)) { + if (prefer_binaural_audio && + binaural_audio_element_ids_.find(audio_element_id) != + binaural_audio_element_ids_.end()) { + info->mix_presentation_id = mix_presentation_id; + binaural_mix_selection_ = kBinauralMixSelectionLoudspeakerLayout; + } else if (prefer_surround_audio && + surround_audio_element_ids_.find(audio_element_id) != + surround_audio_element_ids_.end()) { + info->mix_presentation_id = mix_presentation_id; + } + } + + for (int k = 0; k < count_label; ++k) { + // MixPresentationElementAnnotatoions + RCHECK(reader->SkipString()); + } + + // The following fields are for the RenderingConfig + // headphones_rendering_mode + RCHECK(reader->SkipBytes(1)); + uint32_t rendering_config_extension_size; + RCHECK(reader->ReadLeb128(&rendering_config_extension_size)); + // rendering_config_extension_bytes + RCHECK(reader->SkipBytes(rendering_config_extension_size)); + + // The following fields are for the ElementMixConfig + SkipParamDefinition(reader); + // default_mix_gain + RCHECK(reader->SkipBytes(2)); + } + + // The following fields are for the OutputMixConfig + SkipParamDefinition(reader); + // default_mix_gain + RCHECK(reader->SkipBytes(2)); + + uint32_t num_layouts; + RCHECK(reader->ReadLeb128(&num_layouts)); + for (int j = 0; j < num_layouts; ++j) { + uint8_t layout_type; + RCHECK(reader->Read1(&layout_type)); + layout_type = layout_type >> 6; + // If a binaural mix presentation is preferred and the mix + // presentation id has not yet been set, set the mix presentation id + // if the current mix presentation has a binaural loudness layout. The + // mix presentation id will change if a different mix presentation is + // found that uses an audio element with a binaural loudspeaker + // layout, as that is higher priority. + if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && + prefer_binaural_audio && + (!info->mix_presentation_id.has_value() || + binaural_mix_selection_ > kBinauralMixSelectionLoudnessLayout)) { + info->mix_presentation_id = mix_presentation_id; + binaural_mix_selection_ = kBinauralMixSelectionLoudnessLayout; + } + + // The following fields are for the LoudnessInfo + uint8_t info_type; + RCHECK(reader->Read1(&info_type)); + // integrated_loudness and digital_loudness + RCHECK(reader->SkipBytes(4)); + if (info_type & 1) { + // true_peak + RCHECK(reader->SkipBytes(2)); + } + if (info_type & 2) { + uint8_t num_anchored_loudness; + RCHECK(reader->Read1(&num_anchored_loudness)); + for (uint8_t k = 0; k < num_anchored_loudness; ++k) { + // anchor_element and anchored_loudness + RCHECK(reader->SkipBytes(3)); + } + } + if ((info_type & 0b11111100) > 0) { + uint32_t info_type_size; + RCHECK(reader->ReadLeb128(&info_type_size)); + // info_type_bytes + RCHECK(reader->SkipBytes(info_type_size)); + } + } + } + + // If the mix presentation id is unassigned at this point, the stream is + // stereo, or a proper mix presentation for binaural or surround preferred + // streams hasn't yet been parsed. Default to the first read mix + // presentation in case a preferred mix does not exist. + if (!info->mix_presentation_id.has_value()) { + info->mix_presentation_id = mix_presentation_id; + } + + return true; +} + +bool IamfBufferParser::SkipParamDefinition(BufferReader* reader) const { + // parameter_id + RCHECK(reader->SkipLeb128()); + // parameter_rate + RCHECK(reader->SkipLeb128()); + uint8_t param_definition_mode; + RCHECK(reader->Read1(¶m_definition_mode)); + param_definition_mode = param_definition_mode >> 7; + if (param_definition_mode == static_cast(0)) { + // duration + RCHECK(reader->SkipLeb128()); + uint32_t constant_subblock_duration; + RCHECK(reader->ReadLeb128(&constant_subblock_duration)); + if (constant_subblock_duration == 0) { + uint32_t num_subblocks; + RCHECK(reader->ReadLeb128(&num_subblocks)); + for (int i = 0; i < num_subblocks; ++i) { + // subblock_duration + RCHECK(reader->SkipLeb128()); + } + } + } + return true; +} + +} // namespace libiamf +} // namespace shared +} // namespace starboard diff --git a/starboard/shared/libiamf/iamf_buffer_parser.h b/starboard/shared/libiamf/iamf_buffer_parser.h new file mode 100644 index 000000000000..83154fd0825a --- /dev/null +++ b/starboard/shared/libiamf/iamf_buffer_parser.h @@ -0,0 +1,109 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ +#define STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ + +#include +#include +#include +#include + +#include "starboard/common/log.h" +#include "starboard/common/ref_counted.h" +#include "starboard/shared/internal_only.h" +#include "starboard/shared/starboard/player/input_buffer_internal.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +class BufferReader; + +// TODO: Skip parsing if the OBUs are redundant +class IamfBufferParser { + public: + typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; + + struct IamfBufferInfo { + bool is_valid() const { + return mix_presentation_id.has_value() && sample_rate > 0 && + num_samples > 0; + } + + uint32_t num_samples; + int sample_rate; + std::optional mix_presentation_id; + std::vector config_obus; + size_t config_obus_size; + std::vector data; + size_t data_size; + const scoped_refptr input_buffer; + }; + + IamfBufferParser(); + + bool ParseInputBuffer(const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + + private: + // Used in the selection of a binaural mix presentation, using the strategy + // defined in + // https://aomediacodec.github.io/iamf/#processing-mixpresentation-selection. + // The preferred methods of choosing a binaural mix presentation are listed + // from high to low. + enum BinauralMixSelection { + kBinauralMixSelectionLoudspeakerLayout, + kBinauralMixSelectionLoudnessLayout, + kBinauralMixSelectionNotFound + }; + + bool ParseInputBufferInternal(const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + // Reads a single Descriptor OBU. Returns false on error. + bool ParseDescriptorOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + bool ParseOBUHeader(BufferReader* reader, + uint8_t* obu_type, + uint32_t* obu_size) const; + bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info); + bool ParseAudioElementOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + bool ParseMixPresentationOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + // Helper function to skip parsing ParamDefinitions found in the config OBUs + // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#paramdefinition + bool SkipParamDefinition(BufferReader* reader) const; + + std::unordered_set binaural_audio_element_ids_; + std::unordered_set surround_audio_element_ids_; + + BinauralMixSelection binaural_mix_selection_ = kBinauralMixSelectionNotFound; +}; + +} // namespace libiamf +} // namespace shared +} // namespace starboard + +#endif // STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc deleted file mode 100644 index 342b62706967..000000000000 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ /dev/null @@ -1,531 +0,0 @@ -// Copyright 2024 The Cobalt Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "starboard/shared/libiamf/iamf_config_reader.h" - -#include -#include - -#include "starboard/common/string.h" -#include "third_party/libiamf/source/code/include/IAMF_defines.h" - -namespace starboard { -namespace shared { -namespace libiamf { - -namespace { - -// From //media/formats/mp4/rcheck.h. -#define RCHECK(condition) \ - do { \ - if (!(condition)) { \ - SB_DLOG(ERROR) << "Failure while parsing IAMF config: " #condition; \ - return false; \ - } \ - } while (0) - -// From -// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax. -constexpr int kObuTypeCodecConfig = 0; -constexpr int kObuTypeAudioElement = 1; -constexpr int kObuTypeMixPresentation = 2; -constexpr int kObuTypeSequenceHeader = 31; - -// From -// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig. -constexpr int kFourccOpus = 0x4f707573; -constexpr int kFourccIpcm = 0x6970636d; - -} // namespace - -IamfConfigReader::BufferReader::BufferReader(const uint8_t* buf, size_t size) - : buf_(buf), pos_(0), size_(size), error_(false) {} - -bool IamfConfigReader::BufferReader::Read1(uint8_t* ptr) { - if (!HasBytes(sizeof(uint8_t)) || !ptr) { - error_ = true; - return false; - } - *ptr = buf_[pos_++]; - return true; -} - -bool IamfConfigReader::BufferReader::Read4(uint32_t* ptr) { - if (!HasBytes(sizeof(uint32_t)) || !ptr) { - error_ = true; - return false; - } - std::memcpy(ptr, &buf_[pos_], sizeof(uint32_t)); - *ptr = ByteSwap(*ptr); - pos_ += sizeof(uint32_t); - return true; -} - -bool IamfConfigReader::BufferReader::ReadLeb128(uint32_t* ptr) { - if (!HasBytes(sizeof(uint32_t)) || !ptr) { - error_ = true; - return false; - } - int bytes_read = ReadLeb128Internal(buf_ + pos_, ptr); - if (bytes_read < 0) { - error_ = true; - return false; - } - pos_ += bytes_read; - return true; -} - -bool IamfConfigReader::BufferReader::ReadString(std::string* str) { - int bytes_read = ReadStringInternal(buf_ + pos_, str); - if (bytes_read < 0) { - error_ = true; - return false; - } - pos_ += bytes_read; - return true; -} - -bool IamfConfigReader::BufferReader::SkipBytes(size_t size) { - if (!HasBytes(size)) { - error_ = true; - return false; - } - pos_ += size; - return true; -} - -bool IamfConfigReader::BufferReader::SkipLeb128() { - uint32_t val; - return ReadLeb128(&val); -} - -bool IamfConfigReader::BufferReader::SkipString() { - std::string str; - return ReadString(&str); -} - -int IamfConfigReader::BufferReader::ReadLeb128Internal(const uint8_t* buf, - uint32_t* value) { - SB_DCHECK(buf); - SB_DCHECK(value); - *value = 0; - bool error = true; - size_t i = 0; - for (; i < sizeof(uint32_t); ++i) { - uint8_t byte = buf[i]; - *value |= ((byte & 0x7f) << (i * 7)); - if (!(byte & 0x80)) { - error = false; - break; - } - } - - if (error) { - return -1; - } - return i + 1; -} - -int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, - std::string* str) { - SB_DCHECK(buf); - - int remaining_size = static_cast(size_) - pos_; - if (remaining_size <= 0) { - return -1; - } - - // The size of the string is capped to 128 bytes. - int max_str_size = std::min(remaining_size, 128); - str->clear(); - - size_t size = 0; - while (buf[size] != '\0' && size < max_str_size) { - size++; - } - - if (buf[size] != '\0') { - return -1; - } - - if (size > 0) { - str->resize(size); - std::memcpy(str->data(), reinterpret_cast(buf), size); - } - - // Account for null terminator byte. - return ++size; -} - -IamfConfigReader::IamfConfigReader( - const scoped_refptr& input_buffer, - bool prefer_binaural_audio, - bool prefer_surround_audio) - : prefer_binaural_audio_(prefer_binaural_audio), - prefer_surround_audio_(prefer_surround_audio) { -#if SB_IS_BIG_ENDIAN -#error IamfConfigReader assumes little-endianness. -#endif // SB_IS_BIG_ENDIAN - SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); - Read(input_buffer); -} - -bool IamfConfigReader::Read(const scoped_refptr& input_buffer) { - SB_DCHECK(input_buffer->data()); - BufferReader reader(input_buffer->data(), input_buffer->size()); - - while (!is_valid() && reader.pos() < reader.size()) { - RCHECK(ReadOBU(&reader)); - } - - if (reader.error()) { - return false; - } - - data_size_ = reader.size() - config_size_; - config_obus_.assign(reader.buf(), reader.buf() + config_size_); - data_.assign(reader.buf() + config_size_, reader.buf() + reader.size()); - - return true; -} - -bool IamfConfigReader::ReadOBU(BufferReader* reader) { - SB_DCHECK(reader); - uint8_t obu_type = 0; - uint32_t obu_size = 0; - if (!ReadOBUHeader(reader, &obu_type, &obu_size)) { - SB_DLOG(ERROR) << "Error reading OBU header"; - return false; - } - - int next_obu_pos = reader->pos() + obu_size; - - auto skip_param_definition = [&]() { - // parameter_id - RCHECK(reader->SkipLeb128()); - // parameter_rate - RCHECK(reader->SkipLeb128()); - uint8_t param_definition_mode; - RCHECK(reader->Read1(¶m_definition_mode)); - param_definition_mode = param_definition_mode >> 7; - if (param_definition_mode == static_cast(0)) { - // duration - RCHECK(reader->SkipLeb128()); - uint32_t constant_subblock_duration; - RCHECK(reader->ReadLeb128(&constant_subblock_duration)); - if (constant_subblock_duration == 0) { - uint32_t num_subblocks; - RCHECK(reader->ReadLeb128(&num_subblocks)); - for (int i = 0; i < num_subblocks; ++i) { - // subblock_duration - RCHECK(reader->SkipLeb128()); - } - } - } - return true; - }; - - switch (static_cast(obu_type)) { - case kObuTypeCodecConfig: { - RCHECK(reader->SkipLeb128()); - - uint32_t codec_id = 0; - RCHECK(reader->Read4(&codec_id)); - - RCHECK(reader->ReadLeb128(&samples_per_buffer_)); - - // audio_roll_distance - RCHECK(reader->SkipBytes(2)); - - switch (codec_id) { - case kFourccOpus: - sample_rate_ = 48000; - break; - case kFourccIpcm: { - // sample_format_flags - RCHECK(reader->SkipBytes(1)); - - uint8_t sample_size_unsigned; - RCHECK(reader->Read1(&sample_size_unsigned)); - sample_size_ = static_cast(sample_size_unsigned); - - uint32_t sample_rate_unsigned; - RCHECK(reader->Read4(&sample_rate_unsigned)); - sample_rate_ = static_cast(sample_rate_unsigned); - break; - } - default: - SB_NOTREACHED(); - return false; - } - - break; - } - case kObuTypeAudioElement: { - uint32_t audio_element_id; - RCHECK(reader->ReadLeb128(&audio_element_id)); - - uint8_t audio_element_type; - RCHECK(reader->Read1(&audio_element_type)); - audio_element_type = audio_element_type >> 5; - - // codec_config_id - RCHECK(reader->SkipLeb128()); - - uint32_t num_substreams; - RCHECK(reader->ReadLeb128(&num_substreams)); - - for (int i = 0; i < num_substreams; ++i) { - // audio_substream_id - RCHECK(reader->SkipLeb128()); - } - - uint32_t num_parameters; - RCHECK(reader->ReadLeb128(&num_parameters)); - - for (int i = 0; i < num_parameters; ++i) { - uint32_t param_definition_type; - RCHECK(reader->ReadLeb128(¶m_definition_type)); - - if (param_definition_type == IAMF_PARAMETER_TYPE_DEMIXING) { - skip_param_definition(); - // DemixingParamDefintion - RCHECK(reader->SkipBytes(1)); - } else if (param_definition_type == IAMF_PARAMETER_TYPE_RECON_GAIN) { - skip_param_definition(); - } else if (param_definition_type > 2) { - uint32_t param_definition_size; - RCHECK(reader->ReadLeb128(¶m_definition_size)); - RCHECK(reader->SkipBytes(param_definition_size)); - } - } - - if (static_cast(audio_element_type) == - AUDIO_ELEMENT_CHANNEL_BASED && - (prefer_binaural_audio_ || prefer_surround_audio_)) { - // Parse ScalableChannelLayoutConfig for binaural and surround - // loudspeaker layouts - uint8_t num_layers; - RCHECK(reader->Read1(&num_layers)); - num_layers = num_layers >> 5; - // Read ChannelAudioLayerConfigs - for (int i = 0; i < static_cast(num_layers); ++i) { - uint8_t loudspeaker_layout; - bool output_gain_is_present_flag; - RCHECK(reader->Read1(&loudspeaker_layout)); - output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; - loudspeaker_layout = loudspeaker_layout >> 4; - if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL) { - binaural_audio_element_ids_.insert(audio_element_id); - } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && - loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT) { - surround_audio_element_ids_.insert(audio_element_id); - } - - // substream_count and coupled_substream_count - RCHECK(reader->SkipBytes(2)); - - if (output_gain_is_present_flag) { - // output_gain_flags and output_gain - RCHECK(reader->SkipBytes(3)); - } - - if (i == 1 && loudspeaker_layout == static_cast(15)) { - // expanded_loudspeaker_layout - RCHECK(reader->SkipBytes(1)); - } - } - } - break; - } - case kObuTypeSequenceHeader: - break; - case kObuTypeMixPresentation: { - uint32_t mix_presentation_id; - RCHECK(reader->ReadLeb128(&mix_presentation_id)); - - uint32_t count_label; - RCHECK(reader->ReadLeb128(&count_label)); - for (int i = 0; i < count_label; ++i) { - // language_label; - RCHECK(reader->SkipString()); - } - - for (int i = 0; i < count_label; ++i) { - // MixPresentationAnnotations; - RCHECK(reader->SkipString()); - } - - uint32_t num_sub_mixes; - RCHECK(reader->ReadLeb128(&num_sub_mixes)); - for (int i = 0; i < num_sub_mixes; ++i) { - uint32_t num_audio_elements; - RCHECK(reader->ReadLeb128(&num_audio_elements)); - for (int j = 0; j < num_audio_elements; ++j) { - uint32_t audio_element_id; - RCHECK(reader->ReadLeb128(&audio_element_id)); - - // Set a mix presentation for binaural or surround streams. The mix - // presentation is chosen if there exists an audio element that has - // the qualities it requires - such as an audio element with a - // binaural loudspeaker layout. - if (!mix_presentation_id_.has_value() || - (prefer_binaural_audio_ && - binaural_mix_selection_ > - kBinauralMixSelectionLoudspeakerLayout)) { - if (prefer_binaural_audio_ && - binaural_audio_element_ids_.find(audio_element_id) != - binaural_audio_element_ids_.end()) { - mix_presentation_id_ = mix_presentation_id; - binaural_mix_selection_ = kBinauralMixSelectionLoudspeakerLayout; - } else if (prefer_surround_audio_ && - surround_audio_element_ids_.find(audio_element_id) != - surround_audio_element_ids_.end()) { - mix_presentation_id_ = mix_presentation_id; - } - } - - for (int k = 0; k < count_label; ++k) { - // MixPresentationElementAnnotatoions - RCHECK(reader->SkipString()); - } - - // The following fields are for the RenderingConfig - // headphones_rendering_mode - RCHECK(reader->SkipBytes(1)); - uint32_t rendering_config_extension_size; - RCHECK(reader->ReadLeb128(&rendering_config_extension_size)); - // rendering_config_extension_bytes - RCHECK(reader->SkipBytes(rendering_config_extension_size)); - - // The following fields are for the ElementMixConfig - RCHECK(skip_param_definition()); - // default_mix_gain - RCHECK(reader->SkipBytes(2)); - } - // The following fields are for the OutputMixConfig - RCHECK(skip_param_definition()); - // default_mix_gain - RCHECK(reader->SkipBytes(2)); - - uint32_t num_layouts; - RCHECK(reader->ReadLeb128(&num_layouts)); - for (int j = 0; j < num_layouts; ++j) { - uint8_t layout_type; - RCHECK(reader->Read1(&layout_type)); - layout_type = layout_type >> 6; - // If a binaural mix presentation is preferred and the mix - // presentation id has not yet been set, set the mix presentation id - // if the current mix presentation has a binaural loudness layout. The - // mix presentation id will change if a different mix presentation is - // found that uses an audio element with a binaural loudspeaker - // layout, as that is higher priority. - if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && - prefer_binaural_audio_ && - (!mix_presentation_id_.has_value() || - binaural_mix_selection_ > kBinauralMixSelectionLoudnessLayout)) { - mix_presentation_id_ = mix_presentation_id; - binaural_mix_selection_ = kBinauralMixSelectionLoudnessLayout; - } - - // The following fields are for the LoudnessInfo - uint8_t info_type; - RCHECK(reader->Read1(&info_type)); - // integrated_loudness and digital_loudness - RCHECK(reader->SkipBytes(4)); - if (info_type & 1) { - // true_peak - RCHECK(reader->SkipBytes(2)); - } - if (info_type & 2) { - uint8_t num_anchored_loudness; - RCHECK(reader->Read1(&num_anchored_loudness)); - for (uint8_t k = 0; k < num_anchored_loudness; ++k) { - // anchor_element and anchored_loudness - RCHECK(reader->SkipBytes(3)); - } - } - if ((info_type & 0b11111100) > 0) { - uint32_t info_type_size; - RCHECK(reader->ReadLeb128(&info_type_size)); - // info_type_bytes - RCHECK(reader->SkipBytes(info_type_size)); - } - } - } - - // If the mix presentation id is unassigned at this point, the stream is - // stereo, or a proper mix presentation for binaural or surround preferred - // streams hasn't yet been parsed. Default to the first read mix - // presentation in case a preferred mix does not exist. - if (!mix_presentation_id_.has_value()) { - mix_presentation_id_ = mix_presentation_id; - } - - break; - } - default: - // Once an OBU is read that is not a descriptor, descriptor parsing is - // assumed to be complete. - SB_DCHECK(mix_presentation_id_.has_value()); - return true; - } - - // Skip to the next OBU. - const size_t remaining_size = next_obu_pos - reader->pos(); - RCHECK(reader->SkipBytes(remaining_size)); - config_size_ = reader->pos(); - return true; -} - -bool IamfConfigReader::ReadOBUHeader(BufferReader* reader, - uint8_t* obu_type, - uint32_t* obu_size) { - uint8_t header_flags; - RCHECK(reader->Read1(&header_flags)); - *obu_type = (header_flags >> 3) & 0x1f; - - const bool obu_redundant_copy = (header_flags >> 2) & 1; - const bool obu_trimming_status_flag = (header_flags >> 1) & 1; - const bool obu_extension_flag = header_flags & 1; - - *obu_size = 0; - - RCHECK(reader->ReadLeb128(obu_size)); - - // |obu_size| contains the size of the OBU after its own field. - // If either of the flags are set, subtract the number of bytes read - // from the flags from |obu_size| before returning to ReadOBU(). - size_t reader_pos_before_flags = reader->pos(); - - if (obu_trimming_status_flag) { - RCHECK(reader->SkipLeb128()); - RCHECK(reader->SkipLeb128()); - } - - if (obu_extension_flag) { - RCHECK(reader->SkipLeb128()); - } - - size_t flag_bytes_read = reader->pos() - reader_pos_before_flags; - if (flag_bytes_read >= *obu_size) { - return false; - } - *obu_size -= flag_bytes_read; - return true; -} - -} // namespace libiamf -} // namespace shared -} // namespace starboard diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h deleted file mode 100644 index 0a7db26bba9a..000000000000 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2024 The Cobalt Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ -#define STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ - -#include -#include -#include -#include - -#include "starboard/common/log.h" -#include "starboard/common/ref_counted.h" -#include "starboard/shared/internal_only.h" -#include "starboard/shared/starboard/player/input_buffer_internal.h" - -namespace starboard { -namespace shared { -namespace libiamf { - -// TODO: Add handling for non-redundant OBUs -class IamfConfigReader { - public: - typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; - - IamfConfigReader(const scoped_refptr& input_buffer, - bool prefer_binaural_audio, - bool prefer_surround_audio); - - bool is_valid() const { - return mix_presentation_id_.has_value() && sample_rate_ > 0 && - samples_per_buffer_ > 0; - } - int sample_rate() const { return sample_rate_; } - uint32_t samples_per_buffer() const { return samples_per_buffer_; } - uint32_t config_size() const { return config_size_; } - uint32_t mix_presentation_id() const { - SB_DCHECK(is_valid()) - << "Called mix_presentation_id() on invalid IamfConfigReader."; - if (prefer_binaural_audio_ && - binaural_mix_selection_ == kBinauralMixSelectionNotFound) { - SB_LOG(INFO) << "Could not find binaural mix presentation."; - } - return mix_presentation_id_.value(); - } - - const std::vector& config_obus() const { return config_obus_; } - const std::vector& data() const { return data_; } - - private: - class BufferReader { - public: - BufferReader(const uint8_t* buf, size_t size); - - bool Read1(uint8_t* ptr); - bool Read4(uint32_t* ptr); - bool ReadLeb128(uint32_t* ptr); - bool ReadString(std::string* str); - - bool SkipBytes(size_t size); - bool SkipLeb128(); - bool SkipString(); - - size_t size() const { return size_; } - size_t pos() const { return pos_; } - const uint8_t* buf() const { return buf_; } - bool error() const { return error_; } - - private: - bool HasBytes(size_t size) const { return size + pos_ <= size_; } - inline uint32_t ByteSwap(uint32_t x) { -#if defined(COMPILER_MSVC) - return _byteswap_ulong(x); -#else - return __builtin_bswap32(x); -#endif - } - // Decodes an Leb128 value and stores it in |value|. Returns the number of - // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, - // or -1 on error. - int ReadLeb128Internal(const uint8_t* buf, uint32_t* value); - // Reads a c-string into |str|. Returns the number of bytes read, capped to - // 128 bytes, or -1 on error. - int ReadStringInternal(const uint8_t* buf, std::string* str); - - int pos_ = 0; - const uint8_t* buf_; - const size_t size_ = 0; - bool error_ = false; - }; - - // Used in the selection of a binaural mix presentation, using the strategy - // defined in - // https://aomediacodec.github.io/iamf/#processing-mixpresentation-selection. - // The preferred methods of choosing a binaural mix presentation are listed - // from high to low. - enum BinauralMixSelection { - kBinauralMixSelectionLoudspeakerLayout, - kBinauralMixSelectionLoudnessLayout, - kBinauralMixSelectionNotFound - }; - - bool Read(const scoped_refptr& input_buffer); - // Reads a single Descriptor OBU. Returns false on error. - bool ReadOBU(BufferReader* reader); - bool ReadOBUHeader(BufferReader* reader, - uint8_t* obu_type, - uint32_t* obu_size); - - int sample_rate_ = 0; - int sample_size_ = 0; - uint32_t samples_per_buffer_ = 0; - uint32_t config_size_ = 0; - uint32_t data_size_ = 0; - - std::optional mix_presentation_id_; - std::unordered_set binaural_audio_element_ids_; - std::unordered_set surround_audio_element_ids_; - const bool prefer_binaural_audio_; - const bool prefer_surround_audio_; - - BinauralMixSelection binaural_mix_selection_ = kBinauralMixSelectionNotFound; - - std::vector config_obus_; - std::vector data_; -}; - -} // namespace libiamf -} // namespace shared -} // namespace starboard - -#endif // STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ From 3efad63315e689cb031d7e0301b93a36024e005c Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Fri, 6 Sep 2024 11:40:12 -0700 Subject: [PATCH 13/13] Update handling of surround audio configurations --- .../shared/libiamf/iamf_audio_decoder.cc | 50 +++++-------------- 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 58db3987d272..43db908041b0 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -27,7 +27,8 @@ namespace { using shared::starboard::player::DecodedAudio; constexpr int kForceBinauralAudio = false; -constexpr int kForceSurroundAudio = false; +constexpr int kForce6ChannelAudio = false; +constexpr int kForce8ChannelAudio = false; std::string ErrorCodeToString(int code) { switch (code) { @@ -115,8 +116,9 @@ bool IamfAudioDecoder::DecodeInternal( } IamfBufferParser::IamfBufferInfo info; - IamfBufferParser().ParseInputBuffer(input_buffer, &info, kForceBinauralAudio, - kForceSurroundAudio); + IamfBufferParser().ParseInputBuffer( + input_buffer, &info, kForceBinauralAudio, + kForce6ChannelAudio | kForce8ChannelAudio); if (!info.is_valid()) { ReportError("Failed to parse IA Descriptors"); return false; @@ -209,43 +211,15 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, return false; } } else { - // Default to stereo output. If kForceSurroundAudio is true, set to a sound - // system matching, the platform's audio configuration, if available. IAMF_SoundSystem sound_system = SOUND_SYSTEM_A; - if (kForceSurroundAudio) { - SbMediaAudioConfiguration out_config; - SbMediaGetAudioConfiguration(0, &out_config); - int channels = std::max(out_config.number_of_channels, 2); - if (channels > 8 || channels < 1) { - ReportError(::starboard::FormatString( - "Can't create decoder with %i channels", channels)); - return false; - } - switch (channels) { - case 1: - sound_system = SOUND_SYSTEM_MONO; - break; - case 2: - // Stereo output. - sound_system = SOUND_SYSTEM_A; - SB_DLOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; - break; - case 6: - // 5.1 output. - sound_system = SOUND_SYSTEM_B; - SB_DLOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; - break; - case 8: - // 7.1 output. - sound_system = SOUND_SYSTEM_C; - SB_DLOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; - break; - default: - SB_NOTREACHED(); - return false; - } + if (kForce6ChannelAudio) { + SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; + sound_system = SOUND_SYSTEM_B; + } else if (kForce8ChannelAudio) { + SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; + sound_system = SOUND_SYSTEM_C; } else { - SB_DLOG(INFO) << "Defaulting to stereo output."; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; } error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system);