From c7039d101d0f791806438b4bc854a0d13ab742c2 Mon Sep 17 00:00:00 2001 From: Jason <2697724+jasonzhangxx@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:11:53 -0700 Subject: [PATCH] [nplb] Add tests for partial audio frames (#545) b/274020873 --- starboard/nplb/multiple_player_test.cc | 10 +- .../player_get_audio_configuration_test.cc | 22 +- starboard/nplb/player_test_fixture.cc | 387 +++++++++++++----- starboard/nplb/player_test_fixture.h | 89 ++-- starboard/nplb/player_test_util.cc | 47 ++- starboard/nplb/player_test_util.h | 7 +- starboard/nplb/player_write_sample_test.cc | 211 +++++++++- starboard/nplb/vertical_video_test.cc | 6 +- .../player/decoded_audio_internal.cc | 2 - .../player/decoded_audio_test_internal.cc | 11 +- 10 files changed, 622 insertions(+), 170 deletions(-) diff --git a/starboard/nplb/multiple_player_test.cc b/starboard/nplb/multiple_player_test.cc index 0bcd732e21fa..ee097b62ea36 100644 --- a/starboard/nplb/multiple_player_test.cc +++ b/starboard/nplb/multiple_player_test.cc @@ -78,10 +78,10 @@ void NoInput(const SbPlayerTestConfig& player_config, GroupedSamples samples; if (player_fixture.HasAudio()) { - samples.AddAudioSamplesWithEOS(0, 0); + samples.AddAudioEOS(); } if (player_fixture.HasVideo()) { - samples.AddVideoSamplesWithEOS(0, 0); + samples.AddVideoEOS(); } ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerEndOfStream()); @@ -103,12 +103,14 @@ void WriteSamples(const SbPlayerTestConfig& player_config, GroupedSamples samples; if (player_fixture.HasAudio()) { - samples.AddAudioSamplesWithEOS( + samples.AddAudioSamples( 0, player_fixture.ConvertDurationToAudioBufferCount(kDurationToPlay)); + samples.AddAudioEOS(); } if (player_fixture.HasVideo()) { - samples.AddVideoSamplesWithEOS( + samples.AddVideoSamples( 0, player_fixture.ConvertDurationToVideoBufferCount(kDurationToPlay)); + samples.AddVideoEOS(); } ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); diff --git a/starboard/nplb/player_get_audio_configuration_test.cc b/starboard/nplb/player_get_audio_configuration_test.cc index f982ff08e3c1..d89fd62022ea 100644 --- a/starboard/nplb/player_get_audio_configuration_test.cc +++ b/starboard/nplb/player_get_audio_configuration_test.cc @@ -120,10 +120,12 @@ TEST_P(SbPlayerGetAudioConfigurationTest, SunnyDay) { GroupedSamples samples; if (player_fixture.HasAudio()) { - samples.AddAudioSamplesWithEOS(0, kSamplesToWrite); + samples.AddAudioSamples(0, kSamplesToWrite); + samples.AddAudioEOS(); } if (player_fixture.HasVideo()) { - samples.AddVideoSamplesWithEOS(0, kSamplesToWrite); + samples.AddVideoSamples(0, kSamplesToWrite); + samples.AddVideoEOS(); } ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerPresenting()); @@ -165,10 +167,10 @@ TEST_P(SbPlayerGetAudioConfigurationTest, NoInput) { GroupedSamples samples; if (player_fixture.HasAudio()) { - samples.AddAudioSamplesWithEOS(0, 0); + samples.AddAudioEOS(); } if (player_fixture.HasVideo()) { - samples.AddVideoSamplesWithEOS(0, 0); + samples.AddVideoEOS(); } ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerPresenting()); @@ -206,10 +208,12 @@ TEST_P(SbPlayerGetAudioConfigurationTest, MultipleSeeks) { GroupedSamples samples; if (player_fixture.HasAudio()) { - samples.AddAudioSamplesWithEOS(0, kSamplesToWrite); + samples.AddAudioSamples(0, kSamplesToWrite); + samples.AddAudioEOS(); } if (player_fixture.HasVideo()) { - samples.AddVideoSamplesWithEOS(0, kSamplesToWrite); + samples.AddVideoSamples(0, kSamplesToWrite); + samples.AddVideoEOS(); } ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerPresenting()); @@ -236,14 +240,16 @@ TEST_P(SbPlayerGetAudioConfigurationTest, MultipleSeeks) { samples = GroupedSamples(); if (player_fixture.HasAudio()) { - samples.AddAudioSamplesWithEOS( + samples.AddAudioSamples( 0, player_fixture.ConvertDurationToAudioBufferCount(seek_to_time) + kSamplesToWrite); + samples.AddAudioEOS(); } if (player_fixture.HasVideo()) { - samples.AddVideoSamplesWithEOS( + samples.AddVideoSamples( 0, player_fixture.ConvertDurationToVideoBufferCount(seek_to_time) + kSamplesToWrite); + samples.AddVideoEOS(); } ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerPresenting()); diff --git a/starboard/nplb/player_test_fixture.cc b/starboard/nplb/player_test_fixture.cc index 47c0935396c0..58f1dfe80836 100644 --- a/starboard/nplb/player_test_fixture.cc +++ b/starboard/nplb/player_test_fixture.cc @@ -15,6 +15,7 @@ #include "starboard/nplb/player_test_fixture.h" #include +#include #include "starboard/common/string.h" #include "starboard/nplb/drm_helpers.h" @@ -26,6 +27,162 @@ namespace nplb { using shared::starboard::player::video_dmp::VideoDmpReader; using testing::FakeGraphicsContextProvider; +using GroupedSamples = SbPlayerTestFixture::GroupedSamples; +using AudioSamplesDescriptor = GroupedSamples::AudioSamplesDescriptor; +using VideoSamplesDescriptor = GroupedSamples::VideoSamplesDescriptor; + +// TODO: Refine the implementation. +class SbPlayerTestFixture::GroupedSamplesIterator { + public: + explicit GroupedSamplesIterator(const GroupedSamples& grouped_samples) + : grouped_samples_(grouped_samples) {} + + bool HasMoreAudio() const { + return audio_samples_index_ < grouped_samples_.audio_samples_.size(); + } + + bool HasMoreVideo() const { + return video_samples_index_ < grouped_samples_.video_samples_.size(); + } + + AudioSamplesDescriptor GetCurrentAudioSamplesToWrite() const { + SB_DCHECK(HasMoreAudio()); + AudioSamplesDescriptor descriptor = + grouped_samples_.audio_samples_[audio_samples_index_]; + descriptor.start_index += current_written_audio_samples_; + descriptor.samples_count -= current_written_audio_samples_; + return descriptor; + } + + VideoSamplesDescriptor GetCurrentVideoSamplesToWrite() const { + SB_DCHECK(HasMoreVideo()); + VideoSamplesDescriptor descriptor = + grouped_samples_.video_samples_[video_samples_index_]; + descriptor.start_index += current_written_video_samples_; + descriptor.samples_count -= current_written_video_samples_; + return descriptor; + } + + void AdvanceAudio(int samples_count) { + SB_DCHECK(HasMoreAudio()); + if (grouped_samples_.audio_samples_[audio_samples_index_] + .is_end_of_stream) { + // For EOS, |samples_count| must be 1. + SB_DCHECK(samples_count == 1); + SB_DCHECK(current_written_audio_samples_ == 0); + audio_samples_index_++; + return; + } + + SB_DCHECK( + current_written_audio_samples_ + samples_count <= + grouped_samples_.audio_samples_[audio_samples_index_].samples_count); + + current_written_audio_samples_ += samples_count; + if (current_written_audio_samples_ == + grouped_samples_.audio_samples_[audio_samples_index_].samples_count) { + audio_samples_index_++; + current_written_audio_samples_ = 0; + } + } + + void AdvanceVideo(int samples_count) { + SB_DCHECK(HasMoreVideo()); + if (grouped_samples_.video_samples_[video_samples_index_] + .is_end_of_stream) { + // For EOS, |samples_count| must be 1. + SB_DCHECK(samples_count == 1); + SB_DCHECK(current_written_video_samples_ == 0); + video_samples_index_++; + return; + } + + SB_DCHECK( + current_written_video_samples_ + samples_count <= + grouped_samples_.video_samples_[video_samples_index_].samples_count); + + current_written_video_samples_ += samples_count; + if (current_written_video_samples_ == + grouped_samples_.video_samples_[video_samples_index_].samples_count) { + video_samples_index_++; + current_written_video_samples_ = 0; + } + } + + private: + const GroupedSamples& grouped_samples_; + int audio_samples_index_ = 0; + int current_written_audio_samples_ = 0; + int video_samples_index_ = 0; + int current_written_video_samples_ = 0; +}; + +GroupedSamples& GroupedSamples::AddAudioSamples(int start_index, + int number_of_samples) { + AddAudioSamples(start_index, number_of_samples, 0, 0, 0); + return *this; +} + +GroupedSamples& GroupedSamples::AddAudioSamples( + int start_index, + int number_of_samples, + SbTime timestamp_offset, + SbTime discarded_duration_from_front, + SbTime discarded_duration_from_back) { + SB_DCHECK(start_index >= 0); + SB_DCHECK(number_of_samples >= 0); + SB_DCHECK(audio_samples_.empty() || !audio_samples_.back().is_end_of_stream); + // Currently, the implementation only supports writing one sample at a time + // if |discarded_duration_from_front| or |discarded_duration_from_back| is not + // 0. + SB_DCHECK(discarded_duration_from_front == 0 || number_of_samples == 1); + SB_DCHECK(discarded_duration_from_back == 0 || number_of_samples == 1); + + AudioSamplesDescriptor descriptor; + descriptor.start_index = start_index; + descriptor.samples_count = number_of_samples; + descriptor.timestamp_offset = timestamp_offset; + descriptor.discarded_duration_from_front = discarded_duration_from_front; + descriptor.discarded_duration_from_back = discarded_duration_from_back; + audio_samples_.push_back(descriptor); + + return *this; +} + +GroupedSamples& GroupedSamples::AddAudioEOS() { + SB_DCHECK(audio_samples_.empty() || !audio_samples_.back().is_end_of_stream); + + AudioSamplesDescriptor descriptor; + descriptor.is_end_of_stream = true; + audio_samples_.push_back(descriptor); + + return *this; +} + +GroupedSamples& GroupedSamples::AddVideoSamples(int start_index, + int number_of_samples) { + SB_DCHECK(start_index >= 0); + SB_DCHECK(number_of_samples >= 0); + SB_DCHECK(video_samples_.empty() || !video_samples_.back().is_end_of_stream); + + VideoSamplesDescriptor descriptor; + descriptor.start_index = start_index; + descriptor.samples_count = number_of_samples; + video_samples_.push_back(descriptor); + + return *this; +} + +GroupedSamples& GroupedSamples::AddVideoEOS() { + SB_DCHECK(video_samples_.empty() || !video_samples_.back().is_end_of_stream); + + VideoSamplesDescriptor descriptor; + descriptor.is_end_of_stream = true; + video_samples_.push_back(descriptor); + + return *this; +} + SbPlayerTestFixture::CallbackEvent::CallbackEvent() : event_type(kEmptyEvent) {} SbPlayerTestFixture::CallbackEvent::CallbackEvent(SbPlayer player, @@ -106,75 +263,69 @@ void SbPlayerTestFixture::Write(const GroupedSamples& grouped_samples) { ASSERT_FALSE(error_occurred_); - int audio_start_index = grouped_samples.audio_start_index(); - int audio_samples_to_write = grouped_samples.audio_samples_to_write(); - int video_start_index = grouped_samples.video_start_index(); - int video_samples_to_write = grouped_samples.video_samples_to_write(); - bool write_audio_eos = grouped_samples.write_audio_eos(); - bool write_video_eos = grouped_samples.write_video_eos(); - - SB_DCHECK(audio_start_index >= 0); - SB_DCHECK(audio_samples_to_write >= 0); - SB_DCHECK(video_start_index >= 0); - SB_DCHECK(video_samples_to_write >= 0); - if (audio_samples_to_write > 0 || write_audio_eos) { - SB_DCHECK(audio_dmp_reader_); - } - if (video_samples_to_write > 0 || write_video_eos) { - SB_DCHECK(video_dmp_reader_); - } - int max_audio_samples_per_write = SbPlayerGetMaximumNumberOfSamplesPerWrite(player_, kSbMediaTypeAudio); int max_video_samples_per_write = SbPlayerGetMaximumNumberOfSamplesPerWrite(player_, kSbMediaTypeVideo); - // Cap the samples to write to the end of the dmp files. - if (audio_samples_to_write > 0) { - audio_samples_to_write = std::min( - audio_samples_to_write, - audio_dmp_reader_->number_of_audio_buffers() - audio_start_index); - } - if (video_samples_to_write > 0) { - video_samples_to_write = std::min( - video_samples_to_write, - video_dmp_reader_->number_of_video_buffers() - video_start_index); - } + GroupedSamplesIterator iterator(grouped_samples); + SB_DCHECK(!iterator.HasMoreAudio() || audio_dmp_reader_); + SB_DCHECK(!iterator.HasMoreVideo() || video_dmp_reader_); - bool has_more_audio = audio_samples_to_write > 0 || write_audio_eos; - bool has_more_video = video_samples_to_write > 0 || write_video_eos; - while (has_more_audio || has_more_video) { - ASSERT_NO_FATAL_FAILURE(WaitForDecoderStateNeedsData()); - if (can_accept_more_audio_data_ && has_more_audio) { - if (audio_samples_to_write > 0) { - auto samples_to_write = - std::min(max_audio_samples_per_write, audio_samples_to_write); - ASSERT_NO_FATAL_FAILURE(WriteSamples( - kSbMediaTypeAudio, audio_start_index, samples_to_write)); - audio_start_index += samples_to_write; - audio_samples_to_write -= samples_to_write; - } else if (!audio_end_of_stream_written_ && write_audio_eos) { + const SbTime kDefaultWriteTimeout = kSbTimeSecond * 5; + + SbTimeMonotonic start = SbTimeGetMonotonicNow(); + while (SbTimeGetMonotonicNow() - start < kDefaultWriteTimeout) { + if (CanWriteMoreAudioData() && iterator.HasMoreAudio()) { + auto descriptor = iterator.GetCurrentAudioSamplesToWrite(); + if (descriptor.is_end_of_stream) { + SB_DCHECK(!audio_end_of_stream_written_); ASSERT_NO_FATAL_FAILURE(WriteEndOfStream(kSbMediaTypeAudio)); + iterator.AdvanceAudio(1); + } else { + SB_DCHECK(descriptor.samples_count > 0); + SB_DCHECK(descriptor.start_index + descriptor.samples_count < + audio_dmp_reader_->number_of_audio_buffers()) + << "Audio dmp file is not long enough to finish the test."; + + auto samples_to_write = + std::min(max_audio_samples_per_write, descriptor.samples_count); + ASSERT_NO_FATAL_FAILURE( + WriteAudioSamples(descriptor.start_index, samples_to_write, + descriptor.timestamp_offset, + descriptor.discarded_duration_from_front, + descriptor.discarded_duration_from_back)); + iterator.AdvanceAudio(samples_to_write); } - has_more_audio = audio_samples_to_write > 0 || - (!audio_end_of_stream_written_ && write_audio_eos); } + if (CanWriteMoreVideoData() && iterator.HasMoreVideo()) { + auto descriptor = iterator.GetCurrentVideoSamplesToWrite(); + if (descriptor.is_end_of_stream) { + SB_DCHECK(!video_end_of_stream_written_); + ASSERT_NO_FATAL_FAILURE(WriteEndOfStream(kSbMediaTypeVideo)); + iterator.AdvanceVideo(1); + } else { + SB_DCHECK(descriptor.samples_count > 0); + SB_DCHECK(descriptor.start_index + descriptor.samples_count < + video_dmp_reader_->number_of_video_buffers()) + << "Video dmp file is not long enough to finish the test."; - if (can_accept_more_video_data_ && has_more_video) { - if (video_samples_to_write > 0) { auto samples_to_write = - std::min(max_video_samples_per_write, video_samples_to_write); - ASSERT_NO_FATAL_FAILURE(WriteSamples( - kSbMediaTypeVideo, video_start_index, samples_to_write)); - video_start_index += samples_to_write; - video_samples_to_write -= samples_to_write; - } else if (!video_end_of_stream_written_ && write_video_eos) { - ASSERT_NO_FATAL_FAILURE(WriteEndOfStream(kSbMediaTypeVideo)); + std::min(max_video_samples_per_write, descriptor.samples_count); + ASSERT_NO_FATAL_FAILURE( + WriteVideoSamples(descriptor.start_index, samples_to_write)); + iterator.AdvanceVideo(samples_to_write); } - has_more_video = video_samples_to_write > 0 || - (!video_end_of_stream_written_ && write_video_eos); + } + + if (iterator.HasMoreAudio() || iterator.HasMoreVideo()) { + ASSERT_NO_FATAL_FAILURE(WaitForDecoderStateNeedsData()); + } else { + return; } } + + FAIL() << "Failed to write all samples."; } void SbPlayerTestFixture::WaitForPlayerPresenting() { @@ -195,14 +346,33 @@ void SbPlayerTestFixture::WaitForPlayerEndOfStream() { ASSERT_NO_FATAL_FAILURE(WaitForPlayerState(kSbPlayerStateEndOfStream)); } +SbTime SbPlayerTestFixture::GetCurrentMediaTime() const { +#if SB_API_VERSION >= 15 + SbPlayerInfo info = {}; + SbPlayerGetInfo(player_, &info); +#else // SB_API_VERSION >= 15 + SbPlayerInfo2 info = {}; + SbPlayerGetInfo2(player_, &info); +#endif // SB_API_VERSION >= 15 + return info.current_media_timestamp; +} + +void SbPlayerTestFixture::SetAudioWriteDuration(SbTime duration) { + SB_DCHECK(thread_checker_.CalledOnValidThread()); + SB_DCHECK(duration > 0); + audio_write_duration_ = duration; +} + +SbTime SbPlayerTestFixture::GetAudioSampleTimestamp(int index) const { + SB_DCHECK(HasAudio()); + SB_DCHECK(index < audio_dmp_reader_->number_of_audio_buffers()); + return audio_dmp_reader_->GetPlayerSampleInfo(kSbMediaTypeAudio, index) + .timestamp; +} + int SbPlayerTestFixture::ConvertDurationToAudioBufferCount( SbTime duration) const { - if (!HasAudio()) { - SB_DLOG(ERROR) - << "Unable to calculate buffer count without a valid audio dmp file."; - return 0; - } - + SB_DCHECK(HasAudio()); SB_DCHECK(audio_dmp_reader_->number_of_audio_buffers()); return duration * audio_dmp_reader_->number_of_audio_buffers() / audio_dmp_reader_->audio_duration(); @@ -210,12 +380,7 @@ int SbPlayerTestFixture::ConvertDurationToAudioBufferCount( int SbPlayerTestFixture::ConvertDurationToVideoBufferCount( SbTime duration) const { - if (!HasVideo()) { - SB_DLOG(ERROR) - << "Unable to calculate buffer count without a valid video dmp file."; - return 0; - } - + SB_DCHECK(HasVideo()); SB_DCHECK(video_dmp_reader_->number_of_video_buffers()); return duration * video_dmp_reader_->number_of_video_buffers() / video_dmp_reader_->video_duration(); @@ -331,34 +496,71 @@ void SbPlayerTestFixture::TearDown() { drm_system_ = kSbDrmSystemInvalid; } -void SbPlayerTestFixture::WriteSamples(SbMediaType media_type, - int start_index, - int samples_to_write) { +bool SbPlayerTestFixture::CanWriteMoreAudioData() { + if (!can_accept_more_audio_data_) { + return false; + } + + if (!audio_write_duration_) { + return true; + } + + return last_written_audio_timestamp_ - GetCurrentMediaTime() < + audio_write_duration_; +} + +bool SbPlayerTestFixture::CanWriteMoreVideoData() { + return can_accept_more_video_data_; +} + +void SbPlayerTestFixture::WriteAudioSamples( + int start_index, + int samples_to_write, + SbTime timestamp_offset, + SbTime discarded_duration_from_front, + SbTime discarded_duration_from_back) { SB_DCHECK(thread_checker_.CalledOnValidThread()); + SB_DCHECK(SbPlayerIsValid(player_)); + SB_DCHECK(audio_dmp_reader_); SB_DCHECK(start_index >= 0); SB_DCHECK(samples_to_write > 0); - SB_DCHECK(SbPlayerIsValid(player_)); - SB_DCHECK(samples_to_write <= - SbPlayerGetMaximumNumberOfSamplesPerWrite(player_, media_type)); + SB_DCHECK(samples_to_write <= SbPlayerGetMaximumNumberOfSamplesPerWrite( + player_, kSbMediaTypeAudio)); + SB_DCHECK(start_index + samples_to_write + 1 < + audio_dmp_reader_->number_of_audio_buffers()); + SB_DCHECK(discarded_duration_from_front == 0 || samples_to_write == 1); + SB_DCHECK(discarded_duration_from_back == 0 || samples_to_write == 1); + + CallSbPlayerWriteSamples( + player_, kSbMediaTypeAudio, audio_dmp_reader_.get(), start_index, + samples_to_write, timestamp_offset, + std::vector(samples_to_write, discarded_duration_from_front), + std::vector(samples_to_write, discarded_duration_from_back)); + + last_written_audio_timestamp_ = + audio_dmp_reader_ + ->GetPlayerSampleInfo(kSbMediaTypeAudio, + start_index + samples_to_write) + .timestamp; - if (media_type == kSbMediaTypeAudio) { - SB_DCHECK(audio_dmp_reader_); - SB_DCHECK(start_index + samples_to_write <= - audio_dmp_reader_->number_of_audio_buffers()); - CallSbPlayerWriteSamples(player_, kSbMediaTypeAudio, - audio_dmp_reader_.get(), start_index, - samples_to_write); - can_accept_more_audio_data_ = false; - } else { - SB_DCHECK(media_type == kSbMediaTypeVideo); - SB_DCHECK(video_dmp_reader_); - SB_DCHECK(start_index + samples_to_write <= - video_dmp_reader_->number_of_video_buffers()); - CallSbPlayerWriteSamples(player_, kSbMediaTypeVideo, - video_dmp_reader_.get(), start_index, - samples_to_write); - can_accept_more_video_data_ = false; - } + can_accept_more_audio_data_ = false; +} + +void SbPlayerTestFixture::WriteVideoSamples(int start_index, + int samples_to_write) { + SB_DCHECK(thread_checker_.CalledOnValidThread()); + SB_DCHECK(start_index >= 0); + SB_DCHECK(samples_to_write > 0); + SB_DCHECK(SbPlayerIsValid(player_)); + SB_DCHECK(samples_to_write <= SbPlayerGetMaximumNumberOfSamplesPerWrite( + player_, kSbMediaTypeVideo)); + SB_DCHECK(video_dmp_reader_); + SB_DCHECK(start_index + samples_to_write < + video_dmp_reader_->number_of_video_buffers()); + + CallSbPlayerWriteSamples(player_, kSbMediaTypeVideo, video_dmp_reader_.get(), + start_index, samples_to_write); + can_accept_more_video_data_ = false; } void SbPlayerTestFixture::WriteEndOfStream(SbMediaType media_type) { @@ -426,7 +628,6 @@ void SbPlayerTestFixture::WaitAndProcessNextEvent(SbTime timeout) { void SbPlayerTestFixture::WaitForDecoderStateNeedsData(const SbTime timeout) { SB_DCHECK(thread_checker_.CalledOnValidThread()); - SB_DCHECK(!can_accept_more_audio_data_ || !can_accept_more_video_data_); bool old_can_accept_more_audio_data = can_accept_more_audio_data_; bool old_can_accept_more_video_data = can_accept_more_video_data_; @@ -441,8 +642,6 @@ void SbPlayerTestFixture::WaitForDecoderStateNeedsData(const SbTime timeout) { return; } } while (SbTimeGetMonotonicNow() - start < timeout); - - FAIL() << "WaitForDecoderStateNeedsData() did not receive expected state."; } void SbPlayerTestFixture::WaitForPlayerState(const SbPlayerState desired_state, diff --git a/starboard/nplb/player_test_fixture.h b/starboard/nplb/player_test_fixture.h index 7fdf1444c15c..6e7e1ef64b8d 100644 --- a/starboard/nplb/player_test_fixture.h +++ b/starboard/nplb/player_test_fixture.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "starboard/common/queue.h" #include "starboard/common/scoped_ptr.h" @@ -35,43 +36,39 @@ namespace nplb { class SbPlayerTestFixture { public: // A simple encapsulation of grouped samples. + class GroupedSamplesIterator; class GroupedSamples { public: - int audio_start_index() const { return audio_start_index_; } - int audio_samples_to_write() const { return audio_samples_to_write_; } - bool write_audio_eos() const { return write_audio_eos_; } - - int video_start_index() const { return video_start_index_; } - int video_samples_to_write() const { return video_samples_to_write_; } - bool write_video_eos() const { return write_video_eos_; } - - void AddAudioSamples(int audio_start_index, int audio_samples_to_write) { - audio_start_index_ = audio_start_index; - audio_samples_to_write_ = audio_samples_to_write; - } - void AddAudioSamplesWithEOS(int audio_start_index, - int audio_samples_to_write) { - AddAudioSamples(audio_start_index, audio_samples_to_write); - write_audio_eos_ = true; - } - void AddVideoSamples(int video_start_index, int video_samples_to_write) { - video_start_index_ = video_start_index; - video_samples_to_write_ = video_samples_to_write; - } - - void AddVideoSamplesWithEOS(int video_start_index, - int video_samples_to_write) { - AddVideoSamples(video_start_index, video_samples_to_write); - write_video_eos_ = true; - } + struct AudioSamplesDescriptor { + int start_index = 0; + int samples_count = 0; + SbTime timestamp_offset = 0; + SbTime discarded_duration_from_front = 0; + SbTime discarded_duration_from_back = 0; + bool is_end_of_stream = false; + }; + + struct VideoSamplesDescriptor { + int start_index = 0; + int samples_count = 0; + bool is_end_of_stream = false; + }; + + GroupedSamples& AddAudioSamples(int start_index, int number_of_samples); + GroupedSamples& AddAudioSamples(int start_index, + int number_of_samples, + SbTime timestamp_offset, + SbTime discarded_duration_from_front, + SbTime discarded_duration_from_back); + GroupedSamples& AddAudioEOS(); + GroupedSamples& AddVideoSamples(int start_index, int number_of_samples); + GroupedSamples& AddVideoEOS(); + + friend class GroupedSamplesIterator; private: - int audio_start_index_ = 0; - int audio_samples_to_write_ = 0; - bool write_audio_eos_ = false; - int video_start_index_ = 0; - int video_samples_to_write_ = 0; - bool write_video_eos_ = false; + std::vector audio_samples_; + std::vector video_samples_; }; SbPlayerTestFixture( @@ -89,16 +86,19 @@ class SbPlayerTestFixture { void WaitForPlayerPresenting(); // Wait until kSbPlayerStateEndOfStream received. void WaitForPlayerEndOfStream(); + SbTime GetCurrentMediaTime() const; + + void SetAudioWriteDuration(SbTime duration); SbPlayer GetPlayer() const { return player_; } bool HasAudio() const { return audio_dmp_reader_; } bool HasVideo() const { return video_dmp_reader_; } + + SbTime GetAudioSampleTimestamp(int index) const; int ConvertDurationToAudioBufferCount(SbTime duration) const; int ConvertDurationToVideoBufferCount(SbTime duration) const; private: - static constexpr SbTime kDefaultWaitForDecoderStateNeedsDataTimeout = - 5 * kSbTimeSecond; static constexpr SbTime kDefaultWaitForPlayerStateTimeout = 5 * kSbTimeSecond; static constexpr SbTime kDefaultWaitForCallbackEventTimeout = 15 * kSbTimeMillisecond; @@ -155,9 +155,15 @@ class SbPlayerTestFixture { void Initialize(); void TearDown(); - void WriteSamples(SbMediaType media_type, - int start_index, - int samples_to_write); + bool CanWriteMoreAudioData(); + bool CanWriteMoreVideoData(); + + void WriteAudioSamples(int start_index, + int samples_to_write, + SbTime timestamp_offset, + SbTime discarded_duration_from_front, + SbTime discarded_duration_from_back); + void WriteVideoSamples(int start_index, int samples_to_write); void WriteEndOfStream(SbMediaType media_type); // Checks if there are pending callback events and, if so, logs the received @@ -170,7 +176,7 @@ class SbPlayerTestFixture { // Waits for |kSbPlayerDecoderStateNeedsData| to be sent. void WaitForDecoderStateNeedsData( - const SbTime timeout = kDefaultWaitForDecoderStateNeedsDataTimeout); + const SbTime timeout = kDefaultWaitForCallbackEventTimeout); // Waits for desired player state update to be sent. void WaitForPlayerState( @@ -206,6 +212,11 @@ class SbPlayerTestFixture { bool can_accept_more_audio_data_ = false; bool can_accept_more_video_data_ = false; + // The duration of how far past the current playback position we will write + // audio samples. + SbTime audio_write_duration_ = 0; + SbTime last_written_audio_timestamp_ = 0; + // Set of received player state updates from the underlying player. This is // used to check that the state updates occur in a valid order during normal // playback. diff --git a/starboard/nplb/player_test_util.cc b/starboard/nplb/player_test_util.cc index 4ce2f1d41471..3ae56c568cd7 100644 --- a/starboard/nplb/player_test_util.cc +++ b/starboard/nplb/player_test_util.cc @@ -228,10 +228,25 @@ void CallSbPlayerWriteSamples( SbMediaType sample_type, shared::starboard::player::video_dmp::VideoDmpReader* dmp_reader, int start_index, - int number_of_samples_to_write) { + int number_of_samples_to_write, + SbTime timestamp_offset, + const std::vector& discarded_durations_from_front, + const std::vector& discarded_durations_from_back) { SB_DCHECK(start_index >= 0); SB_DCHECK(number_of_samples_to_write > 0); + if (sample_type == kSbMediaTypeAudio) { + SB_DCHECK(discarded_durations_from_front.empty() || + discarded_durations_from_front.size() == + number_of_samples_to_write); + SB_DCHECK(discarded_durations_from_front.size() == + discarded_durations_from_back.size()); + } else { + SB_DCHECK(sample_type == kSbMediaTypeVideo); + SB_DCHECK(discarded_durations_from_front.empty()); + SB_DCHECK(discarded_durations_from_back.empty()); + } + static auto const* enhanced_audio_extension = static_cast( SbSystemGetExtension(kCobaltExtensionEnhancedAudioName)); @@ -258,7 +273,7 @@ void CallSbPlayerWriteSamples( sample_infos.back().type = source.type; sample_infos.back().buffer = source.buffer; sample_infos.back().buffer_size = source.buffer_size; - sample_infos.back().timestamp = source.timestamp; + sample_infos.back().timestamp = source.timestamp + timestamp_offset; sample_infos.back().side_data = source.side_data; sample_infos.back().side_data_count = source.side_data_count; sample_infos.back().drm_info = source.drm_info; @@ -267,8 +282,15 @@ void CallSbPlayerWriteSamples( audio_sample_infos.emplace_back(source.audio_sample_info); audio_sample_infos.back().ConvertTo( &sample_infos.back().audio_sample_info); + if (!discarded_durations_from_front.empty()) { + sample_infos.back().audio_sample_info.discarded_duration_from_front = + discarded_durations_from_front[i]; + } + if (!discarded_durations_from_back.empty()) { + sample_infos.back().audio_sample_info.discarded_duration_from_back = + discarded_durations_from_back[i]; + } } else { - SB_DCHECK(sample_type == kSbMediaTypeVideo); video_sample_infos.emplace_back(source.video_sample_info); video_sample_infos.back().ConvertTo( &sample_infos.back().video_sample_info); @@ -285,6 +307,17 @@ void CallSbPlayerWriteSamples( for (int i = 0; i < number_of_samples_to_write; ++i) { sample_infos.push_back( dmp_reader->GetPlayerSampleInfo(sample_type, start_index++)); + sample_infos.back().timestamp += timestamp_offset; +#if SB_API_VERSION >= 15 + if (!discarded_durations_from_front.empty()) { + sample_infos.back().audio_sample_info.discarded_duration_from_front = + discarded_durations_from_front[i]; + } + if (!discarded_durations_from_back.empty()) { + sample_infos.back().audio_sample_info.discarded_duration_from_back = + discarded_durations_from_back[i]; + } +#endif // SB_API_VERSION >= 15 } #if SB_API_VERSION >= 15 SbPlayerWriteSamples(player, sample_type, sample_infos.data(), @@ -325,5 +358,13 @@ bool IsOutputModeSupported(SbPlayerOutputMode output_mode, return supported; } +bool IsPartialAudioSupported() { +#if SB_API_VERSION >= 15 + return true; +#else // SB_API_VERSION >= 15 + return SbSystemGetExtension(kCobaltExtensionEnhancedAudioName) != nullptr; +#endif // SB_API_VERSION >= 15 +} + } // namespace nplb } // namespace starboard diff --git a/starboard/nplb/player_test_util.h b/starboard/nplb/player_test_util.h index f2ee7bcf483e..c2fdeeaad4be 100644 --- a/starboard/nplb/player_test_util.h +++ b/starboard/nplb/player_test_util.h @@ -89,13 +89,18 @@ void CallSbPlayerWriteSamples( SbMediaType sample_type, shared::starboard::player::video_dmp::VideoDmpReader* dmp_reader, int start_index, - int number_of_samples_to_write); + int number_of_samples_to_write, + SbTime timestamp_offset = 0, + const std::vector& discarded_durations_from_front = {}, + const std::vector& discarded_durations_from_back = {}); bool IsOutputModeSupported(SbPlayerOutputMode output_mode, SbMediaAudioCodec audio_codec, SbMediaVideoCodec video_codec, const char* key_system = ""); +bool IsPartialAudioSupported(); + } // namespace nplb } // namespace starboard diff --git a/starboard/nplb/player_write_sample_test.cc b/starboard/nplb/player_write_sample_test.cc index a10d1b02fb77..bca1ec9babd2 100644 --- a/starboard/nplb/player_write_sample_test.cc +++ b/starboard/nplb/player_write_sample_test.cc @@ -52,10 +52,10 @@ TEST_P(SbPlayerWriteSampleTest, NoInput) { GroupedSamples samples; if (player_fixture.HasAudio()) { - samples.AddAudioSamplesWithEOS(0, 0); + samples.AddAudioEOS(); } if (player_fixture.HasVideo()) { - samples.AddVideoSamplesWithEOS(0, 0); + samples.AddVideoEOS(); } ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerEndOfStream()); @@ -72,12 +72,14 @@ TEST_P(SbPlayerWriteSampleTest, WriteSingleBatch) { if (player_fixture.HasAudio()) { int samples_to_write = SbPlayerGetMaximumNumberOfSamplesPerWrite( player_fixture.GetPlayer(), kSbMediaTypeAudio); - samples.AddAudioSamplesWithEOS(0, samples_to_write); + samples.AddAudioSamples(0, samples_to_write); + samples.AddAudioEOS(); } if (player_fixture.HasVideo()) { int samples_to_write = SbPlayerGetMaximumNumberOfSamplesPerWrite( player_fixture.GetPlayer(), kSbMediaTypeVideo); - samples.AddVideoSamplesWithEOS(0, samples_to_write); + samples.AddVideoSamples(0, samples_to_write); + samples.AddVideoEOS(); } ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); @@ -94,10 +96,9 @@ TEST_P(SbPlayerWriteSampleTest, WriteMultipleBatches) { int samples_to_write = 0; // Try to write multiple batches for both audio and video. if (player_fixture.HasAudio()) { - samples_to_write = std::max( - samples_to_write, SbPlayerGetMaximumNumberOfSamplesPerWrite( - player_fixture.GetPlayer(), kSbMediaTypeAudio) + - 1); + samples_to_write = SbPlayerGetMaximumNumberOfSamplesPerWrite( + player_fixture.GetPlayer(), kSbMediaTypeAudio) + + 1; } if (player_fixture.HasVideo()) { samples_to_write = std::max( @@ -111,14 +112,204 @@ TEST_P(SbPlayerWriteSampleTest, WriteMultipleBatches) { GroupedSamples samples; if (player_fixture.HasAudio()) { - samples.AddAudioSamplesWithEOS(0, samples_to_write); + samples.AddAudioSamples(0, samples_to_write); + samples.AddAudioEOS(); + } + if (player_fixture.HasVideo()) { + samples.AddVideoSamples(0, samples_to_write); + samples.AddVideoEOS(); + } + + ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); + ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerEndOfStream()); +} + +TEST_P(SbPlayerWriteSampleTest, LimitedAudioInput) { + SbPlayerTestFixture player_fixture(GetParam(), + &fake_graphics_context_provider_); + if (HasFatalFailure()) { + return; + } + + // TODO: we simply set audio write duration to 0.5 second. Ideally, we should + // set the audio write duration to 10 seconds if audio connectors are remote. + player_fixture.SetAudioWriteDuration(kSbTimeSecond / 2); + + GroupedSamples samples; + if (player_fixture.HasAudio()) { + samples.AddAudioSamples( + 0, player_fixture.ConvertDurationToAudioBufferCount(kSbTimeSecond)); + samples.AddAudioEOS(); + } + if (player_fixture.HasVideo()) { + samples.AddVideoSamples( + 0, player_fixture.ConvertDurationToVideoBufferCount(kSbTimeSecond)); + samples.AddVideoEOS(); + } + + ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); + ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerEndOfStream()); +} + +TEST_P(SbPlayerWriteSampleTest, PartialAudio) { + if (!IsPartialAudioSupported()) { + // TODO: Use GTEST_SKIP when we have a newer version of gtest. + SB_LOG(INFO) + << "The platform doesn't support partial audio. Skip the tests."; + return; + } + + SbPlayerTestFixture player_fixture(GetParam(), + &fake_graphics_context_provider_); + if (HasFatalFailure()) { + return; + } + if (!player_fixture.HasAudio()) { + // TODO: Use GTEST_SKIP when we have a newer version of gtest. + SB_LOG(INFO) << "Skip PartialAudio test for audioless content."; + return; + } + + const SbTime kDurationToPlay = kSbTimeSecond; + const float kSegmentSize = 0.1f; + + GroupedSamples samples; + if (player_fixture.HasVideo()) { + samples.AddVideoSamples( + 0, player_fixture.ConvertDurationToVideoBufferCount(kDurationToPlay)); + samples.AddVideoEOS(); + } + + int total_buffers_to_write = + player_fixture.ConvertDurationToAudioBufferCount(kDurationToPlay); + for (int i = 0; i < total_buffers_to_write; i++) { + SbTime current_timestamp = player_fixture.GetAudioSampleTimestamp(i); + SbTime next_timestamp = player_fixture.GetAudioSampleTimestamp(i + 1); + SbTime buffer_duration = next_timestamp - current_timestamp; + SbTime segment_duration = buffer_duration * kSegmentSize; + SbTime written_duration = 0; + while (written_duration < buffer_duration) { + samples.AddAudioSamples( + i, 1, written_duration, written_duration, + std::max( + 0, buffer_duration - written_duration - segment_duration)); + written_duration += segment_duration; + } + } + samples.AddAudioEOS(); + + ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); + ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerPresenting()); + + SbTime start_system_time = SbTimeGetMonotonicNow(); + SbTime start_media_time = player_fixture.GetCurrentMediaTime(); + + ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerEndOfStream()); + + SbTime end_system_time = SbTimeGetMonotonicNow(); + SbTime end_media_time = player_fixture.GetCurrentMediaTime(); + + const SbTime kDurationDifferenceAllowance = 500 * kSbTimeMillisecond; + EXPECT_NEAR(end_media_time, kDurationToPlay, kDurationDifferenceAllowance); + EXPECT_NEAR(end_system_time - start_system_time + start_media_time, + kDurationToPlay, kDurationDifferenceAllowance); + + SB_DLOG(INFO) << "The expected media time should be " << kDurationToPlay + << ", the actual media time is " << end_media_time + << ", with difference " + << std::abs(end_media_time - kDurationToPlay) << "."; + SB_DLOG(INFO) << "The expected total playing time should be " + << kDurationToPlay << ", the actual playing time is " + << end_system_time - start_system_time + start_media_time + << ", with difference " + << std::abs(end_system_time - start_system_time + + start_media_time - kDurationToPlay) + << "."; +} + +TEST_P(SbPlayerWriteSampleTest, PartialAudioDiscardAll) { + if (!IsPartialAudioSupported()) { + // TODO: Use GTEST_SKIP when we have a newer version of gtest. + SB_LOG(INFO) + << "The platform doesn't support partial audio. Skip the tests."; + return; + } + + SbPlayerTestFixture player_fixture(GetParam(), + &fake_graphics_context_provider_); + if (HasFatalFailure()) { + return; } + if (!player_fixture.HasAudio()) { + // TODO: Use GTEST_SKIP when we have a newer version of gtest. + SB_LOG(INFO) << "Skip PartialAudio test for audioless content."; + return; + } + + const SbTime kDurationToPlay = kSbTimeSecond; + const SbTime kDurationPerWrite = 100 * kSbTimeMillisecond; + const SbTime kNumberOfBuffersToDiscard = 20; + + GroupedSamples samples; if (player_fixture.HasVideo()) { - samples.AddVideoSamplesWithEOS(0, samples_to_write); + samples.AddVideoSamples( + 0, player_fixture.ConvertDurationToVideoBufferCount(kDurationToPlay)); + samples.AddVideoEOS(); } + int written_buffer_index = 0; + SbTime current_time_offset = 0; + int num_of_buffers_per_write = + player_fixture.ConvertDurationToAudioBufferCount(kDurationPerWrite); + while (current_time_offset < kDurationToPlay) { + // Discard from front. + for (int i = 0; i < kNumberOfBuffersToDiscard; i++) { + samples.AddAudioSamples(written_buffer_index, 1, current_time_offset, + kSbTimeSecond, 0); + } + + samples.AddAudioSamples(written_buffer_index, num_of_buffers_per_write); + written_buffer_index += num_of_buffers_per_write; + current_time_offset += kDurationPerWrite; + + // Discard from back. + for (int i = 0; i < kNumberOfBuffersToDiscard; i++) { + samples.AddAudioSamples(written_buffer_index, 1, current_time_offset, 0, + kSbTimeSecond); + } + } + samples.AddAudioEOS(); + ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); + ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerPresenting()); + + SbTime start_system_time = SbTimeGetMonotonicNow(); + SbTime start_media_time = player_fixture.GetCurrentMediaTime(); + ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerEndOfStream()); + + SbTime end_system_time = SbTimeGetMonotonicNow(); + SbTime end_media_time = player_fixture.GetCurrentMediaTime(); + + const SbTime kDurationDifferenceAllowance = 500 * kSbTimeMillisecond; + SbTime total_written_duration = + player_fixture.GetAudioSampleTimestamp(written_buffer_index); + EXPECT_NEAR(end_media_time, total_written_duration, + kDurationDifferenceAllowance); + EXPECT_NEAR(end_system_time - start_system_time + start_media_time, + total_written_duration, kDurationDifferenceAllowance); + + SB_DLOG(INFO) << "The expected media time should be " + << total_written_duration << ", the actual media time is " + << end_media_time << ", with difference " + << std::abs(end_media_time - total_written_duration) << "."; + SB_DLOG(INFO) << "The expected total playing time should be " + << total_written_duration << ", the actual playing time is " + << end_system_time - start_system_time + start_media_time + << ", with difference " + << std::abs(end_system_time - start_system_time + + start_media_time - total_written_duration) + << "."; } std::vector GetSupportedTestConfigs() { diff --git a/starboard/nplb/vertical_video_test.cc b/starboard/nplb/vertical_video_test.cc index 56fda9900798..61b0b4637564 100644 --- a/starboard/nplb/vertical_video_test.cc +++ b/starboard/nplb/vertical_video_test.cc @@ -135,8 +135,10 @@ TEST_P(VerticalVideoTest, WriteSamples) { 200 * kSbTimeMillisecond); GroupedSamples samples; - samples.AddVideoSamplesWithEOS(0, audio_samples_to_write); - samples.AddAudioSamplesWithEOS(0, video_samples_to_write); + samples.AddVideoSamples(0, audio_samples_to_write); + samples.AddVideoEOS(); + samples.AddAudioSamples(0, video_samples_to_write); + samples.AddAudioEOS(); ASSERT_NO_FATAL_FAILURE(player_fixture.Write(samples)); ASSERT_NO_FATAL_FAILURE(player_fixture.WaitForPlayerEndOfStream()); diff --git a/starboard/shared/starboard/player/decoded_audio_internal.cc b/starboard/shared/starboard/player/decoded_audio_internal.cc index 79952d7f6fdf..2834bc225e53 100644 --- a/starboard/shared/starboard/player/decoded_audio_internal.cc +++ b/starboard/shared/starboard/player/decoded_audio_internal.cc @@ -149,8 +149,6 @@ void DecodedAudio::AdjustForDiscardedDurations( discarded_frames_from_front = std::min(discarded_frames_from_front, frames()); offset_in_bytes_ += bytes_per_frame * discarded_frames_from_front; size_in_bytes_ -= bytes_per_frame * discarded_frames_from_front; - timestamp_ += - media::AudioFramesToDuration(discarded_frames_from_front, sample_rate); auto discarded_frames_from_back = AudioDurationToFrames(discarded_duration_from_back, sample_rate); diff --git a/starboard/shared/starboard/player/decoded_audio_test_internal.cc b/starboard/shared/starboard/player/decoded_audio_test_internal.cc index a4eae16f5dec..a1e2cd20b37f 100644 --- a/starboard/shared/starboard/player/decoded_audio_test_internal.cc +++ b/starboard/shared/starboard/player/decoded_audio_test_internal.cc @@ -286,19 +286,16 @@ TEST(DecodedAudioTest, AdjustForDiscardedDurations) { kSampleRate, quarter_duration, quarter_duration); ASSERT_NEAR(adjusted_decoded_audio->frames(), original_decoded_audio->frames() / 2, 2); - ASSERT_NEAR(adjusted_decoded_audio->timestamp(), - original_decoded_audio->timestamp() + quarter_duration, - duration_of_one_frame * 2); + ASSERT_EQ(adjusted_decoded_audio->timestamp(), + original_decoded_audio->timestamp()); adjusted_decoded_audio = original_decoded_audio->Clone(); // Adjust more frames than it has from front adjusted_decoded_audio->AdjustForDiscardedDurations( kSampleRate, duration_of_decoded_audio * 2, 0); ASSERT_EQ(adjusted_decoded_audio->frames(), 0); - ASSERT_NEAR( - adjusted_decoded_audio->timestamp(), - original_decoded_audio->timestamp() + duration_of_decoded_audio, - duration_of_one_frame * 2); + ASSERT_EQ(adjusted_decoded_audio->timestamp(), + original_decoded_audio->timestamp()); adjusted_decoded_audio = original_decoded_audio->Clone(); // Adjust more frames than it has from back