Cherry pick PR #2287: [android] Improve video/audio seeking via flush…

… MediaCodec (#2916) Refer to the original PR: #2287 1. Currently, Cobalt resets MediaCodec during seek, which is very slow on some atv devices. 2. This eliminates the visible black screen that appears during seeking, caused by resetting MediaCodec, an issue especially common in DRM-protected content and on older devices. 3. Adjusting the process to flush MediaCodec rather than dismantling and re-establishing it during seek operations prevents the platform from mistakenly concluding that playback has ended (b/296530538). 4. This improvement cuts down the InBufferSeek duration by approximately 45%. b/320568573 Co-authored-by: Bo-Rong Chen <[email protected]>
youtube · Apr 12, 2024 · d55e4a8 · d55e4a8
1 parent 713d81c
commit d55e4a8
Show file tree

Hide file tree

Showing 15 changed files with 311 additions and 28 deletions.
diff --git a/starboard/android/shared/audio_decoder.cc b/starboard/android/shared/audio_decoder.cc
@@ -70,12 +70,14 @@ void* IncrementPointerByBytes(void* pointer, int offset) {
 }  // namespace
 
 AudioDecoder::AudioDecoder(const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system)
+                           SbDrmSystem drm_system,
+                           bool enable_flush_during_seek)
     : audio_stream_info_(audio_stream_info),
       sample_type_(GetSupportedSampleType()),
       output_sample_rate_(audio_stream_info.samples_per_second),
       output_channel_count_(audio_stream_info.number_of_channels),
-      drm_system_(static_cast<DrmSystem*>(drm_system)) {
+      drm_system_(static_cast<DrmSystem*>(drm_system)),
+      enable_flush_during_seek_(enable_flush_during_seek) {
   if (!InitializeCodec()) {
     SB_LOG(ERROR) << "Failed to initialize audio decoder.";
   }
@@ -167,13 +169,16 @@ void AudioDecoder::Reset() {
   SB_DCHECK(BelongsToCurrentThread());
   SB_DCHECK(output_cb_);
 
-  media_decoder_.reset();
-  audio_frame_discarder_.Reset();
+  // If fail to flush |media_decoder_|, then re-create |media_decoder_|.
+  if (!enable_flush_during_seek_ || !media_decoder_->Flush()) {
+    media_decoder_.reset();
 
-  if (!InitializeCodec()) {
-    // TODO: Communicate this failure to our clients somehow.
-    SB_LOG(ERROR) << "Failed to initialize codec after reset.";
+    if (!InitializeCodec()) {
+      // TODO: Communicate this failure to our clients somehow.
+      SB_LOG(ERROR) << "Failed to initialize codec after reset.";
+    }
   }
+  audio_frame_discarder_.Reset();
 
   consumed_cb_ = nullptr;
 

diff --git a/starboard/android/shared/audio_decoder.h b/starboard/android/shared/audio_decoder.h
@@ -45,7 +45,8 @@ class AudioDecoder
       AudioStreamInfo;
 
   AudioDecoder(const AudioStreamInfo& audio_stream_info,
-               SbDrmSystem drm_system);
+               SbDrmSystem drm_system,
+               bool enable_flush_during_seek);
   ~AudioDecoder() override;
 
   void Initialize(const OutputCB& output_cb, const ErrorCB& error_cb) override;
@@ -77,6 +78,7 @@ class AudioDecoder
 
   const AudioStreamInfo audio_stream_info_;
   const SbMediaAudioSampleType sample_type_;
+  const bool enable_flush_during_seek_;
 
   jint output_sample_rate_;
   jint output_channel_count_;

diff --git a/starboard/android/shared/audio_renderer_passthrough.cc b/starboard/android/shared/audio_renderer_passthrough.cc
@@ -74,14 +74,15 @@ int ParseAc3SyncframeAudioSampleCount(const uint8_t* buffer, int size) {
 
 AudioRendererPassthrough::AudioRendererPassthrough(
     const AudioStreamInfo& audio_stream_info,
-    SbDrmSystem drm_system)
+    SbDrmSystem drm_system,
+    bool enable_flush_during_seek)
     : audio_stream_info_(audio_stream_info) {
   SB_DCHECK(audio_stream_info_.codec == kSbMediaAudioCodecAc3 ||
             audio_stream_info_.codec == kSbMediaAudioCodecEac3);
   if (SbDrmSystemIsValid(drm_system)) {
     SB_LOG(INFO) << "Creating AudioDecoder as decryptor.";
-    scoped_ptr<AudioDecoder> audio_decoder(
-        new AudioDecoder(audio_stream_info, drm_system));
+    scoped_ptr<AudioDecoder> audio_decoder(new AudioDecoder(
+        audio_stream_info, drm_system, enable_flush_during_seek));
     if (audio_decoder->is_valid()) {
       decoder_.reset(audio_decoder.release());
     }

diff --git a/starboard/android/shared/audio_renderer_passthrough.h b/starboard/android/shared/audio_renderer_passthrough.h
@@ -54,7 +54,8 @@ class AudioRendererPassthrough
       AudioStreamInfo;
 
   AudioRendererPassthrough(const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system);
+                           SbDrmSystem drm_system,
+                           bool enable_flush_during_seek);
   ~AudioRendererPassthrough() override;
 
   bool is_valid() const { return decoder_ != nullptr; }

diff --git a/starboard/android/shared/audio_track_audio_sink_type.h b/starboard/android/shared/audio_track_audio_sink_type.h
@@ -77,6 +77,8 @@ class AudioTrackAudioSinkType : public SbAudioSinkPrivate::Type {
   }
 
   void Destroy(SbAudioSink audio_sink) override {
+    // TODO(b/330793785): Use audio_sink.flush() instead of re-creating a new
+    // audio_sink.
     if (audio_sink != kSbAudioSinkInvalid && !IsValid(audio_sink)) {
       SB_LOG(WARNING) << "audio_sink is invalid.";
       return;

diff --git a/starboard/android/shared/media_codec_bridge.cc b/starboard/android/shared/media_codec_bridge.cc
@@ -446,6 +446,11 @@ void MediaCodecBridge::SetPlaybackRate(double playback_rate) {
       j_media_codec_bridge_, "setPlaybackRate", "(D)V", playback_rate);
 }
 
+bool MediaCodecBridge::Start() {
+  return JniEnvExt::Get()->CallBooleanMethodOrAbort(j_media_codec_bridge_,
+                                                    "start", "()Z") == JNI_TRUE;
+}
+
 jint MediaCodecBridge::Flush() {
   return JniEnvExt::Get()->CallIntMethodOrAbort(j_media_codec_bridge_, "flush",
                                                 "()I");

diff --git a/starboard/android/shared/media_codec_bridge.h b/starboard/android/shared/media_codec_bridge.h
@@ -198,6 +198,7 @@ class MediaCodecBridge {
   void ReleaseOutputBufferAtTimestamp(jint index, jlong render_timestamp_ns);
 
   void SetPlaybackRate(double playback_rate);
+  bool Start();
   jint Flush();
   FrameSize GetOutputSize();
   AudioOutputFormatResult GetAudioOutputFormat();

diff --git a/starboard/android/shared/media_decoder.cc b/starboard/android/shared/media_decoder.cc
@@ -265,6 +265,9 @@ void MediaDecoder::DecoderThreadFunc() {
           }
         }
         SB_DCHECK(dequeue_output_results.empty());
+        if (destroying_.load()) {
+          break;
+        }
         CollectPendingData_Locked(&pending_tasks, &input_buffer_indices,
                                   &dequeue_output_results);
       }
@@ -312,6 +315,9 @@ void MediaDecoder::DecoderThreadFunc() {
         collect_pending_data = !has_input || !has_output;
       }
 
+      if (destroying_.load()) {
+        break;
+      }
       if (collect_pending_data) {
         ScopedLock scoped_lock(mutex_);
         CollectPendingData_Locked(&pending_tasks, &input_buffer_indices,
@@ -616,6 +622,12 @@ void MediaDecoder::OnMediaCodecOutputBufferAvailable(
   SB_DCHECK(media_codec_bridge_);
   SB_DCHECK(buffer_index >= 0);
 
+  // TODO(b/291959069): After |decoder_thread_| is destroyed, it may still
+  // receive output buffer, discard this invalid output buffer.
+  if (destroying_.load() || !SbThreadIsValid(decoder_thread_)) {
+    return;
+  }
+
   DequeueOutputResult dequeue_output_result;
   dequeue_output_result.status = 0;
   dequeue_output_result.index = buffer_index;
@@ -644,6 +656,64 @@ void MediaDecoder::OnMediaCodecFrameRendered(SbTime frame_timestamp) {
   frame_rendered_cb_(frame_timestamp);
 }
 
+bool MediaDecoder::Flush() {
+  // Try to flush if we can, otherwise return |false| to recreate the codec
+  // completely. Flush() is called by `player_worker` thread,
+  // but MediaDecoder is on `audio_decoder` and `video_decoder`
+  // threads, let `player_worker` destroy `audio_decoder` and
+  // `video_decoder` threads to clean up all pending tasks,
+  // and Flush()/Start() |media_codec_bridge_|.
+
+  // 1. Destroy `audio_decoder` and `video_decoder` threads.
+  destroying_.store(true);
+  {
+    ScopedLock scoped_lock(mutex_);
+    condition_variable_.Signal();
+  }
+  if (SbThreadIsValid(decoder_thread_)) {
+    SbThreadJoin(decoder_thread_, NULL);
+    decoder_thread_ = kSbThreadInvalid;
+  }
+
+  // 2. Flush()/Start() |media_codec_bridge_| and clean up pending tasks.
+  if (is_valid()) {
+    // 2.1. Flush() |media_codec_bridge_|.
+    host_->OnFlushing();
+    jint status = media_codec_bridge_->Flush();
+    if (status != MEDIA_CODEC_OK) {
+      SB_LOG(ERROR) << "Failed to flush media codec.";
+      return false;
+    }
+
+    // 2.2. Clean up pending_tasks and input_buffer/output_buffer indices.
+    number_of_pending_tasks_.store(0);
+    pending_tasks_.clear();
+    input_buffer_indices_.clear();
+    dequeue_output_results_.clear();
+    pending_queue_input_buffer_task_ = nullopt_t();
+
+    // 2.3. Add OutputFormatChanged to get current output format after Flush().
+    DequeueOutputResult dequeue_output_result = {};
+    dequeue_output_result.index = -1;
+    dequeue_output_results_.push_back(dequeue_output_result);
+
+    // 2.4. Start() |media_codec_bridge_|. As the codec is configured in
+    // asynchronous mode, call Start() after Flush() has returned to
+    // resume codec operations. After Start(), input_buffer_index should
+    // start with 0.
+    if (!media_codec_bridge_->Start()) {
+      SB_LOG(ERROR) << "Failed to start media codec.";
+      return false;
+    }
+  }
+
+  // 3. Recreate `audio_decoder` and `video_decoder` threads in
+  // WriteInputBuffers().
+  stream_ended_.store(false);
+  destroying_.store(false);
+  return true;
+}
+
 }  // namespace shared
 }  // namespace android
 }  // namespace starboard
diff --git a/starboard/android/shared/media_decoder.h b/starboard/android/shared/media_decoder.h
@@ -113,6 +113,8 @@ class MediaDecoder
 
   bool is_valid() const { return media_codec_bridge_ != NULL; }
 
+  bool Flush();
+
  private:
   struct Event {
     enum Type {

diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h
@@ -70,6 +70,11 @@ constexpr bool kForceSecurePipelineInTunnelModeWhenRequired = true;
 // video distortion on some platforms.
 constexpr bool kForceResetSurfaceUnderTunnelMode = true;
 
+// By default, Cobalt recreates MediaCodec when Reset() during Seek().
+// Set the following variable to true to force it Flush() MediaCodec
+// during Seek().
+constexpr bool kForceFlushDecoderDuringReset = false;
+
 // This class allows us to force int16 sample type when tunnel mode is enabled.
 class AudioRendererSinkAndroid : public ::starboard::shared::starboard::player::
                                      filter::AudioRendererSinkImpl {
@@ -221,12 +226,28 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player::
       }
     }
 
+    bool enable_flush_during_seek = false;
+    if (!creation_parameters.video_mime().empty()) {
+      MimeType video_mime_type(creation_parameters.video_mime());
+      if (video_mime_type.ValidateBoolParameter("enableflushduringseek")) {
+        enable_flush_during_seek =
+            video_mime_type.GetParamBoolValue("enableflushduringseek", false);
+      }
+    }
+
+    if (kForceFlushDecoderDuringReset && !enable_flush_during_seek) {
+      SB_LOG(INFO)
+          << "`kForceFlushDecoderDuringReset` is set to true, force flushing"
+          << " audio passthrough decoder during Reset().";
+      enable_flush_during_seek = true;
+    }
+
     SB_LOG(INFO) << "Creating passthrough components.";
     // TODO: Enable tunnel mode for passthrough
     scoped_ptr<AudioRendererPassthrough> audio_renderer;
-    audio_renderer.reset(
-        new AudioRendererPassthrough(creation_parameters.audio_stream_info(),
-                                     creation_parameters.drm_system()));
+    audio_renderer.reset(new AudioRendererPassthrough(
+        creation_parameters.audio_stream_info(),
+        creation_parameters.drm_system(), enable_flush_during_seek));
     if (!audio_renderer->is_valid()) {
       return scoped_ptr<PlayerComponents>();
     }
@@ -298,7 +319,8 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player::
     MimeType video_mime_type(video_mime);
     if (!video_mime.empty()) {
       if (!video_mime_type.is_valid() ||
-          !video_mime_type.ValidateBoolParameter("tunnelmode")) {
+          !video_mime_type.ValidateBoolParameter("tunnelmode") ||
+          !video_mime_type.ValidateBoolParameter("enableflushduringseek")) {
         *error_message =
             "Invalid video MIME: '" + std::string(video_mime) + "'";
         return false;
@@ -359,13 +381,30 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player::
                    << tunnel_mode_audio_session_id << '.';
     }
 
+    bool enable_flush_during_seek =
+        video_mime_type.GetParamBoolValue("enableflushduringseek", false);
+    SB_LOG(INFO) << "Flush MediaCodec during Reset(): "
+                 << (enable_flush_during_seek ? "enabled. " : "disabled. ")
+                 << "Video mime parameter \"enableflushduringseek\" value: "
+                 << video_mime_type.GetParamStringValue("enableflushduringseek",
+                                                        "<not provided>")
+                 << ".";
+
+    if (kForceFlushDecoderDuringReset && !enable_flush_during_seek) {
+      SB_LOG(INFO)
+          << "`kForceFlushDecoderDuringReset` is set to true, force flushing"
+          << " audio decoder during Reset().";
+      enable_flush_during_seek = true;
+    }
+
     if (creation_parameters.audio_codec() != kSbMediaAudioCodecNone) {
       SB_DCHECK(audio_decoder);
       SB_DCHECK(audio_renderer_sink);
 
       using starboard::shared::starboard::media::AudioStreamInfo;
-      auto decoder_creator = [](const AudioStreamInfo& audio_stream_info,
-                                SbDrmSystem drm_system) {
+      auto decoder_creator = [enable_flush_during_seek](
+                                 const AudioStreamInfo& audio_stream_info,
+                                 SbDrmSystem drm_system) {
         bool use_libopus_decoder =
             audio_stream_info.codec == kSbMediaAudioCodecOpus &&
             !SbDrmSystemIsValid(drm_system) && !kForcePlatformOpusDecoder;
@@ -377,8 +416,8 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player::
           }
         } else if (audio_stream_info.codec == kSbMediaAudioCodecAac ||
                    audio_stream_info.codec == kSbMediaAudioCodecOpus) {
-          scoped_ptr<AudioDecoder> audio_decoder_impl(
-              new AudioDecoder(audio_stream_info, drm_system));
+          scoped_ptr<AudioDecoder> audio_decoder_impl(new AudioDecoder(
+              audio_stream_info, drm_system, enable_flush_during_seek));
           if (audio_decoder_impl->is_valid()) {
             return audio_decoder_impl.PassAs<AudioDecoderBase>();
           }
@@ -471,6 +510,7 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player::
       int max_video_input_size,
       std::string* error_message) {
     bool force_big_endian_hdr_metadata = false;
+    bool enable_flush_during_seek = false;
     if (!creation_parameters.video_mime().empty()) {
       // Use mime param to determine endianness of HDR metadata. If param is
       // missing or invalid it defaults to Little Endian.
@@ -481,6 +521,16 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player::
           video_mime_type.GetParamStringValue("hdrinfoendianness",
                                               /*default=*/"little");
       force_big_endian_hdr_metadata = hdr_info_endianness == "big";
+
+      video_mime_type.ValidateBoolParameter("enableflushduringseek");
+      enable_flush_during_seek =
+          video_mime_type.GetParamBoolValue("enableflushduringseek", false);
+    }
+    if (kForceFlushDecoderDuringReset && !enable_flush_during_seek) {
+      SB_LOG(INFO)
+          << "`kForceFlushDecoderDuringReset` is set to true, force flushing"
+          << " video decoder during Reset().";
+      enable_flush_during_seek = true;
     }
 
     scoped_ptr<VideoDecoder> video_decoder(new VideoDecoder(
@@ -490,7 +540,7 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player::
         creation_parameters.max_video_capabilities(),
         tunnel_mode_audio_session_id, force_secure_pipeline_under_tunnel_mode,
         kForceResetSurfaceUnderTunnelMode, force_big_endian_hdr_metadata,
-        max_video_input_size, error_message));
+        max_video_input_size, enable_flush_during_seek, error_message));
     if (creation_parameters.video_codec() == kSbMediaVideoCodecAv1 ||
         video_decoder->is_decoder_created()) {
       return video_decoder.Pass();