Add state setters to MediaSourceAttachment (C25) (#4104)

In preparation for cross-thread MediaSource objects, a small amount of
state and associated setters are being added to the
MediaSourceAttachment interface. This will allow for a push model,
instead of a pull model, for state such as recent media timestamps, and
the error status of the HTMLMediaElement.

Usage of these setters is gated through the
"MediaElement.EnableUsingMediaSourceAttachmentMethods" H5VCC flag. This
flag needs to be coordinated between a HTMLMediaElement, the attached
MediaSource, and all SourceBuffers used. As such, the value of the flag
when the HTMLMediaElement was created is used to coordinate the state of
the rest of the related objects. This prevents a flag mismatch, such as
if the H5VCC value is updated after the HTMLMediaElement has been
created but before the MediaSource is created.

This is based off of the following Chromium changes:

https://chromium-review.googlesource.com/c/chromium/src/+/2391934
https://chromium-review.googlesource.com/c/chromium/src/+/2401808

b/338425449

cobalt/dom/html_media_element.cc

@@ -431,9 +431,11 @@ void HTMLMediaElement::set_current_time(
     LOG(ERROR) << "invalid state error";
     web::DOMException::Raise(web::DOMException::kInvalidStateErr,
                              exception_state);
-    return;
+  } else {
+    Seek(time);
   }
-  Seek(time);
+
+  ReportCurrentTimeToMediaSource();
 }
 
 double HTMLMediaElement::duration() const {
@@ -816,7 +818,7 @@ void HTMLMediaElement::PrepareForLoad() {
   set_playback_rate(default_playback_rate());
 
   // 6 - Set the error attribute to null and the autoplaying flag to true.
-  error_ = NULL;
+  SetError(NULL);
   autoplaying_ = true;
 
   // 7 - Invoke the media element's resource selection algorithm.
@@ -1050,8 +1052,8 @@ void HTMLMediaElement::NoneSupported(const std::string& message) {
 
   // 6.1 - Set the error attribute to a new MediaError object whose code
   // attribute is set to MEDIA_ERR_SRC_NOT_SUPPORTED.
-  error_ = new MediaError(MediaError::kMediaErrSrcNotSupported,
-                          message.empty() ? "Source not supported." : message);
+  SetError(new MediaError(MediaError::kMediaErrSrcNotSupported,
+                          message.empty() ? "Source not supported." : message));
   // 6.2 - Forget the media element's media-resource-specific text tracks.
 
   // 6.3 - Set the element's networkState attribute to the kNetworkNoSource
@@ -1129,6 +1131,10 @@ void HTMLMediaElement::OnPlaybackProgressTimer() {
   }
 
   ScheduleTimeupdateEvent(true);
+
+  // The playback progress timer is used here to provide a steady clock that
+  // allows the attached MediaSource to have access to a recent media time.
+  ReportCurrentTimeToMediaSource();
 }
 
 void HTMLMediaElement::StartPlaybackProgressTimer() {
@@ -1511,6 +1517,8 @@ void HTMLMediaElement::UpdatePlayState() {
       AddPlayedRange(last_seek_time_, time);
     }
   }
+
+  ReportCurrentTimeToMediaSource();
 }
 
 bool HTMLMediaElement::PotentiallyPlaying() const {
@@ -1575,6 +1583,32 @@ void HTMLMediaElement::ConfigureMediaControls() {
   DLOG_IF(WARNING, controls_) << "media control is not supported";
 }
 
+void HTMLMediaElement::ReportCurrentTimeToMediaSource() {
+  if (!is_using_media_source_attachment_methods_) {
+    return;
+  }
+
+  if (!media_source_attachment_) {
+    return;
+  }
+
+  media_source_attachment_->OnElementTimeUpdate(current_time(NULL));
+}
+
+void HTMLMediaElement::SetError(scoped_refptr<MediaError> error) {
+  error_ = error;
+
+  if (!is_using_media_source_attachment_methods_) {
+    return;
+  }
+
+  if (!error || !media_source_attachment_) {
+    return;
+  }
+
+  media_source_attachment_->OnElementError();
+}
+
 void HTMLMediaElement::MediaEngineError(scoped_refptr<MediaError> error) {
   if (error->message().empty()) {
     LOG(WARNING) << "HTMLMediaElement::MediaEngineError " << error->code();
@@ -1589,7 +1623,7 @@ void HTMLMediaElement::MediaEngineError(scoped_refptr<MediaError> error) {
 
   // 2 - Set the error attribute to a new MediaError object whose code attribute
   // is set to MEDIA_ERR_NETWORK/MEDIA_ERR_DECODE.
-  error_ = error;
+  SetError(error);
 
   // 3 - Queue a task to fire a simple event named error at the media element.
   ScheduleOwnEvent(base::Tokens::error());

cobalt/dom/html_media_element.h

@@ -229,7 +229,13 @@ class HTMLMediaElement : public HTMLElement,
 
   void ConfigureMediaControls();
 
+  // Pushes the current media time to the attached MediaSourceAttachment to
+  // avoid having to asynchronously pull media time from a cross-thread
+  // MediaSource object.
+  void ReportCurrentTimeToMediaSource();
+
   // Error report
+  void SetError(scoped_refptr<MediaError> error);
   void MediaEngineError(scoped_refptr<MediaError> error);
 
   // WebMediaPlayerClient methods
@@ -313,6 +319,7 @@ class HTMLMediaElement : public HTMLElement,
   // Time has not changed since sending an "ended" event.
   bool sent_end_event_;
 
+  // See SetError().
   scoped_refptr<MediaError> error_;
 
   // Helper object to reduce the image capacity while a video is playing.

cobalt/dom/media_source.cc

@@ -145,17 +145,6 @@ bool IsMediaElementUsingMediaSourceBufferedRangeEnabled(
       .value_or(false);
 }
 
-// If this function returns true, MediaSource will proxy calls to the
-// attached HTMLMediaElement object through the MediaSourceAttachment interface
-// instead of directly calling against the HTMLMediaElement object.
-// The default value is false.
-bool IsMediaElementUsingMediaSourceAttachmentMethodsEnabled(
-    web::EnvironmentSettings* settings) {
-  return GetMediaSettings(settings)
-      .IsMediaElementUsingMediaSourceAttachmentMethodsEnabled()
-      .value_or(false);
-}
-
 }  // namespace
 
 MediaSource::MediaSource(script::EnvironmentSettings* settings)
@@ -170,9 +159,6 @@ MediaSource::MediaSource(script::EnvironmentSettings* settings)
       chunk_demuxer_(NULL),
       ready_state_(kMediaSourceReadyStateClosed),
       ALLOW_THIS_IN_INITIALIZER_LIST(event_queue_(this)),
-      is_using_media_source_attachment_methods_(
-          IsMediaElementUsingMediaSourceAttachmentMethodsEnabled(
-              environment_settings())),
       source_buffers_(new SourceBufferList(settings, &event_queue_)),
       active_source_buffers_(new SourceBufferList(settings, &event_queue_)),
       live_seekable_range_(new TimeRanges) {
@@ -294,7 +280,8 @@ scoped_refptr<SourceBuffer> MediaSource::AddSourceBuffer(
   switch (status) {
     case ChunkDemuxer::kOk:
       source_buffer =
-          new SourceBuffer(settings, guid, this, chunk_demuxer_, &event_queue_);
+          new SourceBuffer(settings, guid, this, chunk_demuxer_, &event_queue_,
+                           is_using_media_source_attachment_methods_);
       break;
     case ChunkDemuxer::kNotSupported:
       web::DOMException::Raise(web::DOMException::kNotSupportedErr,
@@ -439,6 +426,8 @@ bool MediaSource::IsTypeSupported(script::EnvironmentSettings* settings,
 
 bool MediaSource::StartAttachingToMediaElement(
     HTMLMediaElement* media_element) {
+  is_using_media_source_attachment_methods_ = false;
+
   if (attached_element_) {
     return false;
   }
@@ -472,6 +461,8 @@ bool MediaSource::StartAttachingToMediaElement(
 
 bool MediaSource::StartAttachingToMediaElement(
     MediaSourceAttachmentSupplement* media_source_attachment) {
+  is_using_media_source_attachment_methods_ = true;
+
   if (media_source_attachment_) {
     return false;
   }
@@ -685,10 +676,12 @@ void MediaSource::SetSourceBufferActive(SourceBuffer* source_buffer,
 }
 
 HTMLMediaElement* MediaSource::GetMediaElement() const {
+  DCHECK(!is_using_media_source_attachment_methods_);
   return attached_element_;
 }
 
 MediaSourceAttachmentSupplement* MediaSource::GetMediaSourceAttachment() const {
+  DCHECK(is_using_media_source_attachment_methods_);
   return media_source_attachment_;
 }
 

cobalt/dom/media_source.h

@@ -170,7 +170,7 @@ class MediaSource : public web::EventTarget {
   MediaSourceReadyState ready_state_;
   EventQueue event_queue_;
   // TODO(b/338425449): Remove direct references to HTMLMediaElement.
-  const bool is_using_media_source_attachment_methods_;
+  bool is_using_media_source_attachment_methods_ = false;
   base::WeakPtr<HTMLMediaElement> attached_element_;
   base::WeakPtr<MediaSourceAttachmentSupplement> media_source_attachment_;
 

cobalt/dom/media_source_attachment.h

@@ -71,6 +71,22 @@ class MediaSourceAttachment
   // flag MediaElement.EnableUsingMediaSourceAttachmentMethods is disabled.
   virtual scoped_refptr<MediaSource> media_source() const = 0;
 
+  // Provide state updates to the MediaSource that are necessary for its
+  // operation. These are pushed rather than pulled to reduce complexity and
+  // latency, especially when the MediaSource is in a Worker context.
+  // OnElementTimeUpdate() gives the MediaSource a notion of the recent media
+  // element currentTime so that it can more effectively prevent evicting
+  // buffered media near to playback and/or seek target time in its heuristic.
+  // Alternatives such as pumping this via the media pipeline are insufficient,
+  // as the media pipeline may not be aware of overrides to the playback start
+  // position.
+  virtual void OnElementTimeUpdate(double time) = 0;
+
+  // Needed as a precondition in the Prepare Append algorithm, OnElementError()
+  // lets the MediaSource know if the attached media element has transitioned to
+  // having an error.
+  virtual void OnElementError() = 0;
+
  private:
   friend class base::RefCountedThreadSafe<MediaSourceAttachment>;
 

cobalt/dom/same_thread_media_source_attachment.cc

@@ -34,7 +34,9 @@ namespace dom {
 SameThreadMediaSourceAttachment::SameThreadMediaSourceAttachment(
     scoped_refptr<MediaSource> media_source)
     : media_source_(media_source),
-      task_runner_(base::SequencedTaskRunner::GetCurrentDefault()) {}
+      task_runner_(base::SequencedTaskRunner::GetCurrentDefault()),
+      recent_element_time_(0.0),
+      element_has_error_(false) {}
 
 void SameThreadMediaSourceAttachment::TraceMembers(script::Tracer* tracer) {
   DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault());
@@ -108,14 +110,23 @@ double SameThreadMediaSourceAttachment::GetRecentMediaTime() {
   DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault());
   DCHECK(attached_element_);
 
-  return attached_element_->current_time(NULL);
+  double result = attached_element_->current_time(NULL);
+
+  DVLOG(2) << __func__ << ": recent time=" << recent_element_time_
+           << ", actual current time=" << result;
+
+  return result;
 }
 
 bool SameThreadMediaSourceAttachment::GetElementError() {
   DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault());
   DCHECK(attached_element_);
 
-  return static_cast<bool>(attached_element_->error());
+  bool result = static_cast<bool>(attached_element_->error());
+
+  DCHECK_EQ(result, element_has_error_);
+
+  return result;
 }
 
 scoped_refptr<AudioTrackList>
@@ -136,5 +147,16 @@ SameThreadMediaSourceAttachment::CreateVideoTrackList(
   return base::MakeRefCounted<VideoTrackList>(settings, attached_element_);
 }
 
+void SameThreadMediaSourceAttachment::OnElementTimeUpdate(double time) {
+  recent_element_time_ = time;
+}
+
+void SameThreadMediaSourceAttachment::OnElementError() {
+  DCHECK(!element_has_error_)
+      << "At most one transition to element error per attachment is expected";
+
+  element_has_error_ = true;
+}
+
 }  // namespace dom
 }  // namespace cobalt

cobalt/dom/same_thread_media_source_attachment.h

@@ -51,6 +51,9 @@ class SameThreadMediaSourceAttachment : public MediaSourceAttachmentSupplement {
   scoped_refptr<TimeRanges> GetBufferedRange() const override;
   MediaSourceReadyState GetReadyState() const override;
 
+  void OnElementTimeUpdate(double time) override;
+  void OnElementError() override;
+
   // MediaSourceAttachmentSupplement
   void NotifyDurationChanged(double duration) override;
   bool HasMaxVideoCapabilities() const override;
@@ -84,6 +87,9 @@ class SameThreadMediaSourceAttachment : public MediaSourceAttachmentSupplement {
   // Used to ensure all calls are made on the thread that created this object.
   base::SequencedTaskRunner* task_runner_;
 
+  double recent_element_time_;  // See OnElementTimeUpdate().
+  bool element_has_error_;      // See OnElementError().
+
   DISALLOW_COPY_AND_ASSIGN(SameThreadMediaSourceAttachment);
 };
 

cobalt/dom/source_buffer.cc

@@ -129,17 +129,6 @@ bool IsAvoidCopyingArrayBufferEnabled(web::EnvironmentSettings* settings) {
   return media_settings.IsAvoidCopyingArrayBufferEnabled().value_or(false);
 }
 
-// If this function returns true, MediaSource will proxy calls to the
-// attached HTMLMediaElement object through the MediaSourceAttachment interface
-// instead of directly calling against the HTMLMediaElement object.
-// The default value is false.
-bool IsMediaElementUsingMediaSourceAttachmentMethodsEnabled(
-    web::EnvironmentSettings* settings) {
-  return GetMediaSettings(settings)
-      .IsMediaElementUsingMediaSourceAttachmentMethodsEnabled()
-      .value_or(false);
-}
-
 }  // namespace
 
 SourceBuffer::OnInitSegmentReceivedHelper::OnInitSegmentReceivedHelper(
@@ -170,7 +159,8 @@ void SourceBuffer::OnInitSegmentReceivedHelper::TryToRunOnInitSegmentReceived(
 
 SourceBuffer::SourceBuffer(script::EnvironmentSettings* settings,
                            const std::string& id, MediaSource* media_source,
-                           ChunkDemuxer* chunk_demuxer, EventQueue* event_queue)
+                           ChunkDemuxer* chunk_demuxer, EventQueue* event_queue,
+                           const bool is_using_media_source_attachment_methods)
     : web::EventTarget(settings),
       on_init_segment_received_helper_(new OnInitSegmentReceivedHelper(this)),
       id_(id),
@@ -181,8 +171,7 @@ SourceBuffer::SourceBuffer(script::EnvironmentSettings* settings,
       chunk_demuxer_(chunk_demuxer),
       event_queue_(event_queue),
       is_using_media_source_attachment_methods_(
-          IsMediaElementUsingMediaSourceAttachmentMethodsEnabled(
-              environment_settings())),
+          is_using_media_source_attachment_methods),
       audio_tracks_(
           is_using_media_source_attachment_methods_
               ? media_source->GetMediaSourceAttachment()->CreateAudioTrackList(

cobalt/dom/source_buffer.h

@@ -90,9 +90,11 @@ class SourceBuffer : public web::EventTarget {
 
   // Custom, not in any spec.
   //
+  // TODO(b/338425449): Remove is_using_media_source_attachment_methods.
   SourceBuffer(script::EnvironmentSettings* settings, const std::string& id,
                MediaSource* media_source, ChunkDemuxer* chunk_demuxer,
-               EventQueue* event_queue);
+               EventQueue* event_queue,
+               const bool is_using_media_source_attachment_methods);
 
   // Web API: SourceBuffer
   //