From 6f2089c77b8a0c00786c99f4fd7f2fae4697f936 Mon Sep 17 00:00:00 2001 From: Drew Thomas Date: Fri, 27 Sep 2024 09:20:35 -0700 Subject: [PATCH] Add state setters to MediaSourceAttachment (C25) (#4104) In preparation for cross-thread MediaSource objects, a small amount of state and associated setters are being added to the MediaSourceAttachment interface. This will allow for a push model, instead of a pull model, for state such as recent media timestamps, and the error status of the HTMLMediaElement. Usage of these setters is gated through the "MediaElement.EnableUsingMediaSourceAttachmentMethods" H5VCC flag. This flag needs to be coordinated between a HTMLMediaElement, the attached MediaSource, and all SourceBuffers used. As such, the value of the flag when the HTMLMediaElement was created is used to coordinate the state of the rest of the related objects. This prevents a flag mismatch, such as if the H5VCC value is updated after the HTMLMediaElement has been created but before the MediaSource is created. This is based off of the following Chromium changes: https://chromium-review.googlesource.com/c/chromium/src/+/2391934 https://chromium-review.googlesource.com/c/chromium/src/+/2401808 b/338425449 --- cobalt/dom/html_media_element.cc | 46 ++++++++++++++++--- cobalt/dom/html_media_element.h | 7 +++ cobalt/dom/media_source.cc | 23 ++++------ cobalt/dom/media_source.h | 2 +- cobalt/dom/media_source_attachment.h | 16 +++++++ .../same_thread_media_source_attachment.cc | 28 +++++++++-- .../dom/same_thread_media_source_attachment.h | 6 +++ cobalt/dom/source_buffer.cc | 17 ++----- cobalt/dom/source_buffer.h | 4 +- 9 files changed, 109 insertions(+), 40 deletions(-) diff --git a/cobalt/dom/html_media_element.cc b/cobalt/dom/html_media_element.cc index 6aa4a1713616..2be5f86f59ba 100644 --- a/cobalt/dom/html_media_element.cc +++ b/cobalt/dom/html_media_element.cc @@ -431,9 +431,11 @@ void HTMLMediaElement::set_current_time( LOG(ERROR) << "invalid state error"; web::DOMException::Raise(web::DOMException::kInvalidStateErr, exception_state); - return; + } else { + Seek(time); } - Seek(time); + + ReportCurrentTimeToMediaSource(); } double HTMLMediaElement::duration() const { @@ -816,7 +818,7 @@ void HTMLMediaElement::PrepareForLoad() { set_playback_rate(default_playback_rate()); // 6 - Set the error attribute to null and the autoplaying flag to true. - error_ = NULL; + SetError(NULL); autoplaying_ = true; // 7 - Invoke the media element's resource selection algorithm. @@ -1050,8 +1052,8 @@ void HTMLMediaElement::NoneSupported(const std::string& message) { // 6.1 - Set the error attribute to a new MediaError object whose code // attribute is set to MEDIA_ERR_SRC_NOT_SUPPORTED. - error_ = new MediaError(MediaError::kMediaErrSrcNotSupported, - message.empty() ? "Source not supported." : message); + SetError(new MediaError(MediaError::kMediaErrSrcNotSupported, + message.empty() ? "Source not supported." : message)); // 6.2 - Forget the media element's media-resource-specific text tracks. // 6.3 - Set the element's networkState attribute to the kNetworkNoSource @@ -1129,6 +1131,10 @@ void HTMLMediaElement::OnPlaybackProgressTimer() { } ScheduleTimeupdateEvent(true); + + // The playback progress timer is used here to provide a steady clock that + // allows the attached MediaSource to have access to a recent media time. + ReportCurrentTimeToMediaSource(); } void HTMLMediaElement::StartPlaybackProgressTimer() { @@ -1511,6 +1517,8 @@ void HTMLMediaElement::UpdatePlayState() { AddPlayedRange(last_seek_time_, time); } } + + ReportCurrentTimeToMediaSource(); } bool HTMLMediaElement::PotentiallyPlaying() const { @@ -1575,6 +1583,32 @@ void HTMLMediaElement::ConfigureMediaControls() { DLOG_IF(WARNING, controls_) << "media control is not supported"; } +void HTMLMediaElement::ReportCurrentTimeToMediaSource() { + if (!is_using_media_source_attachment_methods_) { + return; + } + + if (!media_source_attachment_) { + return; + } + + media_source_attachment_->OnElementTimeUpdate(current_time(NULL)); +} + +void HTMLMediaElement::SetError(scoped_refptr error) { + error_ = error; + + if (!is_using_media_source_attachment_methods_) { + return; + } + + if (!error || !media_source_attachment_) { + return; + } + + media_source_attachment_->OnElementError(); +} + void HTMLMediaElement::MediaEngineError(scoped_refptr error) { if (error->message().empty()) { LOG(WARNING) << "HTMLMediaElement::MediaEngineError " << error->code(); @@ -1589,7 +1623,7 @@ void HTMLMediaElement::MediaEngineError(scoped_refptr error) { // 2 - Set the error attribute to a new MediaError object whose code attribute // is set to MEDIA_ERR_NETWORK/MEDIA_ERR_DECODE. - error_ = error; + SetError(error); // 3 - Queue a task to fire a simple event named error at the media element. ScheduleOwnEvent(base::Tokens::error()); diff --git a/cobalt/dom/html_media_element.h b/cobalt/dom/html_media_element.h index 6a8554862506..6e93f9e9e0bc 100644 --- a/cobalt/dom/html_media_element.h +++ b/cobalt/dom/html_media_element.h @@ -229,7 +229,13 @@ class HTMLMediaElement : public HTMLElement, void ConfigureMediaControls(); + // Pushes the current media time to the attached MediaSourceAttachment to + // avoid having to asynchronously pull media time from a cross-thread + // MediaSource object. + void ReportCurrentTimeToMediaSource(); + // Error report + void SetError(scoped_refptr error); void MediaEngineError(scoped_refptr error); // WebMediaPlayerClient methods @@ -313,6 +319,7 @@ class HTMLMediaElement : public HTMLElement, // Time has not changed since sending an "ended" event. bool sent_end_event_; + // See SetError(). scoped_refptr error_; // Helper object to reduce the image capacity while a video is playing. diff --git a/cobalt/dom/media_source.cc b/cobalt/dom/media_source.cc index 2967676882aa..16dbea235dfe 100644 --- a/cobalt/dom/media_source.cc +++ b/cobalt/dom/media_source.cc @@ -145,17 +145,6 @@ bool IsMediaElementUsingMediaSourceBufferedRangeEnabled( .value_or(false); } -// If this function returns true, MediaSource will proxy calls to the -// attached HTMLMediaElement object through the MediaSourceAttachment interface -// instead of directly calling against the HTMLMediaElement object. -// The default value is false. -bool IsMediaElementUsingMediaSourceAttachmentMethodsEnabled( - web::EnvironmentSettings* settings) { - return GetMediaSettings(settings) - .IsMediaElementUsingMediaSourceAttachmentMethodsEnabled() - .value_or(false); -} - } // namespace MediaSource::MediaSource(script::EnvironmentSettings* settings) @@ -170,9 +159,6 @@ MediaSource::MediaSource(script::EnvironmentSettings* settings) chunk_demuxer_(NULL), ready_state_(kMediaSourceReadyStateClosed), ALLOW_THIS_IN_INITIALIZER_LIST(event_queue_(this)), - is_using_media_source_attachment_methods_( - IsMediaElementUsingMediaSourceAttachmentMethodsEnabled( - environment_settings())), source_buffers_(new SourceBufferList(settings, &event_queue_)), active_source_buffers_(new SourceBufferList(settings, &event_queue_)), live_seekable_range_(new TimeRanges) { @@ -294,7 +280,8 @@ scoped_refptr MediaSource::AddSourceBuffer( switch (status) { case ChunkDemuxer::kOk: source_buffer = - new SourceBuffer(settings, guid, this, chunk_demuxer_, &event_queue_); + new SourceBuffer(settings, guid, this, chunk_demuxer_, &event_queue_, + is_using_media_source_attachment_methods_); break; case ChunkDemuxer::kNotSupported: web::DOMException::Raise(web::DOMException::kNotSupportedErr, @@ -439,6 +426,8 @@ bool MediaSource::IsTypeSupported(script::EnvironmentSettings* settings, bool MediaSource::StartAttachingToMediaElement( HTMLMediaElement* media_element) { + is_using_media_source_attachment_methods_ = false; + if (attached_element_) { return false; } @@ -472,6 +461,8 @@ bool MediaSource::StartAttachingToMediaElement( bool MediaSource::StartAttachingToMediaElement( MediaSourceAttachmentSupplement* media_source_attachment) { + is_using_media_source_attachment_methods_ = true; + if (media_source_attachment_) { return false; } @@ -685,10 +676,12 @@ void MediaSource::SetSourceBufferActive(SourceBuffer* source_buffer, } HTMLMediaElement* MediaSource::GetMediaElement() const { + DCHECK(!is_using_media_source_attachment_methods_); return attached_element_; } MediaSourceAttachmentSupplement* MediaSource::GetMediaSourceAttachment() const { + DCHECK(is_using_media_source_attachment_methods_); return media_source_attachment_; } diff --git a/cobalt/dom/media_source.h b/cobalt/dom/media_source.h index c30d7d7f053a..1cae5b40b17f 100644 --- a/cobalt/dom/media_source.h +++ b/cobalt/dom/media_source.h @@ -170,7 +170,7 @@ class MediaSource : public web::EventTarget { MediaSourceReadyState ready_state_; EventQueue event_queue_; // TODO(b/338425449): Remove direct references to HTMLMediaElement. - const bool is_using_media_source_attachment_methods_; + bool is_using_media_source_attachment_methods_ = false; base::WeakPtr attached_element_; base::WeakPtr media_source_attachment_; diff --git a/cobalt/dom/media_source_attachment.h b/cobalt/dom/media_source_attachment.h index 83aa86245aac..870e1e97dcae 100644 --- a/cobalt/dom/media_source_attachment.h +++ b/cobalt/dom/media_source_attachment.h @@ -71,6 +71,22 @@ class MediaSourceAttachment // flag MediaElement.EnableUsingMediaSourceAttachmentMethods is disabled. virtual scoped_refptr media_source() const = 0; + // Provide state updates to the MediaSource that are necessary for its + // operation. These are pushed rather than pulled to reduce complexity and + // latency, especially when the MediaSource is in a Worker context. + // OnElementTimeUpdate() gives the MediaSource a notion of the recent media + // element currentTime so that it can more effectively prevent evicting + // buffered media near to playback and/or seek target time in its heuristic. + // Alternatives such as pumping this via the media pipeline are insufficient, + // as the media pipeline may not be aware of overrides to the playback start + // position. + virtual void OnElementTimeUpdate(double time) = 0; + + // Needed as a precondition in the Prepare Append algorithm, OnElementError() + // lets the MediaSource know if the attached media element has transitioned to + // having an error. + virtual void OnElementError() = 0; + private: friend class base::RefCountedThreadSafe; diff --git a/cobalt/dom/same_thread_media_source_attachment.cc b/cobalt/dom/same_thread_media_source_attachment.cc index 1bb911596ea5..8f85a02d26e6 100644 --- a/cobalt/dom/same_thread_media_source_attachment.cc +++ b/cobalt/dom/same_thread_media_source_attachment.cc @@ -34,7 +34,9 @@ namespace dom { SameThreadMediaSourceAttachment::SameThreadMediaSourceAttachment( scoped_refptr media_source) : media_source_(media_source), - task_runner_(base::SequencedTaskRunner::GetCurrentDefault()) {} + task_runner_(base::SequencedTaskRunner::GetCurrentDefault()), + recent_element_time_(0.0), + element_has_error_(false) {} void SameThreadMediaSourceAttachment::TraceMembers(script::Tracer* tracer) { DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); @@ -108,14 +110,23 @@ double SameThreadMediaSourceAttachment::GetRecentMediaTime() { DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); DCHECK(attached_element_); - return attached_element_->current_time(NULL); + double result = attached_element_->current_time(NULL); + + DVLOG(2) << __func__ << ": recent time=" << recent_element_time_ + << ", actual current time=" << result; + + return result; } bool SameThreadMediaSourceAttachment::GetElementError() { DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); DCHECK(attached_element_); - return static_cast(attached_element_->error()); + bool result = static_cast(attached_element_->error()); + + DCHECK_EQ(result, element_has_error_); + + return result; } scoped_refptr @@ -136,5 +147,16 @@ SameThreadMediaSourceAttachment::CreateVideoTrackList( return base::MakeRefCounted(settings, attached_element_); } +void SameThreadMediaSourceAttachment::OnElementTimeUpdate(double time) { + recent_element_time_ = time; +} + +void SameThreadMediaSourceAttachment::OnElementError() { + DCHECK(!element_has_error_) + << "At most one transition to element error per attachment is expected"; + + element_has_error_ = true; +} + } // namespace dom } // namespace cobalt diff --git a/cobalt/dom/same_thread_media_source_attachment.h b/cobalt/dom/same_thread_media_source_attachment.h index 15033c171d65..abaf41c1f136 100644 --- a/cobalt/dom/same_thread_media_source_attachment.h +++ b/cobalt/dom/same_thread_media_source_attachment.h @@ -51,6 +51,9 @@ class SameThreadMediaSourceAttachment : public MediaSourceAttachmentSupplement { scoped_refptr GetBufferedRange() const override; MediaSourceReadyState GetReadyState() const override; + void OnElementTimeUpdate(double time) override; + void OnElementError() override; + // MediaSourceAttachmentSupplement void NotifyDurationChanged(double duration) override; bool HasMaxVideoCapabilities() const override; @@ -84,6 +87,9 @@ class SameThreadMediaSourceAttachment : public MediaSourceAttachmentSupplement { // Used to ensure all calls are made on the thread that created this object. base::SequencedTaskRunner* task_runner_; + double recent_element_time_; // See OnElementTimeUpdate(). + bool element_has_error_; // See OnElementError(). + DISALLOW_COPY_AND_ASSIGN(SameThreadMediaSourceAttachment); }; diff --git a/cobalt/dom/source_buffer.cc b/cobalt/dom/source_buffer.cc index 42679e43ba18..2865b74062e5 100644 --- a/cobalt/dom/source_buffer.cc +++ b/cobalt/dom/source_buffer.cc @@ -129,17 +129,6 @@ bool IsAvoidCopyingArrayBufferEnabled(web::EnvironmentSettings* settings) { return media_settings.IsAvoidCopyingArrayBufferEnabled().value_or(false); } -// If this function returns true, MediaSource will proxy calls to the -// attached HTMLMediaElement object through the MediaSourceAttachment interface -// instead of directly calling against the HTMLMediaElement object. -// The default value is false. -bool IsMediaElementUsingMediaSourceAttachmentMethodsEnabled( - web::EnvironmentSettings* settings) { - return GetMediaSettings(settings) - .IsMediaElementUsingMediaSourceAttachmentMethodsEnabled() - .value_or(false); -} - } // namespace SourceBuffer::OnInitSegmentReceivedHelper::OnInitSegmentReceivedHelper( @@ -170,7 +159,8 @@ void SourceBuffer::OnInitSegmentReceivedHelper::TryToRunOnInitSegmentReceived( SourceBuffer::SourceBuffer(script::EnvironmentSettings* settings, const std::string& id, MediaSource* media_source, - ChunkDemuxer* chunk_demuxer, EventQueue* event_queue) + ChunkDemuxer* chunk_demuxer, EventQueue* event_queue, + const bool is_using_media_source_attachment_methods) : web::EventTarget(settings), on_init_segment_received_helper_(new OnInitSegmentReceivedHelper(this)), id_(id), @@ -181,8 +171,7 @@ SourceBuffer::SourceBuffer(script::EnvironmentSettings* settings, chunk_demuxer_(chunk_demuxer), event_queue_(event_queue), is_using_media_source_attachment_methods_( - IsMediaElementUsingMediaSourceAttachmentMethodsEnabled( - environment_settings())), + is_using_media_source_attachment_methods), audio_tracks_( is_using_media_source_attachment_methods_ ? media_source->GetMediaSourceAttachment()->CreateAudioTrackList( diff --git a/cobalt/dom/source_buffer.h b/cobalt/dom/source_buffer.h index 11c42d4cf986..ae9efef8bda7 100644 --- a/cobalt/dom/source_buffer.h +++ b/cobalt/dom/source_buffer.h @@ -90,9 +90,11 @@ class SourceBuffer : public web::EventTarget { // Custom, not in any spec. // + // TODO(b/338425449): Remove is_using_media_source_attachment_methods. SourceBuffer(script::EnvironmentSettings* settings, const std::string& id, MediaSource* media_source, ChunkDemuxer* chunk_demuxer, - EventQueue* event_queue); + EventQueue* event_queue, + const bool is_using_media_source_attachment_methods); // Web API: SourceBuffer //