Cherry pick PR #1663: [android] Support KEY_MAX_INPUT_SIZE configurab…

…le from the webapp (#2314) Refer to the original PR: youtube/cobalt#1663 1. Add HTMLVideoElement.setMaxVideoInputSize(max_video_input_size) to allow the web app to explicitly set the maximum size in bytes of a buffer of data for video described by android MediaFormat. 2. If the setting value is 0, it uses the default value. 3. This uses the SbThreadLocal to store max_video_input_size, which allows each player to acquire its input size from webapp. 4. This implementation includes setting max_video_input_size per player, i.e., each player can configure its max_video_input_size. b/176923480 Co-authored-by: Bo-Rong Chen <[email protected]>
youtube · Jan 28, 2024 · 5902022 · 5902022
1 parent 54ff8b9
commit 5902022
Show file tree

Hide file tree

Showing 40 changed files with 630 additions and 79 deletions.
diff --git a/cobalt/demos/content/configure-max-video-input-size/configure-max-video-input-size.html b/cobalt/demos/content/configure-max-video-input-size/configure-max-video-input-size.html
@@ -0,0 +1,45 @@
+<!DOCTYPE html>
+<!--
+ | Copyright 2024 The Cobalt Authors. All Rights Reserved.
+ |
+ | Licensed under the Apache License, Version 2.0 (the "License");
+ | you may not use this file except in compliance with the License.
+ | You may obtain a copy of the License at
+ |
+ |     http://www.apache.org/licenses/LICENSE-2.0
+ |
+ | Unless required by applicable law or agreed to in writing, software
+ | distributed under the License is distributed on an "AS IS" BASIS,
+ | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ | See the License for the specific language governing permissions and
+ | limitations under the License.
+ -->
+
+<html>
+<head>
+  <title>Configure Max Video Input Size</title>
+  <style>
+    body {
+      background-color: #0f0;
+    }
+    video {
+      height: 240px;
+      width: 426px;
+    }
+  </style>
+</head>
+<body>
+  <script type="text/javascript" src="configure-max-video-input-size.js"></script>
+  <div id="ui-layer">
+    <div class="item">
+      <video id="video1"></video>
+    </div>
+    <div class="item">
+      <video id="video2" muted="1" style="background-color: #4285F4">
+      </video>
+    </div>
+  </div>
+  <br>
+  <div id="status"></div>
+</body>
+</html>
diff --git a/cobalt/demos/content/configure-max-video-input-size/configure-max-video-input-size.js b/cobalt/demos/content/configure-max-video-input-size/configure-max-video-input-size.js
@@ -0,0 +1,98 @@
+// Copyright 2024 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+var audioData;
+var videoData;
+
+let status_div;
+
+function downloadMediaData(downloadedCallback) {
+  var xhr = new XMLHttpRequest;
+
+  xhr.onload = function() {
+    audioData = xhr.response;
+    console.log("Downloaded " + audioData.byteLength + " of audio data.");
+
+    xhr.onload = function() {
+      videoData = xhr.response;
+      console.log("Downloaded " + videoData.byteLength + " of video data.");
+      downloadedCallback();
+    }
+
+    xhr.open("GET", "vp9-720p.webm", true);
+    xhr.send();
+  }
+
+  xhr.open("GET", "dash-audio.mp4", true);
+  xhr.responseType = "arraybuffer";
+  xhr.send();
+}
+
+function playVideoOn(videoElement) {
+  var ms = new MediaSource;
+  ms.addEventListener('sourceopen', function() {
+    console.log("Creating SourceBuffer objects.");
+    var audioBuffer = ms.addSourceBuffer('audio/mp4; codecs="mp4a.40.2"');
+    var videoBuffer = ms.addSourceBuffer('video/webm; codecs="vp9"');
+    audioBuffer.addEventListener("updateend", function() {
+      audioBuffer.abort();
+      videoBuffer.addEventListener("updateend", function() {
+        setTimeout(function() {
+          videoBuffer.addEventListener("updateend", function() {
+            videoBuffer.abort();
+            ms.endOfStream();
+            videoElement.ontimeupdate = function() {
+              if (videoElement.currentTime > 10) {
+                console.log("Stop playback.");
+                videoElement.src = '';
+                videoElement.load();
+                videoElement.ontimeupdate = null;
+              }
+            }
+            console.log("Start playback.");
+            videoElement.play();
+          });
+          videoBuffer.appendBuffer(videoData.slice(1024));
+        }, 5000);
+      });
+      videoBuffer.appendBuffer(videoData.slice(0, 1024));
+    });
+    audioBuffer.appendBuffer(audioData);
+  });
+
+  console.log("Attaching MediaSource to video element.");
+  videoElement.src = URL.createObjectURL(ms);
+}
+
+function setupPlayerHandler() {
+  // Setup one primary player and one sub player.
+  // i = 0: primary player, buffer size is 3110500
+  // i = 1: sub player, buffer size is 777000
+  const maxVideoInputSizes = [3110500, 777000];
+  videoElements = document.getElementsByTagName('video');
+  status_div = document.getElementById('status');
+  status_div.innerHTML += 'Video Get Element By Id...<br>';
+  for(let i = 0; i < maxVideoInputSizes.length; i++) {
+    if (videoElements[i].setMaxVideoInputSize && i < videoElements.length) {
+      videoElements[i].setMaxVideoInputSize(maxVideoInputSizes[i]);
+      status_div.innerHTML += 'Set Max Video Input Size of Video Element '+ i + ' to '+ maxVideoInputSizes[i] + '<br>';
+    }
+    if (i == 1 && videoElements[i].setMaxVideoCapabilities) {
+      videoElements[i].setMaxVideoCapabilities("width=1920; height=1080");
+    }
+    playVideoOn(videoElements[i]);
+  }
+}
+
+downloadMediaData(setupPlayerHandler);
diff --git a/cobalt/demos/content/configure-max-video-input-size/dash-audio.mp4 b/cobalt/demos/content/configure-max-video-input-size/dash-audio.mp4
diff --git a/cobalt/demos/content/configure-max-video-input-size/vp9-720p.webm b/cobalt/demos/content/configure-max-video-input-size/vp9-720p.webm
diff --git a/cobalt/dom/html_media_element.cc b/cobalt/dom/html_media_element.cc
@@ -130,6 +130,7 @@ bool OriginIsSafe(loader::RequestMode request_mode, const GURL& resource_url,
 
 HTMLMediaElement::HTMLMediaElement(Document* document, base::Token tag_name)
     : HTMLElement(document, tag_name),
+      max_video_input_size_(0),
       load_state_(kWaitingForSource),
       ALLOW_THIS_IN_INITIALIZER_LIST(event_queue_(this)),
       playback_rate_(1.f),
@@ -1667,6 +1668,10 @@ std::string HTMLMediaElement::MaxVideoCapabilities() const {
   return max_video_capabilities_;
 }
 
+int HTMLMediaElement::MaxVideoInputSize() const {
+  return max_video_input_size_;
+}
+
 bool HTMLMediaElement::PreferDecodeToTexture() {
   TRACE_EVENT0("cobalt::dom", "HTMLMediaElement::PreferDecodeToTexture()");
 
@@ -1762,5 +1767,20 @@ void HTMLMediaElement::SetMaxVideoCapabilities(
   max_video_capabilities_ = max_video_capabilities;
 }
 
+void HTMLMediaElement::SetMaxVideoInputSize(
+    unsigned int max_video_input_size,
+    script::ExceptionState* exception_state) {
+  if (GetAttribute("src").value_or("").length() > 0) {
+    LOG(WARNING) << "Cannot set max_video_input_size after src is defined.";
+    web::DOMException::Raise(web::DOMException::kInvalidStateErr,
+                             exception_state);
+    return;
+  }
+
+  LOG(INFO) << "max_video_input_size is changed from " << max_video_input_size_
+            << " to " << max_video_input_size;
+  max_video_input_size_ = static_cast<int>(max_video_input_size);
+}
+
 }  // namespace dom
 }  // namespace cobalt
diff --git a/cobalt/dom/html_media_element.h b/cobalt/dom/html_media_element.h
@@ -159,6 +159,10 @@ class HTMLMediaElement : public HTMLElement,
     return !max_video_capabilities_.empty();
   }
 
+  // Set max video input size.
+  void SetMaxVideoInputSize(unsigned int max_video_input_size,
+                            script::ExceptionState* exception_state);
+
   DEFINE_WRAPPABLE_TYPE(HTMLMediaElement);
   void TraceMembers(script::Tracer* tracer) override;
 
@@ -240,6 +244,7 @@ class HTMLMediaElement : public HTMLElement,
   void SourceOpened(ChunkDemuxer* chunk_demuxer) override;
   std::string SourceURL() const override;
   std::string MaxVideoCapabilities() const override;
+  int MaxVideoInputSize() const override;
   bool PreferDecodeToTexture() override;
   void EncryptedMediaInitDataEncountered(
       const char* init_data_type, const unsigned char* init_data,
@@ -252,6 +257,8 @@ class HTMLMediaElement : public HTMLElement,
 
   std::string max_video_capabilities_;
 
+  int max_video_input_size_;
+
   // Loading state.
   enum LoadState { kWaitingForSource, kLoadingFromSrcAttr };
   LoadState load_state_;

diff --git a/cobalt/dom/html_video_element.idl b/cobalt/dom/html_video_element.idl
@@ -33,4 +33,12 @@ interface HTMLVideoElement : HTMLMediaElement {
   // is the same as the format for the string passed in to
   // HTMLMediaElement.canPlayType().
   [RaisesException] void setMaxVideoCapabilities(DOMString max_video_capabilities);
+
+  // Non standard interface (b/176923480). Set the maximum size in bytes of an
+  // input buffer for video. This communicates to the platform that the size of
+  // a single input buffer will not exceed the size passed in this function.
+  // This function must be called before the src attribute is set on the element,
+  // otherwise an INVALID_STATE_ERR exception is raised. Use default buffer size
+  // if set to 0.
+  [RaisesException] void setMaxVideoInputSize(unsigned long maxVideoInputSize);
 };
diff --git a/cobalt/media/base/pipeline.h b/cobalt/media/base/pipeline.h
@@ -126,7 +126,8 @@ class MEDIA_EXPORT Pipeline : public base::RefCountedThreadSafe<Pipeline> {
                      const base::Closure& duration_change_cb,
                      const base::Closure& output_mode_change_cb,
                      const base::Closure& content_size_change_cb,
-                     const std::string& max_video_capabilities) = 0;
+                     const std::string& max_video_capabilities,
+                     const int max_video_input_size) = 0;
 
 #if SB_HAS(PLAYER_WITH_URL)
   // Build a pipeline with an url-base player.

diff --git a/cobalt/media/base/sbplayer_bridge.cc b/cobalt/media/base/sbplayer_bridge.cc
@@ -30,6 +30,7 @@
 #include "starboard/common/player.h"
 #include "starboard/common/string.h"
 #include "starboard/configuration.h"
+#include "starboard/extension/player_set_max_video_input_size.h"
 #include "starboard/memory.h"
 #include "starboard/once.h"
 #include "third_party/chromium/media/base/starboard_utils.h"
@@ -226,7 +227,8 @@ SbPlayerBridge::SbPlayerBridge(
     SbPlayerSetBoundsHelper* set_bounds_helper, bool allow_resume_after_suspend,
     SbPlayerOutputMode default_output_mode,
     DecodeTargetProvider* const decode_target_provider,
-    const std::string& max_video_capabilities, std::string pipeline_identifier)
+    const std::string& max_video_capabilities, int max_video_input_size,
+    std::string pipeline_identifier)
     : sbplayer_interface_(interface),
       task_runner_(task_runner),
       get_decode_target_graphics_context_provider_func_(
@@ -242,6 +244,7 @@ SbPlayerBridge::SbPlayerBridge(
       video_config_(video_config),
       decode_target_provider_(decode_target_provider),
       max_video_capabilities_(max_video_capabilities),
+      max_video_input_size_(max_video_input_size),
       cval_stats_(&interface->cval_stats_),
       pipeline_identifier_(pipeline_identifier)
 #if SB_HAS(PLAYER_WITH_URL)
@@ -768,6 +771,18 @@ void SbPlayerBridge::CreatePlayer() {
   DCHECK_EQ(sbplayer_interface_->GetPreferredOutputMode(&creation_param),
             output_mode_);
   cval_stats_->StartTimer(MediaTiming::SbPlayerCreate, pipeline_identifier_);
+  const StarboardExtensionPlayerSetMaxVideoInputSizeApi*
+      player_set_max_video_input_size_extension =
+          static_cast<const StarboardExtensionPlayerSetMaxVideoInputSizeApi*>(
+              SbSystemGetExtension(
+                  kStarboardExtensionPlayerSetMaxVideoInputSizeName));
+  if (player_set_max_video_input_size_extension &&
+      strcmp(player_set_max_video_input_size_extension->name,
+             kStarboardExtensionPlayerSetMaxVideoInputSizeName) == 0 &&
+      player_set_max_video_input_size_extension->version >= 1) {
+    player_set_max_video_input_size_extension
+        ->SetMaxVideoInputSizeForCurrentThread(max_video_input_size_);
+  }
   player_ = sbplayer_interface_->Create(
       window_, &creation_param, &SbPlayerBridge::DeallocateSampleCB,
       &SbPlayerBridge::DecoderStatusCB, &SbPlayerBridge::PlayerStatusCB,

diff --git a/cobalt/media/base/sbplayer_bridge.h b/cobalt/media/base/sbplayer_bridge.h
@@ -94,7 +94,7 @@ class SbPlayerBridge {
                  SbPlayerOutputMode default_output_mode,
                  DecodeTargetProvider* const decode_target_provider,
                  const std::string& max_video_capabilities,
-                 std::string pipeline_identifier);
+                 int max_video_input_size, std::string pipeline_identifier);
 
   ~SbPlayerBridge();
 
@@ -301,6 +301,9 @@ class SbPlayerBridge {
   // A string of video maximum capabilities.
   std::string max_video_capabilities_;
 
+  // Set the maximum size in bytes of an input buffer for video.
+  int max_video_input_size_;
+
   // Keep track of errors during player creation.
   bool is_creating_player_ = false;
   std::string player_creation_error_message_;

diff --git a/cobalt/media/base/sbplayer_pipeline.cc b/cobalt/media/base/sbplayer_pipeline.cc
@@ -229,7 +229,8 @@ void SbPlayerPipeline::Start(Demuxer* demuxer,
                              const base::Closure& duration_change_cb,
                              const base::Closure& output_mode_change_cb,
                              const base::Closure& content_size_change_cb,
-                             const std::string& max_video_capabilities) {
+                             const std::string& max_video_capabilities,
+                             const int max_video_input_size) {
   TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::Start");
 
   DCHECK(!ended_cb.is_null());
@@ -252,6 +253,7 @@ void SbPlayerPipeline::Start(Demuxer* demuxer,
   parameters.output_mode_change_cb = output_mode_change_cb;
   parameters.content_size_change_cb = content_size_change_cb;
   parameters.max_video_capabilities = max_video_capabilities;
+  parameters.max_video_input_size = max_video_input_size;
 #if SB_HAS(PLAYER_WITH_URL)
   parameters.is_url_based = false;
 #endif  // SB_HAS(PLAYER_WITH_URL)
@@ -640,6 +642,7 @@ void SbPlayerPipeline::StartTask(StartTaskParameters parameters) {
   output_mode_change_cb_ = parameters.output_mode_change_cb;
   content_size_change_cb_ = parameters.content_size_change_cb;
   max_video_capabilities_ = parameters.max_video_capabilities;
+  max_video_input_size_ = parameters.max_video_input_size;
 #if SB_HAS(PLAYER_WITH_URL)
   is_url_based_ = parameters.is_url_based;
   if (is_url_based_) {
@@ -837,7 +840,7 @@ void SbPlayerPipeline::CreatePlayer(SbDrmSystem drm_system) {
         audio_mime_type, video_config, video_mime_type, window_, drm_system,
         this, set_bounds_helper_.get(), allow_resume_after_suspend_,
         default_output_mode_, decode_target_provider_, max_video_capabilities_,
-        pipeline_identifier_));
+        max_video_input_size_, pipeline_identifier_));
     if (player_bridge_->IsValid()) {
 #if SB_API_VERSION >= 15
       // TODO(b/267678497): When `player_bridge_->GetAudioConfigurations()`

diff --git a/cobalt/media/base/sbplayer_pipeline.h b/cobalt/media/base/sbplayer_pipeline.h
@@ -82,7 +82,8 @@ class MEDIA_EXPORT SbPlayerPipeline : public Pipeline,
              const base::Closure& duration_change_cb,
              const base::Closure& output_mode_change_cb,
              const base::Closure& content_size_change_cb,
-             const std::string& max_video_capabilities) override;
+             const std::string& max_video_capabilities,
+             const int max_video_input_size) override;
 #if SB_HAS(PLAYER_WITH_URL)
   void Start(const SetDrmSystemReadyCB& set_drm_system_ready_cb,
              const OnEncryptedMediaInitDataEncounteredCB&
@@ -133,6 +134,7 @@ class MEDIA_EXPORT SbPlayerPipeline : public Pipeline,
     base::Closure output_mode_change_cb;
     base::Closure content_size_change_cb;
     std::string max_video_capabilities;
+    int max_video_input_size;
 #if SB_HAS(PLAYER_WITH_URL)
     std::string source_url;
     bool is_url_based;
@@ -344,6 +346,8 @@ class MEDIA_EXPORT SbPlayerPipeline : public Pipeline,
   size_t retrograde_media_time_counter_ = 0;
   // The maximum video playback capabilities required for the playback.
   base::CVal<std::string> max_video_capabilities_;
+  // Set the maximum size in bytes of an input buffer for video.
+  int max_video_input_size_;
 
   PlaybackStatistics playback_statistics_;
 

diff --git a/cobalt/media/player/web_media_player.h b/cobalt/media/player/web_media_player.h
@@ -217,6 +217,7 @@ class WebMediaPlayerClient {
   virtual void SourceOpened(::media::ChunkDemuxer* chunk_demuxer) = 0;
   virtual std::string SourceURL() const = 0;
   virtual std::string MaxVideoCapabilities() const = 0;
+  virtual int MaxVideoInputSize() const = 0;
 
   // Clients should implement this in order to indicate a preference for whether
   // a video should be decoded to a texture or through a punch out system.  If

diff --git a/cobalt/media/player/web_media_player_impl.cc b/cobalt/media/player/web_media_player_impl.cc
@@ -851,7 +851,7 @@ void WebMediaPlayerImpl::StartPipeline(::media::Demuxer* demuxer) {
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnDurationChanged),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnOutputModeChanged),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnContentSizeChanged),
-      GetClient()->MaxVideoCapabilities());
+      GetClient()->MaxVideoCapabilities(), GetClient()->MaxVideoInputSize());
 }
 
 void WebMediaPlayerImpl::SetNetworkState(WebMediaPlayer::NetworkState state) {

diff --git a/cobalt/media/sandbox/web_media_player_helper.cc b/cobalt/media/sandbox/web_media_player_helper.cc
@@ -54,6 +54,7 @@ class WebMediaPlayerHelper::WebMediaPlayerClientStub
   }
   std::string SourceURL() const override { return ""; }
   std::string MaxVideoCapabilities() const override { return std::string(); };
+  int MaxVideoInputSize() const override { return 0; };
 
   void EncryptedMediaInitDataEncountered(const char*, const unsigned char*,
                                          unsigned) override {}