[android] Support playback rate in VideoRendererAlgorithm (#3136)

b/333790056 b/332858454 (cherry picked from commit efb0625)
youtube · May 2, 2024 · a594a4b · a594a4b
1 parent ecdcfdb
commit a594a4b
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 13 deletions.
diff --git a/starboard/android/apk/app/src/main/java/dev/cobalt/media/VideoFrameReleaseTimeHelper.java b/starboard/android/apk/app/src/main/java/dev/cobalt/media/VideoFrameReleaseTimeHelper.java
@@ -59,6 +59,7 @@ public final class VideoFrameReleaseTimeHelper {
   private long lastFramePresentationTimeUs;
   private long adjustedLastFrameTimeNs;
   private long pendingAdjustedFrameTimeNs;
+  private double lastPlaybackRate;
 
   private boolean haveSync;
   private long syncUnadjustedReleaseTimeNs;
@@ -93,6 +94,7 @@ private VideoFrameReleaseTimeHelper(double defaultDisplayRefreshRate) {
   @UsedByNative
   public void enable() {
     haveSync = false;
+    lastPlaybackRate = -1;
     if (useDefaultDisplayVsync) {
       vsyncSampler.addObserver();
     }
@@ -118,7 +120,17 @@ public void disable() {
    */
   @SuppressWarnings("unused")
   @UsedByNative
-  public long adjustReleaseTime(long framePresentationTimeUs, long unadjustedReleaseTimeNs) {
+  public long adjustReleaseTime(
+      long framePresentationTimeUs, long unadjustedReleaseTimeNs, double playbackRate) {
+    if (playbackRate == 0) {
+      return unadjustedReleaseTimeNs;
+    }
+    if (playbackRate != lastPlaybackRate) {
+      // Resync if playback rate has changed.
+      haveSync = false;
+      lastPlaybackRate = playbackRate;
+    }
+
     long framePresentationTimeNs = framePresentationTimeUs * 1000;
 
     // Until we know better, the adjustment will be a no-op.
@@ -140,18 +152,18 @@ public long adjustReleaseTime(long framePresentationTimeUs, long unadjustedRelea
             (framePresentationTimeNs - syncFramePresentationTimeNs) / frameCount;
         // Project the adjusted frame time forward using the average.
         long candidateAdjustedFrameTimeNs = adjustedLastFrameTimeNs + averageFrameDurationNs;
-
-        if (isDriftTooLarge(candidateAdjustedFrameTimeNs, unadjustedReleaseTimeNs)) {
+        if (isDriftTooLarge(candidateAdjustedFrameTimeNs, unadjustedReleaseTimeNs, playbackRate)) {
           haveSync = false;
         } else {
           adjustedFrameTimeNs = candidateAdjustedFrameTimeNs;
           adjustedReleaseTimeNs =
-              syncUnadjustedReleaseTimeNs + adjustedFrameTimeNs - syncFramePresentationTimeNs;
+              syncUnadjustedReleaseTimeNs
+                  + (long) ((adjustedFrameTimeNs - syncFramePresentationTimeNs) / playbackRate);
         }
       } else {
         // We're synced but haven't waited the required number of frames to apply an adjustment.
         // Check drift anyway.
-        if (isDriftTooLarge(framePresentationTimeNs, unadjustedReleaseTimeNs)) {
+        if (isDriftTooLarge(framePresentationTimeNs, unadjustedReleaseTimeNs, playbackRate)) {
           haveSync = false;
         }
       }
@@ -184,10 +196,11 @@ protected void onSynced() {
     // Do nothing.
   }
 
-  private boolean isDriftTooLarge(long frameTimeNs, long releaseTimeNs) {
+  private boolean isDriftTooLarge(long frameTimeNs, long releaseTimeNs, double playbackRate) {
     long elapsedFrameTimeNs = frameTimeNs - syncFramePresentationTimeNs;
     long elapsedReleaseTimeNs = releaseTimeNs - syncUnadjustedReleaseTimeNs;
-    return Math.abs(elapsedReleaseTimeNs - elapsedFrameTimeNs) > MAX_ALLOWED_DRIFT_NS;
+    return Math.abs(elapsedReleaseTimeNs - elapsedFrameTimeNs / playbackRate)
+        > MAX_ALLOWED_DRIFT_NS;
   }
 
   private static long closestVsync(long releaseTime, long sampledVsyncTime, long vsyncDuration) {

diff --git a/starboard/android/shared/video_render_algorithm.cc b/starboard/android/shared/video_render_algorithm.cc
@@ -91,15 +91,17 @@ void VideoRenderAlgorithm::Render(
       }
     }
 
-    jlong early_us = frames->front()->timestamp() - playback_time;
+    jlong early_us = (frames->front()->timestamp() - playback_time) /
+                     (playback_rate != 0 ? playback_rate : 1);
 
     auto system_time_ns = GetSystemNanoTime();
     auto unadjusted_frame_release_time_ns =
         system_time_ns + (early_us * kSbTimeNanosecondsPerMicrosecond);
 
     auto adjusted_release_time_ns =
         video_frame_release_time_helper_.AdjustReleaseTime(
-            frames->front()->timestamp(), unadjusted_frame_release_time_ns);
+            frames->front()->timestamp(), unadjusted_frame_release_time_ns,
+            playback_rate);
 
     early_us = (adjusted_release_time_ns - system_time_ns) /
                kSbTimeNanosecondsPerMicrosecond;
@@ -151,12 +153,13 @@ VideoRenderAlgorithm::VideoFrameReleaseTimeHelper::
 
 jlong VideoRenderAlgorithm::VideoFrameReleaseTimeHelper::AdjustReleaseTime(
     jlong frame_presentation_time_us,
-    jlong unadjusted_release_time_ns) {
+    jlong unadjusted_release_time_ns,
+    double playback_rate) {
   SB_DCHECK(j_video_frame_release_time_helper_);
   auto* env = JniEnvExt::Get();
   return env->CallLongMethodOrAbort(
-      j_video_frame_release_time_helper_, "adjustReleaseTime", "(JJ)J",
-      frame_presentation_time_us, unadjusted_release_time_ns);
+      j_video_frame_release_time_helper_, "adjustReleaseTime", "(JJD)J",
+      frame_presentation_time_us, unadjusted_release_time_ns, playback_rate);
 }
 
 }  // namespace shared

diff --git a/starboard/android/shared/video_render_algorithm.h b/starboard/android/shared/video_render_algorithm.h
@@ -44,7 +44,8 @@ class VideoRenderAlgorithm : public ::starboard::shared::starboard::player::
     VideoFrameReleaseTimeHelper();
     ~VideoFrameReleaseTimeHelper();
     jlong AdjustReleaseTime(jlong frame_presentation_time_us,
-                            jlong unadjusted_release_time_ns);
+                            jlong unadjusted_release_time_ns,
+                            double playback_rate);
 
    private:
     jobject j_video_frame_release_time_helper_ = nullptr;