From a594a4bd035fa5409db007005b9979c37c5efc51 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 2 May 2024 16:29:26 -0700 Subject: [PATCH] [android] Support playback rate in VideoRendererAlgorithm (#3136) b/333790056 b/332858454 (cherry picked from commit efb062519541de7cc6b8c6e228dc9b962cb81fe0) --- .../media/VideoFrameReleaseTimeHelper.java | 27 ++++++++++++++----- .../android/shared/video_render_algorithm.cc | 13 +++++---- .../android/shared/video_render_algorithm.h | 3 ++- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/starboard/android/apk/app/src/main/java/dev/cobalt/media/VideoFrameReleaseTimeHelper.java b/starboard/android/apk/app/src/main/java/dev/cobalt/media/VideoFrameReleaseTimeHelper.java index 622e92b2be7e..e334a3d2a7e1 100644 --- a/starboard/android/apk/app/src/main/java/dev/cobalt/media/VideoFrameReleaseTimeHelper.java +++ b/starboard/android/apk/app/src/main/java/dev/cobalt/media/VideoFrameReleaseTimeHelper.java @@ -59,6 +59,7 @@ public final class VideoFrameReleaseTimeHelper { private long lastFramePresentationTimeUs; private long adjustedLastFrameTimeNs; private long pendingAdjustedFrameTimeNs; + private double lastPlaybackRate; private boolean haveSync; private long syncUnadjustedReleaseTimeNs; @@ -93,6 +94,7 @@ private VideoFrameReleaseTimeHelper(double defaultDisplayRefreshRate) { @UsedByNative public void enable() { haveSync = false; + lastPlaybackRate = -1; if (useDefaultDisplayVsync) { vsyncSampler.addObserver(); } @@ -118,7 +120,17 @@ public void disable() { */ @SuppressWarnings("unused") @UsedByNative - public long adjustReleaseTime(long framePresentationTimeUs, long unadjustedReleaseTimeNs) { + public long adjustReleaseTime( + long framePresentationTimeUs, long unadjustedReleaseTimeNs, double playbackRate) { + if (playbackRate == 0) { + return unadjustedReleaseTimeNs; + } + if (playbackRate != lastPlaybackRate) { + // Resync if playback rate has changed. + haveSync = false; + lastPlaybackRate = playbackRate; + } + long framePresentationTimeNs = framePresentationTimeUs * 1000; // Until we know better, the adjustment will be a no-op. @@ -140,18 +152,18 @@ public long adjustReleaseTime(long framePresentationTimeUs, long unadjustedRelea (framePresentationTimeNs - syncFramePresentationTimeNs) / frameCount; // Project the adjusted frame time forward using the average. long candidateAdjustedFrameTimeNs = adjustedLastFrameTimeNs + averageFrameDurationNs; - - if (isDriftTooLarge(candidateAdjustedFrameTimeNs, unadjustedReleaseTimeNs)) { + if (isDriftTooLarge(candidateAdjustedFrameTimeNs, unadjustedReleaseTimeNs, playbackRate)) { haveSync = false; } else { adjustedFrameTimeNs = candidateAdjustedFrameTimeNs; adjustedReleaseTimeNs = - syncUnadjustedReleaseTimeNs + adjustedFrameTimeNs - syncFramePresentationTimeNs; + syncUnadjustedReleaseTimeNs + + (long) ((adjustedFrameTimeNs - syncFramePresentationTimeNs) / playbackRate); } } else { // We're synced but haven't waited the required number of frames to apply an adjustment. // Check drift anyway. - if (isDriftTooLarge(framePresentationTimeNs, unadjustedReleaseTimeNs)) { + if (isDriftTooLarge(framePresentationTimeNs, unadjustedReleaseTimeNs, playbackRate)) { haveSync = false; } } @@ -184,10 +196,11 @@ protected void onSynced() { // Do nothing. } - private boolean isDriftTooLarge(long frameTimeNs, long releaseTimeNs) { + private boolean isDriftTooLarge(long frameTimeNs, long releaseTimeNs, double playbackRate) { long elapsedFrameTimeNs = frameTimeNs - syncFramePresentationTimeNs; long elapsedReleaseTimeNs = releaseTimeNs - syncUnadjustedReleaseTimeNs; - return Math.abs(elapsedReleaseTimeNs - elapsedFrameTimeNs) > MAX_ALLOWED_DRIFT_NS; + return Math.abs(elapsedReleaseTimeNs - elapsedFrameTimeNs / playbackRate) + > MAX_ALLOWED_DRIFT_NS; } private static long closestVsync(long releaseTime, long sampledVsyncTime, long vsyncDuration) { diff --git a/starboard/android/shared/video_render_algorithm.cc b/starboard/android/shared/video_render_algorithm.cc index dc22c357f934..5bc2bd006777 100644 --- a/starboard/android/shared/video_render_algorithm.cc +++ b/starboard/android/shared/video_render_algorithm.cc @@ -91,7 +91,8 @@ void VideoRenderAlgorithm::Render( } } - jlong early_us = frames->front()->timestamp() - playback_time; + jlong early_us = (frames->front()->timestamp() - playback_time) / + (playback_rate != 0 ? playback_rate : 1); auto system_time_ns = GetSystemNanoTime(); auto unadjusted_frame_release_time_ns = @@ -99,7 +100,8 @@ void VideoRenderAlgorithm::Render( auto adjusted_release_time_ns = video_frame_release_time_helper_.AdjustReleaseTime( - frames->front()->timestamp(), unadjusted_frame_release_time_ns); + frames->front()->timestamp(), unadjusted_frame_release_time_ns, + playback_rate); early_us = (adjusted_release_time_ns - system_time_ns) / kSbTimeNanosecondsPerMicrosecond; @@ -151,12 +153,13 @@ VideoRenderAlgorithm::VideoFrameReleaseTimeHelper:: jlong VideoRenderAlgorithm::VideoFrameReleaseTimeHelper::AdjustReleaseTime( jlong frame_presentation_time_us, - jlong unadjusted_release_time_ns) { + jlong unadjusted_release_time_ns, + double playback_rate) { SB_DCHECK(j_video_frame_release_time_helper_); auto* env = JniEnvExt::Get(); return env->CallLongMethodOrAbort( - j_video_frame_release_time_helper_, "adjustReleaseTime", "(JJ)J", - frame_presentation_time_us, unadjusted_release_time_ns); + j_video_frame_release_time_helper_, "adjustReleaseTime", "(JJD)J", + frame_presentation_time_us, unadjusted_release_time_ns, playback_rate); } } // namespace shared diff --git a/starboard/android/shared/video_render_algorithm.h b/starboard/android/shared/video_render_algorithm.h index 67f4868b33b8..7ad9f1decd9e 100644 --- a/starboard/android/shared/video_render_algorithm.h +++ b/starboard/android/shared/video_render_algorithm.h @@ -44,7 +44,8 @@ class VideoRenderAlgorithm : public ::starboard::shared::starboard::player:: VideoFrameReleaseTimeHelper(); ~VideoFrameReleaseTimeHelper(); jlong AdjustReleaseTime(jlong frame_presentation_time_us, - jlong unadjusted_release_time_ns); + jlong unadjusted_release_time_ns, + double playback_rate); private: jobject j_video_frame_release_time_helper_ = nullptr;