Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add new consecutiveDroppedFrames callback #1917

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add new consecutiveDroppedFrames callback #1917

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Libgav1VideoRenderer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,22 @@ public class Libgav1VideoRenderer extends DecoderVideoRenderer {
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
* invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
* @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
* be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
* be called.
*/
public Libgav1VideoRenderer(
long allowedJoiningTimeMs,
@Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify) {
int maxDroppedFramesToNotify,
int minConsecutiveDroppedFramesToNotify) {
this(
allowedJoiningTimeMs,
eventHandler,
eventListener,
maxDroppedFramesToNotify,
minConsecutiveDroppedFramesToNotify,
THREAD_COUNT_AUTODETECT,
DEFAULT_NUM_OF_INPUT_BUFFERS,
DEFAULT_NUM_OF_OUTPUT_BUFFERS);
Expand All @@ -104,6 +109,9 @@ public Libgav1VideoRenderer(
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
* invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
* @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
* be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
* be called.
* @param threads Number of threads libgav1 will use to decode. If {@link
* #THREAD_COUNT_AUTODETECT} is passed, then the number of threads to use is autodetected
* based on CPU capabilities.
Expand All @@ -115,10 +123,16 @@ public Libgav1VideoRenderer(
@Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify,
int minConsecutiveDroppedFramesToNotify,
int threads,
int numInputBuffers,
int numOutputBuffers) {
super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify);
super(
allowedJoiningTimeMs,
eventHandler,
eventListener,
maxDroppedFramesToNotify,
minConsecutiveDroppedFramesToNotify);
this.threads = threads;
this.numInputBuffers = numInputBuffers;
this.numOutputBuffers = numOutputBuffers;
Expand Down
13 changes: 11 additions & 2 deletions ..._ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/ExperimentalFfmpegVideoRenderer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,22 @@ public final class ExperimentalFfmpegVideoRenderer extends DecoderVideoRenderer
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
* invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
* @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
* be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
* be called.
*/
public ExperimentalFfmpegVideoRenderer(
long allowedJoiningTimeMs,
@Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify) {
super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify);
int maxDroppedFramesToNotify,
int minConsecutiveDroppedFramesToNotify) {
super(
allowedJoiningTimeMs,
eventHandler,
eventListener,
maxDroppedFramesToNotify,
minConsecutiveDroppedFramesToNotify);
// TODO: Implement.
}

Expand Down
20 changes: 17 additions & 3 deletions libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/LibvpxVideoRenderer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer {
* can attempt to seamlessly join an ongoing playback.
*/
public LibvpxVideoRenderer(long allowedJoiningTimeMs) {
this(allowedJoiningTimeMs, null, null, 0);
this(allowedJoiningTimeMs, null, null, 0, 0);
}

/**
Expand All @@ -78,17 +78,22 @@ public LibvpxVideoRenderer(long allowedJoiningTimeMs) {
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
* invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
* @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
* be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
* be called.
*/
public LibvpxVideoRenderer(
long allowedJoiningTimeMs,
@Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify) {
int maxDroppedFramesToNotify,
int minConsecutiveDroppedFramesToNotify) {
this(
allowedJoiningTimeMs,
eventHandler,
eventListener,
maxDroppedFramesToNotify,
minConsecutiveDroppedFramesToNotify,
getRuntime().availableProcessors(),
/* numInputBuffers= */ 4,
/* numOutputBuffers= */ 4);
Expand All @@ -104,6 +109,9 @@ public LibvpxVideoRenderer(
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
* invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
* @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
* be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
* be called.
* @param threads Number of threads libvpx will use to decode.
* @param numInputBuffers Number of input buffers.
* @param numOutputBuffers Number of output buffers.
Expand All @@ -113,10 +121,16 @@ public LibvpxVideoRenderer(
@Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify,
int minConsecutiveDroppedFramesToNotify,
int threads,
int numInputBuffers,
int numOutputBuffers) {
super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify);
super(
allowedJoiningTimeMs,
eventHandler,
eventListener,
maxDroppedFramesToNotify,
minConsecutiveDroppedFramesToNotify);
this.threads = threads;
this.numInputBuffers = numInputBuffers;
this.numOutputBuffers = numOutputBuffers;
Expand Down
21 changes: 17 additions & 4 deletions libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,12 @@ public class DefaultRenderersFactory implements RenderersFactory {
*/
public static final int MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY = 50;

/**
* The minimum number of consecutive video frames that would be dropped to
* consider invoking {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)}.
*/
public static final int MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY = 5;

private static final String TAG = "DefaultRenderersFactory";

private final Context context;
Expand Down Expand Up @@ -347,7 +353,8 @@ protected void buildVideoRenderers(
enableDecoderFallback,
eventHandler,
eventListener,
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
out.add(videoRenderer);

if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {
Expand All @@ -366,14 +373,16 @@ protected void buildVideoRenderers(
long.class,
android.os.Handler.class,
androidx.media3.exoplayer.video.VideoRendererEventListener.class,
int.class,
int.class);
Renderer renderer =
(Renderer)
constructor.newInstance(
allowedVideoJoiningTimeMs,
eventHandler,
eventListener,
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
out.add(extensionRendererIndex++, renderer);
Log.i(TAG, "Loaded LibvpxVideoRenderer.");
} catch (ClassNotFoundException e) {
Expand All @@ -391,14 +400,16 @@ protected void buildVideoRenderers(
long.class,
android.os.Handler.class,
androidx.media3.exoplayer.video.VideoRendererEventListener.class,
int.class,
int.class);
Renderer renderer =
(Renderer)
constructor.newInstance(
allowedVideoJoiningTimeMs,
eventHandler,
eventListener,
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
out.add(extensionRendererIndex++, renderer);
Log.i(TAG, "Loaded Libgav1VideoRenderer.");
} catch (ClassNotFoundException e) {
Expand All @@ -417,14 +428,16 @@ protected void buildVideoRenderers(
long.class,
android.os.Handler.class,
androidx.media3.exoplayer.video.VideoRendererEventListener.class,
int.class,
int.class);
Renderer renderer =
(Renderer)
constructor.newInstance(
allowedVideoJoiningTimeMs,
eventHandler,
eventListener,
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
out.add(extensionRendererIndex++, renderer);
Log.i(TAG, "Loaded FfmpegVideoRenderer.");
} catch (ClassNotFoundException e) {
Expand Down
5 changes: 5 additions & 0 deletions libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3033,6 +3033,11 @@ public void onDroppedFrames(int count, long elapsed) {
analyticsCollector.onDroppedFrames(count, elapsed);
}

@Override
public void onConsecutiveDroppedFrames(int count, long elapsed) {
analyticsCollector.onConsecutiveDroppedFrames(count, elapsed);
}

@Override
public void onVideoSizeChanged(VideoSize newVideoSize) {
videoSize = newVideoSize;
Expand Down
12 changes: 12 additions & 0 deletions ...aries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,18 @@ void onVideoInputFormatChanged(
*/
void onDroppedFrames(int count, long elapsedMs);

/**
* Called to report the number of consecutive frames dropped by the video renderer. Consecutive
* dropped frames are reported once a frame is renderered after a period in which more frames were
* dropped consecutively than the specified threshold.
*
* @param count The number of consecutive dropped frames.
* @param elapsedMs The duration in milliseconds over which the consecutive frames were dropped.
* This duration is timed from when the first frame was dropped, until the time the renderer
* succesfully rendered a frame or the rendered was interrupted (stopped, seeked, disabled).
*/
void onConsecutiveDroppedFrames(int count, long elapsedMs);

/**
* Called when a video decoder is released.
*
Expand Down
18 changes: 17 additions & 1 deletion libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,8 @@ public int size() {
EVENT_VIDEO_CODEC_ERROR,
EVENT_AUDIO_TRACK_INITIALIZED,
EVENT_AUDIO_TRACK_RELEASED,
EVENT_RENDERER_READY_CHANGED
EVENT_RENDERER_READY_CHANGED,
EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES,
})
@interface EventFlags {}

Expand Down Expand Up @@ -448,6 +449,9 @@ public int size() {
/** A renderer changed its readiness for playback. */
@UnstableApi int EVENT_RENDERER_READY_CHANGED = 1033;

/** Consecutive video frames have been dropped. */
@UnstableApi int EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES = 1034;

/** Time information of an event. */
@UnstableApi
final class EventTime {
Expand Down Expand Up @@ -1244,6 +1248,18 @@ default void onVideoInputFormatChanged(
@UnstableApi
default void onDroppedVideoFrames(EventTime eventTime, int droppedFrames, long elapsedMs) {}

/**
* Called after consecutive video frames have been dropped.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that this event is called after an 'end' to the run of consecutive dropped frames.

*
* @param eventTime The event time.
* @param consecutiveDroppedFrames The number of consecutive frames that have been dropped before the
* last rendered frame.
* @param elapsedMs The duration in milliseconds over which the frames were dropped. This duration
* is timed from the first dropped framed in the sequence.
*/
@UnstableApi
default void onConsecutiveDroppedVideoFrames(EventTime eventTime, int droppedFrames, long elapsedMs) {}

/**
* Called when a video renderer releases a decoder.
*
Expand Down
9 changes: 9 additions & 0 deletions ...xoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,15 @@ public final void onDroppedFrames(int count, long elapsedMs) {
listener -> listener.onDroppedVideoFrames(eventTime, count, elapsedMs));
}

@Override
public final void onConsecutiveDroppedFrames(int count, long elapsedMs) {
EventTime eventTime = generatePlayingMediaPeriodEventTime();
sendEvent(
eventTime,
AnalyticsListener.EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES,
listener -> listener.onConsecutiveDroppedVideoFrames(eventTime, count, elapsedMs));
}

@Override
public final void onVideoDecoderReleased(String decoderName) {
EventTime eventTime = generateReadingMediaPeriodEventTime();
Expand Down
7 changes: 7 additions & 0 deletions libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,13 @@ public void onDroppedVideoFrames(EventTime eventTime, int droppedFrames, long el
logd(eventTime, "droppedFrames", Integer.toString(droppedFrames));
}

@UnstableApi
@Override
public void onConsecutiveDroppedVideoFrames(
EventTime eventTime, int consecutiveDroppedFrames, long elapsedMs) {
logd(eventTime, "consecutiveDroppedFrames", Integer.toString(consecutiveDroppedFrames));
}

@UnstableApi
@Override
public void onVideoDecoderReleased(EventTime eventTime, String decoderName) {
Expand Down
25 changes: 22 additions & 3 deletions libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {

private final long allowedJoiningTimeMs;
private final int maxDroppedFramesToNotify;
private final int minConsecutiveDroppedFramesToNotify;
private final EventDispatcher eventDispatcher;
private final TimedValueQueue<Format> formatQueue;
private final DecoderInputBuffer flagsOnlyBuffer;
Expand Down Expand Up @@ -150,6 +151,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {

private long droppedFrameAccumulationStartTimeMs;
private int droppedFrames;
private long consecutiveDroppedFrameAccumulationStartTimeMs;
private int consecutiveDroppedFrameCount;
private int buffersInCodecCount;
private long lastRenderTimeUs;
Expand All @@ -165,15 +167,20 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
* invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
* @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
* be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
* be called.
*/
protected DecoderVideoRenderer(
long allowedJoiningTimeMs,
@Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify) {
int maxDroppedFramesToNotify,
int minConsecutiveDroppedFramesToNotify) {
super(C.TRACK_TYPE_VIDEO);
this.allowedJoiningTimeMs = allowedJoiningTimeMs;
this.maxDroppedFramesToNotify = maxDroppedFramesToNotify;
this.minConsecutiveDroppedFramesToNotify = minConsecutiveDroppedFramesToNotify;
joiningDeadlineMs = C.TIME_UNSET;
formatQueue = new TimedValueQueue<>();
flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance();
Expand Down Expand Up @@ -296,7 +303,7 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb
outputStreamEnded = false;
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
initialPositionUs = C.TIME_UNSET;
consecutiveDroppedFrameCount = 0;
maybeNotifyConsecutiveDroppedFrames();
if (decoder != null) {
flushDecoder();
}
Expand All @@ -319,6 +326,7 @@ protected void onStarted() {
protected void onStopped() {
joiningDeadlineMs = C.TIME_UNSET;
maybeNotifyDroppedFrames();
maybeNotifyConsecutiveDroppedFrames();
}

@Override
Expand Down Expand Up @@ -603,8 +611,8 @@ protected void renderOutputBuffer(
} else {
renderOutputBufferToSurface(outputBuffer, checkNotNull(outputSurface));
}
consecutiveDroppedFrameCount = 0;
decoderCounters.renderedOutputBufferCount++;
maybeNotifyConsecutiveDroppedFrames();
maybeNotifyRenderedFirstFrame();
}
}
Expand Down Expand Up @@ -996,6 +1004,17 @@ private void maybeNotifyDroppedFrames() {
}
}

private void maybeNotifyConsecutiveDroppedFrames() {
if (consecutiveDroppedFrameCount > 0
&& consecutiveDroppedFrameCount >= minConsecutiveDroppedFramesToNotify) {
long elapsedMs = SystemClock.elapsedRealtime() - consecutiveDroppedFrameAccumulationStartTimeMs;
eventDispatcher.consecutiveDroppedFrames(consecutiveDroppedFrameCount, elapsedMs);
}
// Always reset the counter to 0, even if the threshold is not reached.
consecutiveDroppedFrameCount = 0;
consecutiveDroppedFrameAccumulationStartTimeMs = SystemClock.elapsedRealtime();
}

private static boolean isBufferLate(long earlyUs) {
// Class a buffer as late if it should have been presented more than 30 ms ago.
return earlyUs < -30000;
Expand Down
Loading