From 31560a034f3c15981ccc7b58d59b1e380f7eb443 Mon Sep 17 00:00:00 2001 From: Felipe Cavalcanti Date: Wed, 16 Nov 2022 21:54:03 -0300 Subject: [PATCH 1/3] Use vt for manually decoding frames. Fixes #533 use pts from moonlight server to schedule frame display use decompression callback unused frameRef field to propagate frameType information Use obj-c cb for decode session Revert to direct decode, use PTS correctly --- Limelight/Stream/Connection.m | 15 +-- Limelight/Stream/VideoDecoderRenderer.h | 5 +- Limelight/Stream/VideoDecoderRenderer.m | 164 ++++++++++++++++-------- 3 files changed, 118 insertions(+), 66 deletions(-) diff --git a/Limelight/Stream/Connection.m b/Limelight/Stream/Connection.m index 7cc84e3d..95e0506e 100644 --- a/Limelight/Stream/Connection.m +++ b/Limelight/Stream/Connection.m @@ -55,14 +55,9 @@ int DrDecoderSetup(int videoFormat, int width, int height, int redrawRate, void* return 0; } -void DrStart(void) +void DrCleanup(void) { - [renderer start]; -} - -void DrStop(void) -{ - [renderer stop]; + [renderer cleanup]; } -(BOOL) getVideoStats:(video_stats_t*)stats @@ -433,9 +428,9 @@ -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer LiInitializeVideoCallbacks(&_drCallbacks); _drCallbacks.setup = DrDecoderSetup; - _drCallbacks.start = DrStart; - _drCallbacks.stop = DrStop; - _drCallbacks.capabilities = CAPABILITY_PULL_RENDERER | CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC; + _drCallbacks.cleanup = DrCleanup; + _drCallbacks.submitDecodeUnit = DrSubmitDecodeUnit; + _drCallbacks.capabilities = CAPABILITY_DIRECT_SUBMIT | CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC; LiInitializeAudioCallbacks(&_arCallbacks); _arCallbacks.init = ArInit; diff --git a/Limelight/Stream/VideoDecoderRenderer.h b/Limelight/Stream/VideoDecoderRenderer.h index 768db534..aeeabc28 100644 --- a/Limelight/Stream/VideoDecoderRenderer.h +++ b/Limelight/Stream/VideoDecoderRenderer.h @@ -15,9 +15,10 @@ - (id)initWithView:(UIView*)view callbacks:(id)callbacks streamAspectRatio:(float)aspectRatio useFramePacing:(BOOL)useFramePacing; - (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate; -- (void)start; -- (void)stop; +- (void)cleanup; - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(int)bufferType frameType:(int)frameType pts:(unsigned int)pts; +- (OSStatus)decodeFrameWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameType:(int)frameType; + @end diff --git a/Limelight/Stream/VideoDecoderRenderer.m b/Limelight/Stream/VideoDecoderRenderer.m index 7333c650..db623666 100644 --- a/Limelight/Stream/VideoDecoderRenderer.m +++ b/Limelight/Stream/VideoDecoderRenderer.m @@ -6,6 +6,8 @@ // Copyright (c) 2014 Moonlight Stream. All rights reserved. // +@import VideoToolbox; + #import "VideoDecoderRenderer.h" #import "StreamView.h" @@ -23,6 +25,7 @@ @implementation VideoDecoderRenderer { NSData *spsData, *ppsData, *vpsData; CMVideoFormatDescriptionRef formatDesc; + VTDecompressionSessionRef decompressionSession; CADisplayLink* _displayLink; BOOL framePacing; @@ -74,6 +77,12 @@ - (void)reinitializeDisplayLayer CFRelease(formatDesc); formatDesc = nil; } + + if (decompressionSession != nil){ + VTDecompressionSessionInvalidate(decompressionSession); + CFRelease(decompressionSession); + decompressionSession = nil; + } } - (id)initWithView:(StreamView*)view callbacks:(id)callbacks streamAspectRatio:(float)aspectRatio useFramePacing:(BOOL)useFramePacing @@ -94,10 +103,7 @@ - (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate { self->videoFormat = videoFormat; self->frameRate = frameRate; -} - -- (void)start -{ + _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkCallback:)]; if (@available(iOS 15.0, tvOS 15.0, *)) { _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(self->frameRate, self->frameRate, self->frameRate); @@ -106,6 +112,26 @@ - (void)start _displayLink.preferredFramesPerSecond = self->frameRate; } [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode]; + +} + +- (void) setupDecompressionSession { + if (decompressionSession != NULL){ + VTDecompressionSessionInvalidate(decompressionSession); + CFRelease(decompressionSession); + decompressionSession = nil; + } + + int status = VTDecompressionSessionCreate(kCFAllocatorDefault, + formatDesc, + nil, + nil, + nil, + &decompressionSession); + if (status != noErr) { + NSLog(@"Failed to instance VTDecompressionSessionRef, status %d", status); + } + } // TODO: Refactor this @@ -113,31 +139,10 @@ - (void)start - (void)displayLinkCallback:(CADisplayLink *)sender { - VIDEO_FRAME_HANDLE handle; - PDECODE_UNIT du; - - while (LiPollNextVideoFrame(&handle, &du)) { - LiCompleteVideoFrame(handle, DrSubmitDecodeUnit(du)); - - if (framePacing) { - // Calculate the actual display refresh rate - double displayRefreshRate = 1 / (_displayLink.targetTimestamp - _displayLink.timestamp); - - // Only pace frames if the display refresh rate is >= 90% of our stream frame rate. - // Battery saver, accessibility settings, or device thermals can cause the actual - // refresh rate of the display to drop below the physical maximum. - if (displayRefreshRate >= frameRate * 0.9f) { - // Keep one pending frame to smooth out gaps due to - // network jitter at the cost of 1 frame of latency - if (LiGetPendingVideoFrames() == 1) { - break; - } - } - } - } + // Do Nothing } -- (void)stop +- (void)cleanup { [_displayLink invalidate]; } @@ -262,6 +267,8 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i formatDesc = NULL; } } + + [self setupDecompressionSession]; } // Data is NOT to be freed here. It's a direct usage of the caller's buffer. @@ -330,11 +337,12 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i CMSampleBufferRef sampleBuffer; - status = CMSampleBufferCreate(kCFAllocatorDefault, + CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, CMTimeMake(pts, 1000), kCMTimeInvalid}; + + status = CMSampleBufferCreateReady(kCFAllocatorDefault, frameBlockBuffer, - true, NULL, - NULL, formatDesc, 1, 0, - NULL, 0, NULL, + formatDesc, 1, 1, + &sampleTiming, 0, NULL, &sampleBuffer); if (status != noErr) { Log(LOG_E, @"CMSampleBufferCreate failed: %d", (int)status); @@ -342,34 +350,21 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i CFRelease(frameBlockBuffer); return DR_NEED_IDR; } + + OSStatus decodeStatus = [self decodeFrameWithSampleBuffer: sampleBuffer frameType: frameType]; - CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); - CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); - - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_IsDependedOnByOthers, kCFBooleanTrue); - - if (frameType == FRAME_TYPE_PFRAME) { - // P-frame - CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanTrue); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanTrue); + if (decodeStatus != noErr){ + NSLog(@"Failed to decompress frame"); } else { - // I-frame - CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanFalse); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse); + } - - // Enqueue the next frame - [self->displayLayer enqueueSampleBuffer:sampleBuffer]; - if (frameType == FRAME_TYPE_IDR) { - // Ensure the layer is visible now - self->displayLayer.hidden = NO; - - // Tell our parent VC to hide the progress indicator - [self->_callbacks videoContentShown]; - } + /* Flush in-process frames. */ + //VTDecompressionSessionFinishDelayedFrames(decompressionSession); + /* Block until our callback has been called with the last frame. */ + //VTDecompressionSessionWaitForAsynchronousFrames(decompressionSession); + // Dereference the buffers CFRelease(dataBlockBuffer); CFRelease(frameBlockBuffer); @@ -378,4 +373,65 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i return DR_OK; } +- (OSStatus) decodeFrameWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameType:(int)frameType{ + VTDecodeFrameFlags flags = kVTDecodeFrame_EnableAsynchronousDecompression; + VTDecodeInfoFlags flagOut = 0; + + return VTDecompressionSessionDecodeFrameWithOutputHandler(decompressionSession, sampleBuffer, flags, &flagOut, ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) { + if (status != noErr) + { + NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil]; + NSLog(@"Decompression session error: %@", error); + } + + CMVideoFormatDescriptionRef formatDescriptionRef; + + OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &formatDescriptionRef); + if (res != noErr){ + NSLog(@"Failed to create video format description from imageBuffer"); + } + + CMSampleBufferRef sampleBuffer; + CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, kCMTimeInvalid}; + + OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, formatDescriptionRef, &sampleTiming, &sampleBuffer); + + if (err != noErr){ + NSLog(@"Error creating sample buffer for decompressed image buffer %d", (int)err); + return; + } + + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); + CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); + + CFDictionarySetValue(dict, kCMSampleAttachmentKey_IsDependedOnByOthers, kCFBooleanTrue); + + if (frameType == FRAME_TYPE_PFRAME) { + // P-frame + CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanTrue); + CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanTrue); + } else { + // I-frame + CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanFalse); + CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse); + } + + // Enqueue the next frame + [self->displayLayer enqueueSampleBuffer:sampleBuffer]; + + dispatch_async(dispatch_get_main_queue(), ^{ + if (frameType == FRAME_TYPE_IDR) { + // Ensure the layer is visible now + self->displayLayer.hidden = NO; + + // Tell our parent VC to hide the progress indicator + [self->_callbacks videoContentShown]; + } + }); + + CFRelease(sampleBuffer); + CFRelease(formatDescriptionRef); + }); +} + @end From c004cd132e34ba79bafa1a0be8a4658a7404fe19 Mon Sep 17 00:00:00 2001 From: Felipe Cavalcanti Date: Sun, 27 Nov 2022 23:03:09 -0300 Subject: [PATCH 2/3] Improve rendering code --- Limelight/Stream/VideoDecoderRenderer.m | 70 ++++++++++++------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/Limelight/Stream/VideoDecoderRenderer.m b/Limelight/Stream/VideoDecoderRenderer.m index db623666..69f89567 100644 --- a/Limelight/Stream/VideoDecoderRenderer.m +++ b/Limelight/Stream/VideoDecoderRenderer.m @@ -25,6 +25,7 @@ @implementation VideoDecoderRenderer { NSData *spsData, *ppsData, *vpsData; CMVideoFormatDescriptionRef formatDesc; + CMVideoFormatDescriptionRef formatDescImageBuffer; VTDecompressionSessionRef decompressionSession; CADisplayLink* _displayLink; @@ -129,7 +130,7 @@ - (void) setupDecompressionSession { nil, &decompressionSession); if (status != noErr) { - NSLog(@"Failed to instance VTDecompressionSessionRef, status %d", status); + Log(LOG_E, @"Failed to instance VTDecompressionSessionRef, status %d", status); } } @@ -350,21 +351,29 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i CFRelease(frameBlockBuffer); return DR_NEED_IDR; } + + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); + CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); + + CFDictionarySetValue(dict, kCMSampleAttachmentKey_IsDependedOnByOthers, kCFBooleanTrue); + + if (frameType == FRAME_TYPE_PFRAME) { + // P-frame + CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanTrue); + CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanTrue); + } else { + // I-frame + CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanFalse); + CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse); + } OSStatus decodeStatus = [self decodeFrameWithSampleBuffer: sampleBuffer frameType: frameType]; if (decodeStatus != noErr){ - NSLog(@"Failed to decompress frame"); - } else { - + Log(LOG_E, @"Failed to decompress frame: %d", decodeStatus); + return DR_NEED_IDR; } - /* Flush in-process frames. */ - //VTDecompressionSessionFinishDelayedFrames(decompressionSession); - - /* Block until our callback has been called with the last frame. */ - //VTDecompressionSessionWaitForAsynchronousFrames(decompressionSession); - // Dereference the buffers CFRelease(dataBlockBuffer); CFRelease(frameBlockBuffer); @@ -375,47 +384,35 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i - (OSStatus) decodeFrameWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameType:(int)frameType{ VTDecodeFrameFlags flags = kVTDecodeFrame_EnableAsynchronousDecompression; - VTDecodeInfoFlags flagOut = 0; - return VTDecompressionSessionDecodeFrameWithOutputHandler(decompressionSession, sampleBuffer, flags, &flagOut, ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) { + return VTDecompressionSessionDecodeFrameWithOutputHandler(decompressionSession, sampleBuffer, flags, NULL, ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) { if (status != noErr) { NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil]; - NSLog(@"Decompression session error: %@", error); + Log(LOG_E, @"Decompression session error: %@", error); + LiRequestIdrFrame(); + return; } - CMVideoFormatDescriptionRef formatDescriptionRef; - - OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &formatDescriptionRef); - if (res != noErr){ - NSLog(@"Failed to create video format description from imageBuffer"); + if (self->formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->formatDescImageBuffer, imageBuffer)){ + + OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->formatDescImageBuffer)); + if (res != noErr){ + Log(LOG_E, @"Failed to create video format description from imageBuffer"); + return; + } } CMSampleBufferRef sampleBuffer; - CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, kCMTimeInvalid}; + CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration}; - OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, formatDescriptionRef, &sampleTiming, &sampleBuffer); + OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->formatDescImageBuffer, &sampleTiming, &sampleBuffer); if (err != noErr){ - NSLog(@"Error creating sample buffer for decompressed image buffer %d", (int)err); + Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err); return; } - CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); - CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); - - CFDictionarySetValue(dict, kCMSampleAttachmentKey_IsDependedOnByOthers, kCFBooleanTrue); - - if (frameType == FRAME_TYPE_PFRAME) { - // P-frame - CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanTrue); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanTrue); - } else { - // I-frame - CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanFalse); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse); - } - // Enqueue the next frame [self->displayLayer enqueueSampleBuffer:sampleBuffer]; @@ -430,7 +427,6 @@ - (OSStatus) decodeFrameWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameTy }); CFRelease(sampleBuffer); - CFRelease(formatDescriptionRef); }); } From df707d23d73cf6491784f002cb0a40c6e92323ca Mon Sep 17 00:00:00 2001 From: Felipe Cavalcanti Date: Mon, 28 Nov 2022 08:25:02 -0300 Subject: [PATCH 3/3] Fix decoding --- Limelight/Stream/VideoDecoderRenderer.m | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/Limelight/Stream/VideoDecoderRenderer.m b/Limelight/Stream/VideoDecoderRenderer.m index 69f89567..a4c6f396 100644 --- a/Limelight/Stream/VideoDecoderRenderer.m +++ b/Limelight/Stream/VideoDecoderRenderer.m @@ -79,6 +79,11 @@ - (void)reinitializeDisplayLayer formatDesc = nil; } + if (formatDescImageBuffer != nil) { + CFRelease(formatDescImageBuffer); + formatDescImageBuffer = nil; + } + if (decompressionSession != nil){ VTDecompressionSessionInvalidate(decompressionSession); CFRelease(decompressionSession); @@ -351,21 +356,6 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i CFRelease(frameBlockBuffer); return DR_NEED_IDR; } - - CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); - CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); - - CFDictionarySetValue(dict, kCMSampleAttachmentKey_IsDependedOnByOthers, kCFBooleanTrue); - - if (frameType == FRAME_TYPE_PFRAME) { - // P-frame - CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanTrue); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanTrue); - } else { - // I-frame - CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanFalse); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse); - } OSStatus decodeStatus = [self decodeFrameWithSampleBuffer: sampleBuffer frameType: frameType];