Example of frame dropping class

- NOTE: THIS IS JUST AN EXAMPLE
  -> But it works
- Video Frame Handler which will perform local preview at capture
- Video Frame Handler will inject frames at at specified frame rate
  -> NOTE: this is HARDCODED to 5
- Adapt the Video Processing handler to also limit fps

DolbyIO/Source/Private/Subsystem/DolbyIOInitialization.cpp

@@ -8,6 +8,7 @@
 #include "Utils/DolbyIOErrorHandler.h"
 #include "Utils/DolbyIOLogging.h"
 #include "Video/DolbyIOVideoFrameHandler.h"
+#include "Video/dummy_frame_dropper.h"
 #include "Video/DolbyIOVideoSink.h"
 
 #include "Engine/GameInstance.h"
@@ -31,8 +32,8 @@ void UDolbyIOSubsystem::Initialize(FSubsystemCollectionBase& Collection)
 		FScopeLock Lock{&VideoSinksLock};
 		VideoSinks.Emplace(LocalCameraTrackID, std::make_shared<FVideoSink>(LocalCameraTrackID));
 		VideoSinks.Emplace(LocalScreenshareTrackID, std::make_shared<FVideoSink>(LocalScreenshareTrackID));
-		LocalCameraFrameHandler = std::make_shared<FVideoFrameHandler>(VideoSinks[LocalCameraTrackID]);
-		LocalScreenshareFrameHandler = std::make_shared<FVideoFrameHandler>(VideoSinks[LocalScreenshareTrackID]);
+		LocalCameraFrameHandler = std::make_shared<dummy_frame_dropper>(VideoSinks[LocalCameraTrackID], 5);
+		LocalScreenshareFrameHandler = std::make_shared<dummy_frame_dropper>(VideoSinks[LocalScreenshareTrackID], 5);
 	}
 
 	FTimerManager& TimerManager = GetGameInstance()->GetTimerManager();

DolbyIO/Source/Private/Subsystem/DolbyIOScreenshare.cpp

@@ -7,6 +7,7 @@
 #include "Utils/DolbyIOErrorHandler.h"
 #include "Utils/DolbyIOLogging.h"
 #include "Video/DolbyIOVideoFrameHandler.h"
+#include "Video/dummy_frame_dropper.h"
 
 using namespace dolbyio::comms;
 using namespace DolbyIO;

DolbyIO/Source/Private/Subsystem/DolbyIOVideo.cpp

@@ -9,6 +9,7 @@
 #include "Utils/DolbyIOLogging.h"
 #include "Video/DolbyIOVideoFrameHandler.h"
 #include "Video/DolbyIOVideoProcessingFrameHandler.h"
+#include "Video/dummy_frame_dropper.h"
 
 using namespace dolbyio::comms;
 using namespace DolbyIO;
@@ -29,7 +30,7 @@ void UDolbyIOSubsystem::EnableVideo(const FDolbyIOVideoDevice& VideoDevice, bool
 #if PLATFORM_WINDOWS | PLATFORM_MAC
 		DLB_UE_LOG("Blurring background");
 		VideoFrameHandler =
-		    std::make_shared<FVideoProcessingFrameHandler>(VideoProcessor, LocalCameraFrameHandler->sink());
+		    std::make_shared<FVideoProcessingFrameHandler>(VideoProcessor, LocalCameraFrameHandler->sink(), 5);
 #else
 		DLB_WARNING(OnEnableVideoError, "Cannot blur background on this platform");
 #endif

DolbyIO/Source/Private/Video/DolbyIOVideoProcessingFrameHandler.h

@@ -13,9 +13,11 @@ namespace DolbyIO
 	{
 	public:
 		FVideoProcessingFrameHandler(std::shared_ptr<dolbyio::comms::video_frame_handler> VideoFrameHandler,
-		                             std::shared_ptr<dolbyio::comms::video_sink> PreviewSink)
-		    : VideoFrameHandler(std::move(VideoFrameHandler)), PreviewSink(std::move(PreviewSink))
+									 std::shared_ptr<dolbyio::comms::video_sink> PreviewSink, int fps)
+			: desired_fps_(fps), VideoFrameHandler(std::move(VideoFrameHandler)), PreviewSink(std::move(PreviewSink))
 		{
+			assert(desired_fps_);
+			pass_every_n_ = 30 / desired_fps_;
 		}
 
 	private:
@@ -31,19 +33,31 @@ namespace DolbyIO
 		void set_sink(const std::shared_ptr<video_sink>& Sink,
 		              const dolbyio::comms::video_source::config& Config) override
 		{
-			SdkSink = Sink;
+			{
+				std::lock_guard<std::mutex> lock(sdk_lock_);
+				SdkSink = Sink;
+			}
 			VideoFrameHandler->source()->set_sink(shared_from_this(), Config);
 		}
 
 		void handle_frame(const dolbyio::comms::video_frame& VideoFrame) override
 		{
+			std::lock_guard<std::mutex> lock(sdk_lock_);
 			if (SdkSink)
 			{
-				SdkSink->handle_frame(VideoFrame);
-				PreviewSink->handle_frame(VideoFrame);
+				if (++frames_counter_ >= pass_every_n_)
+				{
+					SdkSink->handle_frame(VideoFrame);
+					frames_counter_ = 0;
+				}
 			}
+			PreviewSink->handle_frame(VideoFrame);
 		}
 
+		std::mutex sdk_lock_;
+		int desired_fps_{0};
+		int frames_counter_{0};
+		int pass_every_n_{0};
 		std::shared_ptr<dolbyio::comms::video_frame_handler> VideoFrameHandler;
 		std::shared_ptr<dolbyio::comms::video_sink> PreviewSink;
 		std::shared_ptr<dolbyio::comms::video_sink> SdkSink;

DolbyIO/Source/Private/Video/dummy_frame_dropper.h

@@ -0,0 +1,63 @@
+// Copyright 2023 Dolby Laboratories
+
+#pragma once
+
+#include "Utils/DolbyIOCppSdk.h"
+
+#include <mutex>
+
+namespace DolbyIO
+{
+	class dummy_frame_dropper final : public dolbyio::comms::video_source,
+					  public dolbyio::comms::video_sink,
+					  public dolbyio::comms::video_frame_handler,
+					  public std::enable_shared_from_this<dummy_frame_dropper>
+	{
+	public:
+		static constexpr int frame_rate_ = 30;
+		dummy_frame_dropper(std::shared_ptr<dolbyio::comms::video_sink> sink, int fps)
+			: local_sink_(std::move(sink)), desired_frame_rate_(fps)
+		{
+			assert(desired_frame_rate_);
+			assert(local_sink_);
+			frame_to_pass_back_ = frame_rate_ / desired_frame_rate_;
+		}
+		~dummy_frame_dropper() = default;
+
+		void handle_frame(const dolbyio::comms::video_frame& frame) override
+		{
+			std::lock_guard<std::mutex> lock(sink_lock_);
+			if (sdk_video_sink_)
+			{
+				// Only provide frame back to SDK if it the desired fps frame
+				if (++frame_counter_ >= frame_to_pass_back_)
+				{
+					sdk_video_sink_->handle_frame(frame);
+					frame_counter_ = 0;
+				}
+			}
+			local_sink_->handle_frame(frame);
+		}
+		void set_sink(const std::shared_ptr<video_sink>& sink, const config&) override
+		{
+			std::lock_guard<std::mutex> lock(sink_lock_);
+			sdk_video_sink_ = sink;
+		}
+		std::shared_ptr<video_sink> sink() override
+		{
+			return shared_from_this();
+		}
+		std::shared_ptr<video_source> source() override
+		{
+			return shared_from_this();
+		}
+
+	private:
+		std::mutex sink_lock_;
+		std::shared_ptr<dolbyio::comms::video_sink> local_sink_{};
+		std::shared_ptr<dolbyio::comms::video_sink> sdk_video_sink_{};
+		int desired_frame_rate_;
+		int frame_to_pass_back_;
+		int frame_counter_{0};
+	};
+}

DolbyIO/Source/Public/DolbyIO.h

@@ -153,6 +153,7 @@ namespace DolbyIO
 	class FErrorHandler;
 	class FVideoFrameHandler;
 	class FVideoSink;
+	class dummy_frame_dropper;
 }
 
 UCLASS(DisplayName = "Dolby.io Subsystem")
@@ -477,8 +478,8 @@ class DOLBYIO_API UDolbyIOSubsystem : public UGameInstanceSubsystem
 	FCriticalSection VideoSinksLock;
 
 	std::shared_ptr<dolbyio::comms::plugin::video_processor> VideoProcessor;
-	std::shared_ptr<DolbyIO::FVideoFrameHandler> LocalCameraFrameHandler;
-	std::shared_ptr<DolbyIO::FVideoFrameHandler> LocalScreenshareFrameHandler;
+	std::shared_ptr<DolbyIO::dummy_frame_dropper> LocalCameraFrameHandler;
+	std::shared_ptr<DolbyIO::dummy_frame_dropper> LocalScreenshareFrameHandler;
 	TSharedPtr<DolbyIO::FDevices> Devices;
 	TSharedPtr<dolbyio::comms::sdk> Sdk;
 	TSharedPtr<dolbyio::comms::refresh_token> RefreshTokenCb;