diff --git a/DolbyIO/Source/Private/Subsystem/DolbyIOInitialization.cpp b/DolbyIO/Source/Private/Subsystem/DolbyIOInitialization.cpp index 9f59e581..548affd0 100644 --- a/DolbyIO/Source/Private/Subsystem/DolbyIOInitialization.cpp +++ b/DolbyIO/Source/Private/Subsystem/DolbyIOInitialization.cpp @@ -8,6 +8,7 @@ #include "Utils/DolbyIOErrorHandler.h" #include "Utils/DolbyIOLogging.h" #include "Video/DolbyIOVideoFrameHandler.h" +#include "Video/dummy_frame_dropper.h" #include "Video/DolbyIOVideoSink.h" #include "Engine/GameInstance.h" @@ -31,8 +32,8 @@ void UDolbyIOSubsystem::Initialize(FSubsystemCollectionBase& Collection) FScopeLock Lock{&VideoSinksLock}; VideoSinks.Emplace(LocalCameraTrackID, std::make_shared(LocalCameraTrackID)); VideoSinks.Emplace(LocalScreenshareTrackID, std::make_shared(LocalScreenshareTrackID)); - LocalCameraFrameHandler = std::make_shared(VideoSinks[LocalCameraTrackID]); - LocalScreenshareFrameHandler = std::make_shared(VideoSinks[LocalScreenshareTrackID]); + LocalCameraFrameHandler = std::make_shared(VideoSinks[LocalCameraTrackID], 5); + LocalScreenshareFrameHandler = std::make_shared(VideoSinks[LocalScreenshareTrackID], 5); } FTimerManager& TimerManager = GetGameInstance()->GetTimerManager(); diff --git a/DolbyIO/Source/Private/Subsystem/DolbyIOScreenshare.cpp b/DolbyIO/Source/Private/Subsystem/DolbyIOScreenshare.cpp index a81b3c5d..1f88ea8f 100644 --- a/DolbyIO/Source/Private/Subsystem/DolbyIOScreenshare.cpp +++ b/DolbyIO/Source/Private/Subsystem/DolbyIOScreenshare.cpp @@ -7,6 +7,7 @@ #include "Utils/DolbyIOErrorHandler.h" #include "Utils/DolbyIOLogging.h" #include "Video/DolbyIOVideoFrameHandler.h" +#include "Video/dummy_frame_dropper.h" using namespace dolbyio::comms; using namespace DolbyIO; diff --git a/DolbyIO/Source/Private/Subsystem/DolbyIOVideo.cpp b/DolbyIO/Source/Private/Subsystem/DolbyIOVideo.cpp index f1ae8c71..46aa15fd 100644 --- a/DolbyIO/Source/Private/Subsystem/DolbyIOVideo.cpp +++ b/DolbyIO/Source/Private/Subsystem/DolbyIOVideo.cpp @@ -9,6 +9,7 @@ #include "Utils/DolbyIOLogging.h" #include "Video/DolbyIOVideoFrameHandler.h" #include "Video/DolbyIOVideoProcessingFrameHandler.h" +#include "Video/dummy_frame_dropper.h" using namespace dolbyio::comms; using namespace DolbyIO; @@ -29,7 +30,7 @@ void UDolbyIOSubsystem::EnableVideo(const FDolbyIOVideoDevice& VideoDevice, bool #if PLATFORM_WINDOWS | PLATFORM_MAC DLB_UE_LOG("Blurring background"); VideoFrameHandler = - std::make_shared(VideoProcessor, LocalCameraFrameHandler->sink()); + std::make_shared(VideoProcessor, LocalCameraFrameHandler->sink(), 5); #else DLB_WARNING(OnEnableVideoError, "Cannot blur background on this platform"); #endif diff --git a/DolbyIO/Source/Private/Video/DolbyIOVideoProcessingFrameHandler.h b/DolbyIO/Source/Private/Video/DolbyIOVideoProcessingFrameHandler.h index ebe74b3d..4563cfd0 100644 --- a/DolbyIO/Source/Private/Video/DolbyIOVideoProcessingFrameHandler.h +++ b/DolbyIO/Source/Private/Video/DolbyIOVideoProcessingFrameHandler.h @@ -13,9 +13,11 @@ namespace DolbyIO { public: FVideoProcessingFrameHandler(std::shared_ptr VideoFrameHandler, - std::shared_ptr PreviewSink) - : VideoFrameHandler(std::move(VideoFrameHandler)), PreviewSink(std::move(PreviewSink)) + std::shared_ptr PreviewSink, int fps) + : desired_fps_(fps), VideoFrameHandler(std::move(VideoFrameHandler)), PreviewSink(std::move(PreviewSink)) { + assert(desired_fps_); + pass_every_n_ = 30 / desired_fps_; } private: @@ -31,19 +33,31 @@ namespace DolbyIO void set_sink(const std::shared_ptr& Sink, const dolbyio::comms::video_source::config& Config) override { - SdkSink = Sink; + { + std::lock_guard lock(sdk_lock_); + SdkSink = Sink; + } VideoFrameHandler->source()->set_sink(shared_from_this(), Config); } void handle_frame(const dolbyio::comms::video_frame& VideoFrame) override { + std::lock_guard lock(sdk_lock_); if (SdkSink) { - SdkSink->handle_frame(VideoFrame); - PreviewSink->handle_frame(VideoFrame); + if (++frames_counter_ >= pass_every_n_) + { + SdkSink->handle_frame(VideoFrame); + frames_counter_ = 0; + } } + PreviewSink->handle_frame(VideoFrame); } + std::mutex sdk_lock_; + int desired_fps_{0}; + int frames_counter_{0}; + int pass_every_n_{0}; std::shared_ptr VideoFrameHandler; std::shared_ptr PreviewSink; std::shared_ptr SdkSink; diff --git a/DolbyIO/Source/Private/Video/dummy_frame_dropper.h b/DolbyIO/Source/Private/Video/dummy_frame_dropper.h new file mode 100644 index 00000000..08542e90 --- /dev/null +++ b/DolbyIO/Source/Private/Video/dummy_frame_dropper.h @@ -0,0 +1,63 @@ +// Copyright 2023 Dolby Laboratories + +#pragma once + +#include "Utils/DolbyIOCppSdk.h" + +#include + +namespace DolbyIO +{ + class dummy_frame_dropper final : public dolbyio::comms::video_source, + public dolbyio::comms::video_sink, + public dolbyio::comms::video_frame_handler, + public std::enable_shared_from_this + { + public: + static constexpr int frame_rate_ = 30; + dummy_frame_dropper(std::shared_ptr sink, int fps) + : local_sink_(std::move(sink)), desired_frame_rate_(fps) + { + assert(desired_frame_rate_); + assert(local_sink_); + frame_to_pass_back_ = frame_rate_ / desired_frame_rate_; + } + ~dummy_frame_dropper() = default; + + void handle_frame(const dolbyio::comms::video_frame& frame) override + { + std::lock_guard lock(sink_lock_); + if (sdk_video_sink_) + { + // Only provide frame back to SDK if it the desired fps frame + if (++frame_counter_ >= frame_to_pass_back_) + { + sdk_video_sink_->handle_frame(frame); + frame_counter_ = 0; + } + } + local_sink_->handle_frame(frame); + } + void set_sink(const std::shared_ptr& sink, const config&) override + { + std::lock_guard lock(sink_lock_); + sdk_video_sink_ = sink; + } + std::shared_ptr sink() override + { + return shared_from_this(); + } + std::shared_ptr source() override + { + return shared_from_this(); + } + + private: + std::mutex sink_lock_; + std::shared_ptr local_sink_{}; + std::shared_ptr sdk_video_sink_{}; + int desired_frame_rate_; + int frame_to_pass_back_; + int frame_counter_{0}; + }; +} diff --git a/DolbyIO/Source/Public/DolbyIO.h b/DolbyIO/Source/Public/DolbyIO.h index ffee92cc..d768d6f1 100644 --- a/DolbyIO/Source/Public/DolbyIO.h +++ b/DolbyIO/Source/Public/DolbyIO.h @@ -153,6 +153,7 @@ namespace DolbyIO class FErrorHandler; class FVideoFrameHandler; class FVideoSink; + class dummy_frame_dropper; } UCLASS(DisplayName = "Dolby.io Subsystem") @@ -477,8 +478,8 @@ class DOLBYIO_API UDolbyIOSubsystem : public UGameInstanceSubsystem FCriticalSection VideoSinksLock; std::shared_ptr VideoProcessor; - std::shared_ptr LocalCameraFrameHandler; - std::shared_ptr LocalScreenshareFrameHandler; + std::shared_ptr LocalCameraFrameHandler; + std::shared_ptr LocalScreenshareFrameHandler; TSharedPtr Devices; TSharedPtr Sdk; TSharedPtr RefreshTokenCb;