From 4a7d9daa26ab1f04a5e7ba07cd9a2d46599517aa Mon Sep 17 00:00:00 2001 From: tremblap Date: Thu, 23 Feb 2023 10:20:59 +0000 Subject: [PATCH 01/42] ampfeatureclient - default value fix --- include/clients/rt/AmpFeatureClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/clients/rt/AmpFeatureClient.hpp b/include/clients/rt/AmpFeatureClient.hpp index 0a472f8c0..d26c2c920 100644 --- a/include/clients/rt/AmpFeatureClient.hpp +++ b/include/clients/rt/AmpFeatureClient.hpp @@ -38,7 +38,7 @@ constexpr auto AmpFeatureParams = defineParameters( LongParam("fastRampDown", "Fast Envelope Ramp Down Length", 1, Min(1)), LongParam("slowRampUp", "Slow Envelope Ramp Up Length", 100, Min(1)), LongParam("slowRampDown", "Slow Envelope Ramp Down Length", 100, Min(1)), - FloatParam("floor", "Floor value (dB)", -145, Min(-144), Max(144)), + FloatParam("floor", "Floor value (dB)", -144, Min(-144), Max(144)), FloatParam("highPassFreq", "High-Pass Filter Cutoff", 85, Min(0))); class AmpFeatureClient : public FluidBaseClient, public AudioIn, public AudioOut From da1cb2fc823a8f1774c30cfd018dc57edbe4962c Mon Sep 17 00:00:00 2001 From: tremblap Date: Mon, 27 Feb 2023 15:40:18 +0000 Subject: [PATCH 02/42] skeleton of voiceallocator, to troubleshoot multi control in management --- FlucomaClients.cmake | 1 + include/clients/rt/VoiceAllocator.hpp | 107 ++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 include/clients/rt/VoiceAllocator.hpp diff --git a/FlucomaClients.cmake b/FlucomaClients.cmake index 94ccc38be..1bb593562 100644 --- a/FlucomaClients.cmake +++ b/FlucomaClients.cmake @@ -137,6 +137,7 @@ add_client(SineFeature clients/rt/SineFeatureClient.hpp CLASS RTSineFeatureClien add_client(Sines clients/rt/SinesClient.hpp CLASS RTSinesClient ) add_client(SpectralShape clients/rt/SpectralShapeClient.hpp CLASS RTSpectralShapeClient ) add_kr_in_client(Stats clients/rt/RunningStatsClient.hpp CLASS RunningStatsClient ) +add_kr_in_client(VoiceAllocator clients/rt/VoiceAllocator.hpp CLASS VoiceAllocatorClient ) add_client(TransientSlice clients/rt/TransientSliceClient.hpp CLASS RTTransientSliceClient ) add_client(Transients clients/rt/TransientClient.hpp CLASS RTTransientClient ) diff --git a/include/clients/rt/VoiceAllocator.hpp b/include/clients/rt/VoiceAllocator.hpp new file mode 100644 index 000000000..30ab7ac3e --- /dev/null +++ b/include/clients/rt/VoiceAllocator.hpp @@ -0,0 +1,107 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#pragma once + +#include "../common/AudioClient.hpp" +#include "../common/FluidBaseClient.hpp" +#include "../common/FluidSource.hpp" +#include "../common/ParameterConstraints.hpp" +#include "../common/ParameterSet.hpp" +#include "../common/ParameterTypes.hpp" +//#include "../../algorithms/public/RunningStats.hpp" +#include "../../data/TensorTypes.hpp" + +namespace fluid { +namespace client { +namespace voiceallocator { + +template +using HostVector = FluidTensorView; + +constexpr auto VoiceAllocatorParams = + defineParameters(LongParam("history", "History Size", 2, Min(2))); //will be most probably a max num voice and all other params + +class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOut +{ +public: + using ParamDescType = decltype(VoiceAllocatorParams); + + using ParamSetViewType = ParameterSetView; + std::reference_wrapper mParams; + + void setParams(ParamSetViewType& p) { mParams = p; } + + template + auto& get() const + { + return mParams.get().template get(); + } + + static constexpr auto& getParameterDescriptors() + { + return VoiceAllocatorParams; + } + + VoiceAllocatorClient(ParamSetViewType& p, FluidContext&) + : mParams(p), mInputSize{0}, mSizeTracker{0} + { + controlChannelsIn(3); + controlChannelsOut({2, -1}); + setInputLabels({"input stream", "middle", "third"}); + setOutputLabels({"mean", "sample standard deviation"}); + } + + template + void process(std::vector>& input, + std::vector>& output, FluidContext&) + { + + std::cout << "left: " << input[0] << '\n'; + std::cout << "middle: " << input[1] << '\n'; + std::cout << "right: " << input[2] << '\n'; + + bool inputSizeChanged = mInputSize != input[0].size() ; + bool sizeParamChanged = mSizeTracker.changed(get<0>()); + + if(inputSizeChanged|| sizeParamChanged) + { + mInputSize = input[0].size(); +// mAlgorithm.init(get<0>(),mInputSize); + } + +// mAlgorithm.process(input[0],output[0],output[1]); + } + + MessageResult clear() + { +// mAlgorithm.init(get<0>(),mInputSize); + return {}; + } + + static auto getMessageDescriptors() + { + return defineMessages(makeMessage("clear", &VoiceAllocatorClient::clear)); + } + + index latency() { return 0; } + +private: +// algorithm::RunningStats mAlgorithm; + index mInputSize; + ParameterTrackChanges mSizeTracker; +}; + +} // namespace runningstats + +using VoiceAllocatorClient = ClientWrapper; + +} // namespace client +} // namespace fluid From 494418440f2e50806a9508ae97dfe9d206e81ae0 Mon Sep 17 00:00:00 2001 From: tremblap Date: Mon, 27 Feb 2023 19:03:15 +0000 Subject: [PATCH 03/42] added a better monitoring of 3 x 3 input --- include/clients/rt/VoiceAllocator.hpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/include/clients/rt/VoiceAllocator.hpp b/include/clients/rt/VoiceAllocator.hpp index 30ab7ac3e..641d025a5 100644 --- a/include/clients/rt/VoiceAllocator.hpp +++ b/include/clients/rt/VoiceAllocator.hpp @@ -54,20 +54,16 @@ class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOu : mParams(p), mInputSize{0}, mSizeTracker{0} { controlChannelsIn(3); - controlChannelsOut({2, -1}); - setInputLabels({"input stream", "middle", "third"}); - setOutputLabels({"mean", "sample standard deviation"}); + controlChannelsOut({3, -1}); + setInputLabels({"left", "middle", "right"}); + setOutputLabels({"lefto", "middleo", "righto"}); } template void process(std::vector>& input, std::vector>& output, FluidContext&) { - - std::cout << "left: " << input[0] << '\n'; - std::cout << "middle: " << input[1] << '\n'; - std::cout << "right: " << input[2] << '\n'; - + bool inputSizeChanged = mInputSize != input[0].size() ; bool sizeParamChanged = mSizeTracker.changed(get<0>()); @@ -78,6 +74,9 @@ class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOu } // mAlgorithm.process(input[0],output[0],output[1]); + output[2] <<= input[2]; + output[1] <<= input[1]; + output[0] <<= input[0]; } MessageResult clear() From 51376a44cac9975372f6fc53ad8468c0c2979be4 Mon Sep 17 00:00:00 2001 From: tremblap Date: Mon, 27 Feb 2023 19:07:33 +0000 Subject: [PATCH 04/42] clang'd --- include/clients/rt/VoiceAllocator.hpp | 42 +++++++++++++++------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/include/clients/rt/VoiceAllocator.hpp b/include/clients/rt/VoiceAllocator.hpp index 641d025a5..355dc3225 100644 --- a/include/clients/rt/VoiceAllocator.hpp +++ b/include/clients/rt/VoiceAllocator.hpp @@ -16,7 +16,7 @@ under the European Union’s Horizon 2020 research and innovation programme #include "../common/ParameterConstraints.hpp" #include "../common/ParameterSet.hpp" #include "../common/ParameterTypes.hpp" -//#include "../../algorithms/public/RunningStats.hpp" +// #include "../../algorithms/public/RunningStats.hpp" #include "../../data/TensorTypes.hpp" namespace fluid { @@ -26,10 +26,13 @@ namespace voiceallocator { template using HostVector = FluidTensorView; -constexpr auto VoiceAllocatorParams = - defineParameters(LongParam("history", "History Size", 2, Min(2))); //will be most probably a max num voice and all other params +constexpr auto VoiceAllocatorParams = defineParameters(LongParam( + "history", "History Size", 2, + Min(2))); // will be most probably a max num voice and all other params -class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOut +class VoiceAllocatorClient : public FluidBaseClient, + public ControlIn, + ControlOut { public: using ParamDescType = decltype(VoiceAllocatorParams); @@ -50,7 +53,7 @@ class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOu return VoiceAllocatorParams; } - VoiceAllocatorClient(ParamSetViewType& p, FluidContext&) + VoiceAllocatorClient(ParamSetViewType& p, FluidContext&) : mParams(p), mInputSize{0}, mSizeTracker{0} { controlChannelsIn(3); @@ -63,25 +66,25 @@ class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOu void process(std::vector>& input, std::vector>& output, FluidContext&) { - - bool inputSizeChanged = mInputSize != input[0].size() ; + + bool inputSizeChanged = mInputSize != input[0].size(); bool sizeParamChanged = mSizeTracker.changed(get<0>()); - if(inputSizeChanged|| sizeParamChanged) + if (inputSizeChanged || sizeParamChanged) { mInputSize = input[0].size(); -// mAlgorithm.init(get<0>(),mInputSize); + // mAlgorithm.init(get<0>(),mInputSize); } -// mAlgorithm.process(input[0],output[0],output[1]); - output[2] <<= input[2]; - output[1] <<= input[1]; - output[0] <<= input[0]; + // mAlgorithm.process(input[0],output[0],output[1]); + output[2] <<= input[2]; + output[1] <<= input[1]; + output[0] <<= input[0]; } MessageResult clear() - { -// mAlgorithm.init(get<0>(),mInputSize); + { + // mAlgorithm.init(get<0>(),mInputSize); return {}; } @@ -93,14 +96,15 @@ class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOu index latency() { return 0; } private: -// algorithm::RunningStats mAlgorithm; - index mInputSize; + // algorithm::RunningStats mAlgorithm; + index mInputSize; ParameterTrackChanges mSizeTracker; }; -} // namespace runningstats +} // namespace voiceallocator -using VoiceAllocatorClient = ClientWrapper; +using VoiceAllocatorClient = + ClientWrapper; } // namespace client } // namespace fluid From 734c48b7fb6384b8ae369869d16c4fc0efffcf79 Mon Sep 17 00:00:00 2001 From: tremblap Date: Mon, 14 Aug 2023 20:04:32 +0100 Subject: [PATCH 05/42] allocate fixed arrays for the output to see if that helps... --- include/clients/rt/VoiceAllocator.hpp | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/include/clients/rt/VoiceAllocator.hpp b/include/clients/rt/VoiceAllocator.hpp index 355dc3225..09ccc3de3 100644 --- a/include/clients/rt/VoiceAllocator.hpp +++ b/include/clients/rt/VoiceAllocator.hpp @@ -53,12 +53,15 @@ class VoiceAllocatorClient : public FluidBaseClient, return VoiceAllocatorParams; } - VoiceAllocatorClient(ParamSetViewType& p, FluidContext&) - : mParams(p), mInputSize{0}, mSizeTracker{0} + VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) + : mParams(p), mInputSize{0}, mSizeTracker{0}, + mOut0(16, c.allocator()), + mOut1(16, c.allocator()), + mOut2(16, c.allocator()) { - controlChannelsIn(3); - controlChannelsOut({3, -1}); - setInputLabels({"left", "middle", "right"}); + controlChannelsIn(2); + controlChannelsOut({3, 16, 16}); + setInputLabels({"left", "right"}); setOutputLabels({"lefto", "middleo", "righto"}); } @@ -76,10 +79,15 @@ class VoiceAllocatorClient : public FluidBaseClient, // mAlgorithm.init(get<0>(),mInputSize); } + // copy in to fixed output array + mOut2(Slice(0,input[0].size())) <<= input[0]; + mOut1(Slice(0,input[1].size())) <<= input[1]; + mOut0(Slice(0,input[0].size())) <<= input[0]; + // mAlgorithm.process(input[0],output[0],output[1]); - output[2] <<= input[2]; - output[1] <<= input[1]; - output[0] <<= input[0]; + output[2] <<= mOut2; + output[1] <<= mOut1; + output[0] <<= mOut0; } MessageResult clear() @@ -99,6 +107,9 @@ class VoiceAllocatorClient : public FluidBaseClient, // algorithm::RunningStats mAlgorithm; index mInputSize; ParameterTrackChanges mSizeTracker; + FluidTensor mOut0; + FluidTensor mOut1; + FluidTensor mOut2; }; } // namespace voiceallocator From b1af3378b1317dfaed2f044124c554153dd7a5ef Mon Sep 17 00:00:00 2001 From: tremblap Date: Wed, 16 Aug 2023 10:47:26 +0100 Subject: [PATCH 06/42] fix for dynamic mem alloc but breaks stats - pushed at @aharker's request --- include/clients/rt/VoiceAllocator.hpp | 36 ++++++++++++++++----------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/include/clients/rt/VoiceAllocator.hpp b/include/clients/rt/VoiceAllocator.hpp index 09ccc3de3..2c7594046 100644 --- a/include/clients/rt/VoiceAllocator.hpp +++ b/include/clients/rt/VoiceAllocator.hpp @@ -26,9 +26,13 @@ namespace voiceallocator { template using HostVector = FluidTensorView; -constexpr auto VoiceAllocatorParams = defineParameters(LongParam( - "history", "History Size", 2, - Min(2))); // will be most probably a max num voice and all other params +enum VoiceAllocatorParamIndex { + kNVoices +}; + +constexpr auto VoiceAllocatorParams = defineParameters( + LongParamRuntimeMax( "numVoices", "Number of Voices", 1, Min(1)) + ); class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, @@ -55,12 +59,12 @@ class VoiceAllocatorClient : public FluidBaseClient, VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) : mParams(p), mInputSize{0}, mSizeTracker{0}, - mOut0(16, c.allocator()), - mOut1(16, c.allocator()), - mOut2(16, c.allocator()) + mOut0(get().max(), c.allocator()), + mOut1(get().max(), c.allocator()), + mOut2(get().max(), c.allocator()) { controlChannelsIn(2); - controlChannelsOut({3, 16, 16}); + controlChannelsOut({3, get(), get().max()}); setInputLabels({"left", "right"}); setOutputLabels({"lefto", "middleo", "righto"}); } @@ -69,25 +73,27 @@ class VoiceAllocatorClient : public FluidBaseClient, void process(std::vector>& input, std::vector>& output, FluidContext&) { + index nVoices = get(); bool inputSizeChanged = mInputSize != input[0].size(); - bool sizeParamChanged = mSizeTracker.changed(get<0>()); + bool sizeParamChanged = mSizeTracker.changed(nVoices); if (inputSizeChanged || sizeParamChanged) { mInputSize = input[0].size(); - // mAlgorithm.init(get<0>(),mInputSize); + controlChannelsOut({3, nVoices}); //update the dynamic out size + // other initialisation } // copy in to fixed output array - mOut2(Slice(0,input[0].size())) <<= input[0]; + mOut2(Slice(0,input[0].size())) <<= input[0];// dummy code so I don't check that the input is smaller than the output - the voice alloc algo should deal with input more cleverly than just copy mOut1(Slice(0,input[1].size())) <<= input[1]; mOut0(Slice(0,input[0].size())) <<= input[0]; // mAlgorithm.process(input[0],output[0],output[1]); - output[2] <<= mOut2; - output[1] <<= mOut1; - output[0] <<= mOut0; + output[2](Slice(0,nVoices)) <<= mOut2(Slice(0,nVoices)); + output[1](Slice(0,nVoices)) <<= mOut1(Slice(0,nVoices)); + output[0](Slice(0,nVoices)) <<= mOut0(Slice(0,nVoices)); } MessageResult clear() @@ -105,8 +111,8 @@ class VoiceAllocatorClient : public FluidBaseClient, private: // algorithm::RunningStats mAlgorithm; - index mInputSize; - ParameterTrackChanges mSizeTracker; + index mInputSize; + ParameterTrackChanges mSizeTracker; FluidTensor mOut0; FluidTensor mOut1; FluidTensor mOut2; From 26635b37a2afae9d5ed7cb9528db178b7beceaf3 Mon Sep 17 00:00:00 2001 From: tremblap Date: Thu, 17 Aug 2023 09:20:50 +0100 Subject: [PATCH 07/42] new const needs --- include/clients/rt/VoiceAllocator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/clients/rt/VoiceAllocator.hpp b/include/clients/rt/VoiceAllocator.hpp index 2c7594046..fda14b1eb 100644 --- a/include/clients/rt/VoiceAllocator.hpp +++ b/include/clients/rt/VoiceAllocator.hpp @@ -107,7 +107,7 @@ class VoiceAllocatorClient : public FluidBaseClient, return defineMessages(makeMessage("clear", &VoiceAllocatorClient::clear)); } - index latency() { return 0; } + index latency() const { return 0; } private: // algorithm::RunningStats mAlgorithm; From e2ece726b18fa207cb2a1e241f3ccabede0d77c4 Mon Sep 17 00:00:00 2001 From: Alex Harker Date: Thu, 17 Aug 2023 10:26:38 +0100 Subject: [PATCH 08/42] Changes for how clients determine if the output follows the input size --- include/clients/common/AudioClient.hpp | 7 ++++++- include/clients/rt/RunningStatsClient.hpp | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/clients/common/AudioClient.hpp b/include/clients/common/AudioClient.hpp index 03c80cb14..b310acfb1 100644 --- a/include/clients/common/AudioClient.hpp +++ b/include/clients/common/AudioClient.hpp @@ -34,13 +34,18 @@ constexpr bool isAudio = isAudioIn || isAudioOut; struct Control {}; struct ControlIn : Control -{}; +{}; struct ControlOut : Control {}; +struct ControlOutFollowsIn : ControlIn, ControlOut +{}; + template constexpr bool isControlIn = std::is_base_of::value; template +constexpr bool isControlOutFollowsIn = std::is_base_of::value;template +template constexpr bool isControlOut = std::is_base_of::value; template constexpr bool isControl = std::is_base_of::value; diff --git a/include/clients/rt/RunningStatsClient.hpp b/include/clients/rt/RunningStatsClient.hpp index 40350d679..0aab924be 100644 --- a/include/clients/rt/RunningStatsClient.hpp +++ b/include/clients/rt/RunningStatsClient.hpp @@ -29,7 +29,7 @@ using HostVector = FluidTensorView; constexpr auto RunningStatsParams = defineParameters(LongParam("history", "History Size", 2, Min(2))); -class RunningStatsClient : public FluidBaseClient, public ControlIn, ControlOut +class RunningStatsClient : public FluidBaseClient, public ControlOutFollowsIn { public: using ParamDescType = decltype(RunningStatsParams); From d5b4a7c228857b1ad3bbbf477b9aadbe398b15db Mon Sep 17 00:00:00 2001 From: tremblap Date: Thu, 17 Aug 2023 10:34:13 +0100 Subject: [PATCH 09/42] remove duplicate --- include/clients/common/AudioClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/clients/common/AudioClient.hpp b/include/clients/common/AudioClient.hpp index b310acfb1..b28f0ca69 100644 --- a/include/clients/common/AudioClient.hpp +++ b/include/clients/common/AudioClient.hpp @@ -44,7 +44,7 @@ struct ControlOutFollowsIn : ControlIn, ControlOut template constexpr bool isControlIn = std::is_base_of::value; template -constexpr bool isControlOutFollowsIn = std::is_base_of::value;template +constexpr bool isControlOutFollowsIn = std::is_base_of::value; template constexpr bool isControlOut = std::is_base_of::value; template From d6c2c20774907efbe7d1355059e3ba6625fd0ae2 Mon Sep 17 00:00:00 2001 From: fearne Date: Sun, 20 Aug 2023 16:57:39 +0100 Subject: [PATCH 10/42] VoiceAllocator.hpp -> VoiceAllocatorClient.hpp --- FlucomaClients.cmake | 2 +- .../clients/rt/{VoiceAllocator.hpp => VoiceAllocatorClient.hpp} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename include/clients/rt/{VoiceAllocator.hpp => VoiceAllocatorClient.hpp} (100%) diff --git a/FlucomaClients.cmake b/FlucomaClients.cmake index 9db471d3a..3d41c8dd0 100644 --- a/FlucomaClients.cmake +++ b/FlucomaClients.cmake @@ -137,7 +137,7 @@ add_client(SineFeature clients/rt/SineFeatureClient.hpp CLASS RTSineFeatureClien add_client(Sines clients/rt/SinesClient.hpp CLASS RTSinesClient ) add_client(SpectralShape clients/rt/SpectralShapeClient.hpp CLASS RTSpectralShapeClient ) add_kr_in_client(Stats clients/rt/RunningStatsClient.hpp CLASS RunningStatsClient ) -add_kr_in_client(VoiceAllocator clients/rt/VoiceAllocator.hpp CLASS VoiceAllocatorClient ) +add_kr_in_client(VoiceAllocator clients/rt/VoiceAllocatorClient.hpp CLASS VoiceAllocatorClient ) add_client(TransientSlice clients/rt/TransientSliceClient.hpp CLASS RTTransientSliceClient ) add_client(Transients clients/rt/TransientClient.hpp CLASS RTTransientClient ) diff --git a/include/clients/rt/VoiceAllocator.hpp b/include/clients/rt/VoiceAllocatorClient.hpp similarity index 100% rename from include/clients/rt/VoiceAllocator.hpp rename to include/clients/rt/VoiceAllocatorClient.hpp From 77c04d4a32015fe052729aafcb2428bf79006f6c Mon Sep 17 00:00:00 2001 From: fearne Date: Sun, 20 Aug 2023 17:08:30 +0100 Subject: [PATCH 11/42] names & boilerplate --- include/clients/rt/VoiceAllocatorClient.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index fda14b1eb..3a9789ef9 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -38,6 +38,9 @@ class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOut { + template + using vector = rt::vector; + public: using ParamDescType = decltype(VoiceAllocatorParams); @@ -65,13 +68,13 @@ class VoiceAllocatorClient : public FluidBaseClient, { controlChannelsIn(2); controlChannelsOut({3, get(), get().max()}); - setInputLabels({"left", "right"}); - setOutputLabels({"lefto", "middleo", "righto"}); + setInputLabels({"frequencies", "magnitudes"}); + setOutputLabels({"frequencies", "magnitudes", "voice IDs"}); } template void process(std::vector>& input, - std::vector>& output, FluidContext&) + std::vector>& output, FluidContext& c) { index nVoices = get(); From 8954b291891ae6254ee22c092234cdc66d8fe248 Mon Sep 17 00:00:00 2001 From: fearne Date: Sun, 20 Aug 2023 18:37:57 +0100 Subject: [PATCH 12/42] partialTracking new getActiveVoices function --- include/algorithms/util/PartialTracking.hpp | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/include/algorithms/util/PartialTracking.hpp b/include/algorithms/util/PartialTracking.hpp index a4ac7926b..54329a939 100644 --- a/include/algorithms/util/PartialTracking.hpp +++ b/include/algorithms/util/PartialTracking.hpp @@ -32,6 +32,13 @@ struct SinePeak bool assigned; }; +struct VoicePeak +{ + double freq; + double logMag; + index voiceID; +}; + struct SineTrack { @@ -134,6 +141,26 @@ class PartialTracking return sinePeaks; } + // todo - refactor this function with the one above + vector getActiveVoices(Allocator& alloc) + { + vector voicePeaks(0, alloc); + index latencyFrame = mCurrentFrame - mMinTrackLength; + if (latencyFrame < 0) return voicePeaks; + for (auto&& track : mTracks) + { + if (track.startFrame > latencyFrame) continue; + if (track.endFrame >= 0 && track.endFrame <= latencyFrame) continue; + if (track.endFrame >= 0 && + track.endFrame - track.startFrame < mMinTrackLength) + continue; + voicePeaks.push_back({track.peaks[asUnsigned(latencyFrame - track.startFrame)].freq, + track.peaks[asUnsigned(latencyFrame - track.startFrame)].logMag, + track.trackId}); + } + return voicePeaks; + } + private: void updateVariances() { From 246eb25e648eab43132b0407089e162fd3b1dd74 Mon Sep 17 00:00:00 2001 From: fearne Date: Sun, 20 Aug 2023 18:43:22 +0100 Subject: [PATCH 13/42] parameter list for voiceallocator --- include/clients/rt/VoiceAllocatorClient.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 3a9789ef9..af0223294 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -27,11 +27,25 @@ template using HostVector = FluidTensorView; enum VoiceAllocatorParamIndex { - kNVoices + kNVoices, + kBirthLowThreshold, + kBirthHighTreshold, + kMinTrackLen, + kTrackMethod, + kTrackMagRange, + kTrackFreqRange, + kTrackProb }; constexpr auto VoiceAllocatorParams = defineParameters( - LongParamRuntimeMax( "numVoices", "Number of Voices", 1, Min(1)) + LongParamRuntimeMax( "numVoices", "Number of Voices", 1, Min(1)), + FloatParam("birthLowThreshold", "Track Birth Low Frequency Threshold", -24, Min(-144), Max(0)), + FloatParam("birthHighThreshold", "Track Birth High Frequency Threshold", -60, Min(-144), Max(0)), + LongParam("minTrackLen", "Minimum Track Length", 1, Min(1)), + EnumParam("trackMethod", "Tracking Method", 0, "Greedy", "Hungarian"), + FloatParam("trackMagRange", "Tracking Magnitude Range (dB)", 15., Min(1.), Max(200.)), + FloatParam("trackFreqRange", "Tracking Frequency Range (Hz)", 50., Min(1.), Max(10000.)), + FloatParam("trackProb", "Tracking Matching Probability", 0.5, Min(0.0), Max(1.0)) ); class VoiceAllocatorClient : public FluidBaseClient, From d6813d68fa54cbac783330df0fb6d853e2d0cac5 Mon Sep 17 00:00:00 2001 From: fearne Date: Sun, 20 Aug 2023 18:46:10 +0100 Subject: [PATCH 14/42] functional allocator but still bad track IDs --- include/clients/rt/VoiceAllocatorClient.hpp | 102 +++++++++++++++++--- 1 file changed, 86 insertions(+), 16 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index af0223294..c6f4e5ce8 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -18,6 +18,7 @@ under the European Union’s Horizon 2020 research and innovation programme #include "../common/ParameterTypes.hpp" // #include "../../algorithms/public/RunningStats.hpp" #include "../../data/TensorTypes.hpp" +#include "../../algorithms/util/PartialTracking.hpp" namespace fluid { namespace client { @@ -75,15 +76,18 @@ class VoiceAllocatorClient : public FluidBaseClient, } VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) - : mParams(p), mInputSize{0}, mSizeTracker{0}, - mOut0(get().max(), c.allocator()), - mOut1(get().max(), c.allocator()), - mOut2(get().max(), c.allocator()) + : mParams(p), mVoices(c.allocator()), mTracking(c.allocator()), + mInputSize{ 0 }, mSizeTracker{ 0 }, + mFreqs(get().max(), c.allocator()), + mLogMags(get().max(), c.allocator()), + mVoiceIDs(get().max(), c.allocator()) { controlChannelsIn(2); controlChannelsOut({3, get(), get().max()}); setInputLabels({"frequencies", "magnitudes"}); setOutputLabels({"frequencies", "magnitudes", "voice IDs"}); + mTracking.init(); + mVoices.clear(); } template @@ -102,15 +106,79 @@ class VoiceAllocatorClient : public FluidBaseClient, // other initialisation } - // copy in to fixed output array - mOut2(Slice(0,input[0].size())) <<= input[0];// dummy code so I don't check that the input is smaller than the output - the voice alloc algo should deal with input more cleverly than just copy - mOut1(Slice(0,input[1].size())) <<= input[1]; - mOut0(Slice(0,input[0].size())) <<= input[0]; - - // mAlgorithm.process(input[0],output[0],output[1]); - output[2](Slice(0,nVoices)) <<= mOut2(Slice(0,nVoices)); - output[1](Slice(0,nVoices)) <<= mOut1(Slice(0,nVoices)); - output[0](Slice(0,nVoices)) <<= mOut0(Slice(0,nVoices)); + index lowerSize; + bool unfilledVoices = false; + if (input[0].size() >= nVoices) + { + lowerSize = nVoices; + } + else + { + lowerSize = input[0].size(); + unfilledVoices = true; + } + + //mOut1(Slice(0, lowerSize)) <<= input[1](Slice(0, lowerSize)); + //mOut0(Slice(0, lowerSize)) <<= input[0](Slice(0, lowerSize)); + + vector incomingVoices(0, c.allocator()); + for (index i = 0; i < lowerSize; ++i) + { + if (input[1].row(i) != 0 && input[0].row(i) != 0) + { + incomingVoices.push_back({ input[0].row(i), input[1].row(i), false }); + } + } + + if (true) //change this to IF INPUT = TYPE MAGNITUDE, if dB skip + { + for (algorithm::SinePeak voice : incomingVoices) + { + voice.logMag = 20 * log10(std::max(voice.logMag, algorithm::epsilon)); + } + } + + double maxAmp = -144; + for (algorithm::SinePeak voice : incomingVoices) + { + if (voice.logMag > maxAmp) { maxAmp = voice.logMag; } + } + + mTracking.processFrame(incomingVoices, maxAmp, get(), get(), get(), get(), get(), get(), get(), c.allocator()); + + vector badIDVoices(0, c.allocator()); + badIDVoices = mTracking.getActiveVoices(c.allocator()); + + if (badIDVoices.size() < lowerSize) + { + lowerSize = badIDVoices.size(); + } + + for (index i = 0; i < lowerSize; ++i) + { + output[2].row(i) = badIDVoices[i].voiceID; + output[1].row(i) = badIDVoices[i].logMag; + output[0].row(i) = badIDVoices[i].freq; + } + + for (index i = lowerSize; i < nVoices; ++i) + { + output[2].row(i) = -1; + output[1].row(i) = 0; + output[0].row(i) = 0; + } + + //output[2](Slice(0, lowerSize)) <<= mVoiceIDs(Slice(0, lowerSize)); + //output[1](Slice(0, lowerSize)) <<= mLogMags(Slice(0, lowerSize)); + //output[0](Slice(0, lowerSize)) <<= mFreqs(Slice(0, lowerSize)); + // + //if (unfilledVoices) + //{ + // index unfilledVoicesLength = nVoices - lowerSize; + // output[2](Slice(lowerSize, unfilledVoicesLength)).fill(-1); + // output[1](Slice(lowerSize, unfilledVoicesLength)).fill(0); + // output[0](Slice(lowerSize, unfilledVoicesLength)).fill(0); + //} } MessageResult clear() @@ -128,11 +196,13 @@ class VoiceAllocatorClient : public FluidBaseClient, private: // algorithm::RunningStats mAlgorithm; + vector mVoices; + algorithm::PartialTracking mTracking; index mInputSize; ParameterTrackChanges mSizeTracker; - FluidTensor mOut0; - FluidTensor mOut1; - FluidTensor mOut2; + FluidTensor mFreqs; + FluidTensor mLogMags; + FluidTensor mVoiceIDs; }; } // namespace voiceallocator From 6a9ae431dc247ace89a848f53d72a67595380a52 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 01:38:25 +0100 Subject: [PATCH 15/42] allocator algo somewhat implemented, compiles but weird behaviour --- include/clients/rt/VoiceAllocatorClient.hpp | 94 +++++++++++++++------ 1 file changed, 69 insertions(+), 25 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index c6f4e5ce8..043d83bb4 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -55,6 +55,8 @@ class VoiceAllocatorClient : public FluidBaseClient, { template using vector = rt::vector; + using VoicePeak = algorithm::VoicePeak; + using SinePeak = algorithm::SinePeak; public: using ParamDescType = decltype(VoiceAllocatorParams); @@ -76,8 +78,9 @@ class VoiceAllocatorClient : public FluidBaseClient, } VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) - : mParams(p), mVoices(c.allocator()), mTracking(c.allocator()), + : mParams(p), mTracking(c.allocator()), mInputSize{ 0 }, mSizeTracker{ 0 }, + mFreeVoices(), mActiveVoices(), mActiveVoiceData(0, c.allocator()), //todo - need allocator for queue/deque? mFreqs(get().max(), c.allocator()), mLogMags(get().max(), c.allocator()), mVoiceIDs(get().max(), c.allocator()) @@ -87,7 +90,15 @@ class VoiceAllocatorClient : public FluidBaseClient, setInputLabels({"frequencies", "magnitudes"}); setOutputLabels({"frequencies", "magnitudes", "voice IDs"}); mTracking.init(); - mVoices.clear(); + } + + void init(index nVoices) + { + if (!mActiveVoices.empty()) { mActiveVoices.pop_back(); } + if (!mFreeVoices.empty()) { mFreeVoices.pop(); } + for (index i = 0; i < nVoices; ++i) { mFreeVoices.push(i); } + mActiveVoiceData.resize(nVoices); + for (VoicePeak each : mActiveVoiceData) { each = { 0, 0, 0 }; } } template @@ -103,7 +114,7 @@ class VoiceAllocatorClient : public FluidBaseClient, { mInputSize = input[0].size(); controlChannelsOut({3, nVoices}); //update the dynamic out size - // other initialisation + init(nVoices); } index lowerSize; @@ -121,7 +132,7 @@ class VoiceAllocatorClient : public FluidBaseClient, //mOut1(Slice(0, lowerSize)) <<= input[1](Slice(0, lowerSize)); //mOut0(Slice(0, lowerSize)) <<= input[0](Slice(0, lowerSize)); - vector incomingVoices(0, c.allocator()); + vector incomingVoices(0, c.allocator()); for (index i = 0; i < lowerSize; ++i) { if (input[1].row(i) != 0 && input[0].row(i) != 0) @@ -132,40 +143,28 @@ class VoiceAllocatorClient : public FluidBaseClient, if (true) //change this to IF INPUT = TYPE MAGNITUDE, if dB skip { - for (algorithm::SinePeak voice : incomingVoices) + for (SinePeak voice : incomingVoices) { voice.logMag = 20 * log10(std::max(voice.logMag, algorithm::epsilon)); } } double maxAmp = -144; - for (algorithm::SinePeak voice : incomingVoices) + for (SinePeak voice : incomingVoices) { if (voice.logMag > maxAmp) { maxAmp = voice.logMag; } } mTracking.processFrame(incomingVoices, maxAmp, get(), get(), get(), get(), get(), get(), get(), c.allocator()); - vector badIDVoices(0, c.allocator()); - badIDVoices = mTracking.getActiveVoices(c.allocator()); + vector outgoingVoices(0, c.allocator()); + outgoingVoices = allocatorAlgorithm(mTracking.getActiveVoices(c.allocator()), nVoices, c.allocator()); - if (badIDVoices.size() < lowerSize) - { - lowerSize = badIDVoices.size(); - } - - for (index i = 0; i < lowerSize; ++i) + for (index i = 0; i < nVoices; ++i) { - output[2].row(i) = badIDVoices[i].voiceID; - output[1].row(i) = badIDVoices[i].logMag; - output[0].row(i) = badIDVoices[i].freq; - } - - for (index i = lowerSize; i < nVoices; ++i) - { - output[2].row(i) = -1; - output[1].row(i) = 0; - output[0].row(i) = 0; + output[2].row(i) = outgoingVoices[i].voiceID; + output[1].row(i) = outgoingVoices[i].logMag; + output[0].row(i) = outgoingVoices[i].freq; } //output[2](Slice(0, lowerSize)) <<= mVoiceIDs(Slice(0, lowerSize)); @@ -181,6 +180,49 @@ class VoiceAllocatorClient : public FluidBaseClient, //} } + vector allocatorAlgorithm(vector& incomingVoices, index nVoices, Allocator& alloc) + { + //handle existing voices - killing or sustaining + for (index existing = 0; existing < mActiveVoices.size(); ++existing) + { + bool killVoice = true; + for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) + { + //remove incoming voice events & allows corresponding voice to live if it already exists + if (mActiveVoiceData[mActiveVoices[existing]].voiceID == incomingVoices[incoming].voiceID) + { + killVoice = false; + incomingVoices.erase(incomingVoices.begin() + incoming); + break; + } + } + if (killVoice) //note off + { + mActiveVoiceData[mActiveVoices[existing]] = { 0, 0, 0 }; + mFreeVoices.push(mActiveVoices[existing]); + mActiveVoices.erase(mActiveVoices.begin() + existing); + } + } + + //handle new voice allocation + for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) + { + if (!mFreeVoices.empty()) + { + index newVoiceIndex = mFreeVoices.front(); + mFreeVoices.pop(); + mActiveVoices.push_back(newVoiceIndex); + mActiveVoiceData[newVoiceIndex] = incomingVoices[incoming]; + } + else //voice stealing + { + ; + } + } + + return mActiveVoiceData; + } + MessageResult clear() { // mAlgorithm.init(get<0>(),mInputSize); @@ -196,7 +238,9 @@ class VoiceAllocatorClient : public FluidBaseClient, private: // algorithm::RunningStats mAlgorithm; - vector mVoices; + std::queue mFreeVoices; + std::deque mActiveVoices; + vector mActiveVoiceData; algorithm::PartialTracking mTracking; index mInputSize; ParameterTrackChanges mSizeTracker; From 0a456015f9128ac062b507fa0428a0bcef6ebc63 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:07:59 +0100 Subject: [PATCH 16/42] past me apparently hoping a non-looped IF can clear a queue --- include/clients/rt/VoiceAllocatorClient.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 043d83bb4..5f0b9a118 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -94,8 +94,8 @@ class VoiceAllocatorClient : public FluidBaseClient, void init(index nVoices) { - if (!mActiveVoices.empty()) { mActiveVoices.pop_back(); } - if (!mFreeVoices.empty()) { mFreeVoices.pop(); } + while (!mActiveVoices.empty()) { mActiveVoices.pop_back(); } + while (!mFreeVoices.empty()) { mFreeVoices.pop(); } for (index i = 0; i < nVoices; ++i) { mFreeVoices.push(i); } mActiveVoiceData.resize(nVoices); for (VoicePeak each : mActiveVoiceData) { each = { 0, 0, 0 }; } From be7af8e0fbd8397c1ac381548f017f3f535b4112 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:36:10 +0100 Subject: [PATCH 17/42] proper conversion from amp to dB + convert back to amp --- include/algorithms/util/PartialTracking.hpp | 2 +- include/clients/rt/VoiceAllocatorClient.hpp | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/include/algorithms/util/PartialTracking.hpp b/include/algorithms/util/PartialTracking.hpp index 54329a939..32bdc4db9 100644 --- a/include/algorithms/util/PartialTracking.hpp +++ b/include/algorithms/util/PartialTracking.hpp @@ -155,7 +155,7 @@ class PartialTracking track.endFrame - track.startFrame < mMinTrackLength) continue; voicePeaks.push_back({track.peaks[asUnsigned(latencyFrame - track.startFrame)].freq, - track.peaks[asUnsigned(latencyFrame - track.startFrame)].logMag, + pow(10, track.peaks[asUnsigned(latencyFrame - track.startFrame)].logMag / 20), track.trackId}); } return voicePeaks; diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 5f0b9a118..3aec6a325 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -137,17 +137,20 @@ class VoiceAllocatorClient : public FluidBaseClient, { if (input[1].row(i) != 0 && input[0].row(i) != 0) { - incomingVoices.push_back({ input[0].row(i), input[1].row(i), false }); + incomingVoices.push_back({ input[0].row(i), + 20 * log10(std::max(static_cast(input[1].row(i)), algorithm::epsilon)), + false }); } } - if (true) //change this to IF INPUT = TYPE MAGNITUDE, if dB skip - { - for (SinePeak voice : incomingVoices) - { - voice.logMag = 20 * log10(std::max(voice.logMag, algorithm::epsilon)); - } - } + //if (true) //todo - change this to IF INPUT = TYPE MAGNITUDE, if dB skip + //{ + // for (SinePeak voice : incomingVoices) + // { + // //todo - doesn't actually do anything???? + // voice.logMag = 20 * log10(std::max(voice.logMag, algorithm::epsilon)); + // } + //} double maxAmp = -144; for (SinePeak voice : incomingVoices) From 61d6cb500a1d5f7d2334038e007681402e6789d7 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:38:50 +0100 Subject: [PATCH 18/42] removing mInputSize --- include/clients/rt/VoiceAllocatorClient.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 3aec6a325..eae677e39 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -79,7 +79,7 @@ class VoiceAllocatorClient : public FluidBaseClient, VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) : mParams(p), mTracking(c.allocator()), - mInputSize{ 0 }, mSizeTracker{ 0 }, + mSizeTracker{ 0 }, mFreeVoices(), mActiveVoices(), mActiveVoiceData(0, c.allocator()), //todo - need allocator for queue/deque? mFreqs(get().max(), c.allocator()), mLogMags(get().max(), c.allocator()), @@ -107,12 +107,8 @@ class VoiceAllocatorClient : public FluidBaseClient, { index nVoices = get(); - bool inputSizeChanged = mInputSize != input[0].size(); - bool sizeParamChanged = mSizeTracker.changed(nVoices); - - if (inputSizeChanged || sizeParamChanged) + if (mSizeTracker.changed(nVoices)) { - mInputSize = input[0].size(); controlChannelsOut({3, nVoices}); //update the dynamic out size init(nVoices); } From 4713d3cd2518b2087703a9d628c4d542257afd6b Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:39:38 +0100 Subject: [PATCH 19/42] comments --- include/clients/rt/VoiceAllocatorClient.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index eae677e39..5948d331b 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -195,7 +195,7 @@ class VoiceAllocatorClient : public FluidBaseClient, break; } } - if (killVoice) //note off + if (killVoice) //voice off { mActiveVoiceData[mActiveVoices[existing]] = { 0, 0, 0 }; mFreeVoices.push(mActiveVoices[existing]); @@ -206,7 +206,7 @@ class VoiceAllocatorClient : public FluidBaseClient, //handle new voice allocation for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) { - if (!mFreeVoices.empty()) + if (!mFreeVoices.empty()) //voice on { index newVoiceIndex = mFreeVoices.front(); mFreeVoices.pop(); From af2c89643b4b4ddf0b38d4fa023c2e745451f974 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:41:07 +0100 Subject: [PATCH 20/42] allows existing voices to change their freq/mag --- include/clients/rt/VoiceAllocatorClient.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 5948d331b..137ad6713 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -191,6 +191,7 @@ class VoiceAllocatorClient : public FluidBaseClient, if (mActiveVoiceData[mActiveVoices[existing]].voiceID == incomingVoices[incoming].voiceID) { killVoice = false; + mActiveVoiceData[mActiveVoices[existing]] = incomingVoices[incoming]; //update freq/mag incomingVoices.erase(incomingVoices.begin() + incoming); break; } From d12ae68d11d101c6392c387a1a5edd66cfbebb79 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:41:46 +0100 Subject: [PATCH 21/42] fully removed mInputSize --- include/clients/rt/VoiceAllocatorClient.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 137ad6713..0855f02c6 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -242,7 +242,6 @@ class VoiceAllocatorClient : public FluidBaseClient, std::deque mActiveVoices; vector mActiveVoiceData; algorithm::PartialTracking mTracking; - index mInputSize; ParameterTrackChanges mSizeTracker; FluidTensor mFreqs; FluidTensor mLogMags; From 51cde4d21de512ee1f2aaa3bc4a7c9d48bc8b4a6 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:42:27 +0100 Subject: [PATCH 22/42] fixed bug with allocator loop for removing voices ending too early --- include/clients/rt/VoiceAllocatorClient.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 0855f02c6..1eeb9cb4c 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -201,6 +201,7 @@ class VoiceAllocatorClient : public FluidBaseClient, mActiveVoiceData[mActiveVoices[existing]] = { 0, 0, 0 }; mFreeVoices.push(mActiveVoices[existing]); mActiveVoices.erase(mActiveVoices.begin() + existing); + --existing; } } From dca9460c3c30975f5167eda7636272aa0bae56e9 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:42:46 +0100 Subject: [PATCH 23/42] output proper sorted voiceID instead of partialTracking ID --- include/clients/rt/VoiceAllocatorClient.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 1eeb9cb4c..afd1121fd 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -161,7 +161,8 @@ class VoiceAllocatorClient : public FluidBaseClient, for (index i = 0; i < nVoices; ++i) { - output[2].row(i) = outgoingVoices[i].voiceID; + //output[3].row(i) = outgoingVoices[i].voiceID; + output[2].row(i) = i; output[1].row(i) = outgoingVoices[i].logMag; output[0].row(i) = outgoingVoices[i].freq; } From 15f6b415912fc44d8e0a6dd95759acfaa4d8e830 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 02:47:08 +0100 Subject: [PATCH 24/42] voice stealing comment --- include/clients/rt/VoiceAllocatorClient.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index afd1121fd..d02e16fd1 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -219,6 +219,8 @@ class VoiceAllocatorClient : public FluidBaseClient, else //voice stealing { ; + //probably would require a lot of refactoring, especially with how PartialTracking is currently used + //problem for future homer } } From ab1ccf09e6965715d2a3214441396be42aa7c7e5 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 14:03:56 +0100 Subject: [PATCH 25/42] states & stealing, imperfect --- include/algorithms/util/PartialTracking.hpp | 13 ++++- include/clients/rt/VoiceAllocatorClient.hpp | 64 ++++++++++++++------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/include/algorithms/util/PartialTracking.hpp b/include/algorithms/util/PartialTracking.hpp index 32bdc4db9..4e3228e54 100644 --- a/include/algorithms/util/PartialTracking.hpp +++ b/include/algorithms/util/PartialTracking.hpp @@ -25,6 +25,15 @@ Capability through Linear Programming". Proceedings of DAFx-2018. namespace fluid { namespace algorithm { +enum class VoiceState +{ + kFreeState, + kAttackState, + kSustainState, + kReleaseState, + kStolenState +}; + struct SinePeak { double freq; @@ -37,6 +46,7 @@ struct VoicePeak double freq; double logMag; index voiceID; + VoiceState state; }; struct SineTrack @@ -156,7 +166,8 @@ class PartialTracking continue; voicePeaks.push_back({track.peaks[asUnsigned(latencyFrame - track.startFrame)].freq, pow(10, track.peaks[asUnsigned(latencyFrame - track.startFrame)].logMag / 20), - track.trackId}); + track.trackId, + VoiceState::kAttackState}); } return voicePeaks; } diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index d02e16fd1..fe9ca4dcb 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -29,6 +29,7 @@ using HostVector = FluidTensorView; enum VoiceAllocatorParamIndex { kNVoices, + kStealMethod, kBirthLowThreshold, kBirthHighTreshold, kMinTrackLen, @@ -40,6 +41,7 @@ enum VoiceAllocatorParamIndex { constexpr auto VoiceAllocatorParams = defineParameters( LongParamRuntimeMax( "numVoices", "Number of Voices", 1, Min(1)), + EnumParam("stealMethod", "Voice Stealing Priority", 0, "Oldest", "Quietest"), FloatParam("birthLowThreshold", "Track Birth Low Frequency Threshold", -24, Min(-144), Max(0)), FloatParam("birthHighThreshold", "Track Birth High Frequency Threshold", -60, Min(-144), Max(0)), LongParam("minTrackLen", "Minimum Track Length", 1, Min(1)), @@ -88,7 +90,7 @@ class VoiceAllocatorClient : public FluidBaseClient, controlChannelsIn(2); controlChannelsOut({3, get(), get().max()}); setInputLabels({"frequencies", "magnitudes"}); - setOutputLabels({"frequencies", "magnitudes", "voice IDs"}); + setOutputLabels({"frequencies", "magnitudes", "state"}); mTracking.init(); } @@ -109,27 +111,26 @@ class VoiceAllocatorClient : public FluidBaseClient, if (mSizeTracker.changed(nVoices)) { - controlChannelsOut({3, nVoices}); //update the dynamic out size + controlChannelsOut({4, nVoices}); //update the dynamic out size init(nVoices); } - index lowerSize; - bool unfilledVoices = false; - if (input[0].size() >= nVoices) - { - lowerSize = nVoices; - } - else - { - lowerSize = input[0].size(); - unfilledVoices = true; - } + //index lowerSize; + //if (input[0].size() >= nVoices) + //{ + // lowerSize = nVoices; + //} + //else + //{ + // lowerSize = input[0].size(); + // unfilledVoices = true; + //} //mOut1(Slice(0, lowerSize)) <<= input[1](Slice(0, lowerSize)); //mOut0(Slice(0, lowerSize)) <<= input[0](Slice(0, lowerSize)); vector incomingVoices(0, c.allocator()); - for (index i = 0; i < lowerSize; ++i) + for (index i = 0; i < input[0].size(); ++i) { if (input[1].row(i) != 0 && input[0].row(i) != 0) { @@ -157,12 +158,11 @@ class VoiceAllocatorClient : public FluidBaseClient, mTracking.processFrame(incomingVoices, maxAmp, get(), get(), get(), get(), get(), get(), get(), c.allocator()); vector outgoingVoices(0, c.allocator()); - outgoingVoices = allocatorAlgorithm(mTracking.getActiveVoices(c.allocator()), nVoices, c.allocator()); + outgoingVoices = allocatorAlgorithm(mTracking.getActiveVoices(c.allocator()), get(), c.allocator()); for (index i = 0; i < nVoices; ++i) { - //output[3].row(i) = outgoingVoices[i].voiceID; - output[2].row(i) = i; + output[2].row(i) = static_cast(outgoingVoices[i].state); output[1].row(i) = outgoingVoices[i].logMag; output[0].row(i) = outgoingVoices[i].freq; } @@ -180,8 +180,15 @@ class VoiceAllocatorClient : public FluidBaseClient, //} } - vector allocatorAlgorithm(vector& incomingVoices, index nVoices, Allocator& alloc) + vector allocatorAlgorithm(vector& incomingVoices, bool stealQuietest, Allocator& alloc) { + //move released/stolen to free + for (index existing = 0; existing < mActiveVoiceData.size(); ++existing) + { + if (mActiveVoiceData[existing].state == algorithm::VoiceState::kReleaseState || mActiveVoiceData[existing].state == algorithm::VoiceState::kStolenState) + mActiveVoiceData[existing].state = algorithm::VoiceState::kFreeState; + } + //handle existing voices - killing or sustaining for (index existing = 0; existing < mActiveVoices.size(); ++existing) { @@ -193,13 +200,14 @@ class VoiceAllocatorClient : public FluidBaseClient, { killVoice = false; mActiveVoiceData[mActiveVoices[existing]] = incomingVoices[incoming]; //update freq/mag + mActiveVoiceData[mActiveVoices[existing]].state = algorithm::VoiceState::kSustainState; incomingVoices.erase(incomingVoices.begin() + incoming); break; } } if (killVoice) //voice off { - mActiveVoiceData[mActiveVoices[existing]] = { 0, 0, 0 }; + mActiveVoiceData[mActiveVoices[existing]].state = algorithm::VoiceState::kReleaseState; mFreeVoices.push(mActiveVoices[existing]); mActiveVoices.erase(mActiveVoices.begin() + existing); --existing; @@ -218,9 +226,21 @@ class VoiceAllocatorClient : public FluidBaseClient, } else //voice stealing { - ; - //probably would require a lot of refactoring, especially with how PartialTracking is currently used - //problem for future homer + index stolenVoiceIndex; + if (stealQuietest) + { + auto minElement = std::min_element(mActiveVoiceData.begin(), mActiveVoiceData.end(), [](const VoicePeak& voice1, const VoicePeak& voice2) { return voice1.logMag < voice2.logMag; }); + stolenVoiceIndex = std::distance(mActiveVoiceData.begin(), minElement); + mActiveVoices.erase(mActiveVoices.begin() + stolenVoiceIndex); + } + else //steal oldest + { + stolenVoiceIndex = mActiveVoices.front(); + mActiveVoices.pop_front(); + } + mActiveVoices.push_back(stolenVoiceIndex); + mActiveVoiceData[stolenVoiceIndex] = incomingVoices[incoming]; + mActiveVoiceData[stolenVoiceIndex].state = algorithm::VoiceState::kStolenState; } } From d938027af7e0205b4ddda6f288536effa3a428ba Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 18:28:24 +0100 Subject: [PATCH 26/42] proper init for queues/deques --- include/clients/rt/VoiceAllocatorClient.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index fe9ca4dcb..d03b969ea 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -82,7 +82,8 @@ class VoiceAllocatorClient : public FluidBaseClient, VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) : mParams(p), mTracking(c.allocator()), mSizeTracker{ 0 }, - mFreeVoices(), mActiveVoices(), mActiveVoiceData(0, c.allocator()), //todo - need allocator for queue/deque? + mFreeVoices(c.allocator()), mActiveVoices(c.allocator()), + mActiveVoiceData(0, c.allocator()), mFreqs(get().max(), c.allocator()), mLogMags(get().max(), c.allocator()), mVoiceIDs(get().max(), c.allocator()) @@ -262,8 +263,8 @@ class VoiceAllocatorClient : public FluidBaseClient, private: // algorithm::RunningStats mAlgorithm; - std::queue mFreeVoices; - std::deque mActiveVoices; + rt::queue mFreeVoices; + rt::deque mActiveVoices; vector mActiveVoiceData; algorithm::PartialTracking mTracking; ParameterTrackChanges mSizeTracker; From 2757d7a2b3a25b91639c03c464d8351cab02f5ed Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 18:28:52 +0100 Subject: [PATCH 27/42] add pruning --- include/clients/rt/VoiceAllocatorClient.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index d03b969ea..02f91a8d1 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -168,6 +168,8 @@ class VoiceAllocatorClient : public FluidBaseClient, output[0].row(i) = outgoingVoices[i].freq; } + mTracking.prune(); + //output[2](Slice(0, lowerSize)) <<= mVoiceIDs(Slice(0, lowerSize)); //output[1](Slice(0, lowerSize)) <<= mLogMags(Slice(0, lowerSize)); //output[0](Slice(0, lowerSize)) <<= mFreqs(Slice(0, lowerSize)); From f0b1d4a7c8ad006c6b5a008719afd6232fd3dbc2 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 18:29:17 +0100 Subject: [PATCH 28/42] readability --- include/clients/rt/VoiceAllocatorClient.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 02f91a8d1..901ce32c6 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -135,9 +135,8 @@ class VoiceAllocatorClient : public FluidBaseClient, { if (input[1].row(i) != 0 && input[0].row(i) != 0) { - incomingVoices.push_back({ input[0].row(i), - 20 * log10(std::max(static_cast(input[1].row(i)), algorithm::epsilon)), - false }); + double logMag = 20 * log10(std::max(static_cast(input[1].row(i)), algorithm::epsilon)); + incomingVoices.push_back({ input[0].row(i), logMag, false }); } } From 6360dd195d8ec8f7027da00ce12a7abd163d7126 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 18:31:04 +0100 Subject: [PATCH 29/42] init() call for clear messagess --- include/clients/rt/VoiceAllocatorClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 901ce32c6..3b1542dfd 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -251,7 +251,7 @@ class VoiceAllocatorClient : public FluidBaseClient, MessageResult clear() { - // mAlgorithm.init(get<0>(),mInputSize); + init(get()); return {}; } From af0938c46b9c3d1d78afae8962fb16c9db1c9d1b Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 18:34:47 +0100 Subject: [PATCH 30/42] replaced stealing with sorting --- include/clients/rt/VoiceAllocatorClient.hpp | 60 ++++++++++++++++----- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 3b1542dfd..61af6d760 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -29,7 +29,8 @@ using HostVector = FluidTensorView; enum VoiceAllocatorParamIndex { kNVoices, - kStealMethod, + kPrioritisedVoices, + //kStealMethod, kBirthLowThreshold, kBirthHighTreshold, kMinTrackLen, @@ -41,7 +42,8 @@ enum VoiceAllocatorParamIndex { constexpr auto VoiceAllocatorParams = defineParameters( LongParamRuntimeMax( "numVoices", "Number of Voices", 1, Min(1)), - EnumParam("stealMethod", "Voice Stealing Priority", 0, "Oldest", "Quietest"), + EnumParam("prioritisedVoices", "Prioritised Voice Quality", 0, "Loudest Magnitude", "Lowest Frequency"), + //EnumParam("stealMethod", "Voice Stealing Method", 0, "No Stealing", "Oldest", "Quietest"), FloatParam("birthLowThreshold", "Track Birth Low Frequency Threshold", -24, Min(-144), Max(0)), FloatParam("birthHighThreshold", "Track Birth High Frequency Threshold", -60, Min(-144), Max(0)), LongParam("minTrackLen", "Minimum Track Length", 1, Min(1)), @@ -158,7 +160,7 @@ class VoiceAllocatorClient : public FluidBaseClient, mTracking.processFrame(incomingVoices, maxAmp, get(), get(), get(), get(), get(), get(), get(), c.allocator()); vector outgoingVoices(0, c.allocator()); - outgoingVoices = allocatorAlgorithm(mTracking.getActiveVoices(c.allocator()), get(), c.allocator()); + outgoingVoices = allocatorAlgorithm(sortVoices(mTracking.getActiveVoices(c.allocator()), get()), c.allocator()); for (index i = 0; i < nVoices; ++i) { @@ -182,12 +184,31 @@ class VoiceAllocatorClient : public FluidBaseClient, //} } - vector allocatorAlgorithm(vector& incomingVoices, bool stealQuietest, Allocator& alloc) + vector sortVoices(vector& incomingVoices, index sortingMethod) { - //move released/stolen to free + //sortingMethod - 0 loudest - 1 lowest + switch (sortingMethod) + { + case 0: //loudest + std::sort(incomingVoices.begin(), incomingVoices.end(), + [](const VoicePeak& voice1, const VoicePeak& voice2) + { return voice1.logMag > voice2.logMag; }); + break; + case 1: //lowest + std::sort(incomingVoices.begin(), incomingVoices.end(), + [](const VoicePeak& voice1, const VoicePeak& voice2) + { return voice1.freq < voice2.freq; }); + break; + } + return incomingVoices; + } + + vector allocatorAlgorithm(vector& incomingVoices, Allocator& alloc) + { + //move released to free for (index existing = 0; existing < mActiveVoiceData.size(); ++existing) { - if (mActiveVoiceData[existing].state == algorithm::VoiceState::kReleaseState || mActiveVoiceData[existing].state == algorithm::VoiceState::kStolenState) + if (mActiveVoiceData[existing].state == algorithm::VoiceState::kReleaseState) mActiveVoiceData[existing].state = algorithm::VoiceState::kFreeState; } @@ -224,26 +245,39 @@ class VoiceAllocatorClient : public FluidBaseClient, index newVoiceIndex = mFreeVoices.front(); mFreeVoices.pop(); mActiveVoices.push_back(newVoiceIndex); + algorithm::VoiceState prevState = mActiveVoiceData[newVoiceIndex].state; mActiveVoiceData[newVoiceIndex] = incomingVoices[incoming]; + if (prevState == algorithm::VoiceState::kReleaseState) //mark as stolen + mActiveVoiceData[newVoiceIndex].state = algorithm::VoiceState::kStolenState; } - else //voice stealing + /*else //voice stealing { index stolenVoiceIndex; - if (stealQuietest) + if (stealingMethod == 1) //steal oldest + { + stolenVoiceIndex = mActiveVoices.front(); + mActiveVoices.pop_front(); + } + else if (stealingMethod == 2 && prioritisedVoices == 0) //steal quietest { - auto minElement = std::min_element(mActiveVoiceData.begin(), mActiveVoiceData.end(), [](const VoicePeak& voice1, const VoicePeak& voice2) { return voice1.logMag < voice2.logMag; }); + auto minElement = std::min_element(mActiveVoiceData.begin(), mActiveVoiceData.end(), + [](const VoicePeak& voice1, const VoicePeak& voice2) + { return voice1.logMag < voice2.logMag; }); stolenVoiceIndex = std::distance(mActiveVoiceData.begin(), minElement); mActiveVoices.erase(mActiveVoices.begin() + stolenVoiceIndex); } - else //steal oldest + else if (stealingMethod == 2 && prioritisedVoices == 1) //steal highest { - stolenVoiceIndex = mActiveVoices.front(); - mActiveVoices.pop_front(); + auto minElement = std::max_element(mActiveVoiceData.begin(), mActiveVoiceData.end(), + [](const VoicePeak& voice1, const VoicePeak& voice2) + { return voice1.freq > voice2.freq; }); + stolenVoiceIndex = std::distance(mActiveVoiceData.begin(), minElement); + mActiveVoices.erase(mActiveVoices.begin() + stolenVoiceIndex); } mActiveVoices.push_back(stolenVoiceIndex); mActiveVoiceData[stolenVoiceIndex] = incomingVoices[incoming]; mActiveVoiceData[stolenVoiceIndex].state = algorithm::VoiceState::kStolenState; - } + }*/ } return mActiveVoiceData; From c6c635163bd3fe901ad9ec366ff98ee64acc6309 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 21 Aug 2023 18:37:26 +0100 Subject: [PATCH 31/42] clearer layout of function calls in process + resize outgoingVoices --- include/clients/rt/VoiceAllocatorClient.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 61af6d760..d798799ab 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -160,7 +160,11 @@ class VoiceAllocatorClient : public FluidBaseClient, mTracking.processFrame(incomingVoices, maxAmp, get(), get(), get(), get(), get(), get(), get(), c.allocator()); vector outgoingVoices(0, c.allocator()); - outgoingVoices = allocatorAlgorithm(sortVoices(mTracking.getActiveVoices(c.allocator()), get()), c.allocator()); + outgoingVoices = mTracking.getActiveVoices(c.allocator()); + outgoingVoices = sortVoices(outgoingVoices, get()); + if (outgoingVoices.size() > nVoices) + outgoingVoices.resize(nVoices); + outgoingVoices = allocatorAlgorithm(outgoingVoices, c.allocator()); for (index i = 0; i < nVoices; ++i) { From 02c26a57bb6e36c037f7d721516fe016b6c46e9e Mon Sep 17 00:00:00 2001 From: fearne Date: Wed, 23 Aug 2023 13:18:22 +0100 Subject: [PATCH 32/42] temp removal of trackMethod param, switching to hung causes issues --- include/clients/rt/VoiceAllocatorClient.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index d798799ab..8fcedd6ff 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -34,7 +34,7 @@ enum VoiceAllocatorParamIndex { kBirthLowThreshold, kBirthHighTreshold, kMinTrackLen, - kTrackMethod, + //kTrackMethod, kTrackMagRange, kTrackFreqRange, kTrackProb @@ -47,7 +47,7 @@ constexpr auto VoiceAllocatorParams = defineParameters( FloatParam("birthLowThreshold", "Track Birth Low Frequency Threshold", -24, Min(-144), Max(0)), FloatParam("birthHighThreshold", "Track Birth High Frequency Threshold", -60, Min(-144), Max(0)), LongParam("minTrackLen", "Minimum Track Length", 1, Min(1)), - EnumParam("trackMethod", "Tracking Method", 0, "Greedy", "Hungarian"), + //EnumParam("trackMethod", "Tracking Method", 0, "Greedy", "Hungarian"), //changing this to hungarian currently spikes the output like crazy for a moment FloatParam("trackMagRange", "Tracking Magnitude Range (dB)", 15., Min(1.), Max(200.)), FloatParam("trackFreqRange", "Tracking Frequency Range (Hz)", 50., Min(1.), Max(10000.)), FloatParam("trackProb", "Tracking Matching Probability", 0.5, Min(0.0), Max(1.0)) @@ -157,7 +157,7 @@ class VoiceAllocatorClient : public FluidBaseClient, if (voice.logMag > maxAmp) { maxAmp = voice.logMag; } } - mTracking.processFrame(incomingVoices, maxAmp, get(), get(), get(), get(), get(), get(), get(), c.allocator()); + mTracking.processFrame(incomingVoices, maxAmp, get(), get(), get(), 0, get(), get(), get(), c.allocator()); vector outgoingVoices(0, c.allocator()); outgoingVoices = mTracking.getActiveVoices(c.allocator()); From 57a5051fa8a7c114fb5ac2a0defd21e9eb480874 Mon Sep 17 00:00:00 2001 From: fearne Date: Wed, 23 Aug 2023 13:18:48 +0100 Subject: [PATCH 33/42] consistency of an enumparam with other flucoma objects --- include/clients/rt/VoiceAllocatorClient.hpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 8fcedd6ff..bb047a566 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -42,7 +42,7 @@ enum VoiceAllocatorParamIndex { constexpr auto VoiceAllocatorParams = defineParameters( LongParamRuntimeMax( "numVoices", "Number of Voices", 1, Min(1)), - EnumParam("prioritisedVoices", "Prioritised Voice Quality", 0, "Loudest Magnitude", "Lowest Frequency"), + EnumParam("prioritisedVoices", "Prioritised Voice Quality", 0, "Lowest Frequency", "Loudest Magnitude"), //EnumParam("stealMethod", "Voice Stealing Method", 0, "No Stealing", "Oldest", "Quietest"), FloatParam("birthLowThreshold", "Track Birth Low Frequency Threshold", -24, Min(-144), Max(0)), FloatParam("birthHighThreshold", "Track Birth High Frequency Threshold", -60, Min(-144), Max(0)), @@ -190,18 +190,17 @@ class VoiceAllocatorClient : public FluidBaseClient, vector sortVoices(vector& incomingVoices, index sortingMethod) { - //sortingMethod - 0 loudest - 1 lowest switch (sortingMethod) { - case 0: //loudest + case 0: //lowest std::sort(incomingVoices.begin(), incomingVoices.end(), [](const VoicePeak& voice1, const VoicePeak& voice2) - { return voice1.logMag > voice2.logMag; }); + { return voice1.freq < voice2.freq; }); break; - case 1: //lowest + case 1: //loudest std::sort(incomingVoices.begin(), incomingVoices.end(), [](const VoicePeak& voice1, const VoicePeak& voice2) - { return voice1.freq < voice2.freq; }); + { return voice1.logMag > voice2.logMag; }); break; } return incomingVoices; From f7ddbf5c2eee8280c197094312a8b7b0415d271f Mon Sep 17 00:00:00 2001 From: fearne Date: Wed, 23 Aug 2023 16:18:37 +0100 Subject: [PATCH 34/42] removing random comments --- include/clients/rt/VoiceAllocatorClient.hpp | 63 --------------------- 1 file changed, 63 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index bb047a566..15afd3cc5 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -118,20 +118,6 @@ class VoiceAllocatorClient : public FluidBaseClient, init(nVoices); } - //index lowerSize; - //if (input[0].size() >= nVoices) - //{ - // lowerSize = nVoices; - //} - //else - //{ - // lowerSize = input[0].size(); - // unfilledVoices = true; - //} - - //mOut1(Slice(0, lowerSize)) <<= input[1](Slice(0, lowerSize)); - //mOut0(Slice(0, lowerSize)) <<= input[0](Slice(0, lowerSize)); - vector incomingVoices(0, c.allocator()); for (index i = 0; i < input[0].size(); ++i) { @@ -141,15 +127,6 @@ class VoiceAllocatorClient : public FluidBaseClient, incomingVoices.push_back({ input[0].row(i), logMag, false }); } } - - //if (true) //todo - change this to IF INPUT = TYPE MAGNITUDE, if dB skip - //{ - // for (SinePeak voice : incomingVoices) - // { - // //todo - doesn't actually do anything???? - // voice.logMag = 20 * log10(std::max(voice.logMag, algorithm::epsilon)); - // } - //} double maxAmp = -144; for (SinePeak voice : incomingVoices) @@ -174,18 +151,6 @@ class VoiceAllocatorClient : public FluidBaseClient, } mTracking.prune(); - - //output[2](Slice(0, lowerSize)) <<= mVoiceIDs(Slice(0, lowerSize)); - //output[1](Slice(0, lowerSize)) <<= mLogMags(Slice(0, lowerSize)); - //output[0](Slice(0, lowerSize)) <<= mFreqs(Slice(0, lowerSize)); - // - //if (unfilledVoices) - //{ - // index unfilledVoicesLength = nVoices - lowerSize; - // output[2](Slice(lowerSize, unfilledVoicesLength)).fill(-1); - // output[1](Slice(lowerSize, unfilledVoicesLength)).fill(0); - // output[0](Slice(lowerSize, unfilledVoicesLength)).fill(0); - //} } vector sortVoices(vector& incomingVoices, index sortingMethod) @@ -253,34 +218,6 @@ class VoiceAllocatorClient : public FluidBaseClient, if (prevState == algorithm::VoiceState::kReleaseState) //mark as stolen mActiveVoiceData[newVoiceIndex].state = algorithm::VoiceState::kStolenState; } - /*else //voice stealing - { - index stolenVoiceIndex; - if (stealingMethod == 1) //steal oldest - { - stolenVoiceIndex = mActiveVoices.front(); - mActiveVoices.pop_front(); - } - else if (stealingMethod == 2 && prioritisedVoices == 0) //steal quietest - { - auto minElement = std::min_element(mActiveVoiceData.begin(), mActiveVoiceData.end(), - [](const VoicePeak& voice1, const VoicePeak& voice2) - { return voice1.logMag < voice2.logMag; }); - stolenVoiceIndex = std::distance(mActiveVoiceData.begin(), minElement); - mActiveVoices.erase(mActiveVoices.begin() + stolenVoiceIndex); - } - else if (stealingMethod == 2 && prioritisedVoices == 1) //steal highest - { - auto minElement = std::max_element(mActiveVoiceData.begin(), mActiveVoiceData.end(), - [](const VoicePeak& voice1, const VoicePeak& voice2) - { return voice1.freq > voice2.freq; }); - stolenVoiceIndex = std::distance(mActiveVoiceData.begin(), minElement); - mActiveVoices.erase(mActiveVoices.begin() + stolenVoiceIndex); - } - mActiveVoices.push_back(stolenVoiceIndex); - mActiveVoiceData[stolenVoiceIndex] = incomingVoices[incoming]; - mActiveVoiceData[stolenVoiceIndex].state = algorithm::VoiceState::kStolenState; - }*/ } return mActiveVoiceData; From 1a536f41bbef856f8144f7bc3fdc193a44404635 Mon Sep 17 00:00:00 2001 From: fearne Date: Mon, 11 Sep 2023 01:43:59 +0100 Subject: [PATCH 35/42] algo-client file code split --- include/algorithms/public/VoiceAllocator.hpp | 148 +++++++++++++++++++ include/clients/rt/VoiceAllocatorClient.hpp | 136 ++--------------- 2 files changed, 162 insertions(+), 122 deletions(-) create mode 100644 include/algorithms/public/VoiceAllocator.hpp diff --git a/include/algorithms/public/VoiceAllocator.hpp b/include/algorithms/public/VoiceAllocator.hpp new file mode 100644 index 000000000..ba54a56d0 --- /dev/null +++ b/include/algorithms/public/VoiceAllocator.hpp @@ -0,0 +1,148 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#pragma once + +#include "../util/PartialTracking.hpp" + +namespace fluid { +namespace algorithm { + +class VoiceAllocator +{ + template + using vector = rt::vector; + +public: + VoiceAllocator(index nVoices, Allocator& alloc) + : mTracking(alloc), mVoices{ nVoices }, + mFreeVoices(alloc), mActiveVoices(alloc), + mActiveVoiceData(0, alloc) + {} + + void init(index nVoices, Allocator& alloc) + { + mVoices = nVoices; + while (!mActiveVoices.empty()) { mActiveVoices.pop_back(); } + while (!mFreeVoices.empty()) { mFreeVoices.pop(); } + for (index i = 0; i < nVoices; ++i) { mFreeVoices.push(i); } + mActiveVoiceData.resize(nVoices); + for (VoicePeak each : mActiveVoiceData) { each = { 0, 0, 0 }; } + mTracking.init(); + mInitialized = true; + } + + void processFrame(vector incomingVoices, vector& outgoingVoices, index minTrackLen, double birthLowTreshold, double birthHighTreshold, index trackMethod, double trackMagRange, double trackFreqRange, double trackProb, index sortMethod, Allocator& alloc) + { + assert(mInitialized); + + double maxAmp = -144; + for (const SinePeak& voice : incomingVoices) + { + if (voice.logMag > maxAmp) { maxAmp = voice.logMag; } + } + + mTracking.processFrame(incomingVoices, maxAmp, minTrackLen, birthLowTreshold, birthHighTreshold, trackMethod, trackMagRange, trackFreqRange, trackProb, alloc); + + outgoingVoices = mTracking.getActiveVoices(alloc); + outgoingVoices = sortVoices(outgoingVoices, sortMethod); + if (outgoingVoices.size() > mVoices) + outgoingVoices.resize(mVoices); + outgoingVoices = assignVoices(outgoingVoices, alloc); + + mTracking.prune(); + } + + void reset() { ; } + + bool initialized() const { return mInitialized; } + +private: + + vector sortVoices(vector& incomingVoices, index sortingMethod) + { + switch (sortingMethod) + { + case 0: //lowest + std::sort(incomingVoices.begin(), incomingVoices.end(), + [](const VoicePeak& voice1, const VoicePeak& voice2) + { return voice1.freq < voice2.freq; }); + break; + case 1: //loudest + std::sort(incomingVoices.begin(), incomingVoices.end(), + [](const VoicePeak& voice1, const VoicePeak& voice2) + { return voice1.logMag > voice2.logMag; }); + break; + } + return incomingVoices; + } + + vector assignVoices(vector& incomingVoices, Allocator& alloc) + { + //move released to free + for (index existing = 0; existing < mActiveVoiceData.size(); ++existing) + { + if (mActiveVoiceData[existing].state == algorithm::VoiceState::kReleaseState) + mActiveVoiceData[existing].state = algorithm::VoiceState::kFreeState; + } + + //handle existing voices - killing or sustaining + for (index existing = 0; existing < mActiveVoices.size(); ++existing) + { + bool killVoice = true; + for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) + { + //remove incoming voice events & allows corresponding voice to live if it already exists + if (mActiveVoiceData[mActiveVoices[existing]].voiceID == incomingVoices[incoming].voiceID) + { + killVoice = false; + mActiveVoiceData[mActiveVoices[existing]] = incomingVoices[incoming]; //update freq/mag + mActiveVoiceData[mActiveVoices[existing]].state = algorithm::VoiceState::kSustainState; + incomingVoices.erase(incomingVoices.begin() + incoming); + break; + } + } + if (killVoice) //voice off + { + mActiveVoiceData[mActiveVoices[existing]].state = algorithm::VoiceState::kReleaseState; + mFreeVoices.push(mActiveVoices[existing]); + mActiveVoices.erase(mActiveVoices.begin() + existing); + --existing; + } + } + + //handle new voice allocation + for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) + { + if (!mFreeVoices.empty()) //voice on + { + index newVoiceIndex = mFreeVoices.front(); + mFreeVoices.pop(); + mActiveVoices.push_back(newVoiceIndex); + algorithm::VoiceState prevState = mActiveVoiceData[newVoiceIndex].state; + mActiveVoiceData[newVoiceIndex] = incomingVoices[incoming]; + if (prevState == algorithm::VoiceState::kReleaseState) //mark as stolen + mActiveVoiceData[newVoiceIndex].state = algorithm::VoiceState::kStolenState; + } + } + + return mActiveVoiceData; + } + + PartialTracking mTracking; + index mVoices; + rt::queue mFreeVoices; + rt::deque mActiveVoices; + vector mActiveVoiceData; + + bool mInitialized{ false }; +}; +} // namespace algorithm +} // namespace fluid diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 15afd3cc5..7334fff54 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -18,7 +18,7 @@ under the European Union’s Horizon 2020 research and innovation programme #include "../common/ParameterTypes.hpp" // #include "../../algorithms/public/RunningStats.hpp" #include "../../data/TensorTypes.hpp" -#include "../../algorithms/util/PartialTracking.hpp" +#include "../../algorithms/public/VoiceAllocator.hpp" namespace fluid { namespace client { @@ -57,8 +57,6 @@ class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOut { - template - using vector = rt::vector; using VoicePeak = algorithm::VoicePeak; using SinePeak = algorithm::SinePeak; @@ -82,43 +80,30 @@ class VoiceAllocatorClient : public FluidBaseClient, } VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) - : mParams(p), mTracking(c.allocator()), - mSizeTracker{ 0 }, - mFreeVoices(c.allocator()), mActiveVoices(c.allocator()), - mActiveVoiceData(0, c.allocator()), - mFreqs(get().max(), c.allocator()), - mLogMags(get().max(), c.allocator()), - mVoiceIDs(get().max(), c.allocator()) + : mParams(p), mSizeTracker{ 0 }, + mVoiceAllocator(get(), c.allocator()) { controlChannelsIn(2); controlChannelsOut({3, get(), get().max()}); setInputLabels({"frequencies", "magnitudes"}); setOutputLabels({"frequencies", "magnitudes", "state"}); - mTracking.init(); - } - - void init(index nVoices) - { - while (!mActiveVoices.empty()) { mActiveVoices.pop_back(); } - while (!mFreeVoices.empty()) { mFreeVoices.pop(); } - for (index i = 0; i < nVoices; ++i) { mFreeVoices.push(i); } - mActiveVoiceData.resize(nVoices); - for (VoicePeak each : mActiveVoiceData) { each = { 0, 0, 0 }; } } template void process(std::vector>& input, std::vector>& output, FluidContext& c) { - index nVoices = get(); - - if (mSizeTracker.changed(nVoices)) + if (!input[0].data()) return; + if (!output[0].data() && !output[1].data()) return; + if (!mVoiceAllocator.initialized() || mSizeTracker.changed(get())) { - controlChannelsOut({4, nVoices}); //update the dynamic out size - init(nVoices); + controlChannelsOut({ 4, get() }); //update the dynamic out size + mVoiceAllocator.init(get(), c.allocator()); } - vector incomingVoices(0, c.allocator()); + rt::vector incomingVoices(0, c.allocator()); + rt::vector outgoingVoices(0, c.allocator()); + for (index i = 0; i < input[0].size(); ++i) { if (input[1].row(i) != 0 && input[0].row(i) != 0) @@ -128,104 +113,18 @@ class VoiceAllocatorClient : public FluidBaseClient, } } - double maxAmp = -144; - for (SinePeak voice : incomingVoices) - { - if (voice.logMag > maxAmp) { maxAmp = voice.logMag; } - } - - mTracking.processFrame(incomingVoices, maxAmp, get(), get(), get(), 0, get(), get(), get(), c.allocator()); + mVoiceAllocator.processFrame(incomingVoices, outgoingVoices, get(), get(), get(), 0, get(), get(), get(), get(), c.allocator()); - vector outgoingVoices(0, c.allocator()); - outgoingVoices = mTracking.getActiveVoices(c.allocator()); - outgoingVoices = sortVoices(outgoingVoices, get()); - if (outgoingVoices.size() > nVoices) - outgoingVoices.resize(nVoices); - outgoingVoices = allocatorAlgorithm(outgoingVoices, c.allocator()); - - for (index i = 0; i < nVoices; ++i) + for (index i = 0; i < static_cast(get()); ++i) { output[2].row(i) = static_cast(outgoingVoices[i].state); output[1].row(i) = outgoingVoices[i].logMag; output[0].row(i) = outgoingVoices[i].freq; } - - mTracking.prune(); - } - - vector sortVoices(vector& incomingVoices, index sortingMethod) - { - switch (sortingMethod) - { - case 0: //lowest - std::sort(incomingVoices.begin(), incomingVoices.end(), - [](const VoicePeak& voice1, const VoicePeak& voice2) - { return voice1.freq < voice2.freq; }); - break; - case 1: //loudest - std::sort(incomingVoices.begin(), incomingVoices.end(), - [](const VoicePeak& voice1, const VoicePeak& voice2) - { return voice1.logMag > voice2.logMag; }); - break; - } - return incomingVoices; - } - - vector allocatorAlgorithm(vector& incomingVoices, Allocator& alloc) - { - //move released to free - for (index existing = 0; existing < mActiveVoiceData.size(); ++existing) - { - if (mActiveVoiceData[existing].state == algorithm::VoiceState::kReleaseState) - mActiveVoiceData[existing].state = algorithm::VoiceState::kFreeState; - } - - //handle existing voices - killing or sustaining - for (index existing = 0; existing < mActiveVoices.size(); ++existing) - { - bool killVoice = true; - for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) - { - //remove incoming voice events & allows corresponding voice to live if it already exists - if (mActiveVoiceData[mActiveVoices[existing]].voiceID == incomingVoices[incoming].voiceID) - { - killVoice = false; - mActiveVoiceData[mActiveVoices[existing]] = incomingVoices[incoming]; //update freq/mag - mActiveVoiceData[mActiveVoices[existing]].state = algorithm::VoiceState::kSustainState; - incomingVoices.erase(incomingVoices.begin() + incoming); - break; - } - } - if (killVoice) //voice off - { - mActiveVoiceData[mActiveVoices[existing]].state = algorithm::VoiceState::kReleaseState; - mFreeVoices.push(mActiveVoices[existing]); - mActiveVoices.erase(mActiveVoices.begin() + existing); - --existing; - } - } - - //handle new voice allocation - for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) - { - if (!mFreeVoices.empty()) //voice on - { - index newVoiceIndex = mFreeVoices.front(); - mFreeVoices.pop(); - mActiveVoices.push_back(newVoiceIndex); - algorithm::VoiceState prevState = mActiveVoiceData[newVoiceIndex].state; - mActiveVoiceData[newVoiceIndex] = incomingVoices[incoming]; - if (prevState == algorithm::VoiceState::kReleaseState) //mark as stolen - mActiveVoiceData[newVoiceIndex].state = algorithm::VoiceState::kStolenState; - } - } - - return mActiveVoiceData; } MessageResult clear() { - init(get()); return {}; } @@ -237,15 +136,8 @@ class VoiceAllocatorClient : public FluidBaseClient, index latency() const { return 0; } private: - // algorithm::RunningStats mAlgorithm; - rt::queue mFreeVoices; - rt::deque mActiveVoices; - vector mActiveVoiceData; - algorithm::PartialTracking mTracking; + algorithm::VoiceAllocator mVoiceAllocator; ParameterTrackChanges mSizeTracker; - FluidTensor mFreqs; - FluidTensor mLogMags; - FluidTensor mVoiceIDs; }; } // namespace voiceallocator From 633ff8f7af87278fd7869973fd486d1e60da616d Mon Sep 17 00:00:00 2001 From: tremblap Date: Thu, 4 Apr 2024 13:41:41 +0100 Subject: [PATCH 36/42] implement reset/init --- include/algorithms/public/VoiceAllocator.hpp | 2 +- include/clients/rt/VoiceAllocatorClient.hpp | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/include/algorithms/public/VoiceAllocator.hpp b/include/algorithms/public/VoiceAllocator.hpp index ba54a56d0..21f3eb8f9 100644 --- a/include/algorithms/public/VoiceAllocator.hpp +++ b/include/algorithms/public/VoiceAllocator.hpp @@ -60,7 +60,7 @@ class VoiceAllocator mTracking.prune(); } - void reset() { ; } + void reset() {mInitialized = false;} bool initialized() const { return mInitialized; } diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 7334fff54..b26286be7 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -16,7 +16,6 @@ under the European Union’s Horizon 2020 research and innovation programme #include "../common/ParameterConstraints.hpp" #include "../common/ParameterSet.hpp" #include "../common/ParameterTypes.hpp" -// #include "../../algorithms/public/RunningStats.hpp" #include "../../data/TensorTypes.hpp" #include "../../algorithms/public/VoiceAllocator.hpp" @@ -30,11 +29,9 @@ using HostVector = FluidTensorView; enum VoiceAllocatorParamIndex { kNVoices, kPrioritisedVoices, - //kStealMethod, kBirthLowThreshold, kBirthHighTreshold, kMinTrackLen, - //kTrackMethod, kTrackMagRange, kTrackFreqRange, kTrackProb @@ -43,11 +40,9 @@ enum VoiceAllocatorParamIndex { constexpr auto VoiceAllocatorParams = defineParameters( LongParamRuntimeMax( "numVoices", "Number of Voices", 1, Min(1)), EnumParam("prioritisedVoices", "Prioritised Voice Quality", 0, "Lowest Frequency", "Loudest Magnitude"), - //EnumParam("stealMethod", "Voice Stealing Method", 0, "No Stealing", "Oldest", "Quietest"), FloatParam("birthLowThreshold", "Track Birth Low Frequency Threshold", -24, Min(-144), Max(0)), FloatParam("birthHighThreshold", "Track Birth High Frequency Threshold", -60, Min(-144), Max(0)), LongParam("minTrackLen", "Minimum Track Length", 1, Min(1)), - //EnumParam("trackMethod", "Tracking Method", 0, "Greedy", "Hungarian"), //changing this to hungarian currently spikes the output like crazy for a moment FloatParam("trackMagRange", "Tracking Magnitude Range (dB)", 15., Min(1.), Max(200.)), FloatParam("trackFreqRange", "Tracking Frequency Range (Hz)", 50., Min(1.), Max(10000.)), FloatParam("trackProb", "Tracking Matching Probability", 0.5, Min(0.0), Max(1.0)) @@ -125,6 +120,7 @@ class VoiceAllocatorClient : public FluidBaseClient, MessageResult clear() { + mVoiceAllocator.reset(); return {}; } From c0c14df470d42cb9fd6acba856e0b5ca0416b139 Mon Sep 17 00:00:00 2001 From: tremblap Date: Sat, 6 Apr 2024 11:37:10 +0100 Subject: [PATCH 37/42] instantiation initialisation was needed --- include/clients/rt/VoiceAllocatorClient.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index b26286be7..c9011e27c 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -81,7 +81,8 @@ class VoiceAllocatorClient : public FluidBaseClient, controlChannelsIn(2); controlChannelsOut({3, get(), get().max()}); setInputLabels({"frequencies", "magnitudes"}); - setOutputLabels({"frequencies", "magnitudes", "state"}); + setOutputLabels({"frequencies", "magnitudes", "states"}); + mVoiceAllocator.init(get(), c.allocator()); } template From c684d83e83c5d29b8d4423b3dc49b4778da47f37 Mon Sep 17 00:00:00 2001 From: tremblap Date: Sun, 7 Apr 2024 11:29:12 +0100 Subject: [PATCH 38/42] cleaned the order of initialisation --- include/clients/rt/VoiceAllocatorClient.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index c9011e27c..44087f115 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -75,8 +75,8 @@ class VoiceAllocatorClient : public FluidBaseClient, } VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) - : mParams(p), mSizeTracker{ 0 }, - mVoiceAllocator(get(), c.allocator()) + : mParams(p), mVoiceAllocator(get().max(), c.allocator()), + mSizeTracker{ 0 } { controlChannelsIn(2); controlChannelsOut({3, get(), get().max()}); From 618fd100a2841489be3a3272043b5759f1903acd Mon Sep 17 00:00:00 2001 From: tremblap Date: Wed, 17 Jul 2024 08:48:08 +0100 Subject: [PATCH 39/42] first working NRT adaptor --- .../clients/common/FluidNRTClientWrapper.hpp | 94 ++++++++++++++++++- include/clients/rt/VoiceAllocatorClient.hpp | 18 ++++ 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/include/clients/common/FluidNRTClientWrapper.hpp b/include/clients/common/FluidNRTClientWrapper.hpp index 25e58f1c2..f9c363896 100644 --- a/include/clients/common/FluidNRTClientWrapper.hpp +++ b/include/clients/common/FluidNRTClientWrapper.hpp @@ -117,6 +117,17 @@ struct IsControlOut> constexpr static bool value{std::is_base_of::value}; }; +template +struct IsControlIn +{ + constexpr static bool value = std::is_base_of::value; +}; + +template +struct IsControlIn> +{ + constexpr static bool value{std::is_base_of::value}; +}; template struct AddPadding @@ -126,11 +137,11 @@ struct AddPadding static constexpr bool HasFFT = impl::FilterTupleIndices::type::size() > 0; static constexpr bool HasControlOut = IsControlOut::value; - // static constexpr size_t value = HasControlOut? 2 : 1; + static constexpr bool HasControlIn = IsControlIn::value; static constexpr size_t value = HasFFT && HasControlOut ? 2 : HasFFT && !HasControlOut ? 1 - : !HasFFT && HasControlOut ? 3 + : !HasFFT && HasControlOut && !HasControlIn ? 3 : 0; @@ -659,6 +670,81 @@ struct StreamingControl } }; +////////////////////////////////////////////////////////////////////////////////////////////////////// +template +struct ControlControl +{ + template + static Result process(Client& client, InputList& inputBuffers, + OutputList& outputBuffers, index nFrames, index nChans, + std::pair userPadding, FluidContext& c) + { + // To account for process latency we need to copy the buffers with padding + std::vector inputData; + index maxFeatures = client.maxControlChannelsOut(); + + inputData.reserve(inputBuffers.size()); + + index startPadding = client.latency() + userPadding.first; + index totalPadding = startPadding + userPadding.first; + + index paddedLength = nFrames + totalPadding; + + std::fill_n(std::back_inserter(inputData), inputBuffers.size(), + HostMatrix(nChans, paddedLength)); + + std::vector outputData; + outputData.reserve(outputBuffers.size()); + std::fill_n(std::back_inserter(outputData), outputBuffers.size(), + HostMatrix(nChans * maxFeatures, paddedLength));// TODO: check padded behaviour for output + + double sampleRate{0}; + + // Copy input data (strangely by time series so we have to iterate later) + for (index i = 0; i < nChans; ++i) + { + for (index j = 0; j < asSigned(inputBuffers.size()); ++j) + { + BufferAdaptor::ReadAccess thisInput(inputBuffers[asUnsigned(j)].buffer); + if (i == 0 && j == 0) sampleRate = thisInput.sampleRate(); + inputData[asUnsigned(j)].row(i)(Slice(userPadding.first, nFrames)) <<= + thisInput.samps(inputBuffers[asUnsigned(j)].startFrame, nFrames, + inputBuffers[asUnsigned(j)].startChan + i); + } + } + + std::vector inputs(inputBuffers.size(), {nullptr, 0, 0}); + std::vector outputs(outputBuffers.size(), {nullptr, 0, 0}); + + // run the algorithm + client.reset(c); + + for (index i = 0; i < nFrames; ++i) // iterate each frame as time series + { + for (std::size_t j = 0; j < inputBuffers.size(); ++j) + inputs[j] = inputData[j].col(i); + for (std::size_t j = 0; j < outputBuffers.size(); ++j) + outputs[j] = outputData[j].col(i); + + client.process(inputs, outputs, c); + } + + // copy to outbuf + for (index i = 0; i < asSigned(outputBuffers.size()); ++i) + { + if (!outputBuffers[asUnsigned(i)]) continue; + BufferAdaptor::Access thisOutput(outputBuffers[asUnsigned(i)]); + Result r = thisOutput.resize(nFrames, nChans, sampleRate); + if (!r.ok()) return r; + for (index j = 0; j < nChans; ++j) + thisOutput.samps(j) <<= + outputData[asUnsigned(i)].row(j)(Slice(startPadding, nFrames)); + } + + return {}; + } +}; + ////////////////////////////////////////////////////////////////////////////////////////////////////// template @@ -740,6 +826,10 @@ using NRTControlAdaptor = impl::NRTClientWrapper; +template +using NRTDualControlAdaptor = + impl::NRTClientWrapper; ////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 44087f115..32e8d47a0 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -12,6 +12,7 @@ under the European Union’s Horizon 2020 research and innovation programme #include "../common/AudioClient.hpp" #include "../common/FluidBaseClient.hpp" +#include "../common/FluidNRTClientWrapper.hpp" #include "../common/FluidSource.hpp" #include "../common/ParameterConstraints.hpp" #include "../common/ParameterSet.hpp" @@ -125,6 +126,11 @@ class VoiceAllocatorClient : public FluidBaseClient, return {}; } + void reset(FluidContext&) + { + clear(); + } + static auto getMessageDescriptors() { return defineMessages(makeMessage("clear", &VoiceAllocatorClient::clear)); @@ -142,5 +148,17 @@ class VoiceAllocatorClient : public FluidBaseClient, using VoiceAllocatorClient = ClientWrapper; +auto constexpr NRTVoiceAllocatorParams = makeNRTParams( + InputBufferParam("frequencies", "Source F Buffer"), + InputBufferParam("magnitudes", "Source M Buffer"), + BufferParam("freqed", "dest f Buffer"), + BufferParam("magned", "dest m Buffer"), + BufferParam("voiced", "dest v Buffer") + ); + +using NRTVoiceAllocatorClient = NRTDualControlAdaptor; + +using NRTThreadedVoiceAllocatorClient = NRTThreadingAdaptor; } // namespace client } // namespace fluid From e15fd185ca328c177c11731ccec9c11583425753 Mon Sep 17 00:00:00 2001 From: tremblap Date: Wed, 17 Jul 2024 08:51:06 +0100 Subject: [PATCH 40/42] forgot the cmake --- FlucomaClients.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FlucomaClients.cmake b/FlucomaClients.cmake index 3d41c8dd0..18c96b880 100644 --- a/FlucomaClients.cmake +++ b/FlucomaClients.cmake @@ -118,6 +118,7 @@ add_client(BufThreadDemo clients/nrt/FluidThreadTestClient.hpp CLASS NRTThreaded add_client(BufThresh clients/nrt/BufThreshClient.hpp CLASS NRTThreadedBufferThreshClient ) add_client(BufTransientSlice clients/rt/TransientSliceClient.hpp CLASS NRTThreadedTransientSliceClient ) add_client(BufTransients clients/rt/TransientClient.hpp CLASS NRTThreadedTransientsClient ) +add_kr_in_client(BufVoiceAllocator clients/rt/VoiceAllocatorClient.hpp CLASS NRTThreadedVoiceAllocatorClient ) add_client(Chroma clients/rt/ChromaClient.hpp CLASS RTChromaClient ) add_client(Gain clients/rt/GainClient.hpp CLASS RTGainClient NOINSTALL) add_client(HPSS clients/rt/HPSSClient.hpp CLASS RTHPSSClient ) @@ -137,9 +138,9 @@ add_client(SineFeature clients/rt/SineFeatureClient.hpp CLASS RTSineFeatureClien add_client(Sines clients/rt/SinesClient.hpp CLASS RTSinesClient ) add_client(SpectralShape clients/rt/SpectralShapeClient.hpp CLASS RTSpectralShapeClient ) add_kr_in_client(Stats clients/rt/RunningStatsClient.hpp CLASS RunningStatsClient ) -add_kr_in_client(VoiceAllocator clients/rt/VoiceAllocatorClient.hpp CLASS VoiceAllocatorClient ) add_client(TransientSlice clients/rt/TransientSliceClient.hpp CLASS RTTransientSliceClient ) add_client(Transients clients/rt/TransientClient.hpp CLASS RTTransientClient ) +add_kr_in_client(VoiceAllocator clients/rt/VoiceAllocatorClient.hpp CLASS VoiceAllocatorClient ) #lib manipulation client group add_client(DataSet clients/nrt/DataSetClient.hpp CLASS NRTThreadedDataSetClient GROUP MANIPULATION) From 17a49b25d22604d9969eb58d57b6dc6f370fea5a Mon Sep 17 00:00:00 2001 From: tremblap Date: Wed, 17 Jul 2024 15:28:41 +0100 Subject: [PATCH 41/42] added task evolution --- include/clients/common/FluidNRTClientWrapper.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/clients/common/FluidNRTClientWrapper.hpp b/include/clients/common/FluidNRTClientWrapper.hpp index f9c363896..d27b2cadc 100644 --- a/include/clients/common/FluidNRTClientWrapper.hpp +++ b/include/clients/common/FluidNRTClientWrapper.hpp @@ -716,6 +716,8 @@ struct ControlControl std::vector inputs(inputBuffers.size(), {nullptr, 0, 0}); std::vector outputs(outputBuffers.size(), {nullptr, 0, 0}); + FluidTask* task = c.task(); + // run the algorithm client.reset(c); @@ -727,6 +729,11 @@ struct ControlControl outputs[j] = outputData[j].col(i); client.process(inputs, outputs, c); + + if (task && !task->processUpdate( + static_cast(i), + static_cast(nFrames))) + break; } // copy to outbuf From 536215d99d66df7156eeef375a89959313c674f4 Mon Sep 17 00:00:00 2001 From: tremblap Date: Tue, 27 Aug 2024 16:27:11 +0100 Subject: [PATCH 42/42] clang_formatted --- include/algorithms/public/VoiceAllocator.hpp | 89 +++++++++-------- include/algorithms/util/PartialTracking.hpp | 59 ++++++------ .../clients/common/FluidNRTClientWrapper.hpp | 90 +++++++++--------- include/clients/rt/VoiceAllocatorClient.hpp | 95 +++++++++++-------- 4 files changed, 179 insertions(+), 154 deletions(-) diff --git a/include/algorithms/public/VoiceAllocator.hpp b/include/algorithms/public/VoiceAllocator.hpp index 21f3eb8f9..93c99b98b 100644 --- a/include/algorithms/public/VoiceAllocator.hpp +++ b/include/algorithms/public/VoiceAllocator.hpp @@ -22,9 +22,8 @@ class VoiceAllocator public: VoiceAllocator(index nVoices, Allocator& alloc) - : mTracking(alloc), mVoices{ nVoices }, - mFreeVoices(alloc), mActiveVoices(alloc), - mActiveVoiceData(0, alloc) + : mTracking(alloc), mVoices{nVoices}, mFreeVoices(alloc), + mActiveVoices(alloc), mActiveVoiceData(0, alloc) {} void init(index nVoices, Allocator& alloc) @@ -34,12 +33,17 @@ class VoiceAllocator while (!mFreeVoices.empty()) { mFreeVoices.pop(); } for (index i = 0; i < nVoices; ++i) { mFreeVoices.push(i); } mActiveVoiceData.resize(nVoices); - for (VoicePeak each : mActiveVoiceData) { each = { 0, 0, 0 }; } + for (VoicePeak each : mActiveVoiceData) { each = {0, 0, 0}; } mTracking.init(); mInitialized = true; } - void processFrame(vector incomingVoices, vector& outgoingVoices, index minTrackLen, double birthLowTreshold, double birthHighTreshold, index trackMethod, double trackMagRange, double trackFreqRange, double trackProb, index sortMethod, Allocator& alloc) + void processFrame(vector incomingVoices, + vector& outgoingVoices, index minTrackLen, + double birthLowTreshold, double birthHighTreshold, + index trackMethod, double trackMagRange, + double trackFreqRange, double trackProb, index sortMethod, + Allocator& alloc) { assert(mInitialized); @@ -49,100 +53,111 @@ class VoiceAllocator if (voice.logMag > maxAmp) { maxAmp = voice.logMag; } } - mTracking.processFrame(incomingVoices, maxAmp, minTrackLen, birthLowTreshold, birthHighTreshold, trackMethod, trackMagRange, trackFreqRange, trackProb, alloc); + mTracking.processFrame(incomingVoices, maxAmp, minTrackLen, + birthLowTreshold, birthHighTreshold, trackMethod, + trackMagRange, trackFreqRange, trackProb, alloc); outgoingVoices = mTracking.getActiveVoices(alloc); outgoingVoices = sortVoices(outgoingVoices, sortMethod); - if (outgoingVoices.size() > mVoices) - outgoingVoices.resize(mVoices); + if (outgoingVoices.size() > mVoices) outgoingVoices.resize(mVoices); outgoingVoices = assignVoices(outgoingVoices, alloc); mTracking.prune(); } - void reset() {mInitialized = false;} + void reset() { mInitialized = false; } bool initialized() const { return mInitialized; } private: - - vector sortVoices(vector& incomingVoices, index sortingMethod) + vector sortVoices(vector& incomingVoices, + index sortingMethod) { switch (sortingMethod) { - case 0: //lowest + case 0: // lowest std::sort(incomingVoices.begin(), incomingVoices.end(), - [](const VoicePeak& voice1, const VoicePeak& voice2) - { return voice1.freq < voice2.freq; }); + [](const VoicePeak& voice1, const VoicePeak& voice2) { + return voice1.freq < voice2.freq; + }); break; - case 1: //loudest + case 1: // loudest std::sort(incomingVoices.begin(), incomingVoices.end(), - [](const VoicePeak& voice1, const VoicePeak& voice2) - { return voice1.logMag > voice2.logMag; }); + [](const VoicePeak& voice1, const VoicePeak& voice2) { + return voice1.logMag > voice2.logMag; + }); break; } return incomingVoices; } - vector assignVoices(vector& incomingVoices, Allocator& alloc) + vector assignVoices(vector& incomingVoices, + Allocator& alloc) { - //move released to free + // move released to free for (index existing = 0; existing < mActiveVoiceData.size(); ++existing) { - if (mActiveVoiceData[existing].state == algorithm::VoiceState::kReleaseState) + if (mActiveVoiceData[existing].state == + algorithm::VoiceState::kReleaseState) mActiveVoiceData[existing].state = algorithm::VoiceState::kFreeState; } - //handle existing voices - killing or sustaining + // handle existing voices - killing or sustaining for (index existing = 0; existing < mActiveVoices.size(); ++existing) { bool killVoice = true; for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) { - //remove incoming voice events & allows corresponding voice to live if it already exists - if (mActiveVoiceData[mActiveVoices[existing]].voiceID == incomingVoices[incoming].voiceID) + // remove incoming voice events & allows corresponding voice to live if + // it already exists + if (mActiveVoiceData[mActiveVoices[existing]].voiceID == + incomingVoices[incoming].voiceID) { killVoice = false; - mActiveVoiceData[mActiveVoices[existing]] = incomingVoices[incoming]; //update freq/mag - mActiveVoiceData[mActiveVoices[existing]].state = algorithm::VoiceState::kSustainState; + mActiveVoiceData[mActiveVoices[existing]] = + incomingVoices[incoming]; // update freq/mag + mActiveVoiceData[mActiveVoices[existing]].state = + algorithm::VoiceState::kSustainState; incomingVoices.erase(incomingVoices.begin() + incoming); break; } } - if (killVoice) //voice off + if (killVoice) // voice off { - mActiveVoiceData[mActiveVoices[existing]].state = algorithm::VoiceState::kReleaseState; + mActiveVoiceData[mActiveVoices[existing]].state = + algorithm::VoiceState::kReleaseState; mFreeVoices.push(mActiveVoices[existing]); mActiveVoices.erase(mActiveVoices.begin() + existing); --existing; } } - //handle new voice allocation + // handle new voice allocation for (index incoming = 0; incoming < incomingVoices.size(); ++incoming) { - if (!mFreeVoices.empty()) //voice on + if (!mFreeVoices.empty()) // voice on { index newVoiceIndex = mFreeVoices.front(); mFreeVoices.pop(); mActiveVoices.push_back(newVoiceIndex); algorithm::VoiceState prevState = mActiveVoiceData[newVoiceIndex].state; mActiveVoiceData[newVoiceIndex] = incomingVoices[incoming]; - if (prevState == algorithm::VoiceState::kReleaseState) //mark as stolen - mActiveVoiceData[newVoiceIndex].state = algorithm::VoiceState::kStolenState; + if (prevState == algorithm::VoiceState::kReleaseState) // mark as stolen + mActiveVoiceData[newVoiceIndex].state = + algorithm::VoiceState::kStolenState; } } return mActiveVoiceData; } - PartialTracking mTracking; - index mVoices; - rt::queue mFreeVoices; - rt::deque mActiveVoices; - vector mActiveVoiceData; + PartialTracking mTracking; + index mVoices; + rt::queue mFreeVoices; + rt::deque mActiveVoices; + vector mActiveVoiceData; - bool mInitialized{ false }; + bool mInitialized{false}; }; } // namespace algorithm } // namespace fluid diff --git a/include/algorithms/util/PartialTracking.hpp b/include/algorithms/util/PartialTracking.hpp index 4e3228e54..6ccfb3fa7 100644 --- a/include/algorithms/util/PartialTracking.hpp +++ b/include/algorithms/util/PartialTracking.hpp @@ -25,13 +25,12 @@ Capability through Linear Programming". Proceedings of DAFx-2018. namespace fluid { namespace algorithm { -enum class VoiceState -{ - kFreeState, - kAttackState, - kSustainState, - kReleaseState, - kStolenState +enum class VoiceState { + kFreeState, + kAttackState, + kSustainState, + kReleaseState, + kStolenState }; struct SinePeak @@ -43,10 +42,10 @@ struct SinePeak struct VoicePeak { - double freq; - double logMag; - index voiceID; - VoiceState state; + double freq; + double logMag; + index voiceID; + VoiceState state; }; struct SineTrack @@ -56,8 +55,8 @@ struct SineTrack SineTrack(rt::vector&& p, index s, index e, bool a, bool ass, index t) - : peaks{p}, startFrame{s}, endFrame{e}, active{a}, assigned{ass}, trackId{ - t} + : peaks{p}, startFrame{s}, endFrame{e}, active{a}, assigned{ass}, + trackId{t} {} rt::vector peaks; @@ -154,22 +153,24 @@ class PartialTracking // todo - refactor this function with the one above vector getActiveVoices(Allocator& alloc) { - vector voicePeaks(0, alloc); - index latencyFrame = mCurrentFrame - mMinTrackLength; - if (latencyFrame < 0) return voicePeaks; - for (auto&& track : mTracks) - { - if (track.startFrame > latencyFrame) continue; - if (track.endFrame >= 0 && track.endFrame <= latencyFrame) continue; - if (track.endFrame >= 0 && - track.endFrame - track.startFrame < mMinTrackLength) - continue; - voicePeaks.push_back({track.peaks[asUnsigned(latencyFrame - track.startFrame)].freq, - pow(10, track.peaks[asUnsigned(latencyFrame - track.startFrame)].logMag / 20), - track.trackId, - VoiceState::kAttackState}); - } - return voicePeaks; + vector voicePeaks(0, alloc); + index latencyFrame = mCurrentFrame - mMinTrackLength; + if (latencyFrame < 0) return voicePeaks; + for (auto&& track : mTracks) + { + if (track.startFrame > latencyFrame) continue; + if (track.endFrame >= 0 && track.endFrame <= latencyFrame) continue; + if (track.endFrame >= 0 && + track.endFrame - track.startFrame < mMinTrackLength) + continue; + voicePeaks.push_back( + {track.peaks[asUnsigned(latencyFrame - track.startFrame)].freq, + pow(10, + track.peaks[asUnsigned(latencyFrame - track.startFrame)].logMag / + 20), + track.trackId, VoiceState::kAttackState}); + } + return voicePeaks; } private: diff --git a/include/clients/common/FluidNRTClientWrapper.hpp b/include/clients/common/FluidNRTClientWrapper.hpp index d27b2cadc..3eefd870b 100644 --- a/include/clients/common/FluidNRTClientWrapper.hpp +++ b/include/clients/common/FluidNRTClientWrapper.hpp @@ -141,8 +141,9 @@ struct AddPadding static constexpr size_t value = HasFFT && HasControlOut ? 2 : HasFFT && !HasControlOut ? 1 - : !HasFFT && HasControlOut && !HasControlIn ? 3 - : 0; + : !HasFFT && HasControlOut && !HasControlIn + ? 3 + : 0; // static constexpr size_t value = std::conditional_t, StreamingControl>(); - + using ParamDescType = ParamType; using ParamSetType = ParameterSet; using ParamSetViewType = ParameterSetView; @@ -230,8 +231,8 @@ class NRTClientWrapper : public OfflineIn, public OfflineOut NRTClientWrapper(NRTClientWrapper&& x) - : mParams{std::move(x.mParams)}, - mNRTContext{std::move(x.mNRTContext)}, mClient{std::move(x.mClient)} + : mParams{std::move(x.mParams)}, mNRTContext{std::move(x.mNRTContext)}, + mClient{std::move(x.mClient)} { mRealTimeParams = RTParamSetViewType(RTClient::getParameterDescriptors(), @@ -592,12 +593,12 @@ struct StreamingControl std::fill_n(std::back_inserter(inputData), inputBuffers.size(), HostMatrix(nChans, paddedLength)); - std::vector outputData; + std::vector outputData; outputData.reserve(outputBuffers.size()); std::fill_n(std::back_inserter(outputData), outputBuffers.size(), HostMatrix(nChans * maxFeatures, nAnalysisFrames)); - double sampleRate{0}; + double sampleRate{0}; // Copy input data for (index i = 0; i < nChans; ++i) { @@ -630,10 +631,9 @@ struct StreamingControl inputData[asUnsigned(k)].row(i)(Slice(t, controlRate))); } - for(auto& out: outputData) + for (auto& out : outputData) { - outputs.push_back( - out.col(j)(Slice(i * maxFeatures, maxFeatures))); + outputs.push_back(out.col(j)(Slice(i * maxFeatures, maxFeatures))); } client.process(inputs, outputs, c); @@ -661,11 +661,11 @@ struct StreamingControl for (index i = 0; i < nFeatures; ++i) { for (index j = 0; j < nChans; ++j) - thisOutput.samps(i + j * nFeatures) <<= - outs.second->row(i + j * maxFeatures)(Slice(latencyHops, keepHops)); + thisOutput.samps(i + j * nFeatures) <<= outs.second->row( + i + j * maxFeatures)(Slice(latencyHops, keepHops)); } } - + return {}; } }; @@ -689,17 +689,19 @@ struct ControlControl index totalPadding = startPadding + userPadding.first; index paddedLength = nFrames + totalPadding; - + std::fill_n(std::back_inserter(inputData), inputBuffers.size(), HostMatrix(nChans, paddedLength)); std::vector outputData; outputData.reserve(outputBuffers.size()); - std::fill_n(std::back_inserter(outputData), outputBuffers.size(), - HostMatrix(nChans * maxFeatures, paddedLength));// TODO: check padded behaviour for output + std::fill_n( + std::back_inserter(outputData), outputBuffers.size(), + HostMatrix(nChans * maxFeatures, + paddedLength)); // TODO: check padded behaviour for output + + double sampleRate{0}; - double sampleRate{0}; - // Copy input data (strangely by time series so we have to iterate later) for (index i = 0; i < nChans; ++i) { @@ -713,13 +715,13 @@ struct ControlControl } } - std::vector inputs(inputBuffers.size(), {nullptr, 0, 0}); - std::vector outputs(outputBuffers.size(), {nullptr, 0, 0}); - - FluidTask* task = c.task(); + std::vector inputs(inputBuffers.size(), {nullptr, 0, 0}); + std::vector outputs(outputBuffers.size(), {nullptr, 0, 0}); + + FluidTask* task = c.task(); - // run the algorithm - client.reset(c); + // run the algorithm + client.reset(c); for (index i = 0; i < nFrames; ++i) // iterate each frame as time series { @@ -729,25 +731,24 @@ struct ControlControl outputs[j] = outputData[j].col(i); client.process(inputs, outputs, c); - - if (task && !task->processUpdate( - static_cast(i), - static_cast(nFrames))) - break; + + if (task && !task->processUpdate(static_cast(i), + static_cast(nFrames))) + break; } // copy to outbuf - for (index i = 0; i < asSigned(outputBuffers.size()); ++i) - { - if (!outputBuffers[asUnsigned(i)]) continue; - BufferAdaptor::Access thisOutput(outputBuffers[asUnsigned(i)]); - Result r = thisOutput.resize(nFrames, nChans, sampleRate); - if (!r.ok()) return r; - for (index j = 0; j < nChans; ++j) - thisOutput.samps(j) <<= - outputData[asUnsigned(i)].row(j)(Slice(startPadding, nFrames)); - } - + for (index i = 0; i < asSigned(outputBuffers.size()); ++i) + { + if (!outputBuffers[asUnsigned(i)]) continue; + BufferAdaptor::Access thisOutput(outputBuffers[asUnsigned(i)]); + Result r = thisOutput.resize(nFrames, nChans, sampleRate); + if (!r.ok()) return r; + for (index j = 0; j < nChans; ++j) + thisOutput.samps(j) <<= + outputData[asUnsigned(i)].row(j)(Slice(startPadding, nFrames)); + } + return {}; } }; @@ -1022,10 +1023,7 @@ class NRTThreadingAdaptor : public OfflineIn, public OfflineOut state = kDoneStillProcessing; mThreadedTask->mState = kDoneStillProcessing; } - else - { - mThreadedTask = nullptr; - } + else { mThreadedTask = nullptr; } } return state; @@ -1123,8 +1121,8 @@ class NRTThreadingAdaptor : public OfflineIn, public OfflineOut }; ThreadedTask(ClientPointer client, NRTJob& job, bool synchronous) - : mProcessParams(job.mParams), mState(kNoProcess), - mClient(client), mContext{mTask}, mCallback{job.mCallback} + : mProcessParams(job.mParams), mState(kNoProcess), mClient(client), + mContext{mTask}, mCallback{job.mCallback} { assert(mClient.get() != nullptr); // right? diff --git a/include/clients/rt/VoiceAllocatorClient.hpp b/include/clients/rt/VoiceAllocatorClient.hpp index 32e8d47a0..21eb50d46 100644 --- a/include/clients/rt/VoiceAllocatorClient.hpp +++ b/include/clients/rt/VoiceAllocatorClient.hpp @@ -17,8 +17,8 @@ under the European Union’s Horizon 2020 research and innovation programme #include "../common/ParameterConstraints.hpp" #include "../common/ParameterSet.hpp" #include "../common/ParameterTypes.hpp" -#include "../../data/TensorTypes.hpp" #include "../../algorithms/public/VoiceAllocator.hpp" +#include "../../data/TensorTypes.hpp" namespace fluid { namespace client { @@ -39,22 +39,27 @@ enum VoiceAllocatorParamIndex { }; constexpr auto VoiceAllocatorParams = defineParameters( - LongParamRuntimeMax( "numVoices", "Number of Voices", 1, Min(1)), - EnumParam("prioritisedVoices", "Prioritised Voice Quality", 0, "Lowest Frequency", "Loudest Magnitude"), - FloatParam("birthLowThreshold", "Track Birth Low Frequency Threshold", -24, Min(-144), Max(0)), - FloatParam("birthHighThreshold", "Track Birth High Frequency Threshold", -60, Min(-144), Max(0)), + LongParamRuntimeMax("numVoices", "Number of Voices", 1, Min(1)), + EnumParam("prioritisedVoices", "Prioritised Voice Quality", 0, + "Lowest Frequency", "Loudest Magnitude"), + FloatParam("birthLowThreshold", "Track Birth Low Frequency Threshold", -24, + Min(-144), Max(0)), + FloatParam("birthHighThreshold", "Track Birth High Frequency Threshold", + -60, Min(-144), Max(0)), LongParam("minTrackLen", "Minimum Track Length", 1, Min(1)), - FloatParam("trackMagRange", "Tracking Magnitude Range (dB)", 15., Min(1.), Max(200.)), - FloatParam("trackFreqRange", "Tracking Frequency Range (Hz)", 50., Min(1.), Max(10000.)), - FloatParam("trackProb", "Tracking Matching Probability", 0.5, Min(0.0), Max(1.0)) - ); + FloatParam("trackMagRange", "Tracking Magnitude Range (dB)", 15., Min(1.), + Max(200.)), + FloatParam("trackFreqRange", "Tracking Frequency Range (Hz)", 50., Min(1.), + Max(10000.)), + FloatParam("trackProb", "Tracking Matching Probability", 0.5, Min(0.0), + Max(1.0))); class VoiceAllocatorClient : public FluidBaseClient, public ControlIn, ControlOut { - using VoicePeak = algorithm::VoicePeak; - using SinePeak = algorithm::SinePeak; + using VoicePeak = algorithm::VoicePeak; + using SinePeak = algorithm::SinePeak; public: using ParamDescType = decltype(VoiceAllocatorParams); @@ -76,8 +81,8 @@ class VoiceAllocatorClient : public FluidBaseClient, } VoiceAllocatorClient(ParamSetViewType& p, FluidContext& c) - : mParams(p), mVoiceAllocator(get().max(), c.allocator()), - mSizeTracker{ 0 } + : mParams(p), mVoiceAllocator(get().max(), c.allocator()), + mSizeTracker{0} { controlChannelsIn(2); controlChannelsOut({3, get(), get().max()}); @@ -94,29 +99,35 @@ class VoiceAllocatorClient : public FluidBaseClient, if (!output[0].data() && !output[1].data()) return; if (!mVoiceAllocator.initialized() || mSizeTracker.changed(get())) { - controlChannelsOut({ 4, get() }); //update the dynamic out size + controlChannelsOut({4, get()}); // update the dynamic out size mVoiceAllocator.init(get(), c.allocator()); } - rt::vector incomingVoices(0, c.allocator()); + rt::vector incomingVoices(0, c.allocator()); rt::vector outgoingVoices(0, c.allocator()); for (index i = 0; i < input[0].size(); ++i) { - if (input[1].row(i) != 0 && input[0].row(i) != 0) - { - double logMag = 20 * log10(std::max(static_cast(input[1].row(i)), algorithm::epsilon)); - incomingVoices.push_back({ input[0].row(i), logMag, false }); - } + if (input[1].row(i) != 0 && input[0].row(i) != 0) + { + double logMag = + 20 * log10(std::max(static_cast(input[1].row(i)), + algorithm::epsilon)); + incomingVoices.push_back({input[0].row(i), logMag, false}); + } } - mVoiceAllocator.processFrame(incomingVoices, outgoingVoices, get(), get(), get(), 0, get(), get(), get(), get(), c.allocator()); + mVoiceAllocator.processFrame( + incomingVoices, outgoingVoices, get(), + get(), get(), 0, + get(), get(), get(), + get(), c.allocator()); for (index i = 0; i < static_cast(get()); ++i) { - output[2].row(i) = static_cast(outgoingVoices[i].state); - output[1].row(i) = outgoingVoices[i].logMag; - output[0].row(i) = outgoingVoices[i].freq; + output[2].row(i) = static_cast(outgoingVoices[i].state); + output[1].row(i) = outgoingVoices[i].logMag; + output[0].row(i) = outgoingVoices[i].freq; } } @@ -126,10 +137,7 @@ class VoiceAllocatorClient : public FluidBaseClient, return {}; } - void reset(FluidContext&) - { - clear(); - } + void reset(FluidContext&) { clear(); } static auto getMessageDescriptors() { @@ -139,8 +147,8 @@ class VoiceAllocatorClient : public FluidBaseClient, index latency() const { return 0; } private: - algorithm::VoiceAllocator mVoiceAllocator; - ParameterTrackChanges mSizeTracker; + algorithm::VoiceAllocator mVoiceAllocator; + ParameterTrackChanges mSizeTracker; }; } // namespace voiceallocator @@ -148,17 +156,20 @@ class VoiceAllocatorClient : public FluidBaseClient, using VoiceAllocatorClient = ClientWrapper; -auto constexpr NRTVoiceAllocatorParams = makeNRTParams( - InputBufferParam("frequencies", "Source F Buffer"), - InputBufferParam("magnitudes", "Source M Buffer"), - BufferParam("freqed", "dest f Buffer"), - BufferParam("magned", "dest m Buffer"), - BufferParam("voiced", "dest v Buffer") - ); - -using NRTVoiceAllocatorClient = NRTDualControlAdaptor; - -using NRTThreadedVoiceAllocatorClient = NRTThreadingAdaptor; +auto constexpr NRTVoiceAllocatorParams = + makeNRTParams( + InputBufferParam("frequencies", "Source F Buffer"), + InputBufferParam("magnitudes", "Source M Buffer"), + BufferParam("freqed", "dest f Buffer"), + BufferParam("magned", "dest m Buffer"), + BufferParam("voiced", "dest v Buffer")); + +using NRTVoiceAllocatorClient = + NRTDualControlAdaptor; + +using NRTThreadedVoiceAllocatorClient = + NRTThreadingAdaptor; } // namespace client } // namespace fluid