From 2762c0232f33768103be609fdb7b0d2ee02b7ee3 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 16 Nov 2020 11:03:30 -0600 Subject: [PATCH 01/21] rebasing to pre9 --- .../interface/TritonConverterBase.h | 35 ++++++++++++++++ .../SonicTriton/interface/TritonData.h | 17 ++++++++ .../SonicTriton/plugins/BuildFile.xml | 11 +++++ .../converters/FloatApFixed16Converter.cc | 40 +++++++++++++++++++ .../converters/FloatStandardConverter.cc | 20 ++++++++++ .../converters/Int64StandardConverter.cc | 20 ++++++++++ .../SonicTriton/src/TritonClient.cc | 7 ++++ .../SonicTriton/src/TritonData.cc | 34 +++++++++++++--- .../SonicTriton/src/pluginFactories.cc | 4 ++ .../SonicTriton/test/tritonTest_cfg.py | 3 ++ 10 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h create mode 100644 HeterogeneousCore/SonicTriton/plugins/BuildFile.xml create mode 100644 HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc create mode 100644 HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc create mode 100644 HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc create mode 100644 HeterogeneousCore/SonicTriton/src/pluginFactories.cc diff --git a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h new file mode 100644 index 0000000000000..2d6b45fa9a0c4 --- /dev/null +++ b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h @@ -0,0 +1,35 @@ +#ifndef __TritonConverterBase_H__ +#define __TritonConverterBase_H__ + +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "DataFormats/Common/interface/Handle.h" + +#include + +template +class TritonConverterBase { +public: + TritonConverterBase(const edm::ParameterSet& conf) : converterName_(conf.getParameter("converterName")), byteSize_(sizeof(DT)) {} + TritonConverterBase(const edm::ParameterSet& conf, size_t byteSize) : converterName_(conf.getParameter("converterName")), byteSize_(byteSize) {} + TritonConverterBase(const TritonConverterBase&) = delete; + virtual ~TritonConverterBase() = default; + TritonConverterBase& operator=(const TritonConverterBase&) = delete; + + virtual const uint8_t* convertIn(const DT* in) = 0; + virtual const DT* convertOut(const uint8_t* in) = 0; + + virtual const int64_t getByteSize() const { return byteSize_; } + + const std::string& name() const { return converterName_; } + +private: + const std::string converterName_; + const int64_t byteSize_; +}; + +#include "FWCore/PluginManager/interface/PluginFactory.h" + +template +using TritonConverterFactory = edmplugin::PluginFactory*(const edm::ParameterSet&)>; + +#endif diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 50808de4a1216..362106b80a8b9 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -4,6 +4,8 @@ #include "FWCore/Utilities/interface/Exception.h" #include "FWCore/Utilities/interface/Span.h" +#include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" + #include #include #include @@ -40,6 +42,13 @@ class TritonData { bool setShape(const ShapeType& newShape) { return setShape(newShape, true); } bool setShape(unsigned loc, int64_t val) { return setShape(loc, val, true); } + void setConverterParams(const edm::ParameterSet& conf) { + converterConf_ = conf; + converterName_ = conf.getParameter("converterName"); + } + template + std::unique_ptr> createConverter() const; + //io accessors template void toServer(std::shared_ptr> ptr); @@ -93,6 +102,8 @@ class TritonData { int64_t byteSize_; std::any holder_; std::shared_ptr result_; + edm::ParameterSet converterConf_; + std::string converterName_; }; using TritonInputData = TritonData; @@ -108,6 +119,12 @@ template <> template TritonOutput
TritonOutputData::fromServer() const; template <> +template +std::unique_ptr> TritonOutputData::createConverter() const; +template <> +template +std::unique_ptr> TritonInputData::createConverter() const; +template <> void TritonInputData::reset(); template <> void TritonOutputData::reset(); diff --git a/HeterogeneousCore/SonicTriton/plugins/BuildFile.xml b/HeterogeneousCore/SonicTriton/plugins/BuildFile.xml new file mode 100644 index 0000000000000..c72beb48f8274 --- /dev/null +++ b/HeterogeneousCore/SonicTriton/plugins/BuildFile.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc new file mode 100644 index 0000000000000..ddf0ba4b78dc3 --- /dev/null +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc @@ -0,0 +1,40 @@ +#include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" + +#include "ap_fixed.h" + +template +class FloatApFixed16Converter : public TritonConverterBase { +public: + FloatApFixed16Converter(const edm::ParameterSet& conf) + : TritonConverterBase(conf, 2) { + } + + const uint8_t* convertIn(const float* in) override; + const float* convertOut(const uint8_t* in) override; + +private: + + std::vector> makeVecIn(const float *in) { + unsigned int nfeat = sizeof(in)/sizeof(float); + std::vector> temp_storage(in, in + nfeat); + return temp_storage; + } + + std::vector makeVecOut(const ap_fixed<16,I> *in) { + unsigned int nfeat = sizeof(in)/sizeof(ap_fixed<16,I>); + std::vector temp_storage(in, in + nfeat); + return temp_storage; + } +}; + +DEFINE_EDM_PLUGIN(TritonConverterFactory, FloatApFixed16Converter<6>, "FloatApFixed16F6Converter"); + +template +const uint8_t* FloatApFixed16Converter::convertIn(const float *in) { + return reinterpret_cast((this->makeVecIn(in)).data()); +} + +template +const float* FloatApFixed16Converter::convertOut(const uint8_t* in) { + return (this->makeVecOut(reinterpret_cast*>(in))).data(); +} diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc new file mode 100644 index 0000000000000..a3a8d1850a882 --- /dev/null +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc @@ -0,0 +1,20 @@ +#include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" + +class FloatStandardConverter : public TritonConverterBase { +public: + FloatStandardConverter(const edm::ParameterSet& conf) + : TritonConverterBase(conf) {} + + const uint8_t* convertIn(const float* in) override; + const float* convertOut(const uint8_t* in) override; +}; + +DEFINE_EDM_PLUGIN(TritonConverterFactory, FloatStandardConverter, "FloatStandardConverter"); + +const uint8_t* FloatStandardConverter::convertIn(const float *in) { + return reinterpret_cast(in); +} + +const float* FloatStandardConverter::convertOut(const uint8_t* in) { + return reinterpret_cast(in); +} diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc new file mode 100644 index 0000000000000..9a237269d0f36 --- /dev/null +++ b/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc @@ -0,0 +1,20 @@ +#include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" + +class Int64StandardConverter : public TritonConverterBase { +public: + Int64StandardConverter(const edm::ParameterSet& conf) + : TritonConverterBase(conf) {} + + const uint8_t* convertIn(const int64_t* in) override; + const int64_t* convertOut(const uint8_t* in) override; +}; + +DEFINE_EDM_PLUGIN(TritonConverterFactory, Int64StandardConverter, "Int64StandardConverter"); + +const uint8_t* Int64StandardConverter::convertIn(const int64_t *in) { + return reinterpret_cast(in); +} + +const int64_t* Int64StandardConverter::convertOut(const uint8_t* in) { + return reinterpret_cast(in); +} diff --git a/HeterogeneousCore/SonicTriton/src/TritonClient.cc b/HeterogeneousCore/SonicTriton/src/TritonClient.cc index 98380e6546f4d..1b288d773ce3c 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonClient.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonClient.cc @@ -79,6 +79,7 @@ TritonClient::TritonClient(const edm::ParameterSet& params) if (!msg_str.empty()) throw cms::Exception("ModelErrors") << msg_str; + const edm::ParameterSet converterDefs = params.getParameterSet("converterDefinition"); //setup input map std::stringstream io_msg; if (verbose_) @@ -90,6 +91,7 @@ TritonClient::TritonClient(const edm::ParameterSet& params) auto [curr_itr, success] = input_.emplace( std::piecewise_construct, std::forward_as_tuple(iname), std::forward_as_tuple(iname, nicInput, noBatch_)); auto& curr_input = curr_itr->second; + curr_input.setConverterParams(converterDefs); inputsTriton_.push_back(curr_input.data()); if (verbose_) { io_msg << " " << iname << " (" << curr_input.dname() << ", " << curr_input.byteSize() @@ -113,6 +115,7 @@ TritonClient::TritonClient(const edm::ParameterSet& params) auto [curr_itr, success] = output_.emplace( std::piecewise_construct, std::forward_as_tuple(oname), std::forward_as_tuple(oname, nicOutput, noBatch_)); auto& curr_output = curr_itr->second; + curr_output.setConverterParams(converterDefs); outputsTriton_.push_back(curr_output.data()); if (verbose_) { io_msg << " " << oname << " (" << curr_output.dname() << ", " << curr_output.byteSize() @@ -336,10 +339,14 @@ inference::ModelStatistics TritonClient::getServerSideStatus() const { //for fillDescriptions void TritonClient::fillPSetDescription(edm::ParameterSetDescription& iDesc) { + edm::ParameterSetDescription descConverter; + fillBasePSetDescription(descConverter); + descConverter.add("converterName"); edm::ParameterSetDescription descClient; fillBasePSetDescription(descClient); descClient.add("modelName"); descClient.add("modelVersion", ""); + descClient.add("converterDefinition", descConverter); //server parameters should not affect the physics results descClient.addUntracked("batchSize"); descClient.addUntracked("address"); diff --git a/HeterogeneousCore/SonicTriton/src/TritonData.cc b/HeterogeneousCore/SonicTriton/src/TritonData.cc index 258671be07691..dea17a833c287 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonData.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonData.cc @@ -1,6 +1,8 @@ #include "HeterogeneousCore/SonicTriton/interface/TritonData.h" #include "HeterogeneousCore/SonicTriton/interface/triton_utils.h" +#include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/PluginManager/interface/PluginFactory.h" #include "model_config.pb.h" @@ -101,6 +103,18 @@ void TritonData::setBatchSize(unsigned bsize) { fullShape_[0] = batchSize_; } +template <> +template +std::unique_ptr> TritonInputData::createConverter() const { + return TritonConverterFactory
::get()->create(converterName_,converterConf_); +} + +template <> +template +std::unique_ptr> TritonOutputData::createConverter() const { + return TritonConverterFactory
::get()->create(converterName_,converterConf_); +} + //io accessors template <> template @@ -116,14 +130,16 @@ void TritonInputData::toServer(std::shared_ptr> ptr) { //shape must be specified for variable dims or if batch size changes data_->SetShape(fullShape_); - if (byteSize_ != sizeof(DT)) - throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << sizeof(DT) + std::unique_ptr> converter = createConverter
(); + + if (byteSize_ != converter->getByteSize()) + throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << converter->getByteSize() << " (should be " << byteSize_ << " for " << dname_ << ")"; int64_t nInput = sizeShape(); for (unsigned i0 = 0; i0 < batchSize_; ++i0) { const DT* arr = data_in[i0].data(); - triton_utils::throwIfError(data_->AppendRaw(reinterpret_cast(arr), nInput * byteSize_), + triton_utils::throwIfError(data_->AppendRaw(converter->convertIn(arr), nInput * byteSize_), name_ + " input(): unable to set data for batch entry " + std::to_string(i0)); } @@ -138,6 +154,8 @@ TritonOutput
TritonOutputData::fromServer() const { throw cms::Exception("TritonDataError") << name_ << " output(): missing result"; } + std::unique_ptr> converter = createConverter
(); + if (byteSize_ != sizeof(DT)) { throw cms::Exception("TritonDataError") << name_ << " output(): inconsistent byte size " << sizeof(DT) << " (should be " << byteSize_ << " for " << dname_ << ")"; @@ -147,14 +165,14 @@ TritonOutput
TritonOutputData::fromServer() const { TritonOutput
dataOut; const uint8_t* r0; size_t contentByteSize; - size_t expectedContentByteSize = nOutput * byteSize_ * batchSize_; + size_t expectedContentByteSize = nOutput * converter->getByteSize() * batchSize_; triton_utils::throwIfError(result_->RawData(name_, &r0, &contentByteSize), "output(): unable to get raw"); if (contentByteSize != expectedContentByteSize) { throw cms::Exception("TritonDataError") << name_ << " output(): unexpected content byte size " << contentByteSize << " (expected " << expectedContentByteSize << ")"; } - const DT* r1 = reinterpret_cast(r0); + const DT* r1 = converter->convertOut(r0); dataOut.reserve(batchSize_); for (unsigned i0 = 0; i0 < batchSize_; ++i0) { auto offset = i0 * nOutput; @@ -183,3 +201,9 @@ template void TritonInputData::toServer(std::shared_ptr> data template void TritonInputData::toServer(std::shared_ptr> data_in); template TritonOutput TritonOutputData::fromServer() const; + +template std::unique_ptr> TritonInputData::createConverter() const; +template std::unique_ptr> TritonInputData::createConverter() const; + +template std::unique_ptr> TritonOutputData::createConverter() const; +template std::unique_ptr> TritonOutputData::createConverter() const; diff --git a/HeterogeneousCore/SonicTriton/src/pluginFactories.cc b/HeterogeneousCore/SonicTriton/src/pluginFactories.cc new file mode 100644 index 0000000000000..101532a5de3c1 --- /dev/null +++ b/HeterogeneousCore/SonicTriton/src/pluginFactories.cc @@ -0,0 +1,4 @@ +#include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" + +EDM_REGISTER_PLUGINFACTORY(TritonConverterFactory, "TritonConverterFloatFactory"); +EDM_REGISTER_PLUGINFACTORY(TritonConverterFactory, "TritonConverterInt64Factory"); diff --git a/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py b/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py index 0646e527d16b0..1ea5a5c0a774a 100644 --- a/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py +++ b/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py @@ -49,6 +49,9 @@ modelVersion = cms.string(""), verbose = cms.untracked.bool(options.verbose), allowedTries = cms.untracked.uint32(0), + converterDefinition = cms.PSet( + converterName = cms.string("FloatStandardConverter"), + ), ) ) if options.producer=="TritonImageProducer": From 05f2ca0dc87e7e364712ccc42b864e26cf2fd5b7 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 16 Nov 2020 11:11:01 -0600 Subject: [PATCH 02/21] formatting and code checks --- .../interface/TritonConverterBase.h | 6 +++-- .../converters/FloatApFixed16Converter.cc | 27 +++++++++---------- .../converters/FloatStandardConverter.cc | 11 +++----- .../converters/Int64StandardConverter.cc | 11 +++----- .../SonicTriton/src/TritonClient.cc | 2 +- .../SonicTriton/src/TritonData.cc | 10 +++---- 6 files changed, 28 insertions(+), 39 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h index 2d6b45fa9a0c4..dc4cc1e1145c9 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h @@ -9,8 +9,10 @@ template class TritonConverterBase { public: - TritonConverterBase(const edm::ParameterSet& conf) : converterName_(conf.getParameter("converterName")), byteSize_(sizeof(DT)) {} - TritonConverterBase(const edm::ParameterSet& conf, size_t byteSize) : converterName_(conf.getParameter("converterName")), byteSize_(byteSize) {} + TritonConverterBase(const edm::ParameterSet& conf) + : converterName_(conf.getParameter("converterName")), byteSize_(sizeof(DT)) {} + TritonConverterBase(const edm::ParameterSet& conf, size_t byteSize) + : converterName_(conf.getParameter("converterName")), byteSize_(byteSize) {} TritonConverterBase(const TritonConverterBase&) = delete; virtual ~TritonConverterBase() = default; TritonConverterBase& operator=(const TritonConverterBase&) = delete; diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc index ddf0ba4b78dc3..9064ce0cbcd04 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc @@ -2,26 +2,23 @@ #include "ap_fixed.h" -template +template class FloatApFixed16Converter : public TritonConverterBase { public: - FloatApFixed16Converter(const edm::ParameterSet& conf) - : TritonConverterBase(conf, 2) { - } + FloatApFixed16Converter(const edm::ParameterSet& conf) : TritonConverterBase(conf, 2) {} const uint8_t* convertIn(const float* in) override; const float* convertOut(const uint8_t* in) override; private: - - std::vector> makeVecIn(const float *in) { - unsigned int nfeat = sizeof(in)/sizeof(float); - std::vector> temp_storage(in, in + nfeat); + std::vector> makeVecIn(const float* in) { + unsigned int nfeat = sizeof(in) / sizeof(float); + std::vector> temp_storage(in, in + nfeat); return temp_storage; } - std::vector makeVecOut(const ap_fixed<16,I> *in) { - unsigned int nfeat = sizeof(in)/sizeof(ap_fixed<16,I>); + std::vector makeVecOut(const ap_fixed<16, I>* in) { + unsigned int nfeat = sizeof(in) / sizeof(ap_fixed<16, I>); std::vector temp_storage(in, in + nfeat); return temp_storage; } @@ -29,12 +26,12 @@ class FloatApFixed16Converter : public TritonConverterBase { DEFINE_EDM_PLUGIN(TritonConverterFactory, FloatApFixed16Converter<6>, "FloatApFixed16F6Converter"); -template -const uint8_t* FloatApFixed16Converter::convertIn(const float *in) { - return reinterpret_cast((this->makeVecIn(in)).data()); +template +const uint8_t* FloatApFixed16Converter::convertIn(const float* in) { + return reinterpret_cast((this->makeVecIn(in)).data()); } -template +template const float* FloatApFixed16Converter::convertOut(const uint8_t* in) { - return (this->makeVecOut(reinterpret_cast*>(in))).data(); + return (this->makeVecOut(reinterpret_cast*>(in))).data(); } diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc index a3a8d1850a882..769c6b344c4b7 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc @@ -2,8 +2,7 @@ class FloatStandardConverter : public TritonConverterBase { public: - FloatStandardConverter(const edm::ParameterSet& conf) - : TritonConverterBase(conf) {} + FloatStandardConverter(const edm::ParameterSet& conf) : TritonConverterBase(conf) {} const uint8_t* convertIn(const float* in) override; const float* convertOut(const uint8_t* in) override; @@ -11,10 +10,6 @@ class FloatStandardConverter : public TritonConverterBase { DEFINE_EDM_PLUGIN(TritonConverterFactory, FloatStandardConverter, "FloatStandardConverter"); -const uint8_t* FloatStandardConverter::convertIn(const float *in) { - return reinterpret_cast(in); -} +const uint8_t* FloatStandardConverter::convertIn(const float* in) { return reinterpret_cast(in); } -const float* FloatStandardConverter::convertOut(const uint8_t* in) { - return reinterpret_cast(in); -} +const float* FloatStandardConverter::convertOut(const uint8_t* in) { return reinterpret_cast(in); } diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc index 9a237269d0f36..632762879a0f8 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc @@ -2,8 +2,7 @@ class Int64StandardConverter : public TritonConverterBase { public: - Int64StandardConverter(const edm::ParameterSet& conf) - : TritonConverterBase(conf) {} + Int64StandardConverter(const edm::ParameterSet& conf) : TritonConverterBase(conf) {} const uint8_t* convertIn(const int64_t* in) override; const int64_t* convertOut(const uint8_t* in) override; @@ -11,10 +10,6 @@ class Int64StandardConverter : public TritonConverterBase { DEFINE_EDM_PLUGIN(TritonConverterFactory, Int64StandardConverter, "Int64StandardConverter"); -const uint8_t* Int64StandardConverter::convertIn(const int64_t *in) { - return reinterpret_cast(in); -} +const uint8_t* Int64StandardConverter::convertIn(const int64_t* in) { return reinterpret_cast(in); } -const int64_t* Int64StandardConverter::convertOut(const uint8_t* in) { - return reinterpret_cast(in); -} +const int64_t* Int64StandardConverter::convertOut(const uint8_t* in) { return reinterpret_cast(in); } diff --git a/HeterogeneousCore/SonicTriton/src/TritonClient.cc b/HeterogeneousCore/SonicTriton/src/TritonClient.cc index 1b288d773ce3c..028d69c9ab1f5 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonClient.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonClient.cc @@ -79,7 +79,7 @@ TritonClient::TritonClient(const edm::ParameterSet& params) if (!msg_str.empty()) throw cms::Exception("ModelErrors") << msg_str; - const edm::ParameterSet converterDefs = params.getParameterSet("converterDefinition"); + const edm::ParameterSet& converterDefs = params.getParameterSet("converterDefinition"); //setup input map std::stringstream io_msg; if (verbose_) diff --git a/HeterogeneousCore/SonicTriton/src/TritonData.cc b/HeterogeneousCore/SonicTriton/src/TritonData.cc index dea17a833c287..3bb48c3897006 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonData.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonData.cc @@ -104,15 +104,15 @@ void TritonData::setBatchSize(unsigned bsize) { } template <> -template +template std::unique_ptr> TritonInputData::createConverter() const { - return TritonConverterFactory
::get()->create(converterName_,converterConf_); + return TritonConverterFactory
::get()->create(converterName_, converterConf_); } template <> -template +template std::unique_ptr> TritonOutputData::createConverter() const { - return TritonConverterFactory
::get()->create(converterName_,converterConf_); + return TritonConverterFactory
::get()->create(converterName_, converterConf_); } //io accessors @@ -130,7 +130,7 @@ void TritonInputData::toServer(std::shared_ptr> ptr) { //shape must be specified for variable dims or if batch size changes data_->SetShape(fullShape_); - std::unique_ptr> converter = createConverter
(); + std::unique_ptr> converter = createConverter
(); if (byteSize_ != converter->getByteSize()) throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << converter->getByteSize() From b4a3f2cbe3414a594ec51c020f8f4e116d31fc4d Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Fri, 20 Nov 2020 18:02:14 -0600 Subject: [PATCH 03/21] cleanup, removing unused PSet arg for converter --- HeterogeneousCore/SonicTriton/BuildFile.xml | 1 + .../interface/TritonConverterBase.h | 24 ++++++++++------- .../SonicTriton/interface/TritonData.h | 12 +++------ .../SonicTriton/plugins/BuildFile.xml | 14 +++------- .../converters/FloatApFixed16Converter.cc | 27 ++++++++----------- .../converters/FloatStandardConverter.cc | 12 +++------ .../converters/Int64StandardConverter.cc | 12 +++------ .../SonicTriton/src/TritonData.cc | 19 +++---------- 8 files changed, 44 insertions(+), 77 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/BuildFile.xml b/HeterogeneousCore/SonicTriton/BuildFile.xml index b574f395f4d12..1961d534af2ef 100644 --- a/HeterogeneousCore/SonicTriton/BuildFile.xml +++ b/HeterogeneousCore/SonicTriton/BuildFile.xml @@ -1,6 +1,7 @@ + diff --git a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h index dc4cc1e1145c9..30782a5cd1997 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h @@ -1,5 +1,5 @@ -#ifndef __TritonConverterBase_H__ -#define __TritonConverterBase_H__ +#ifndef HeterogeneousCore_SonicTriton_TritonConverterBase +#define HeterogeneousCore_SonicTriton_TritonConverterBase #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "DataFormats/Common/interface/Handle.h" @@ -8,19 +8,20 @@ template class TritonConverterBase { +//class needs to be templated since the convert functions require the data type, but need to also be virtual, and virtual member function templates are not allowed ion C++ public: - TritonConverterBase(const edm::ParameterSet& conf) - : converterName_(conf.getParameter("converterName")), byteSize_(sizeof(DT)) {} - TritonConverterBase(const edm::ParameterSet& conf, size_t byteSize) - : converterName_(conf.getParameter("converterName")), byteSize_(byteSize) {} + TritonConverterBase(const std::string convName) + : converterName_(convName), byteSize_(sizeof(DT)) {} + TritonConverterBase(const std::string convName, size_t byteSize) + : converterName_(convName), byteSize_(byteSize) {} TritonConverterBase(const TritonConverterBase&) = delete; virtual ~TritonConverterBase() = default; TritonConverterBase& operator=(const TritonConverterBase&) = delete; - virtual const uint8_t* convertIn(const DT* in) = 0; - virtual const DT* convertOut(const uint8_t* in) = 0; + virtual const uint8_t* convertIn (const DT* in) const = 0; + virtual const DT* convertOut (const uint8_t* in) const = 0; - virtual const int64_t getByteSize() const { return byteSize_; } + const int64_t byteSize() const { return byteSize_; } const std::string& name() const { return converterName_; } @@ -32,6 +33,9 @@ class TritonConverterBase { #include "FWCore/PluginManager/interface/PluginFactory.h" template -using TritonConverterFactory = edmplugin::PluginFactory*(const edm::ParameterSet&)>; +using TritonConverterFactory = edmplugin::PluginFactory*()>; + +#define DEFINE_TRITON_CONVERTER(input, type, name) DEFINE_EDM_PLUGIN(TritonConverterFactory, type, name) +#define DEFINE_TRITON_CONVERTER_SIMPLE(input, type) DEFINE_EDM_PLUGIN(TritonConverterFactory, type, #type) #endif diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 362106b80a8b9..f854d2ec37c76 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -4,6 +4,7 @@ #include "FWCore/Utilities/interface/Exception.h" #include "FWCore/Utilities/interface/Span.h" +#include "FWCore/PluginManager/interface/PluginFactory.h" #include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" #include @@ -43,11 +44,10 @@ class TritonData { bool setShape(unsigned loc, int64_t val) { return setShape(loc, val, true); } void setConverterParams(const edm::ParameterSet& conf) { - converterConf_ = conf; converterName_ = conf.getParameter("converterName"); } template - std::unique_ptr> createConverter() const; + std::unique_ptr> createConverter() const { return TritonConverterFactory
::get()->create(converterName_); } //io accessors template @@ -102,7 +102,7 @@ class TritonData { int64_t byteSize_; std::any holder_; std::shared_ptr result_; - edm::ParameterSet converterConf_; + std::any converter_; std::string converterName_; }; @@ -119,12 +119,6 @@ template <> template TritonOutput
TritonOutputData::fromServer() const; template <> -template -std::unique_ptr> TritonOutputData::createConverter() const; -template <> -template -std::unique_ptr> TritonInputData::createConverter() const; -template <> void TritonInputData::reset(); template <> void TritonOutputData::reset(); diff --git a/HeterogeneousCore/SonicTriton/plugins/BuildFile.xml b/HeterogeneousCore/SonicTriton/plugins/BuildFile.xml index c72beb48f8274..0427e58fdb43a 100644 --- a/HeterogeneousCore/SonicTriton/plugins/BuildFile.xml +++ b/HeterogeneousCore/SonicTriton/plugins/BuildFile.xml @@ -1,11 +1,5 @@ - - - - - - - - - - + + + + diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc index 9064ce0cbcd04..4e08801ec1024 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc @@ -1,37 +1,32 @@ #include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" +#include #include "ap_fixed.h" template class FloatApFixed16Converter : public TritonConverterBase { public: - FloatApFixed16Converter(const edm::ParameterSet& conf) : TritonConverterBase(conf, 2) {} + FloatApFixed16Converter() : TritonConverterBase("FloatApFixed16F"+std::to_string(I)+"Converter", 2) {} - const uint8_t* convertIn(const float* in) override; - const float* convertOut(const uint8_t* in) override; + const uint8_t* convertIn(const float* in) const { + return reinterpret_cast((this->makeVecIn(in)).data()); + } + const float* convertOut(const uint8_t* in) const { + return (this->makeVecOut(reinterpret_cast*>(in))).data(); + } private: - std::vector> makeVecIn(const float* in) { + std::vector> makeVecIn(const float* in) const { unsigned int nfeat = sizeof(in) / sizeof(float); std::vector> temp_storage(in, in + nfeat); return temp_storage; } - std::vector makeVecOut(const ap_fixed<16, I>* in) { + std::vector makeVecOut(const ap_fixed<16, I>* in) const { unsigned int nfeat = sizeof(in) / sizeof(ap_fixed<16, I>); std::vector temp_storage(in, in + nfeat); return temp_storage; } }; -DEFINE_EDM_PLUGIN(TritonConverterFactory, FloatApFixed16Converter<6>, "FloatApFixed16F6Converter"); - -template -const uint8_t* FloatApFixed16Converter::convertIn(const float* in) { - return reinterpret_cast((this->makeVecIn(in)).data()); -} - -template -const float* FloatApFixed16Converter::convertOut(const uint8_t* in) { - return (this->makeVecOut(reinterpret_cast*>(in))).data(); -} +DEFINE_TRITON_CONVERTER(float, FloatApFixed16Converter<6>, "FloatApFixed16F6Converter"); diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc index 769c6b344c4b7..1071d84a7266b 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc @@ -2,14 +2,10 @@ class FloatStandardConverter : public TritonConverterBase { public: - FloatStandardConverter(const edm::ParameterSet& conf) : TritonConverterBase(conf) {} + FloatStandardConverter() : TritonConverterBase("FloatStandardConverter") {} - const uint8_t* convertIn(const float* in) override; - const float* convertOut(const uint8_t* in) override; + const uint8_t* convertIn(const float* in) const { return reinterpret_cast(in); } + const float* convertOut(const uint8_t* in) const { return reinterpret_cast(in); } }; -DEFINE_EDM_PLUGIN(TritonConverterFactory, FloatStandardConverter, "FloatStandardConverter"); - -const uint8_t* FloatStandardConverter::convertIn(const float* in) { return reinterpret_cast(in); } - -const float* FloatStandardConverter::convertOut(const uint8_t* in) { return reinterpret_cast(in); } +DEFINE_TRITON_CONVERTER_SIMPLE(float, FloatStandardConverter); diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc index 632762879a0f8..2277b1bad47d1 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc @@ -2,14 +2,10 @@ class Int64StandardConverter : public TritonConverterBase { public: - Int64StandardConverter(const edm::ParameterSet& conf) : TritonConverterBase(conf) {} + Int64StandardConverter() : TritonConverterBase("Int64StandardConverter") {} - const uint8_t* convertIn(const int64_t* in) override; - const int64_t* convertOut(const uint8_t* in) override; + const uint8_t* convertIn(const int64_t* in) const { return reinterpret_cast(in); } + const int64_t* convertOut(const uint8_t* in) const { return reinterpret_cast(in); } }; -DEFINE_EDM_PLUGIN(TritonConverterFactory, Int64StandardConverter, "Int64StandardConverter"); - -const uint8_t* Int64StandardConverter::convertIn(const int64_t* in) { return reinterpret_cast(in); } - -const int64_t* Int64StandardConverter::convertOut(const uint8_t* in) { return reinterpret_cast(in); } +DEFINE_TRITON_CONVERTER_SIMPLE(int64_t, Int64StandardConverter); diff --git a/HeterogeneousCore/SonicTriton/src/TritonData.cc b/HeterogeneousCore/SonicTriton/src/TritonData.cc index 3bb48c3897006..1c4683e4626f9 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonData.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonData.cc @@ -2,7 +2,6 @@ #include "HeterogeneousCore/SonicTriton/interface/triton_utils.h" #include "HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "FWCore/PluginManager/interface/PluginFactory.h" #include "model_config.pb.h" @@ -103,18 +102,6 @@ void TritonData::setBatchSize(unsigned bsize) { fullShape_[0] = batchSize_; } -template <> -template -std::unique_ptr> TritonInputData::createConverter() const { - return TritonConverterFactory
::get()->create(converterName_, converterConf_); -} - -template <> -template -std::unique_ptr> TritonOutputData::createConverter() const { - return TritonConverterFactory
::get()->create(converterName_, converterConf_); -} - //io accessors template <> template @@ -132,8 +119,8 @@ void TritonInputData::toServer(std::shared_ptr> ptr) { std::unique_ptr> converter = createConverter
(); - if (byteSize_ != converter->getByteSize()) - throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << converter->getByteSize() + if (byteSize_ != converter->byteSize()) + throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << converter->byteSize() << " (should be " << byteSize_ << " for " << dname_ << ")"; int64_t nInput = sizeShape(); @@ -165,7 +152,7 @@ TritonOutput
TritonOutputData::fromServer() const { TritonOutput
dataOut; const uint8_t* r0; size_t contentByteSize; - size_t expectedContentByteSize = nOutput * converter->getByteSize() * batchSize_; + size_t expectedContentByteSize = nOutput * converter->byteSize() * batchSize_; triton_utils::throwIfError(result_->RawData(name_, &r0, &contentByteSize), "output(): unable to get raw"); if (contentByteSize != expectedContentByteSize) { throw cms::Exception("TritonDataError") << name_ << " output(): unexpected content byte size " << contentByteSize From 5bade57ec18e11532e2ef06193679feaf6665e6d Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 23 Nov 2020 17:59:26 -0600 Subject: [PATCH 04/21] using std::any to store converter --- .../SonicTriton/interface/TritonData.h | 6 ++++-- .../SonicTriton/src/TritonData.cc | 21 +++++++------------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index f854d2ec37c76..20f14a25a6a6c 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -47,7 +47,9 @@ class TritonData { converterName_ = conf.getParameter("converterName"); } template - std::unique_ptr> createConverter() const { return TritonConverterFactory
::get()->create(converterName_); } + void createConverter() const { + if (!converter_.has_value()) converter_ = std::shared_ptr>(TritonConverterFactory
::get()->create(converterName_)); + } //io accessors template @@ -102,7 +104,7 @@ class TritonData { int64_t byteSize_; std::any holder_; std::shared_ptr result_; - std::any converter_; + mutable std::any converter_; std::string converterName_; }; diff --git a/HeterogeneousCore/SonicTriton/src/TritonData.cc b/HeterogeneousCore/SonicTriton/src/TritonData.cc index 1c4683e4626f9..d7a3df83d6ba3 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonData.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonData.cc @@ -117,16 +117,16 @@ void TritonInputData::toServer(std::shared_ptr> ptr) { //shape must be specified for variable dims or if batch size changes data_->SetShape(fullShape_); - std::unique_ptr> converter = createConverter
(); + createConverter
(); - if (byteSize_ != converter->byteSize()) - throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << converter->byteSize() + if (byteSize_ != std::any_cast>>(converter_)->byteSize()) + throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << std::any_cast>>(converter_)->byteSize() << " (should be " << byteSize_ << " for " << dname_ << ")"; int64_t nInput = sizeShape(); for (unsigned i0 = 0; i0 < batchSize_; ++i0) { const DT* arr = data_in[i0].data(); - triton_utils::throwIfError(data_->AppendRaw(converter->convertIn(arr), nInput * byteSize_), + triton_utils::throwIfError(data_->AppendRaw(std::any_cast>>(converter_)->convertIn(arr), nInput * byteSize_), name_ + " input(): unable to set data for batch entry " + std::to_string(i0)); } @@ -141,7 +141,8 @@ TritonOutput
TritonOutputData::fromServer() const { throw cms::Exception("TritonDataError") << name_ << " output(): missing result"; } - std::unique_ptr> converter = createConverter
(); + createConverter
(); + //std::unique_ptr> converter = std::any_cast; if (byteSize_ != sizeof(DT)) { throw cms::Exception("TritonDataError") << name_ << " output(): inconsistent byte size " << sizeof(DT) @@ -152,14 +153,14 @@ TritonOutput
TritonOutputData::fromServer() const { TritonOutput
dataOut; const uint8_t* r0; size_t contentByteSize; - size_t expectedContentByteSize = nOutput * converter->byteSize() * batchSize_; + size_t expectedContentByteSize = nOutput * std::any_cast>>(converter_)->byteSize() * batchSize_; triton_utils::throwIfError(result_->RawData(name_, &r0, &contentByteSize), "output(): unable to get raw"); if (contentByteSize != expectedContentByteSize) { throw cms::Exception("TritonDataError") << name_ << " output(): unexpected content byte size " << contentByteSize << " (expected " << expectedContentByteSize << ")"; } - const DT* r1 = converter->convertOut(r0); + const DT* r1 = std::any_cast>>(converter_)->convertOut(r0); dataOut.reserve(batchSize_); for (unsigned i0 = 0; i0 < batchSize_; ++i0) { auto offset = i0 * nOutput; @@ -188,9 +189,3 @@ template void TritonInputData::toServer(std::shared_ptr> data template void TritonInputData::toServer(std::shared_ptr> data_in); template TritonOutput TritonOutputData::fromServer() const; - -template std::unique_ptr> TritonInputData::createConverter() const; -template std::unique_ptr> TritonInputData::createConverter() const; - -template std::unique_ptr> TritonOutputData::createConverter() const; -template std::unique_ptr> TritonOutputData::createConverter() const; From d9bf227739a3c9af7b1fc82e9385ace034e48801 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Tue, 24 Nov 2020 13:28:43 -0600 Subject: [PATCH 05/21] adjusting to handle possibility of multiple converters --- HeterogeneousCore/SonicTriton/interface/TritonData.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 20f14a25a6a6c..0d94e6465a0a2 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -48,7 +48,9 @@ class TritonData { } template void createConverter() const { - if (!converter_.has_value()) converter_ = std::shared_ptr>(TritonConverterFactory
::get()->create(converterName_)); + using ConverterType = std::shared_ptr>; + //this contruction catches bad any_cast without throwing std exception + if (auto ptr = std::any_cast(&converter_)) {} else { converter_ = ConverterType(TritonConverterFactory
::get()->create(converterName_)); } } //io accessors From 920ee4f8a17c43427d317e0cd0cadcc6a7ef3d7a Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Tue, 24 Nov 2020 13:39:24 -0600 Subject: [PATCH 06/21] fixing typo --- HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h index 30782a5cd1997..438d518806873 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h @@ -8,7 +8,7 @@ template class TritonConverterBase { -//class needs to be templated since the convert functions require the data type, but need to also be virtual, and virtual member function templates are not allowed ion C++ +//class needs to be templated since the convert functions require the data type, but need to also be virtual, and virtual member function templates are not allowed in C++ public: TritonConverterBase(const std::string convName) : converterName_(convName), byteSize_(sizeof(DT)) {} From 47fb54c38cf3c0bf6c67a4615715f6fb153bc997 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Tue, 24 Nov 2020 13:39:37 -0600 Subject: [PATCH 07/21] removing old comment --- HeterogeneousCore/SonicTriton/src/TritonData.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/HeterogeneousCore/SonicTriton/src/TritonData.cc b/HeterogeneousCore/SonicTriton/src/TritonData.cc index d7a3df83d6ba3..c6ae0128f65ac 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonData.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonData.cc @@ -142,7 +142,6 @@ TritonOutput
TritonOutputData::fromServer() const { } createConverter
(); - //std::unique_ptr> converter = std::any_cast; if (byteSize_ != sizeof(DT)) { throw cms::Exception("TritonDataError") << name_ << " output(): inconsistent byte size " << sizeof(DT) From 70a522b9ed51b53135c1bd4180a0d86d2efa7dfa Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Tue, 24 Nov 2020 13:52:14 -0600 Subject: [PATCH 08/21] fixing another typo --- HeterogeneousCore/SonicTriton/interface/TritonData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 0d94e6465a0a2..411ecad995a6c 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -49,7 +49,7 @@ class TritonData { template void createConverter() const { using ConverterType = std::shared_ptr>; - //this contruction catches bad any_cast without throwing std exception + //this construction catches bad any_cast without throwing std exception if (auto ptr = std::any_cast(&converter_)) {} else { converter_ = ConverterType(TritonConverterFactory
::get()->create(converterName_)); } } From 037c8b05037601ff944046fbb7120d965ceb07de Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 30 Nov 2020 14:22:37 -0600 Subject: [PATCH 09/21] adding clear for more complicated conversions --- .../interface/TritonConverterBase.h | 2 ++ .../SonicTriton/interface/TritonData.h | 27 ++++++++++++------- .../converters/FloatApFixed16Converter.cc | 16 +++++++++-- .../SonicTriton/src/TritonData.cc | 16 ++++++----- 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h index 438d518806873..dc21bb8a8a415 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonConverterBase.h @@ -25,6 +25,8 @@ class TritonConverterBase { const std::string& name() const { return converterName_; } + virtual void clear() const {} + private: const std::string converterName_; const int64_t byteSize_; diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 411ecad995a6c..ac681f3c3983a 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -43,16 +43,6 @@ class TritonData { bool setShape(const ShapeType& newShape) { return setShape(newShape, true); } bool setShape(unsigned loc, int64_t val) { return setShape(loc, val, true); } - void setConverterParams(const edm::ParameterSet& conf) { - converterName_ = conf.getParameter("converterName"); - } - template - void createConverter() const { - using ConverterType = std::shared_ptr>; - //this construction catches bad any_cast without throwing std exception - if (auto ptr = std::any_cast(&converter_)) {} else { converter_ = ConverterType(TritonConverterFactory
::get()->create(converterName_)); } - } - //io accessors template void toServer(std::shared_ptr> ptr); @@ -82,6 +72,22 @@ class TritonData { void setResult(std::shared_ptr result) { result_ = result; } IO* data() { return data_.get(); } + void setConverterParams(const edm::ParameterSet& conf) { + converterName_ = conf.getParameter("converterName"); + } + + template + std::shared_ptr> createConverter() const { + using ConverterType = std::shared_ptr>; + //this construction catches bad any_cast without throwing std exception + if (auto ptr = std::any_cast(&converter_)) { + } else { + converter_ = ConverterType(TritonConverterFactory
::get()->create(converterName_)); + converter_clear = std::bind(&TritonConverterBase
::clear, std::any_cast(converter_).get()); + } + return std::any_cast(converter_); + } + //helpers bool anyNeg(const ShapeView& vec) const { return std::any_of(vec.begin(), vec.end(), [](int64_t i) { return i < 0; }); @@ -108,6 +114,7 @@ class TritonData { std::shared_ptr result_; mutable std::any converter_; std::string converterName_; + mutable std::function converter_clear; }; using TritonInputData = TritonData; diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc index 4e08801ec1024..b6593edd7de6e 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc @@ -9,10 +9,19 @@ class FloatApFixed16Converter : public TritonConverterBase { FloatApFixed16Converter() : TritonConverterBase("FloatApFixed16F"+std::to_string(I)+"Converter", 2) {} const uint8_t* convertIn(const float* in) const { - return reinterpret_cast((this->makeVecIn(in)).data()); + auto temp_vec = std::make_shared>>(std::move(this->makeVecIn(in))); + inputHolder_.push_back(temp_vec); + return reinterpret_cast(temp_vec->data()); } const float* convertOut(const uint8_t* in) const { - return (this->makeVecOut(reinterpret_cast*>(in))).data(); + auto temp_vec = std::make_shared>(std::move(this->makeVecOut(reinterpret_cast*>(in)))); + outputHolder_.push_back(temp_vec); + return temp_vec->data(); + } + + void clear() const { + inputHolder_.clear(); + outputHolder_.clear(); } private: @@ -27,6 +36,9 @@ class FloatApFixed16Converter : public TritonConverterBase { std::vector temp_storage(in, in + nfeat); return temp_storage; } + + mutable std::vector>>> inputHolder_; + mutable std::vector>> outputHolder_; }; DEFINE_TRITON_CONVERTER(float, FloatApFixed16Converter<6>, "FloatApFixed16F6Converter"); diff --git a/HeterogeneousCore/SonicTriton/src/TritonData.cc b/HeterogeneousCore/SonicTriton/src/TritonData.cc index c6ae0128f65ac..8fd0d9479a578 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonData.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonData.cc @@ -117,16 +117,16 @@ void TritonInputData::toServer(std::shared_ptr> ptr) { //shape must be specified for variable dims or if batch size changes data_->SetShape(fullShape_); - createConverter
(); + auto converter = createConverter
(); - if (byteSize_ != std::any_cast>>(converter_)->byteSize()) - throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << std::any_cast>>(converter_)->byteSize() + if (byteSize_ != converter->byteSize()) + throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << converter->byteSize() << " (should be " << byteSize_ << " for " << dname_ << ")"; int64_t nInput = sizeShape(); for (unsigned i0 = 0; i0 < batchSize_; ++i0) { const DT* arr = data_in[i0].data(); - triton_utils::throwIfError(data_->AppendRaw(std::any_cast>>(converter_)->convertIn(arr), nInput * byteSize_), + triton_utils::throwIfError(data_->AppendRaw(converter->convertIn(arr), nInput * byteSize_), name_ + " input(): unable to set data for batch entry " + std::to_string(i0)); } @@ -141,7 +141,7 @@ TritonOutput
TritonOutputData::fromServer() const { throw cms::Exception("TritonDataError") << name_ << " output(): missing result"; } - createConverter
(); + auto converter = createConverter
(); if (byteSize_ != sizeof(DT)) { throw cms::Exception("TritonDataError") << name_ << " output(): inconsistent byte size " << sizeof(DT) @@ -152,14 +152,14 @@ TritonOutput
TritonOutputData::fromServer() const { TritonOutput
dataOut; const uint8_t* r0; size_t contentByteSize; - size_t expectedContentByteSize = nOutput * std::any_cast>>(converter_)->byteSize() * batchSize_; + size_t expectedContentByteSize = nOutput * converter->byteSize() * batchSize_; triton_utils::throwIfError(result_->RawData(name_, &r0, &contentByteSize), "output(): unable to get raw"); if (contentByteSize != expectedContentByteSize) { throw cms::Exception("TritonDataError") << name_ << " output(): unexpected content byte size " << contentByteSize << " (expected " << expectedContentByteSize << ")"; } - const DT* r1 = std::any_cast>>(converter_)->convertOut(r0); + const DT* r1 = converter->convertOut(r0); dataOut.reserve(batchSize_); for (unsigned i0 = 0; i0 < batchSize_; ++i0) { auto offset = i0 * nOutput; @@ -173,11 +173,13 @@ template <> void TritonInputData::reset() { data_->Reset(); holder_.reset(); + converter_clear(); } template <> void TritonOutputData::reset() { result_.reset(); + converter_clear(); } //explicit template instantiation declarations From 90ce2ed2ee68f7186fbe610af69d9d4fee7443a0 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Fri, 11 Dec 2020 13:49:51 -0600 Subject: [PATCH 10/21] adding override --- .../plugins/converters/FloatApFixed16Converter.cc | 6 +++--- .../plugins/converters/FloatStandardConverter.cc | 4 ++-- .../plugins/converters/Int64StandardConverter.cc | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc index b6593edd7de6e..62b97d405eac9 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatApFixed16Converter.cc @@ -8,18 +8,18 @@ class FloatApFixed16Converter : public TritonConverterBase { public: FloatApFixed16Converter() : TritonConverterBase("FloatApFixed16F"+std::to_string(I)+"Converter", 2) {} - const uint8_t* convertIn(const float* in) const { + const uint8_t* convertIn(const float* in) const override { auto temp_vec = std::make_shared>>(std::move(this->makeVecIn(in))); inputHolder_.push_back(temp_vec); return reinterpret_cast(temp_vec->data()); } - const float* convertOut(const uint8_t* in) const { + const float* convertOut(const uint8_t* in) const override { auto temp_vec = std::make_shared>(std::move(this->makeVecOut(reinterpret_cast*>(in)))); outputHolder_.push_back(temp_vec); return temp_vec->data(); } - void clear() const { + void clear() const override { inputHolder_.clear(); outputHolder_.clear(); } diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc index 1071d84a7266b..b2056bb85129f 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/FloatStandardConverter.cc @@ -4,8 +4,8 @@ class FloatStandardConverter : public TritonConverterBase { public: FloatStandardConverter() : TritonConverterBase("FloatStandardConverter") {} - const uint8_t* convertIn(const float* in) const { return reinterpret_cast(in); } - const float* convertOut(const uint8_t* in) const { return reinterpret_cast(in); } + const uint8_t* convertIn(const float* in) const override { return reinterpret_cast(in); } + const float* convertOut(const uint8_t* in) const override { return reinterpret_cast(in); } }; DEFINE_TRITON_CONVERTER_SIMPLE(float, FloatStandardConverter); diff --git a/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc b/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc index 2277b1bad47d1..8e1381834c369 100644 --- a/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc +++ b/HeterogeneousCore/SonicTriton/plugins/converters/Int64StandardConverter.cc @@ -4,8 +4,8 @@ class Int64StandardConverter : public TritonConverterBase { public: Int64StandardConverter() : TritonConverterBase("Int64StandardConverter") {} - const uint8_t* convertIn(const int64_t* in) const { return reinterpret_cast(in); } - const int64_t* convertOut(const uint8_t* in) const { return reinterpret_cast(in); } + const uint8_t* convertIn(const int64_t* in) const override { return reinterpret_cast(in); } + const int64_t* convertOut(const uint8_t* in) const override { return reinterpret_cast(in); } }; DEFINE_TRITON_CONVERTER_SIMPLE(int64_t, Int64StandardConverter); From e65347172608aee69dfd0342f91f431ab96f1a52 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Fri, 11 Dec 2020 13:50:24 -0600 Subject: [PATCH 11/21] fixing bytesize check --- HeterogeneousCore/SonicTriton/src/TritonData.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HeterogeneousCore/SonicTriton/src/TritonData.cc b/HeterogeneousCore/SonicTriton/src/TritonData.cc index 8fd0d9479a578..67a7fe22a5192 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonData.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonData.cc @@ -143,7 +143,7 @@ TritonOutput
TritonOutputData::fromServer() const { auto converter = createConverter
(); - if (byteSize_ != sizeof(DT)) { + if (byteSize_ != converter->byteSize()) { throw cms::Exception("TritonDataError") << name_ << " output(): inconsistent byte size " << sizeof(DT) << " (should be " << byteSize_ << " for " << dname_ << ")"; } From 6e336a37e4f507cbb228809564111f56c3d47310 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Fri, 11 Dec 2020 13:50:44 -0600 Subject: [PATCH 12/21] removing unnecessary line --- HeterogeneousCore/SonicTriton/src/TritonClient.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/HeterogeneousCore/SonicTriton/src/TritonClient.cc b/HeterogeneousCore/SonicTriton/src/TritonClient.cc index 028d69c9ab1f5..930641b331156 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonClient.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonClient.cc @@ -340,7 +340,6 @@ inference::ModelStatistics TritonClient::getServerSideStatus() const { //for fillDescriptions void TritonClient::fillPSetDescription(edm::ParameterSetDescription& iDesc) { edm::ParameterSetDescription descConverter; - fillBasePSetDescription(descConverter); descConverter.add("converterName"); edm::ParameterSetDescription descClient; fillBasePSetDescription(descClient); From 8ebdcd0dc8757a7581bf12f99372ed25421c46e2 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 14 Dec 2020 13:03:23 -0600 Subject: [PATCH 13/21] adding underscore, expanding to multiple converters for inputs and outputs separately --- .../SonicTriton/interface/TritonData.h | 4 +- .../SonicTriton/src/TritonClient.cc | 33 +++++++++++++--- .../SonicTriton/src/TritonData.cc | 4 +- .../SonicTriton/test/tritonTest_cfg.py | 39 +++++++++++++++++-- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index ac681f3c3983a..8e54e172ba9d5 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -83,7 +83,7 @@ class TritonData { if (auto ptr = std::any_cast(&converter_)) { } else { converter_ = ConverterType(TritonConverterFactory
::get()->create(converterName_)); - converter_clear = std::bind(&TritonConverterBase
::clear, std::any_cast(converter_).get()); + converter_clear_ = std::bind(&TritonConverterBase
::clear, std::any_cast(converter_).get()); } return std::any_cast(converter_); } @@ -114,7 +114,7 @@ class TritonData { std::shared_ptr result_; mutable std::any converter_; std::string converterName_; - mutable std::function converter_clear; + mutable std::function converter_clear_; }; using TritonInputData = TritonData; diff --git a/HeterogeneousCore/SonicTriton/src/TritonClient.cc b/HeterogeneousCore/SonicTriton/src/TritonClient.cc index 930641b331156..d53018adff6b9 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonClient.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonClient.cc @@ -79,7 +79,7 @@ TritonClient::TritonClient(const edm::ParameterSet& params) if (!msg_str.empty()) throw cms::Exception("ModelErrors") << msg_str; - const edm::ParameterSet& converterDefs = params.getParameterSet("converterDefinition"); + const std::vector& inputConverterDefs = params.getParameter>("inputConverters"); //setup input map std::stringstream io_msg; if (verbose_) @@ -91,7 +91,14 @@ TritonClient::TritonClient(const edm::ParameterSet& params) auto [curr_itr, success] = input_.emplace( std::piecewise_construct, std::forward_as_tuple(iname), std::forward_as_tuple(iname, nicInput, noBatch_)); auto& curr_input = curr_itr->second; - curr_input.setConverterParams(converterDefs); + bool foundConverter = false; + for (const auto converterDef : inputConverterDefs) { + if (converterDef.getParameter("inputName") == iname) { + curr_input.setConverterParams(converterDef); + foundConverter = true; + } + } + if (!foundConverter) throw cms::Exception("ModelErrors") << "No matching converter definition for input: " << iname; inputsTriton_.push_back(curr_input.data()); if (verbose_) { io_msg << " " << iname << " (" << curr_input.dname() << ", " << curr_input.byteSize() @@ -103,6 +110,8 @@ TritonClient::TritonClient(const edm::ParameterSet& params) const auto& v_outputs = params.getUntrackedParameter>("outputs"); std::unordered_set s_outputs(v_outputs.begin(), v_outputs.end()); + const std::vector& outputConverterDefs = params.getParameter>("outputConverters"); + //setup output map if (verbose_) io_msg << "Model outputs: " @@ -115,7 +124,14 @@ TritonClient::TritonClient(const edm::ParameterSet& params) auto [curr_itr, success] = output_.emplace( std::piecewise_construct, std::forward_as_tuple(oname), std::forward_as_tuple(oname, nicOutput, noBatch_)); auto& curr_output = curr_itr->second; - curr_output.setConverterParams(converterDefs); + bool foundConverter = false; + for (const auto converterDef : outputConverterDefs) { + if (converterDef.getParameter("outputName") == oname) { + curr_output.setConverterParams(converterDef); + foundConverter = true; + } + } + if (!foundConverter) throw cms::Exception("ModelErrors") << "No matching converter definition for output: " << oname; outputsTriton_.push_back(curr_output.data()); if (verbose_) { io_msg << " " << oname << " (" << curr_output.dname() << ", " << curr_output.byteSize() @@ -339,13 +355,18 @@ inference::ModelStatistics TritonClient::getServerSideStatus() const { //for fillDescriptions void TritonClient::fillPSetDescription(edm::ParameterSetDescription& iDesc) { - edm::ParameterSetDescription descConverter; - descConverter.add("converterName"); + edm::ParameterSetDescription descInConverter; + descInConverter.add("converterName"); + descInConverter.add("inputName"); + edm::ParameterSetDescription descOutConverter; + descOutConverter.add("converterName"); + descOutConverter.add("outputName"); edm::ParameterSetDescription descClient; fillBasePSetDescription(descClient); descClient.add("modelName"); descClient.add("modelVersion", ""); - descClient.add("converterDefinition", descConverter); + descClient.addVPSet("inputConverters", descInConverter); + descClient.addVPSet("outputConverters", descOutConverter); //server parameters should not affect the physics results descClient.addUntracked("batchSize"); descClient.addUntracked("address"); diff --git a/HeterogeneousCore/SonicTriton/src/TritonData.cc b/HeterogeneousCore/SonicTriton/src/TritonData.cc index 67a7fe22a5192..a885aa0cc45fb 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonData.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonData.cc @@ -173,13 +173,13 @@ template <> void TritonInputData::reset() { data_->Reset(); holder_.reset(); - converter_clear(); + converter_clear_(); } template <> void TritonOutputData::reset() { result_.reset(); - converter_clear(); + converter_clear_(); } //explicit template instantiation declarations diff --git a/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py b/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py index 1ea5a5c0a774a..eba943103e9f6 100644 --- a/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py +++ b/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py @@ -29,6 +29,40 @@ "TritonGraphProducer": "gat_test", } +inConvs = { + "TritonImageProducer": cms.VPSet( + cms.PSet( + converterName = cms.string("FloatStandardConverter"), + inputName = cms.string("gpu_0/data"), + ), + ), + "TritonGraphProducer": cms.VPSet( + cms.PSet( + converterName = cms.string("FloatStandardConverter"), + inputName = cms.string("x__0"), + ), + cms.PSet( + converterName = cms.string("Int64StandardConverter"), + inputName = cms.string("edgeindex__1"), + ), + ), +} + +outConvs = { + "TritonImageProducer": cms.VPSet( + cms.PSet( + converterName = cms.string("FloatStandardConverter"), + outputName = cms.string("gpu_0/softmax"), + ), + ), + "TritonGraphProducer": cms.VPSet( + cms.PSet( + converterName = cms.string("FloatStandardConverter"), + outputName = cms.string("logits__0"), + ), + ), +} + if options.producer not in models: raise ValueError("Unknown producer: "+options.producer) @@ -49,9 +83,8 @@ modelVersion = cms.string(""), verbose = cms.untracked.bool(options.verbose), allowedTries = cms.untracked.uint32(0), - converterDefinition = cms.PSet( - converterName = cms.string("FloatStandardConverter"), - ), + inputConverters = inConvs[options.producer], + outputConverters = outConvs[options.producer], ) ) if options.producer=="TritonImageProducer": From 15c99a420fd3443ef55613ee8b5484ad96efc96e Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 14 Dec 2020 17:25:26 -0600 Subject: [PATCH 14/21] setting up default behavior --- .../SonicTriton/interface/TritonData.h | 15 ++++++- .../SonicTriton/src/TritonClient.cc | 40 ++++++++++--------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 8e54e172ba9d5..ccb05d6fbc3f9 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -61,6 +61,17 @@ class TritonData { //default to dims if shape isn't filled int64_t sizeShape() const { return variableDims_ ? dimProduct(shape_) : sizeDims(); } + std::string defaultConverter() const { + std::string base = "StandardConverter"; + if (dname_ == "INT64") { + return "Int64"+base; + } + else if (dname_ == "FP32") { + return "Float"+base; + } + return ""; + } + private: friend class TritonClient; @@ -72,8 +83,8 @@ class TritonData { void setResult(std::shared_ptr result) { result_ = result; } IO* data() { return data_.get(); } - void setConverterParams(const edm::ParameterSet& conf) { - converterName_ = conf.getParameter("converterName"); + void setConverterParams(std::string convName) { + converterName_ = convName; } template diff --git a/HeterogeneousCore/SonicTriton/src/TritonClient.cc b/HeterogeneousCore/SonicTriton/src/TritonClient.cc index d53018adff6b9..b3784876f608c 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonClient.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonClient.cc @@ -79,7 +79,12 @@ TritonClient::TritonClient(const edm::ParameterSet& params) if (!msg_str.empty()) throw cms::Exception("ModelErrors") << msg_str; - const std::vector& inputConverterDefs = params.getParameter>("inputConverters"); + const std::vector& inputConverterDefs = params.getParameterSetVector("inputConverters"); + std::unordered_map inConvMap; + for (const auto converterDef : inputConverterDefs) { + inConvMap[converterDef.getParameter("inputName")] = converterDef.getParameter("converterName"); + } + //setup input map std::stringstream io_msg; if (verbose_) @@ -91,14 +96,11 @@ TritonClient::TritonClient(const edm::ParameterSet& params) auto [curr_itr, success] = input_.emplace( std::piecewise_construct, std::forward_as_tuple(iname), std::forward_as_tuple(iname, nicInput, noBatch_)); auto& curr_input = curr_itr->second; - bool foundConverter = false; - for (const auto converterDef : inputConverterDefs) { - if (converterDef.getParameter("inputName") == iname) { - curr_input.setConverterParams(converterDef); - foundConverter = true; - } + if ( inConvMap.find(iname) == inConvMap.end() ) { + curr_input.setConverterParams(curr_input.defaultConverter()); + } else { + curr_input.setConverterParams(inConvMap[iname]); } - if (!foundConverter) throw cms::Exception("ModelErrors") << "No matching converter definition for input: " << iname; inputsTriton_.push_back(curr_input.data()); if (verbose_) { io_msg << " " << iname << " (" << curr_input.dname() << ", " << curr_input.byteSize() @@ -110,7 +112,11 @@ TritonClient::TritonClient(const edm::ParameterSet& params) const auto& v_outputs = params.getUntrackedParameter>("outputs"); std::unordered_set s_outputs(v_outputs.begin(), v_outputs.end()); - const std::vector& outputConverterDefs = params.getParameter>("outputConverters"); + const std::vector& outputConverterDefs = params.getParameterSetVector("outputConverters"); + std::unordered_map outConvMap; + for (const auto converterDef : outputConverterDefs) { + outConvMap[converterDef.getParameter("outputName")] = converterDef.getParameter("converterName"); + } //setup output map if (verbose_) @@ -124,14 +130,11 @@ TritonClient::TritonClient(const edm::ParameterSet& params) auto [curr_itr, success] = output_.emplace( std::piecewise_construct, std::forward_as_tuple(oname), std::forward_as_tuple(oname, nicOutput, noBatch_)); auto& curr_output = curr_itr->second; - bool foundConverter = false; - for (const auto converterDef : outputConverterDefs) { - if (converterDef.getParameter("outputName") == oname) { - curr_output.setConverterParams(converterDef); - foundConverter = true; - } + if ( outConvMap.find(oname) == outConvMap.end() ) { + curr_output.setConverterParams(curr_output.defaultConverter()); + } else { + curr_output.setConverterParams(outConvMap[oname]); } - if (!foundConverter) throw cms::Exception("ModelErrors") << "No matching converter definition for output: " << oname; outputsTriton_.push_back(curr_output.data()); if (verbose_) { io_msg << " " << oname << " (" << curr_output.dname() << ", " << curr_output.byteSize() @@ -361,12 +364,13 @@ void TritonClient::fillPSetDescription(edm::ParameterSetDescription& iDesc) { edm::ParameterSetDescription descOutConverter; descOutConverter.add("converterName"); descOutConverter.add("outputName"); + std::vector blankVPSet; edm::ParameterSetDescription descClient; fillBasePSetDescription(descClient); descClient.add("modelName"); descClient.add("modelVersion", ""); - descClient.addVPSet("inputConverters", descInConverter); - descClient.addVPSet("outputConverters", descOutConverter); + descClient.addVPSet("inputConverters", descInConverter, blankVPSet); + descClient.addVPSet("outputConverters", descOutConverter, blankVPSet); //server parameters should not affect the physics results descClient.addUntracked("batchSize"); descClient.addUntracked("address"); From 81077f0afd25f5fdd99839083a1a36824eda79ca Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 14 Dec 2020 19:02:34 -0600 Subject: [PATCH 15/21] a few optimizations --- .../SonicTriton/interface/TritonData.h | 25 ++++++++++--------- .../SonicTriton/src/TritonClient.cc | 4 +-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index ccb05d6fbc3f9..41e9b5a927e87 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -61,17 +61,6 @@ class TritonData { //default to dims if shape isn't filled int64_t sizeShape() const { return variableDims_ ? dimProduct(shape_) : sizeDims(); } - std::string defaultConverter() const { - std::string base = "StandardConverter"; - if (dname_ == "INT64") { - return "Int64"+base; - } - else if (dname_ == "FP32") { - return "Float"+base; - } - return ""; - } - private: friend class TritonClient; @@ -83,7 +72,19 @@ class TritonData { void setResult(std::shared_ptr result) { result_ = result; } IO* data() { return data_.get(); } - void setConverterParams(std::string convName) { + std::string defaultConverter() const { + std::string base = "StandardConverter"; + if (dtype_ == inference::DataType::TYPE_INT64) { + return "Int64"+base; + } + else if (dtype_ == inference::DataType::TYPE_FP32) { + return "Float"+base; + } else { + throw cms::Exception("ConverterErrors") << "Unable to create default converter for " << dname_ << " type\n"; + } + } + + void setConverterParams(const std::string& convName) { converterName_ = convName; } diff --git a/HeterogeneousCore/SonicTriton/src/TritonClient.cc b/HeterogeneousCore/SonicTriton/src/TritonClient.cc index b3784876f608c..2a6410db33583 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonClient.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonClient.cc @@ -81,7 +81,7 @@ TritonClient::TritonClient(const edm::ParameterSet& params) const std::vector& inputConverterDefs = params.getParameterSetVector("inputConverters"); std::unordered_map inConvMap; - for (const auto converterDef : inputConverterDefs) { + for (const auto& converterDef : inputConverterDefs) { inConvMap[converterDef.getParameter("inputName")] = converterDef.getParameter("converterName"); } @@ -114,7 +114,7 @@ TritonClient::TritonClient(const edm::ParameterSet& params) const std::vector& outputConverterDefs = params.getParameterSetVector("outputConverters"); std::unordered_map outConvMap; - for (const auto converterDef : outputConverterDefs) { + for (const auto& converterDef : outputConverterDefs) { outConvMap[converterDef.getParameter("outputName")] = converterDef.getParameter("converterName"); } From 3231ad8051499d4d12e3e10708490ca8a22e1e1f Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 14 Dec 2020 19:04:19 -0600 Subject: [PATCH 16/21] small formatting --- HeterogeneousCore/SonicTriton/interface/TritonData.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 41e9b5a927e87..85c5dce8357c1 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -79,7 +79,8 @@ class TritonData { } else if (dtype_ == inference::DataType::TYPE_FP32) { return "Float"+base; - } else { + } + else { throw cms::Exception("ConverterErrors") << "Unable to create default converter for " << dname_ << " type\n"; } } From daabb9473fccc1d57b05703de9cdc727d5a462b3 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 14 Dec 2020 19:07:42 -0600 Subject: [PATCH 17/21] removing unnecessary defs --- .../SonicTriton/test/tritonTest_cfg.py | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py b/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py index eba943103e9f6..b7152140b070c 100644 --- a/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py +++ b/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py @@ -29,40 +29,6 @@ "TritonGraphProducer": "gat_test", } -inConvs = { - "TritonImageProducer": cms.VPSet( - cms.PSet( - converterName = cms.string("FloatStandardConverter"), - inputName = cms.string("gpu_0/data"), - ), - ), - "TritonGraphProducer": cms.VPSet( - cms.PSet( - converterName = cms.string("FloatStandardConverter"), - inputName = cms.string("x__0"), - ), - cms.PSet( - converterName = cms.string("Int64StandardConverter"), - inputName = cms.string("edgeindex__1"), - ), - ), -} - -outConvs = { - "TritonImageProducer": cms.VPSet( - cms.PSet( - converterName = cms.string("FloatStandardConverter"), - outputName = cms.string("gpu_0/softmax"), - ), - ), - "TritonGraphProducer": cms.VPSet( - cms.PSet( - converterName = cms.string("FloatStandardConverter"), - outputName = cms.string("logits__0"), - ), - ), -} - if options.producer not in models: raise ValueError("Unknown producer: "+options.producer) From 2dabb9237d2aad6b65999c0ceb690620b4afba50 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Mon, 14 Dec 2020 19:08:46 -0600 Subject: [PATCH 18/21] removing unnecessary defs --- HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py b/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py index b7152140b070c..0646e527d16b0 100644 --- a/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py +++ b/HeterogeneousCore/SonicTriton/test/tritonTest_cfg.py @@ -49,8 +49,6 @@ modelVersion = cms.string(""), verbose = cms.untracked.bool(options.verbose), allowedTries = cms.untracked.uint32(0), - inputConverters = inConvs[options.producer], - outputConverters = outConvs[options.producer], ) ) if options.producer=="TritonImageProducer": From 9c14bf5c408c4617a6ba481f9375b785018e2303 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Wed, 20 Jan 2021 10:50:34 -0600 Subject: [PATCH 19/21] adding ability to get converter from server --- .../SonicTriton/interface/TritonData.h | 21 ++++++++++++------- .../SonicTriton/src/TritonClient.cc | 10 +++++---- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 85c5dce8357c1..a12e813fb39c0 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -72,16 +72,21 @@ class TritonData { void setResult(std::shared_ptr result) { result_ = result; } IO* data() { return data_.get(); } - std::string defaultConverter() const { - std::string base = "StandardConverter"; - if (dtype_ == inference::DataType::TYPE_INT64) { - return "Int64"+base; + std::string defaultConverter(const std::string name) const { + size_t dcpos = name.find("_DataConverter:"); + if (dcpos != std::string::npos) { + return name.substr(dcpos+15); } - else if (dtype_ == inference::DataType::TYPE_FP32) { - return "Float"+base; - } else { - throw cms::Exception("ConverterErrors") << "Unable to create default converter for " << dname_ << " type\n"; + std::string base = "StandardConverter"; + if (dtype_ == inference::DataType::TYPE_INT64) { + return "Int64"+base; + } + else if (dtype_ == inference::DataType::TYPE_FP32) { + return "Float"+base; + } else { + throw cms::Exception("ConverterErrors") << "Unable to create default converter for " << name_ << " of " << dname_ << " type\n"; + } } } diff --git a/HeterogeneousCore/SonicTriton/src/TritonClient.cc b/HeterogeneousCore/SonicTriton/src/TritonClient.cc index 2a6410db33583..a772e6be1e382 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonClient.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonClient.cc @@ -92,12 +92,13 @@ TritonClient::TritonClient(const edm::ParameterSet& params) << "\n"; inputsTriton_.reserve(nicInputs.size()); for (const auto& nicInput : nicInputs) { - const auto& iname = nicInput.name(); + const std::string iname_full = nicInput.name(); + const auto& iname = iname_full.find("_DataConverter:") != std::string::npos ? iname_full.substr(0,iname_full.find("_DataConverter:")) : iname_full; auto [curr_itr, success] = input_.emplace( std::piecewise_construct, std::forward_as_tuple(iname), std::forward_as_tuple(iname, nicInput, noBatch_)); auto& curr_input = curr_itr->second; if ( inConvMap.find(iname) == inConvMap.end() ) { - curr_input.setConverterParams(curr_input.defaultConverter()); + curr_input.setConverterParams(curr_input.defaultConverter(iname_full)); } else { curr_input.setConverterParams(inConvMap[iname]); } @@ -124,14 +125,15 @@ TritonClient::TritonClient(const edm::ParameterSet& params) << "\n"; outputsTriton_.reserve(nicOutputs.size()); for (const auto& nicOutput : nicOutputs) { - const auto& oname = nicOutput.name(); + const std::string oname_full = nicOutput.name(); + const auto& oname = oname_full.find("_DataConverter:") != std::string::npos ? oname_full.substr(0,oname_full.find("_DataConverter:")) : oname_full; if (!s_outputs.empty() and s_outputs.find(oname) == s_outputs.end()) continue; auto [curr_itr, success] = output_.emplace( std::piecewise_construct, std::forward_as_tuple(oname), std::forward_as_tuple(oname, nicOutput, noBatch_)); auto& curr_output = curr_itr->second; if ( outConvMap.find(oname) == outConvMap.end() ) { - curr_output.setConverterParams(curr_output.defaultConverter()); + curr_output.setConverterParams(curr_output.defaultConverter(oname_full)); } else { curr_output.setConverterParams(outConvMap[oname]); } From 40371151ec3b97d913ac651ba70b27a72871f6a0 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Wed, 20 Jan 2021 12:14:14 -0600 Subject: [PATCH 20/21] reducing duplication --- HeterogeneousCore/SonicTriton/interface/TritonClient.h | 9 +++++++++ HeterogeneousCore/SonicTriton/interface/TritonData.h | 5 ++--- HeterogeneousCore/SonicTriton/src/TritonClient.cc | 8 ++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonClient.h b/HeterogeneousCore/SonicTriton/interface/TritonClient.h index 99ca5f8765fe7..96e30bea35534 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonClient.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonClient.h @@ -40,6 +40,15 @@ class TritonClient : public SonicClient { //for fillDescriptions static void fillPSetDescription(edm::ParameterSetDescription& iDesc); + static std::pair splitNameConverter(const std::string& fullname) { + static const std::string indicator("_DataConverter:"); + size_t dcpos = fullname.find(indicator); + if (dcpos != std::string::npos) + return {fullname.substr(0,dcpos), fullname.substr(dcpos+indicator.size())}; + else + return {fullname, ""}; + } + protected: //helper bool getResults(std::shared_ptr results); diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index a12e813fb39c0..47508c26af617 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -73,9 +73,8 @@ class TritonData { IO* data() { return data_.get(); } std::string defaultConverter(const std::string name) const { - size_t dcpos = name.find("_DataConverter:"); - if (dcpos != std::string::npos) { - return name.substr(dcpos+15); + if (name != "") { + return name; } else { std::string base = "StandardConverter"; diff --git a/HeterogeneousCore/SonicTriton/src/TritonClient.cc b/HeterogeneousCore/SonicTriton/src/TritonClient.cc index a772e6be1e382..cc48385e41d5a 100644 --- a/HeterogeneousCore/SonicTriton/src/TritonClient.cc +++ b/HeterogeneousCore/SonicTriton/src/TritonClient.cc @@ -93,12 +93,12 @@ TritonClient::TritonClient(const edm::ParameterSet& params) inputsTriton_.reserve(nicInputs.size()); for (const auto& nicInput : nicInputs) { const std::string iname_full = nicInput.name(); - const auto& iname = iname_full.find("_DataConverter:") != std::string::npos ? iname_full.substr(0,iname_full.find("_DataConverter:")) : iname_full; + const auto& [iname, iconverter] = TritonClient::splitNameConverter(iname_full); auto [curr_itr, success] = input_.emplace( std::piecewise_construct, std::forward_as_tuple(iname), std::forward_as_tuple(iname, nicInput, noBatch_)); auto& curr_input = curr_itr->second; if ( inConvMap.find(iname) == inConvMap.end() ) { - curr_input.setConverterParams(curr_input.defaultConverter(iname_full)); + curr_input.setConverterParams(curr_input.defaultConverter(iconverter)); } else { curr_input.setConverterParams(inConvMap[iname]); } @@ -126,14 +126,14 @@ TritonClient::TritonClient(const edm::ParameterSet& params) outputsTriton_.reserve(nicOutputs.size()); for (const auto& nicOutput : nicOutputs) { const std::string oname_full = nicOutput.name(); - const auto& oname = oname_full.find("_DataConverter:") != std::string::npos ? oname_full.substr(0,oname_full.find("_DataConverter:")) : oname_full; + const auto& [oname, oconverter] = TritonClient::splitNameConverter(oname_full); if (!s_outputs.empty() and s_outputs.find(oname) == s_outputs.end()) continue; auto [curr_itr, success] = output_.emplace( std::piecewise_construct, std::forward_as_tuple(oname), std::forward_as_tuple(oname, nicOutput, noBatch_)); auto& curr_output = curr_itr->second; if ( outConvMap.find(oname) == outConvMap.end() ) { - curr_output.setConverterParams(curr_output.defaultConverter(oname_full)); + curr_output.setConverterParams(curr_output.defaultConverter(oconverter)); } else { curr_output.setConverterParams(outConvMap[oname]); } From 9faebc090f62c41d5b1da744a7fb5bd69644c5f8 Mon Sep 17 00:00:00 2001 From: Dylan Rankin Date: Wed, 20 Jan 2021 12:20:06 -0600 Subject: [PATCH 21/21] change empty check --- HeterogeneousCore/SonicTriton/interface/TritonData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HeterogeneousCore/SonicTriton/interface/TritonData.h b/HeterogeneousCore/SonicTriton/interface/TritonData.h index 47508c26af617..a3acfb42183b9 100644 --- a/HeterogeneousCore/SonicTriton/interface/TritonData.h +++ b/HeterogeneousCore/SonicTriton/interface/TritonData.h @@ -73,7 +73,7 @@ class TritonData { IO* data() { return data_.get(); } std::string defaultConverter(const std::string name) const { - if (name != "") { + if (!name.empty()) { return name; } else {