From df19efb2ecdf6ec4043ffd568c78dc63aeeb7f81 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Fri, 19 Nov 2021 14:40:27 -0600 Subject: [PATCH 01/11] Add nnapi support in r2inference --- examples/r2i/meson.build | 4 + examples/r2i/nnapi/inception.cc | 198 ++++++++++++++++++++++++++++++++ examples/r2i/nnapi/meson.build | 11 ++ meson_options.txt | 2 + r2i/frameworks.h | 13 ++- r2i/iframeworkfactory.cc | 13 +++ r2i/meson.build | 5 + r2i/nnapi/engine.cc | 45 ++++++++ r2i/nnapi/engine.h | 25 ++++ r2i/nnapi/frameworkfactory.cc | 41 +++++++ r2i/nnapi/frameworkfactory.h | 30 +++++ r2i/nnapi/meson.build | 23 ++++ r2i/runtimeerror.h | 7 +- r2i/tflite/engine.cc | 6 +- r2i/tflite/engine.h | 1 + 15 files changed, 418 insertions(+), 6 deletions(-) create mode 100644 examples/r2i/nnapi/inception.cc create mode 100644 examples/r2i/nnapi/meson.build create mode 100644 r2i/nnapi/engine.cc create mode 100644 r2i/nnapi/engine.h create mode 100644 r2i/nnapi/frameworkfactory.cc create mode 100644 r2i/nnapi/frameworkfactory.h create mode 100644 r2i/nnapi/meson.build diff --git a/examples/r2i/meson.build b/examples/r2i/meson.build index 2925a810..fc28fdd8 100644 --- a/examples/r2i/meson.build +++ b/examples/r2i/meson.build @@ -41,3 +41,7 @@ endif if cdata.get('HAVE_TENSORRT') == true subdir('tensorrt') endif + +if cdata.get('HAVE_NNAPI') == true + subdir('nnapi') +endif diff --git a/examples/r2i/nnapi/inception.cc b/examples/r2i/nnapi/inception.cc new file mode 100644 index 00000000..93f50f23 --- /dev/null +++ b/examples/r2i/nnapi/inception.cc @@ -0,0 +1,198 @@ +/* Copyright (C) 2018-2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#include +#include +#include +#include +#include + +#define STB_IMAGE_IMPLEMENTATION +#include "./stb_image.h" + +#define STB_IMAGE_RESIZE_IMPLEMENTATION +#include "./stb_image_resize.h" + +void PrintTopPrediction(std::shared_ptr prediction) { + r2i::RuntimeError error; + int index = 0; + double max = -1; + int num_labels = prediction->GetResultSize() / sizeof(float); + + for (int i = 0; i < num_labels; ++i) { + double current = prediction->At(i, error); + if (current > max) { + max = current; + index = i; + } + } + + std::cout << "Highest probability is label " << index << " (" << max << ")" + << std::endl; +} + +void PrintUsage() { + std::cerr << "Required arguments: " + << "-i [JPG input_image] " + << "-m [Inception TfLite Model] " + << "-s [Model Input Size] " + << "-I [Input Node] " + << "-O [Output Node] \n" + << " Example: " + << " ./inception -i cat.jpg -m graph_inceptionv2_tensorflow.pb " + << "-s 224" << std::endl; +} + +std::unique_ptr PreProcessImage(const unsigned char *input, int width, + int height, int reqwidth, + int reqheight) { + const int channels = 3; + const int scaled_size = channels * reqwidth * reqheight; + std::unique_ptr scaled(new unsigned char[scaled_size]); + std::unique_ptr adjusted(new float[scaled_size]); + + stbir_resize_uint8(input, width, height, 0, scaled.get(), reqwidth, reqheight, + 0, channels); + + for (int i = 0; i < scaled_size; i += channels) { + /* RGB = (RGB - Mean)*StdDev */ + adjusted[i + 0] = (static_cast(scaled[i + 0]) - 127.5) / 127.5; + adjusted[i + 1] = (static_cast(scaled[i + 1]) - 127.5) / 127.5; + adjusted[i + 2] = (static_cast(scaled[i + 2]) - 127.5) / 127.5; + } + + return adjusted; +} + +std::unique_ptr LoadImage(const std::string &path, int reqwidth, + int reqheight) { + int channels = 3; + int width, height, cp; + + unsigned char *img = stbi_load(path.c_str(), &width, &height, &cp, channels); + if (!img) { + std::cerr << "The picture " << path << " could not be loaded"; + return nullptr; + } + + auto ret = PreProcessImage(img, width, height, reqwidth, reqheight); + free(img); + + return ret; +} + +bool ParseArgs(int &argc, char *argv[], std::string &image_path, + std::string &model_path, int &index, int &size, + std::string &in_node, std::string &out_node) { + int option = 0; + while ((option = getopt(argc, argv, "i:m:p:s:I:O:")) != -1) { + switch (option) { + case 'i': + image_path = optarg; + break; + case 'm': + model_path = optarg; + break; + case 'p': + index = std::stoi(optarg); + break; + case 's': + size = std::stoi(optarg); + break; + case 'I': + in_node = optarg; + break; + case 'O': + out_node = optarg; + break; + default: + return false; + } + } + return true; +} + +int main(int argc, char *argv[]) { + r2i::RuntimeError error; + std::string model_path; + std::string image_path; + std::string in_node; + std::string out_node; + int Index = 0; + int size = 0; + + if (false == ParseArgs(argc, argv, image_path, model_path, Index, size, + in_node, out_node)) { + PrintUsage(); + exit(EXIT_FAILURE); + } + + if (image_path.empty() || model_path.empty()) { + PrintUsage(); + exit(EXIT_FAILURE); + } + + auto factory = + r2i::IFrameworkFactory::MakeFactory(r2i::FrameworkCode::NNAPI, error); + + if (nullptr == factory) { + std::cerr << "TensorFlow backend is not built: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Loading Model: " << model_path << std::endl; + auto loader = factory->MakeLoader(error); + std::shared_ptr model = loader->Load(model_path, error); + if (error.IsError()) { + std::cerr << "Loader error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Setting model to engine" << std::endl; + std::shared_ptr engine = factory->MakeEngine(error); + error = engine->SetModel(model); + + std::cout << "Loading image: " << image_path << std::endl; + std::unique_ptr image_data = LoadImage(image_path, size, size); + + std::cout << "Configuring frame" << std::endl; + std::shared_ptr frame = factory->MakeFrame(error); + + error = frame->Configure(image_data.get(), size, size, + r2i::ImageFormat::Id::RGB, r2i::DataType::Id::FLOAT); + + std::cout << "Starting engine" << std::endl; + error = engine->Start(); + if (error.IsError()) { + std::cerr << "Engine start error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Predicting..." << std::endl; + std::vector> predictions; + error = engine->Predict(frame, predictions); + if (error.IsError()) { + std::cerr << "Engine prediction error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + /* This model only has one output */ + PrintTopPrediction(predictions[0]); + + std::cout << "Stopping engine" << std::endl; + error = engine->Stop(); + if (error.IsError()) { + std::cerr << "Engine stop error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + return EXIT_SUCCESS; +} diff --git a/examples/r2i/nnapi/meson.build b/examples/r2i/nnapi/meson.build new file mode 100644 index 00000000..900ab4c5 --- /dev/null +++ b/examples/r2i/nnapi/meson.build @@ -0,0 +1,11 @@ +# Compile examples +app_examples = [ + 'inception', +] + +foreach app : app_examples + executable(app, '@0@.cc'.format(app), + include_directories: [configinc, common_inc_dir], + dependencies : [r2inference_lib_dep], + install: false) +endforeach \ No newline at end of file diff --git a/meson_options.txt b/meson_options.txt index d8e24c99..7a3a81cf 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -20,3 +20,5 @@ option('enable-onnxrt-acl', type : 'boolean', value: false, description : 'Enable ONNX Runtime backend with ACL execution provider support') option('enable-onnxrt-openvino', type : 'boolean', value: false, description : 'Enable ONNX Runtime backend with OpenVINO execution provider support') +option('enable-nnapi', type : 'boolean', value : false, + description : 'Enable NNAPI delegate for NPU inference execution support') diff --git a/r2i/frameworks.h b/r2i/frameworks.h index adb754b3..350143e9 100644 --- a/r2i/frameworks.h +++ b/r2i/frameworks.h @@ -7,7 +7,7 @@ * RidgeRun, LLC. The user is free to modify the source code after obtaining * a software license from RidgeRun. All source code changes must be provided * back to RidgeRun without any encumbrance. -*/ + */ #ifndef R2I_FRAMEWORKS_H #define R2I_FRAMEWORKS_H @@ -60,9 +60,14 @@ enum FrameworkCode { /** * Number of supported frameworks, mostly for testing purposes. */ - MAX_FRAMEWORK + MAX_FRAMEWORK, + + /** + * Android's NPU delegate + */ + NNAPI }; -} //namespace r2i +} // namespace r2i -#endif //R2I_FRAMEWORKS +#endif // R2I_FRAMEWORKS diff --git a/r2i/iframeworkfactory.cc b/r2i/iframeworkfactory.cc index 9ea5bd19..9f28d21c 100644 --- a/r2i/iframeworkfactory.cc +++ b/r2i/iframeworkfactory.cc @@ -22,6 +22,7 @@ #include "tensorflow/frameworkfactory.h" #include "tensorrt/frameworkfactory.h" #include "tflite/frameworkfactory.h" +#include "nnapi/frameworkfactory.h" namespace r2i { @@ -81,6 +82,14 @@ MakeTensorRTFactory (RuntimeError &error) { } #endif // HAVE_TENSORRT +#ifdef HAVE_NNAPI +static std::unique_ptr +MakeNNAPIFactory (RuntimeError &error) { + return std::unique_ptr (new + nnapi::FrameworkFactory); +} +#endif + typedef std::function(RuntimeError &)> MakeFactory; const std::unordered_map frameworks ({ @@ -113,6 +122,10 @@ const std::unordered_map frameworks ({ {FrameworkCode::TENSORRT, MakeTensorRTFactory}, #endif //HAVE_TENSORRT +#ifdef HAVE_NNAPI + {FrameworkCode::NNAPI, MakeNNAPIFactory}, +#endif + }); std::unique_ptr diff --git a/r2i/meson.build b/r2i/meson.build index d6ea7ac0..97db69a2 100644 --- a/r2i/meson.build +++ b/r2i/meson.build @@ -37,6 +37,11 @@ if cdata.get('HAVE_ONNXRT_OPENVINO') == true r2inference_internal_dep += [internal_onnxrt_openvino_dep] endif +if cdata.get('HAVE_NNAPI') == true + subdir('nnapi') + r2inference_internal_dep += [internal_nnapi_dep] +endif + # Define source code r2inference_sources = [ 'classification.cc', diff --git a/r2i/nnapi/engine.cc b/r2i/nnapi/engine.cc new file mode 100644 index 00000000..825ff43f --- /dev/null +++ b/r2i/nnapi/engine.cc @@ -0,0 +1,45 @@ +/* Copyright (C) 2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#include "r2i/nnapi/engine.h" + + +namespace r2i { +namespace nnapi { +Engine::Engine () : tflite::Engine() { + this->number_of_threads = 1; +} + +void Engine::ConfigureDelegate(::tflite::Interpreter *interpreter) { + RuntimeError error; + ::tflite::StatefulNnApiDelegate::Options options; + options.allow_fp16 = true; + options.allow_dynamic_dimensions = true; + options.disallow_nnapi_cpu = false; + options.accelerator_name = "vsi-npu"; + + auto delegate = ::tflite::evaluation::CreateNNAPIDelegate(options); + + if (!delegate){ + error.Set (RuntimeError::Code::DELEGATE_ERROR, + "NNAPI delegate was not well created");; + } else { + interpreter->ModifyGraphWithDelegate(std::move(delegate)); + } +} + +Engine::~Engine() { + this->Stop(); + this->interpreter.reset(); +} + +} +} \ No newline at end of file diff --git a/r2i/nnapi/engine.h b/r2i/nnapi/engine.h new file mode 100644 index 00000000..7fd79732 --- /dev/null +++ b/r2i/nnapi/engine.h @@ -0,0 +1,25 @@ +#ifndef R2I_NNAPI_H +#define R2I_NNAPI_H + +#include +#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h" +#include "tensorflow/lite/tools/delegates/delegate_provider.h" +#include "tensorflow/lite/tools/evaluation/utils.h" + +namespace r2i { +namespace nnapi { +class Engine : public r2i::tflite::Engine { + public: + Engine (); + ~Engine (); + + protected: + void ConfigureDelegate(::tflite::Interpreter *interpreter) + override; + +}; + + +} +} +#endif \ No newline at end of file diff --git a/r2i/nnapi/frameworkfactory.cc b/r2i/nnapi/frameworkfactory.cc new file mode 100644 index 00000000..8b095f2f --- /dev/null +++ b/r2i/nnapi/frameworkfactory.cc @@ -0,0 +1,41 @@ +/* Copyright (C) 2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#include "frameworkfactory.h" +#include "engine.h" + + +namespace r2i { +namespace nnapi { + +std::unique_ptr FrameworkFactory::MakeEngine ( + RuntimeError &error) { + error.Clean (); + + return std::unique_ptr (new Engine); +} + +r2i::FrameworkMeta FrameworkFactory::GetDescription ( + RuntimeError &error) { + const FrameworkMeta meta { + .code = r2i::FrameworkCode::NNAPI, + .name = "NNAPI", + .label = "nnapi", + .description = "TensorFlow Lite with NNAPI delegate from Android" + }; + + error.Clean (); + + return meta; +} + +} // namespace tflite +} // namespace r2i \ No newline at end of file diff --git a/r2i/nnapi/frameworkfactory.h b/r2i/nnapi/frameworkfactory.h new file mode 100644 index 00000000..5080cea8 --- /dev/null +++ b/r2i/nnapi/frameworkfactory.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#ifndef R2I_NNAPI_FRAMEWORK_FACTORY_H +#define R2I_NNAPI_FRAMEWORK_FACTORY_H + +#include + +namespace r2i { +namespace nnapi { + +class FrameworkFactory : public r2i::tflite::FrameworkFactory { + public: + std::unique_ptr MakeEngine (RuntimeError &error) override; + + r2i::FrameworkMeta GetDescription (RuntimeError &error) override; +}; + +} // namespace nnapi +} // namespace r2i + +#endif //R2I_NNAPI_FRAMEWORK_FACTORY_H \ No newline at end of file diff --git a/r2i/nnapi/meson.build b/r2i/nnapi/meson.build new file mode 100644 index 00000000..b0decf53 --- /dev/null +++ b/r2i/nnapi/meson.build @@ -0,0 +1,23 @@ +# Define source code +nnapi_sources = [ + 'engine.cc', + 'frameworkfactory.cc', +] + +nnapi_headers = [ + 'engine.h', + 'frameworkfactory.h', +] + +# Build library +nnapi_lib = static_library('nnapi', + nnapi_sources, + include_directories : [configinc], + dependencies : [lib_nnapi_dep], +) + +# Install library header files +install_headers(nnapi_headers, subdir : inc_install_dir + '/r2i/nnapi') + +# Define the library as an internal dependency to the current build +internal_nnapi_dep = declare_dependency(link_with: nnapi_lib, dependencies: lib_nnapi_dep) diff --git a/r2i/runtimeerror.h b/r2i/runtimeerror.h index 9fb6d229..7564b2d3 100644 --- a/r2i/runtimeerror.h +++ b/r2i/runtimeerror.h @@ -61,7 +61,7 @@ class RuntimeError { INCOMPATIBLE_MODEL, /** - * The provided Parameters is incompatible with the current operation + * The provided Parameters are incompatible with the current operation */ INCOMPATIBLE_PARAMETERS, @@ -104,6 +104,11 @@ class RuntimeError { * An unknown error has ocurred */ UNKNOWN_ERROR, + + /** + * The delegate was not builded properly + */ + DELEGATE_ERROR, }; /** diff --git a/r2i/tflite/engine.cc b/r2i/tflite/engine.cc index ca22d33b..8fa7fd23 100644 --- a/r2i/tflite/engine.cc +++ b/r2i/tflite/engine.cc @@ -91,6 +91,7 @@ RuntimeError Engine::Start () { } this->SetInterpreterContext(interpreter.get()); + this->ConfigureDelegate(interpreter.get()); std::shared_ptr<::tflite::Interpreter> tflite_interpreter_shared{std::move(interpreter)}; @@ -240,7 +241,6 @@ RuntimeError Engine::PredictAuxiliar(std::shared_ptr in_frame) { "The provided frame input sizes are different to tensor sizes"); return error; } - this->PreprocessInputData(static_cast(frame->GetData()), wanted_width * wanted_height * wanted_channels, this->interpreter.get(), error); if (r2i::RuntimeError::EOK != error.GetCode()) { @@ -355,5 +355,9 @@ void Engine::GetOutputTensorData(::tflite::Interpreter *interpreter, } } +void Engine::ConfigureDelegate(::tflite::Interpreter * /*interpreter*/) { + // No implementation for tflite engine +} + } //namespace tflite } //namepsace r2i diff --git a/r2i/tflite/engine.h b/r2i/tflite/engine.h index 7f79ac99..45bc8191 100644 --- a/r2i/tflite/engine.h +++ b/r2i/tflite/engine.h @@ -62,6 +62,7 @@ class Engine : public IEngine { virtual void SetupResolver(::tflite::ops::builtin::BuiltinOpResolver &resolver); virtual void SetInterpreterContext(::tflite::Interpreter *interpreter); + virtual void ConfigureDelegate(::tflite::Interpreter *interpreter); private: RuntimeError PredictAuxiliar(std::shared_ptr in_frame); From 0bef26358619e35fb931ae3ee6484f588d04ef6b Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Fri, 19 Nov 2021 14:49:55 -0600 Subject: [PATCH 02/11] Add nnapi support in r2inference --- r2i/tflite/engine.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/r2i/tflite/engine.cc b/r2i/tflite/engine.cc index 8fa7fd23..9b435cd1 100644 --- a/r2i/tflite/engine.cc +++ b/r2i/tflite/engine.cc @@ -241,6 +241,7 @@ RuntimeError Engine::PredictAuxiliar(std::shared_ptr in_frame) { "The provided frame input sizes are different to tensor sizes"); return error; } + this->PreprocessInputData(static_cast(frame->GetData()), wanted_width * wanted_height * wanted_channels, this->interpreter.get(), error); if (r2i::RuntimeError::EOK != error.GetCode()) { From 7d20869137b8ee2ea6ddb13a9991bf6bb2e01080 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Mon, 22 Nov 2021 09:58:35 -0600 Subject: [PATCH 03/11] Add space at the end line --- examples/r2i/nnapi/meson.build | 2 +- r2i/nnapi/engine.cc | 2 +- r2i/nnapi/engine.h | 2 +- r2i/nnapi/frameworkfactory.cc | 2 +- r2i/nnapi/frameworkfactory.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/r2i/nnapi/meson.build b/examples/r2i/nnapi/meson.build index 900ab4c5..b568b5f2 100644 --- a/examples/r2i/nnapi/meson.build +++ b/examples/r2i/nnapi/meson.build @@ -8,4 +8,4 @@ foreach app : app_examples include_directories: [configinc, common_inc_dir], dependencies : [r2inference_lib_dep], install: false) -endforeach \ No newline at end of file +endforeach diff --git a/r2i/nnapi/engine.cc b/r2i/nnapi/engine.cc index 825ff43f..7352f4ab 100644 --- a/r2i/nnapi/engine.cc +++ b/r2i/nnapi/engine.cc @@ -42,4 +42,4 @@ Engine::~Engine() { } } -} \ No newline at end of file +} diff --git a/r2i/nnapi/engine.h b/r2i/nnapi/engine.h index 7fd79732..4a944b6d 100644 --- a/r2i/nnapi/engine.h +++ b/r2i/nnapi/engine.h @@ -22,4 +22,4 @@ class Engine : public r2i::tflite::Engine { } } -#endif \ No newline at end of file +#endif diff --git a/r2i/nnapi/frameworkfactory.cc b/r2i/nnapi/frameworkfactory.cc index 8b095f2f..827d5d97 100644 --- a/r2i/nnapi/frameworkfactory.cc +++ b/r2i/nnapi/frameworkfactory.cc @@ -38,4 +38,4 @@ r2i::FrameworkMeta FrameworkFactory::GetDescription ( } } // namespace tflite -} // namespace r2i \ No newline at end of file +} // namespace r2i diff --git a/r2i/nnapi/frameworkfactory.h b/r2i/nnapi/frameworkfactory.h index 5080cea8..e26043f6 100644 --- a/r2i/nnapi/frameworkfactory.h +++ b/r2i/nnapi/frameworkfactory.h @@ -27,4 +27,4 @@ class FrameworkFactory : public r2i::tflite::FrameworkFactory { } // namespace nnapi } // namespace r2i -#endif //R2I_NNAPI_FRAMEWORK_FACTORY_H \ No newline at end of file +#endif //R2I_NNAPI_FRAMEWORK_FACTORY_H From e4753fae2066879dc9d60523df5ece43f3de453e Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Mon, 22 Nov 2021 11:29:38 -0600 Subject: [PATCH 04/11] Add includes for TensorFlow Lite support --- meson.build | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/meson.build b/meson.build index ffa4437d..bb69ed8d 100644 --- a/meson.build +++ b/meson.build @@ -51,7 +51,9 @@ if get_option('enable-tensorflow') endif # Define library dependencies for Tensorflow Lite support -if get_option('enable-tflite') or get_option('enable-coral') +if get_option('enable-tflite') or get_option('enable-coral') or get_option('enable-nnapi') + configinc += include_directories('../recipe-sysroot/usr/include/tensorflow/lite') + configinc += include_directories('../recipe-sysroot/usr/include/tensorflow/lite/tools/make/downloads/flatbuffers/include') dl = cpp.find_library('dl', required: true) dl_dep = declare_dependency(dependencies: dl) tensorflow_lite = cpp.find_library('tensorflow-lite', required: true) @@ -72,6 +74,12 @@ if get_option('enable-coral') cdata.set('HAVE_CORAL', true) endif +# Define library dependencies for NNAPI TensorFlow Lite delegate +if get_option('enable-nnapi') + lib_nnapi_dep = [tensorflow_lite_dep, thread_dep, dl_dep, rt_dep, common_deps] + cdata.set('HAVE_NNAPI', true) +endif + # Define library dependencies for TensorRT support if get_option('enable-tensorrt') From 1302050b86e8d80d81fb03706b1a4543e964ee99 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Mon, 22 Nov 2021 11:39:01 -0600 Subject: [PATCH 05/11] Add compilation support flag for nnapi delegate --- meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/meson.build b/meson.build index bb69ed8d..f39be42f 100644 --- a/meson.build +++ b/meson.build @@ -41,6 +41,7 @@ cdata.set('HAVE_TENSORRT', false) cdata.set('HAVE_ONNXRT', false) cdata.set('HAVE_ONNXRT_ACL', false) cdata.set('HAVE_ONNXRT_OPENVINO', false) +cdata.set('HAVE_NNAPI', false) # Define library dependencies for Tensorflow support if get_option('enable-tensorflow') From cee8e3dda9099d9e0694ac8797671e202775b648 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Fri, 26 Nov 2021 10:30:58 -0600 Subject: [PATCH 06/11] Delete specific includes inside the meson.build --- meson.build | 2 -- 1 file changed, 2 deletions(-) diff --git a/meson.build b/meson.build index f39be42f..4a25c233 100644 --- a/meson.build +++ b/meson.build @@ -53,8 +53,6 @@ endif # Define library dependencies for Tensorflow Lite support if get_option('enable-tflite') or get_option('enable-coral') or get_option('enable-nnapi') - configinc += include_directories('../recipe-sysroot/usr/include/tensorflow/lite') - configinc += include_directories('../recipe-sysroot/usr/include/tensorflow/lite/tools/make/downloads/flatbuffers/include') dl = cpp.find_library('dl', required: true) dl_dep = declare_dependency(dependencies: dl) tensorflow_lite = cpp.find_library('tensorflow-lite', required: true) From 16cab55bc993dbb49408185bb43989fdeaa1ba99 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Fri, 26 Nov 2021 11:23:28 -0600 Subject: [PATCH 07/11] Fix typos and style format --- r2i/frameworks.h | 8 +++--- r2i/iframeworkfactory.cc | 4 +-- r2i/nnapi/engine.cc | 50 +++++++++++++++++------------------ r2i/nnapi/engine.h | 34 ++++++++++++++---------- r2i/nnapi/frameworkfactory.cc | 31 ++++++++++------------ r2i/runtimeerror.h | 8 +++--- r2i/tflite/engine.cc | 3 +-- r2i/tflite/engine.h | 2 +- 8 files changed, 71 insertions(+), 69 deletions(-) diff --git a/r2i/frameworks.h b/r2i/frameworks.h index 350143e9..ffd8af20 100644 --- a/r2i/frameworks.h +++ b/r2i/frameworks.h @@ -58,14 +58,14 @@ enum FrameworkCode { TENSORRT, /** - * Number of supported frameworks, mostly for testing purposes. + * Android's NPU delegate */ - MAX_FRAMEWORK, + NNAPI, /** - * Android's NPU delegate + * Number of supported frameworks, mostly for testing purposes. */ - NNAPI + MAX_FRAMEWORK }; } // namespace r2i diff --git a/r2i/iframeworkfactory.cc b/r2i/iframeworkfactory.cc index 9f28d21c..0550da50 100644 --- a/r2i/iframeworkfactory.cc +++ b/r2i/iframeworkfactory.cc @@ -16,13 +16,13 @@ #include "config.h" #include "coral/frameworkfactory.h" +#include "nnapi/frameworkfactory.h" #include "onnxrt/frameworkfactory.h" #include "onnxrt_acl/frameworkfactory.h" #include "onnxrt_openvino/frameworkfactory.h" #include "tensorflow/frameworkfactory.h" #include "tensorrt/frameworkfactory.h" #include "tflite/frameworkfactory.h" -#include "nnapi/frameworkfactory.h" namespace r2i { @@ -124,7 +124,7 @@ const std::unordered_map frameworks ({ #ifdef HAVE_NNAPI {FrameworkCode::NNAPI, MakeNNAPIFactory}, -#endif +#endif // HAVE_NNAPI }); diff --git a/r2i/nnapi/engine.cc b/r2i/nnapi/engine.cc index 7352f4ab..95d8b0d8 100644 --- a/r2i/nnapi/engine.cc +++ b/r2i/nnapi/engine.cc @@ -11,35 +11,35 @@ #include "r2i/nnapi/engine.h" +#include +#include +#include namespace r2i { namespace nnapi { -Engine::Engine () : tflite::Engine() { - this->number_of_threads = 1; -} -void Engine::ConfigureDelegate(::tflite::Interpreter *interpreter) { - RuntimeError error; - ::tflite::StatefulNnApiDelegate::Options options; - options.allow_fp16 = true; - options.allow_dynamic_dimensions = true; - options.disallow_nnapi_cpu = false; - options.accelerator_name = "vsi-npu"; - - auto delegate = ::tflite::evaluation::CreateNNAPIDelegate(options); - - if (!delegate){ - error.Set (RuntimeError::Code::DELEGATE_ERROR, - "NNAPI delegate was not well created");; - } else { - interpreter->ModifyGraphWithDelegate(std::move(delegate)); - } +Engine::Engine() : tflite::Engine() { this->number_of_threads = 1; } + +RuntimeError Engine::ConfigureDelegate(::tflite::Interpreter *interpreter) { + RuntimeError error; + ::tflite::StatefulNnApiDelegate::Options options; + options.allow_fp16 = true; + options.allow_dynamic_dimensions = true; + options.disallow_nnapi_cpu = false; + options.accelerator_name = "vsi-npu"; + + auto delegate = ::tflite::evaluation::CreateNNAPIDelegate(options); + + if (!delegate) { + error.Set(RuntimeError::Code::DELEGATE_ERROR, + "NNAPI delegate was not well created"); + } else { + interpreter->ModifyGraphWithDelegate(std::move(delegate)); + } + return error; } -Engine::~Engine() { - this->Stop(); - this->interpreter.reset(); -} +Engine::~Engine() { this->Stop(); } -} -} +} // namespace nnapi +} // namespace r2i diff --git a/r2i/nnapi/engine.h b/r2i/nnapi/engine.h index 4a944b6d..a2690512 100644 --- a/r2i/nnapi/engine.h +++ b/r2i/nnapi/engine.h @@ -1,25 +1,31 @@ -#ifndef R2I_NNAPI_H -#define R2I_NNAPI_H +/* Copyright (C) 2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#ifndef R2I_NNAPI_ENGINE_H +#define R2I_NNAPI_ENGINE_H #include -#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h" -#include "tensorflow/lite/tools/delegates/delegate_provider.h" -#include "tensorflow/lite/tools/evaluation/utils.h" namespace r2i { namespace nnapi { + class Engine : public r2i::tflite::Engine { public: - Engine (); - ~Engine (); + Engine(); + ~Engine(); protected: - void ConfigureDelegate(::tflite::Interpreter *interpreter) - override; - + RuntimeError ConfigureDelegate(::tflite::Interpreter *interpreter) override; }; - -} -} -#endif +} // namespace nnapi +} // namespace r2i +#endif // R2I_NNAPI_ENGINE_H diff --git a/r2i/nnapi/frameworkfactory.cc b/r2i/nnapi/frameworkfactory.cc index 827d5d97..2cbfa962 100644 --- a/r2i/nnapi/frameworkfactory.cc +++ b/r2i/nnapi/frameworkfactory.cc @@ -7,35 +7,32 @@ * RidgeRun, LLC. The user is free to modify the source code after obtaining * a software license from RidgeRun. All source code changes must be provided * back to RidgeRun without any encumbrance. -*/ + */ #include "frameworkfactory.h" #include "engine.h" - namespace r2i { namespace nnapi { -std::unique_ptr FrameworkFactory::MakeEngine ( - RuntimeError &error) { - error.Clean (); +std::unique_ptr FrameworkFactory::MakeEngine( + RuntimeError &error) { + error.Clean(); - return std::unique_ptr (new Engine); + return std::unique_ptr(new Engine); } -r2i::FrameworkMeta FrameworkFactory::GetDescription ( - RuntimeError &error) { - const FrameworkMeta meta { - .code = r2i::FrameworkCode::NNAPI, - .name = "NNAPI", - .label = "nnapi", - .description = "TensorFlow Lite with NNAPI delegate from Android" - }; +r2i::FrameworkMeta FrameworkFactory::GetDescription(RuntimeError &error) { + const FrameworkMeta meta{ + .code = r2i::FrameworkCode::NNAPI, + .name = "NNAPI", + .label = "nnapi", + .description = "TensorFlow Lite with NNAPI delegate from Android"}; - error.Clean (); + error.Clean(); return meta; } -} // namespace tflite -} // namespace r2i +} // namespace nnapi +} // namespace r2i diff --git a/r2i/runtimeerror.h b/r2i/runtimeerror.h index 7564b2d3..1069c66c 100644 --- a/r2i/runtimeerror.h +++ b/r2i/runtimeerror.h @@ -101,14 +101,14 @@ class RuntimeError { UNSUPPORTED_FRAMEWORK, /** - * An unknown error has ocurred + * The delegate was not built properly */ - UNKNOWN_ERROR, + DELEGATE_ERROR, /** - * The delegate was not builded properly + * An unknown error has ocurred */ - DELEGATE_ERROR, + UNKNOWN_ERROR, }; /** diff --git a/r2i/tflite/engine.cc b/r2i/tflite/engine.cc index 9b435cd1..7c3dd70a 100644 --- a/r2i/tflite/engine.cc +++ b/r2i/tflite/engine.cc @@ -241,7 +241,6 @@ RuntimeError Engine::PredictAuxiliar(std::shared_ptr in_frame) { "The provided frame input sizes are different to tensor sizes"); return error; } - this->PreprocessInputData(static_cast(frame->GetData()), wanted_width * wanted_height * wanted_channels, this->interpreter.get(), error); if (r2i::RuntimeError::EOK != error.GetCode()) { @@ -356,7 +355,7 @@ void Engine::GetOutputTensorData(::tflite::Interpreter *interpreter, } } -void Engine::ConfigureDelegate(::tflite::Interpreter * /*interpreter*/) { +RuntimeError Engine::ConfigureDelegate(::tflite::Interpreter * /*interpreter*/) { // No implementation for tflite engine } diff --git a/r2i/tflite/engine.h b/r2i/tflite/engine.h index 45bc8191..c09640ca 100644 --- a/r2i/tflite/engine.h +++ b/r2i/tflite/engine.h @@ -62,7 +62,7 @@ class Engine : public IEngine { virtual void SetupResolver(::tflite::ops::builtin::BuiltinOpResolver &resolver); virtual void SetInterpreterContext(::tflite::Interpreter *interpreter); - virtual void ConfigureDelegate(::tflite::Interpreter *interpreter); + virtual RuntimeError ConfigureDelegate(::tflite::Interpreter *interpreter); private: RuntimeError PredictAuxiliar(std::shared_ptr in_frame); From 34e68f1b6c0631f4e4f08ce3e03a934e2c4b7812 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Fri, 26 Nov 2021 11:39:49 -0600 Subject: [PATCH 08/11] Fix backend message and indentation format on nnapi example --- examples/r2i/nnapi/inception.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/r2i/nnapi/inception.cc b/examples/r2i/nnapi/inception.cc index 93f50f23..df1df649 100644 --- a/examples/r2i/nnapi/inception.cc +++ b/examples/r2i/nnapi/inception.cc @@ -144,7 +144,8 @@ int main(int argc, char *argv[]) { r2i::IFrameworkFactory::MakeFactory(r2i::FrameworkCode::NNAPI, error); if (nullptr == factory) { - std::cerr << "TensorFlow backend is not built: " << error << std::endl; + std::cerr << "TFLite NNAPI backend could not be built " << error + << std::endl; exit(EXIT_FAILURE); } From de4039a401ce7430ec6d8b16eb01fb16fae59af2 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Fri, 26 Nov 2021 11:48:13 -0600 Subject: [PATCH 09/11] Fix return error in virtual function for nnapi configure delegate --- r2i/tflite/engine.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/r2i/tflite/engine.cc b/r2i/tflite/engine.cc index 7c3dd70a..45290ad5 100644 --- a/r2i/tflite/engine.cc +++ b/r2i/tflite/engine.cc @@ -357,6 +357,7 @@ void Engine::GetOutputTensorData(::tflite::Interpreter *interpreter, RuntimeError Engine::ConfigureDelegate(::tflite::Interpreter * /*interpreter*/) { // No implementation for tflite engine + return RuntimeError{}; } } //namespace tflite From f787dfd897a2f99a54246219c91048f69ed03f09 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Tue, 30 Nov 2021 16:46:40 -0600 Subject: [PATCH 10/11] Add label of NNAPI and ordering includes --- r2i/nnapi/frameworkfactory.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/r2i/nnapi/frameworkfactory.cc b/r2i/nnapi/frameworkfactory.cc index 2cbfa962..90680d99 100644 --- a/r2i/nnapi/frameworkfactory.cc +++ b/r2i/nnapi/frameworkfactory.cc @@ -8,9 +8,8 @@ * a software license from RidgeRun. All source code changes must be provided * back to RidgeRun without any encumbrance. */ - -#include "frameworkfactory.h" #include "engine.h" +#include "frameworkfactory.h" namespace r2i { namespace nnapi { From 45a5ad61ff5c7d60d158979bc57ef14d3b5fda20 Mon Sep 17 00:00:00 2001 From: Edgar Chaves Date: Tue, 30 Nov 2021 17:20:21 -0600 Subject: [PATCH 11/11] Add label for NNAPI support --- r2i/iframeworkfactory.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r2i/iframeworkfactory.cc b/r2i/iframeworkfactory.cc index 0550da50..f686e7f8 100644 --- a/r2i/iframeworkfactory.cc +++ b/r2i/iframeworkfactory.cc @@ -88,7 +88,7 @@ MakeNNAPIFactory (RuntimeError &error) { return std::unique_ptr (new nnapi::FrameworkFactory); } -#endif +#endif // HAVE_NNAPI typedef std::function(RuntimeError &)> MakeFactory; @@ -124,7 +124,7 @@ const std::unordered_map frameworks ({ #ifdef HAVE_NNAPI {FrameworkCode::NNAPI, MakeNNAPIFactory}, -#endif // HAVE_NNAPI +#endif //HAVE_NNAPI });