diff --git a/examples/r2i/meson.build b/examples/r2i/meson.build index 2925a810..fc28fdd8 100644 --- a/examples/r2i/meson.build +++ b/examples/r2i/meson.build @@ -41,3 +41,7 @@ endif if cdata.get('HAVE_TENSORRT') == true subdir('tensorrt') endif + +if cdata.get('HAVE_NNAPI') == true + subdir('nnapi') +endif diff --git a/examples/r2i/nnapi/inception.cc b/examples/r2i/nnapi/inception.cc new file mode 100644 index 00000000..df1df649 --- /dev/null +++ b/examples/r2i/nnapi/inception.cc @@ -0,0 +1,199 @@ +/* Copyright (C) 2018-2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#include +#include +#include +#include +#include + +#define STB_IMAGE_IMPLEMENTATION +#include "./stb_image.h" + +#define STB_IMAGE_RESIZE_IMPLEMENTATION +#include "./stb_image_resize.h" + +void PrintTopPrediction(std::shared_ptr prediction) { + r2i::RuntimeError error; + int index = 0; + double max = -1; + int num_labels = prediction->GetResultSize() / sizeof(float); + + for (int i = 0; i < num_labels; ++i) { + double current = prediction->At(i, error); + if (current > max) { + max = current; + index = i; + } + } + + std::cout << "Highest probability is label " << index << " (" << max << ")" + << std::endl; +} + +void PrintUsage() { + std::cerr << "Required arguments: " + << "-i [JPG input_image] " + << "-m [Inception TfLite Model] " + << "-s [Model Input Size] " + << "-I [Input Node] " + << "-O [Output Node] \n" + << " Example: " + << " ./inception -i cat.jpg -m graph_inceptionv2_tensorflow.pb " + << "-s 224" << std::endl; +} + +std::unique_ptr PreProcessImage(const unsigned char *input, int width, + int height, int reqwidth, + int reqheight) { + const int channels = 3; + const int scaled_size = channels * reqwidth * reqheight; + std::unique_ptr scaled(new unsigned char[scaled_size]); + std::unique_ptr adjusted(new float[scaled_size]); + + stbir_resize_uint8(input, width, height, 0, scaled.get(), reqwidth, reqheight, + 0, channels); + + for (int i = 0; i < scaled_size; i += channels) { + /* RGB = (RGB - Mean)*StdDev */ + adjusted[i + 0] = (static_cast(scaled[i + 0]) - 127.5) / 127.5; + adjusted[i + 1] = (static_cast(scaled[i + 1]) - 127.5) / 127.5; + adjusted[i + 2] = (static_cast(scaled[i + 2]) - 127.5) / 127.5; + } + + return adjusted; +} + +std::unique_ptr LoadImage(const std::string &path, int reqwidth, + int reqheight) { + int channels = 3; + int width, height, cp; + + unsigned char *img = stbi_load(path.c_str(), &width, &height, &cp, channels); + if (!img) { + std::cerr << "The picture " << path << " could not be loaded"; + return nullptr; + } + + auto ret = PreProcessImage(img, width, height, reqwidth, reqheight); + free(img); + + return ret; +} + +bool ParseArgs(int &argc, char *argv[], std::string &image_path, + std::string &model_path, int &index, int &size, + std::string &in_node, std::string &out_node) { + int option = 0; + while ((option = getopt(argc, argv, "i:m:p:s:I:O:")) != -1) { + switch (option) { + case 'i': + image_path = optarg; + break; + case 'm': + model_path = optarg; + break; + case 'p': + index = std::stoi(optarg); + break; + case 's': + size = std::stoi(optarg); + break; + case 'I': + in_node = optarg; + break; + case 'O': + out_node = optarg; + break; + default: + return false; + } + } + return true; +} + +int main(int argc, char *argv[]) { + r2i::RuntimeError error; + std::string model_path; + std::string image_path; + std::string in_node; + std::string out_node; + int Index = 0; + int size = 0; + + if (false == ParseArgs(argc, argv, image_path, model_path, Index, size, + in_node, out_node)) { + PrintUsage(); + exit(EXIT_FAILURE); + } + + if (image_path.empty() || model_path.empty()) { + PrintUsage(); + exit(EXIT_FAILURE); + } + + auto factory = + r2i::IFrameworkFactory::MakeFactory(r2i::FrameworkCode::NNAPI, error); + + if (nullptr == factory) { + std::cerr << "TFLite NNAPI backend could not be built " << error + << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Loading Model: " << model_path << std::endl; + auto loader = factory->MakeLoader(error); + std::shared_ptr model = loader->Load(model_path, error); + if (error.IsError()) { + std::cerr << "Loader error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Setting model to engine" << std::endl; + std::shared_ptr engine = factory->MakeEngine(error); + error = engine->SetModel(model); + + std::cout << "Loading image: " << image_path << std::endl; + std::unique_ptr image_data = LoadImage(image_path, size, size); + + std::cout << "Configuring frame" << std::endl; + std::shared_ptr frame = factory->MakeFrame(error); + + error = frame->Configure(image_data.get(), size, size, + r2i::ImageFormat::Id::RGB, r2i::DataType::Id::FLOAT); + + std::cout << "Starting engine" << std::endl; + error = engine->Start(); + if (error.IsError()) { + std::cerr << "Engine start error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Predicting..." << std::endl; + std::vector> predictions; + error = engine->Predict(frame, predictions); + if (error.IsError()) { + std::cerr << "Engine prediction error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + /* This model only has one output */ + PrintTopPrediction(predictions[0]); + + std::cout << "Stopping engine" << std::endl; + error = engine->Stop(); + if (error.IsError()) { + std::cerr << "Engine stop error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + return EXIT_SUCCESS; +} diff --git a/examples/r2i/nnapi/meson.build b/examples/r2i/nnapi/meson.build new file mode 100644 index 00000000..b568b5f2 --- /dev/null +++ b/examples/r2i/nnapi/meson.build @@ -0,0 +1,11 @@ +# Compile examples +app_examples = [ + 'inception', +] + +foreach app : app_examples + executable(app, '@0@.cc'.format(app), + include_directories: [configinc, common_inc_dir], + dependencies : [r2inference_lib_dep], + install: false) +endforeach diff --git a/meson.build b/meson.build index a9e345f9..093f1ee2 100644 --- a/meson.build +++ b/meson.build @@ -1,5 +1,5 @@ project('r2inference', ['cpp'], default_options : ['cpp_std=c++11'], - version : '0.11.0', + version : '0.12.0', meson_version : '>= 0.50',) # Set project information @@ -41,6 +41,7 @@ cdata.set('HAVE_TENSORRT', false) cdata.set('HAVE_ONNXRT', false) cdata.set('HAVE_ONNXRT_ACL', false) cdata.set('HAVE_ONNXRT_OPENVINO', false) +cdata.set('HAVE_NNAPI', false) # Define library dependencies for Tensorflow support if get_option('enable-tensorflow') @@ -51,11 +52,13 @@ if get_option('enable-tensorflow') endif # Define library dependencies for Tensorflow Lite support -if get_option('enable-tflite') or get_option('enable-coral') +if get_option('enable-tflite') or get_option('enable-coral') or get_option('enable-nnapi') + dl = cpp.find_library('dl', required: true) + dl_dep = declare_dependency(dependencies: dl) tensorflow_lite = cpp.find_library('tensorflow-lite', required: true) tensorflow_lite_dep = declare_dependency(dependencies: tensorflow_lite) thread_dep = dependency('threads') - lib_tflite_dep = [tensorflow_lite_dep, thread_dep, common_deps] + lib_tflite_dep = [tensorflow_lite_dep, thread_dep, dl_dep, common_deps] cdata.set('HAVE_TFLITE', true) endif @@ -70,6 +73,12 @@ if get_option('enable-coral') cdata.set('HAVE_CORAL', true) endif +# Define library dependencies for NNAPI TensorFlow Lite delegate +if get_option('enable-nnapi') + lib_nnapi_dep = [tensorflow_lite_dep, thread_dep, dl_dep, rt_dep, common_deps] + cdata.set('HAVE_NNAPI', true) +endif + # Define library dependencies for TensorRT support if get_option('enable-tensorrt') diff --git a/meson_options.txt b/meson_options.txt index d8e24c99..7a3a81cf 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -20,3 +20,5 @@ option('enable-onnxrt-acl', type : 'boolean', value: false, description : 'Enable ONNX Runtime backend with ACL execution provider support') option('enable-onnxrt-openvino', type : 'boolean', value: false, description : 'Enable ONNX Runtime backend with OpenVINO execution provider support') +option('enable-nnapi', type : 'boolean', value : false, + description : 'Enable NNAPI delegate for NPU inference execution support') diff --git a/r2i/frameworks.h b/r2i/frameworks.h index adb754b3..ffd8af20 100644 --- a/r2i/frameworks.h +++ b/r2i/frameworks.h @@ -7,7 +7,7 @@ * RidgeRun, LLC. The user is free to modify the source code after obtaining * a software license from RidgeRun. All source code changes must be provided * back to RidgeRun without any encumbrance. -*/ + */ #ifndef R2I_FRAMEWORKS_H #define R2I_FRAMEWORKS_H @@ -57,12 +57,17 @@ enum FrameworkCode { */ TENSORRT, + /** + * Android's NPU delegate + */ + NNAPI, + /** * Number of supported frameworks, mostly for testing purposes. */ MAX_FRAMEWORK }; -} //namespace r2i +} // namespace r2i -#endif //R2I_FRAMEWORKS +#endif // R2I_FRAMEWORKS diff --git a/r2i/iframeworkfactory.cc b/r2i/iframeworkfactory.cc index 9ea5bd19..f686e7f8 100644 --- a/r2i/iframeworkfactory.cc +++ b/r2i/iframeworkfactory.cc @@ -16,6 +16,7 @@ #include "config.h" #include "coral/frameworkfactory.h" +#include "nnapi/frameworkfactory.h" #include "onnxrt/frameworkfactory.h" #include "onnxrt_acl/frameworkfactory.h" #include "onnxrt_openvino/frameworkfactory.h" @@ -81,6 +82,14 @@ MakeTensorRTFactory (RuntimeError &error) { } #endif // HAVE_TENSORRT +#ifdef HAVE_NNAPI +static std::unique_ptr +MakeNNAPIFactory (RuntimeError &error) { + return std::unique_ptr (new + nnapi::FrameworkFactory); +} +#endif // HAVE_NNAPI + typedef std::function(RuntimeError &)> MakeFactory; const std::unordered_map frameworks ({ @@ -113,6 +122,10 @@ const std::unordered_map frameworks ({ {FrameworkCode::TENSORRT, MakeTensorRTFactory}, #endif //HAVE_TENSORRT +#ifdef HAVE_NNAPI + {FrameworkCode::NNAPI, MakeNNAPIFactory}, +#endif //HAVE_NNAPI + }); std::unique_ptr diff --git a/r2i/meson.build b/r2i/meson.build index d6ea7ac0..97db69a2 100644 --- a/r2i/meson.build +++ b/r2i/meson.build @@ -37,6 +37,11 @@ if cdata.get('HAVE_ONNXRT_OPENVINO') == true r2inference_internal_dep += [internal_onnxrt_openvino_dep] endif +if cdata.get('HAVE_NNAPI') == true + subdir('nnapi') + r2inference_internal_dep += [internal_nnapi_dep] +endif + # Define source code r2inference_sources = [ 'classification.cc', diff --git a/r2i/nnapi/engine.cc b/r2i/nnapi/engine.cc new file mode 100644 index 00000000..95d8b0d8 --- /dev/null +++ b/r2i/nnapi/engine.cc @@ -0,0 +1,45 @@ +/* Copyright (C) 2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#include "r2i/nnapi/engine.h" + +#include +#include +#include + +namespace r2i { +namespace nnapi { + +Engine::Engine() : tflite::Engine() { this->number_of_threads = 1; } + +RuntimeError Engine::ConfigureDelegate(::tflite::Interpreter *interpreter) { + RuntimeError error; + ::tflite::StatefulNnApiDelegate::Options options; + options.allow_fp16 = true; + options.allow_dynamic_dimensions = true; + options.disallow_nnapi_cpu = false; + options.accelerator_name = "vsi-npu"; + + auto delegate = ::tflite::evaluation::CreateNNAPIDelegate(options); + + if (!delegate) { + error.Set(RuntimeError::Code::DELEGATE_ERROR, + "NNAPI delegate was not well created"); + } else { + interpreter->ModifyGraphWithDelegate(std::move(delegate)); + } + return error; +} + +Engine::~Engine() { this->Stop(); } + +} // namespace nnapi +} // namespace r2i diff --git a/r2i/nnapi/engine.h b/r2i/nnapi/engine.h new file mode 100644 index 00000000..a2690512 --- /dev/null +++ b/r2i/nnapi/engine.h @@ -0,0 +1,31 @@ +/* Copyright (C) 2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#ifndef R2I_NNAPI_ENGINE_H +#define R2I_NNAPI_ENGINE_H + +#include + +namespace r2i { +namespace nnapi { + +class Engine : public r2i::tflite::Engine { + public: + Engine(); + ~Engine(); + + protected: + RuntimeError ConfigureDelegate(::tflite::Interpreter *interpreter) override; +}; + +} // namespace nnapi +} // namespace r2i +#endif // R2I_NNAPI_ENGINE_H diff --git a/r2i/nnapi/frameworkfactory.cc b/r2i/nnapi/frameworkfactory.cc new file mode 100644 index 00000000..90680d99 --- /dev/null +++ b/r2i/nnapi/frameworkfactory.cc @@ -0,0 +1,37 @@ +/* Copyright (C) 2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ +#include "engine.h" +#include "frameworkfactory.h" + +namespace r2i { +namespace nnapi { + +std::unique_ptr FrameworkFactory::MakeEngine( + RuntimeError &error) { + error.Clean(); + + return std::unique_ptr(new Engine); +} + +r2i::FrameworkMeta FrameworkFactory::GetDescription(RuntimeError &error) { + const FrameworkMeta meta{ + .code = r2i::FrameworkCode::NNAPI, + .name = "NNAPI", + .label = "nnapi", + .description = "TensorFlow Lite with NNAPI delegate from Android"}; + + error.Clean(); + + return meta; +} + +} // namespace nnapi +} // namespace r2i diff --git a/r2i/nnapi/frameworkfactory.h b/r2i/nnapi/frameworkfactory.h new file mode 100644 index 00000000..e26043f6 --- /dev/null +++ b/r2i/nnapi/frameworkfactory.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2021 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#ifndef R2I_NNAPI_FRAMEWORK_FACTORY_H +#define R2I_NNAPI_FRAMEWORK_FACTORY_H + +#include + +namespace r2i { +namespace nnapi { + +class FrameworkFactory : public r2i::tflite::FrameworkFactory { + public: + std::unique_ptr MakeEngine (RuntimeError &error) override; + + r2i::FrameworkMeta GetDescription (RuntimeError &error) override; +}; + +} // namespace nnapi +} // namespace r2i + +#endif //R2I_NNAPI_FRAMEWORK_FACTORY_H diff --git a/r2i/nnapi/meson.build b/r2i/nnapi/meson.build new file mode 100644 index 00000000..b0decf53 --- /dev/null +++ b/r2i/nnapi/meson.build @@ -0,0 +1,23 @@ +# Define source code +nnapi_sources = [ + 'engine.cc', + 'frameworkfactory.cc', +] + +nnapi_headers = [ + 'engine.h', + 'frameworkfactory.h', +] + +# Build library +nnapi_lib = static_library('nnapi', + nnapi_sources, + include_directories : [configinc], + dependencies : [lib_nnapi_dep], +) + +# Install library header files +install_headers(nnapi_headers, subdir : inc_install_dir + '/r2i/nnapi') + +# Define the library as an internal dependency to the current build +internal_nnapi_dep = declare_dependency(link_with: nnapi_lib, dependencies: lib_nnapi_dep) diff --git a/r2i/runtimeerror.h b/r2i/runtimeerror.h index 9fb6d229..1069c66c 100644 --- a/r2i/runtimeerror.h +++ b/r2i/runtimeerror.h @@ -61,7 +61,7 @@ class RuntimeError { INCOMPATIBLE_MODEL, /** - * The provided Parameters is incompatible with the current operation + * The provided Parameters are incompatible with the current operation */ INCOMPATIBLE_PARAMETERS, @@ -100,6 +100,11 @@ class RuntimeError { */ UNSUPPORTED_FRAMEWORK, + /** + * The delegate was not built properly + */ + DELEGATE_ERROR, + /** * An unknown error has ocurred */ diff --git a/r2i/tflite/engine.cc b/r2i/tflite/engine.cc index 546267de..45290ad5 100644 --- a/r2i/tflite/engine.cc +++ b/r2i/tflite/engine.cc @@ -91,6 +91,7 @@ RuntimeError Engine::Start () { } this->SetInterpreterContext(interpreter.get()); + this->ConfigureDelegate(interpreter.get()); std::shared_ptr<::tflite::Interpreter> tflite_interpreter_shared{std::move(interpreter)}; @@ -228,14 +229,18 @@ RuntimeError Engine::PredictAuxiliar(std::shared_ptr in_frame) { int wanted_height = dims->data[1]; int wanted_width = dims->data[2]; int wanted_channels = dims->data[3]; + int total_wanted_size = wanted_height * wanted_width * wanted_channels; - if ((frame->GetWidth() != wanted_width) - or (frame->GetHeight() != wanted_height)) { + int frame_height = frame->GetHeight(); + int frame_width = frame->GetWidth(); + int frame_channels = frame->GetFormat().GetNumPlanes(); + int total_frame_size = frame_height * frame_width * frame_channels; + + if (total_wanted_size != total_frame_size) { error.Set (RuntimeError::Code::FRAMEWORK_ERROR, "The provided frame input sizes are different to tensor sizes"); return error; } - this->PreprocessInputData(static_cast(frame->GetData()), wanted_width * wanted_height * wanted_channels, this->interpreter.get(), error); if (r2i::RuntimeError::EOK != error.GetCode()) { @@ -350,5 +355,10 @@ void Engine::GetOutputTensorData(::tflite::Interpreter *interpreter, } } +RuntimeError Engine::ConfigureDelegate(::tflite::Interpreter * /*interpreter*/) { + // No implementation for tflite engine + return RuntimeError{}; +} + } //namespace tflite } //namepsace r2i diff --git a/r2i/tflite/engine.h b/r2i/tflite/engine.h index 7f79ac99..c09640ca 100644 --- a/r2i/tflite/engine.h +++ b/r2i/tflite/engine.h @@ -62,6 +62,7 @@ class Engine : public IEngine { virtual void SetupResolver(::tflite::ops::builtin::BuiltinOpResolver &resolver); virtual void SetInterpreterContext(::tflite::Interpreter *interpreter); + virtual RuntimeError ConfigureDelegate(::tflite::Interpreter *interpreter); private: RuntimeError PredictAuxiliar(std::shared_ptr in_frame);