diff --git a/INSTALL-Mac.md b/INSTALL-Mac.md index 01aa91f1c..cb4720498 100644 --- a/INSTALL-Mac.md +++ b/INSTALL-Mac.md @@ -1,6 +1,6 @@ # Building the Embedded Learning Library (ELL) on macOS -ELL enables you to design and deploy intelligent machine-learned models onto single-board computers, like Raspberry Pi, Arduino. However, most of your interaction with ELL occurs on a laptop or desktop computer. The steps below describe how to build ELL on a laptop or desktop running macOS. +ELL enables you to design and deploy intelligent machine-learned models onto single-board computers, like Raspberry Pi and Arduino. Most of your interaction with ELL occurs on a laptop or desktop computer, rather than the single-board machine itself. The steps below describe how to build ELL on a laptop or desktop running macOS. ## Cloning the ELL repository @@ -12,7 +12,7 @@ To clone the ELL repository, type git clone https://github.com/Microsoft/ELL.git -## Prerequisites +## Prerequisites for building ELL We recommend using the [*Homebrew*](http://brew.sh/) package manager to download and install prerequisites. Homebrew can be downloaded and installed by @@ -40,7 +40,7 @@ If you already have LLVM installed, ensure that you have version 3.9. Note that ELL uses the [*CMake*](https://cmake.org/) build system, version 3.8 or newer. Optionally, ELL can take advantage of these additional tools: -* [*SWIG*](http://swig.org) version 3.0.12 or newer - a tool that generates Python interaces to C++ libraries. Required if you intend to use ELL from Python +* [*SWIG*](http://swig.org) version 3.0.12 or newer - a tool that generates Python interfaces to C++ libraries. Required if you intend to use ELL from Python * [*OpenBLAS*](http://www.openblas.net/) - fast linear algebra * *Doxygen* - code documentation @@ -51,7 +51,7 @@ To install all of the above, type brew install homebrew/science/openblas brew install doxygen -## Python 3.6 and OpenCV via Miniconda +## Using ELL in Python ELL can optionally be used from Python 3.6. We recommend using the [Miniconda](https://conda.io/miniconda.html) distribution of Python, which makes it easy to install any required Python modules. Download and install Miniconda from here . @@ -63,7 +63,7 @@ Next, activate the environment you just created by typing source activate py36 -You need to repeat this activation command each time you open a new terminal and intend to use ELL from Python. Also, make sure to ativate the `py36` environment before building ELL, to ensure that Python interfaces are created. +You need to repeat this activation command each time you open a new terminal and intend to use ELL from Python. Also, make sure to activate the `py36` environment before building ELL, to ensure that Python interfaces are created. For computer vision tasks, we recommend capturing and preprocessing images using *OpenCV*. To install OpenCV in the current Python environment, type @@ -71,13 +71,13 @@ For computer vision tasks, we recommend capturing and preprocessing images using ## Building ELL -We build ELL by using CMake to create a makefile, invoking that makefile, and optionally building Python interfaces. If you intend to build Python interfaces, make sure to activate the `py36` miniconda environment as described above. +We build ELL by using CMake to create a makefile, invoking that makefile, and optionally building Python interfaces. If you intend to build Python interfaces, make sure to activate the `py36` miniconda environment as described above. In the repository root directory, create a `build` subdirectory and change to that directory. mkdir build cd build - + Invoke CMake by typing cmake .. @@ -88,11 +88,11 @@ Don't forget the two dots (..) at the end of the command! This creates a makefil Optionally, build Python interfaces by typing - make _ELL_python + make _ELL_python The generated executables will appear in `ELL/build/bin`. # Advanced Installation -See our [advanced installation instructions](INSTALL-Advanced.md). +The instructions above are enough to start using ELL. For more advanced topics, like testing and generating documentation, please see our [advanced installation instructions](INSTALL-Advanced.md). diff --git a/INSTALL-Ubuntu.md b/INSTALL-Ubuntu.md index 06fc494b4..317228c1f 100644 --- a/INSTALL-Ubuntu.md +++ b/INSTALL-Ubuntu.md @@ -1,6 +1,6 @@ # Building the Embedded Learning Library (ELL) on Ubuntu Linux -ELL enables you to design and deploy intelligent machine-learned models onto single-board computers, like Raspberry Pi, Arduino. However, most of your interaction with ELL occurs on a laptop or desktop computer. The steps below describe how to build ELL on a laptop or desktop running Ubuntu Linux. +ELL enables you to design and deploy intelligent machine-learned models onto single-board computers, like Raspberry Pi and Arduino. Most of your interaction with ELL occurs on a laptop or desktop computer, rather than the single-board machine itself. The steps below describe how to build ELL on a laptop or desktop running Ubuntu Linux. ## Cloning the ELL repository @@ -12,7 +12,7 @@ To clone the ELL repository, type git clone https://github.com/Microsoft/ELL.git -## Prerequisites +## Prerequisites for building ELL We recommend using the `apt-get` package manager to download and install prerequisites. First, make sure that apt-get is up to date by @@ -21,7 +21,7 @@ We recommend using the `apt-get` package manager to download and install prerequ ### gcc 5, CMake 3.8, libedit, zlib, OpenBLAS, and Doxygen via apt-get ELL requires the following tools and libraries, some of which are installed by default on Ubuntu systems: -* *gcc 5* or newer - C++14 compiler +* *gcc 5* or newer - C++14 compiler * [*CMake*](https://cmake.org/) version 3.8 or newer - build system * *libedit* and *zlib* libraries @@ -50,13 +50,13 @@ and look for the version number. If the version number is 3.9 or greater, you ca ### SWIG 3.0.12 -[*SWIG*](http://swig.org) is a tool that generates Python interaces to C++ libraries. If you intend to use ELL from Python, you must install SWIG version 3.0.12 or newer. At the time of writing this document, `apt-get` doesn't yet have the latest version of `SWIG`, so it must be installed manually +[*SWIG*](http://swig.org) is a tool that generates Python interfaces to C++ libraries. If you intend to use ELL from Python, you must install SWIG version 3.0.12 or newer. At the time of writing this document, `apt-get` doesn't yet have the latest version of `SWIG`, so it must be installed manually wget http://prdownloads.sourceforge.net/swig/swig-3.0.12.tar.gz tar zxvf swig-3.0.12.tar.gz && cd swig-3.0.12 ./configure --without-pcre && make && sudo make install -## Python 3.6 and OpenCV via Miniconda +## Using ELL in Python ELL can optionally be used from Python 3.6. We recommend using the [Miniconda](https://conda.io/miniconda.html) distribution of Python, which makes it easy to install any required Python modules. Download and install Miniconda from here . @@ -68,7 +68,7 @@ Next, activate the environment you just created by typing source activate py36 -You need to repeat this activation command each time you open a new terminal and intend to use ELL from Python. Also, make sure to ativate the `py36` environment before building ELL, to ensure that Python interfaces are created. +You need to repeat this activation command each time you open a new terminal and intend to use ELL from Python. Also, make sure to activate the `py36` environment before building ELL, to ensure that Python interfaces are created. For computer vision tasks, we recommend capturing and preprocessing images using *OpenCV*. To install OpenCV in the current Python environment, type @@ -76,7 +76,7 @@ For computer vision tasks, we recommend capturing and preprocessing images using ## Building ELL -We build ELL by using CMake to create a makefile, invoking that makefile, and optionally building Python interfaces. If you intend to build Python interfaces, make sure to activate the `py36` miniconda environment as described above. +We build ELL by using CMake to create a makefile, invoking that makefile, and optionally building Python interfaces. If you intend to build Python interfaces, make sure to activate the `py36` miniconda environment as described above. In the repository root directory, create a `build` subdirectory and change to that directory. @@ -93,11 +93,10 @@ Don't forget the two dots (..) at the end of the command! This creates a makefil Optionally, build Python interfaces by typing - make _ELL_python + make _ELL_python The generated executables will appear in `ELL/build/bin`. # Advanced Installation -See our [advanced installation instructions](INSTALL-Advanced.md). - +The instructions above are enough to start using ELL. For more advanced topics, like testing and generating documentation, please see our [advanced installation instructions](INSTALL-Advanced.md). diff --git a/INSTALL-Windows.md b/INSTALL-Windows.md index 34f5de1ce..4ab654136 100644 --- a/INSTALL-Windows.md +++ b/INSTALL-Windows.md @@ -1,14 +1,14 @@ # Building a 64-bit version of the Embedded Learning Library (ELL) on Windows -ELL enables you to design and deploy intelligent machine-learned models onto single-board computers, like Raspberry Pi, Arduino. However, most of your interaction with ELL occurs on a laptop or desktop computer. The steps below describe how to build ELL on a laptop or desktop running Windows. +ELL enables you to design and deploy intelligent machine-learned models onto single-board computers, like Raspberry Pi and Arduino. Most of your interaction with ELL occurs on a laptop or desktop computer, rather than the single-board machine itself. The steps below describe how to build ELL on a laptop or desktop running Windows. ## Cloning the ELL repository -The instructions below assume that ELL was obtained from `github.com/Microsoft/ELL` using *git*. For example, one way of doing this is to download and install the git command line tools from and then clone the ELL repository by opening a command prompt and typing +The instructions below assume that ELL was obtained from `github.com/Microsoft/ELL` using *git*. For example, one way of doing this is to download and install the git command line tools from and then clone the ELL repository by opening a command prompt and typing git clone https://github.com/Microsoft/ELL.git -## Prerequisites +## Prerequisites for building ELL ### Visual Studio @@ -20,21 +20,21 @@ ELL uses the [*CMake*](https://cmake.org/) build system, version 3.8 or newer. D ### LLVM 3.9, SWIG 3.0.12, OpenBlas, and Doxygen via NuGet -ELL depends on the [*LLVM*](http://llvm.org/) compiler framework, version 3.9 or newer. +ELL depends on the [*LLVM*](http://llvm.org/) compiler framework, version 3.9 or newer. -SWIG is a tool that generates Python interaces to C++ libraries. If you intend to use ELL from Python, you must install [*SWIG*](http://swig.org) version 3.0.12 or newer. +SWIG is a tool that generates Python interfaces to C++ libraries. If you intend to use ELL from Python, you must install [*SWIG*](http://swig.org) version 3.0.12 or newer. Optionally, ELL can take advantage of these additional tools: * [*OpenBLAS*](http://www.openblas.net/) - fast linear algebra * *Doxygen* - code documentation -We recommend installing all of the above. An easy way to get prebuilt 64-bit versions of these packages is to use the [*NuGet*](https://www.nuget.org/) package manager, version 3.5 or newer. The relevant NuGet packages are specified in `ELL/external/packages.config`. We recommend using the NuGet command line tool called NuGet CLI, which can be downloaded from . After downloading and installing NuGet CLI, open a command prompt, change to the repository's root directory (`ELL`) and type +We recommend installing all of the above. An easy way to get prebuilt 64-bit versions of these packages is to use the [*NuGet*](https://www.nuget.org/) package manager, version 3.5 or newer. The relevant NuGet packages are specified in `ELL/external/packages.config`. We recommend using the NuGet command line tool called NuGet CLI, which can be downloaded from . After downloading and installing NuGet CLI, open a command prompt, change to the repository's root directory (`ELL`) and type nuget.exe restore external/packages.config -PackagesDirectory external NuGet will download the prerequisites into the `ELL/external` directory. -## Python 3.6 and OpenCV via Miniconda +## Using ELL in Python ELL can optionally be used from Python 3.6. We recommend using the [Miniconda](https://conda.io/miniconda.html) distribution of Python, which makes it easy to install any required Python modules. Download and install Miniconda from here . @@ -46,7 +46,7 @@ Next, activate the environment you just created by activate py36 -You need to repeat this activation command each time you open a new terminal and intend to use ELL from Python. Also, make sure to ativate the `py36` environment before building ELL, to ensure that Python interfaces are created. +You need to repeat this activation command each time you open a new terminal and intend to use ELL from Python. Also, make sure to activate the `py36` environment before building ELL, to ensure that Python interfaces are created. For computer vision tasks, we recommend capturing and preprocessing images using *OpenCV*. To install OpenCV in the current Python environment, type @@ -54,7 +54,7 @@ For computer vision tasks, we recommend capturing and preprocessing images using ## Building ELL -We build ELL by using CMake to create a Visual Studio solution, building that solution, and optionally building Python interfaces. If you intend to build Python interfaces, make sure to activate the `py36` environment as described above. +We build ELL by using CMake to create a Visual Studio solution, building that solution, and optionally building Python interfaces. If you intend to build Python interfaces, make sure to activate the `py36` environment as described above. In the repository root directory, create a `build` subdirectory and change to that directory. @@ -65,7 +65,7 @@ If your installed compiler is Visual Studio 2015, invoke CMake as follows cmake -G "Visual Studio 14 2015 Win64" .. -Don't forget the two dots (..) at the end of the command! This command creates a solution file named `ELL.sln`, and other files, in the `build` directory. +Don't forget the two dots (..) at the end of the command! This command creates a solution file named `ELL.sln`, and other files, in the `build` directory. If your compiler is Visual Studio 2017, invoke CMake as follows cmake -G "Visual Studio 15 2017 Win64" .. @@ -76,9 +76,8 @@ Again, don't forget the two dots (..) at the end of the command. After creating The project executables will appear in `ELL/build/bin`. Finally, to build ELL's Python language bindings, type - cmake --build . --target _ELL_python --config Release + cmake --build . --target _ELL_python --config Release # Advanced Installation -See our [advanced installation instructions](INSTALL-Advanced.md). - +The instructions above are enough to start using ELL. For more advanced topics, like testing and generating documentation, please see our [advanced installation instructions](INSTALL-Advanced.md). diff --git a/StyleGuide.md b/StyleGuide.md index 602bb6e4a..745238b1f 100644 --- a/StyleGuide.md +++ b/StyleGuide.md @@ -1,8 +1,5 @@ # Embedded Learning Library (ELL) style guide -This is the C++ coding style guide for the ELL project. - -For Python coding we are using [PEP 8](https://www.python.org/dev/peps/pep-0008/). - +This is the C++ coding style guide for the ELL project. For Python, we use [PEP 8](https://www.python.org/dev/peps/pep-0008/). ## File names and extensions Header files use the extension ".h". Source code files use the extension ".cpp" if they are compiled into a .lib or .exe, and ".tcc" if they contain templated source code. @@ -11,22 +8,22 @@ Each file should typically contain a single class and its name should match the ## Projects A project is a set of source files that compile into a single executable or library, or that implement a single template library. -### Project naming: -Projects have camelCase names, preferrably single word, such as "model", "utilities", "compile". +### Project naming: +Projects have camelCase names, preferably single word, such as "model", "utilities", "compile". -### Directory, namespace, output naming: -All of the source code in a project should be defined in a namespace that shares the project name. All of the source files associated with a project should be contained in a directory that shares the project name. The executable or library generated by a project should share the project name. +### Directory, namespace, output naming: +All of the source code in a project should be defined in a namespace that shares the project name. All of the source files associated with a project should be contained in a directory that shares the project name. The executable or library generated by a project should share the project name. -### Project directory structure: +### Project directory structure: Each project is contained in a directory. The directory contains a CMakeLists.txt file. The directory typically contains the following subdirectories: * "include", for h files * "src" for cpp files (unless the project defines a template library without cpp files) -* "tcc" for tcc files (these define implementaitons of template classes) +* "tcc" for tcc files (these define implementations of template classes) * "test" for source files that define unit tests, invoked via ctest -* "doc" for documentation that does not fit in the source files themselves. For example, complex mathy algorithms may require detailed documentation in a latex file, which would live in the doc directory +* "doc" for documentation that does not fit in the source files themselves. For example, complex math algorithms may require detailed documentation in a latex file, which would live in the doc directory -Additionally, the directory can contain a README.md file. +Additionally, the directory can contain a README.md file. ## Naming Names should be descriptive and self-evident, rather than terse. Use whole words rather than abbreviations. Example: use `GetFunctionDefinition` instead of `GetFuncDef`. Exceptions to this rule are: @@ -39,10 +36,10 @@ use PascalCase, e.g., `class MyClass {};` use camelCase for name of enum values e.g., `enum MyEnum { valueOne, valueTwo };` -Functions, member and non-member: -use PascalCase, e.g., `int GetValue() const { return _value; }` +Functions, member and non-member: +use PascalCase, e.g., `int GetValue() const { return _value; }` -When possible, function names should be imperative mood verbs, e.g., `GetValue()` and `Permute()`. Specifically, accessor member functions should start with `Get`, e.g., `GetOutputVector()`. +When possible, function names should be imperative mood verbs, e.g., `GetValue()` and `Permute()`. Specifically, accessor member functions should start with `Get`, e.g., `GetOutputVector()`. Exceptions to this rule are: * member functions that returns the size of an object, which can simply be `Size()` @@ -54,7 +51,7 @@ Method and Function arguments: camelCase: e.g., `int GetValue(int rowIndex, int Member variables: Use `_` (underscore) followed by camelCase, e.g., `_myMemberVariable` -Template typenames: +Template typenames: Use PascalCase and add the suffix "Type", e.g., `template ` Template variables that are not typenames: Use camelCase, e.g., `template ` @@ -66,13 +63,13 @@ All source code files should start with a header that specifies project name, fi Next, indicate all `#include` statements. First, include files from the local project. Second, include files from other projects, grouping by project and adding the project name as a comment. Finally, include standard libraries. - //////////////////////////////////////////////////////////////////////////////////////////////////// - // + //////////////////////////////////////////////////////////////////////////////////////////////////// + // // Project: Embedded Learning Library (ELL) // File: FileName.h (libraryName) // Authors: Author1, Author2 // - //////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////// #pragma once @@ -114,7 +111,7 @@ Almost all function implementations belong in .cpp and .tcc files. The exception } ## Formatting -Use clang-format to enforce correct formatting. Use the definitions in the file `.clang-format`, which is found in the main project directory. +Use clang-format to enforce correct formatting. Use the definitions in the file `.clang-format`, which is found in the main project directory. ## Documentation All public classes, functions, and variables that are declared in .h files should be preceded by an XML documentation block. The only exception is functions that are defined with =default or =delete, which should not be documented. Each documentation row should begin with three slashes (///) and the first line in a documentation block should contain a summary tag. Follow this example: diff --git a/docs/gallery/Raspberry-Pi-3-Fan-Mount/index.md b/docs/gallery/Raspberry-Pi-3-Fan-Mount/index.md index 837e8e2b3..a14392e54 100644 --- a/docs/gallery/Raspberry-Pi-3-Fan-Mount/index.md +++ b/docs/gallery/Raspberry-Pi-3-Fan-Mount/index.md @@ -29,6 +29,6 @@ See also our tutorial on [Active Cooling your Raspberry Pi 3](/ELL/tutorials/Act |- | License | Distributed under Creative Commons Attribution 4.0 |- -| Notes | Designed by Spencer Persaud
Does not require support material
But it will require extra nylon spacers to get the bar up off the board. +| Notes | Designed by Spencer Persaud
Does not require support material, but requires extra nylon spacers. |= diff --git a/docs/index.md b/docs/index.md index b73fc583b..db352c7e7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,10 +2,10 @@ layout: home title: The Embedded Learning Library --- - -The Embedded Learning Library (ELL) allows you to design and deploy intelligent machine-learned models onto resource constrained platforms and small single-board computers, like Raspberry Pi, Arduino, and micro:bit. The deployed models run locally, without requiring a network connection and without relying on servers in the cloud. ELL is an early preview of the embedded AI and machine learning technologies developed at Microsoft Research. -We built ELL for makers, technology enthusiasts, students, entrepeneurs, and developers who aspire to build intelligent devices and AI-powered gadgets. Our tools, our code, and all of the other resources available on this website are free for anyone to adapt and use (for details, see licensing below). Just keep in mind that ELL is a work in progress and that we expect it change rapidly, including breaking API changes. +The Embedded Learning Library (ELL) allows you to design and deploy intelligent machine-learned models onto resource constrained platforms and small single-board computers, like Raspberry Pi, Arduino, and micro:bit. The deployed models run locally, without requiring a network connection and without relying on servers in the cloud. ELL is an early preview of the embedded AI and machine learning technologies developed at Microsoft Research. + +We built ELL for makers, technology enthusiasts, students, entrepreneurs, and developers who aspire to build intelligent devices and AI-powered gadgets. Our tools, our code, and all of the other resources available on this website are free for anyone to adapt and use (for details, see licensing below). Just keep in mind that ELL is a work in progress and that we expect it change rapidly, including breaking API changes. ELL is a software library and an accompanying set of software tools, written in modern C++, with an optional interface in Python. Download ELL from our [GitHub repository](https://github.com/Microsoft/ELL), either as a [zip file](https://github.com/Microsoft/ELL/archive/master.zip), or with the following clone command: @@ -14,9 +14,9 @@ ELL is a software library and an accompanying set of software tools, written in While the goal of ELL is to deploy software onto resource constrained platforms and small single-board computers, most of the interaction with ELL occurs on a laptop or desktop computer (Windows, Ubuntu Linux, or Mac). Technically, you can think of ELL as a cross-compiler for embedded intelligence - the compiler itself runs on your laptop or desktop computer and the machine code that it generates runs on your single-board computer. ## Installation and Setup -Install ELL on a +Install ELL on a [Windows](https://github.com/Microsoft/ELL/blob/master/INSTALL-Windows.md), [Ubuntu Linux](https://github.com/Microsoft/ELL/blob/master/INSTALL-Ubuntu.md), or [Mac](https://github.com/Microsoft/ELL/blob/master/INSTALL-Mac.md) -laptop or desktop computer. If you intend to deploy models onto a Raspberry Pi, follow our instruction on [setting up the Rasberry Pi](/ELL/tutorials/Setting-Up-your-Raspberry-Pi). +laptop or desktop computer. If you intend to deploy models onto a Raspberry Pi, follow our instruction on [setting up the Raspberry Pi](/ELL/tutorials/Setting-Up-your-Raspberry-Pi). ## Getting Started A great place to start is our [tutorials section](/ELL/tutorials). As we develop and release new functionality in ELL, we publish new tutorials that showcase that functionality. Currently, our tutorials are focused on simple embedded computer vision tasks on Raspberry Pi, but we expect the scope to grow with time. Have fun! @@ -26,7 +26,7 @@ Our [gallery](/ELL/gallery) is a collection of bits and pieces that you can down ## License The ELL code and sample code in our tutorials are released under the [MIT Open Source License](https://github.com/Microsoft/ELL/blob/master/LICENSE.txt). Some of the other content on this website, such as the 3D models in our gallery, are released under the [Creative Commons Attribution 4.0 license](https://creativecommons.org/licenses/by/4.0/). - + ## Code of conduct ELL has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information on this code of conduct, see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/docs/tutorials/Active-Cooling-your-Raspberry-Pi-3/index.md b/docs/tutorials/Active-Cooling-your-Raspberry-Pi-3/index.md index 7c340a53c..544aa6790 100644 --- a/docs/tutorials/Active-Cooling-your-Raspberry-Pi-3/index.md +++ b/docs/tutorials/Active-Cooling-your-Raspberry-Pi-3/index.md @@ -1,22 +1,23 @@ --- layout: default -title: Active Cooling your Raspberry Pi 3 -permalink: /tutorials/Active-Cooling-your-Raspberry-Pi-3/ +title: Active cooling your Raspberry Pi 3 +permalink: /tutorials/Active-cooling-your-Raspberry-Pi-3/ --- -# Active Cooling your Raspberry Pi 3 -The Raspberry Pi 3 tends to overheat when pushed to its limits. When the processor's internal temperature approaches 85 degrees Celsius, it protects itself by clocking down or shutting down completely, and the performance of our AI models takes a hit. +# Active cooling your Raspberry Pi 3 + +The Raspberry Pi 3 tends to overheat when pushed to its limits. When the processor's internal temperature approaches 85 degrees Celsius, it protects itself by clocking down or shutting down completely, and the performance of our AI models takes a hit. ![Idle RPi3 heatmap](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/Pi-3-IR.jpg) -These infra-red images show a Pi running idle (left) and after a few minutes of running a compute-intensive AI model (right). Notice that the main processor heats up much more than any of the other components on the board. Some cooling kits for the Raspberry Pi include heatsinks for the other components, but these infra-red images suggest that we should really focus on cooling the processor. In this tutorial, we will build a simple and effective active cooling solution for the Raspberry Pi 3. +These infra-red images show a Pi running idle (left) and after a few minutes of running a compute-intensive AI model (right). Notice that the main processor heats up much more than any of the other components on the board. Some cooling kits for the Raspberry Pi include heatsinks for the other components, but these infra-red images suggest that we should really focus on cooling the processor. In this tutorial, we will build a simple and effective active cooling solution for the Raspberry Pi 3. -### Materials: +### Materials: -* [Adafruit Aluminum Heat Sink for Raspberry Pi 3 - 15 x 15 x 15mm](https://www.adafruit.com/product/3082) (comes with a thermally conductive sticker), or equivalent. -* [Adafruit Miniature 5V Cooling Fan for Raspberry Pi](https://www.adafruit.com/product/3368) (comes with mouting screws and nuts), or equivalent 5V 0.2A DC brushless fan, 30mm x 30mm, with mounting holes spaced 24mm apart. -* The secret ingredient: our *Pi 3 Fan Mount*. You need to 3D print [this part](/ELL/gallery/Raspberry-Pi-3-Fan-Mount). -* Two M2.5 x 12 pan head machine screws and nuts, to attach the fan mount to the circuit board. +* [Adafruit Aluminum Heat Sink for Raspberry Pi 3 - 15 x 15 x 15mm](https://www.adafruit.com/product/3082) (comes with a thermally conductive sticker), or equivalent. +* [Adafruit Miniature 5V Cooling Fan for Raspberry Pi](https://www.adafruit.com/product/3368) (comes with mouting screws and nuts), or equivalent 5V 0.2A DC brushless fan, 30mm x 30mm, with mounting holes spaced 24mm apart. +* The secret ingredient: our *Pi 3 Fan Mount*. You need to 3D print [this part](/ELL/gallery/Raspberry-Pi-3-Fan-Mount). +* Two M2.5 x 12 pan head machine screws and nuts, to attach the fan mount to the circuit board. ![Pi Active Cooling Materials](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/Pi-Active-Cooling-Materials.jpg) @@ -24,7 +25,7 @@ These infra-red images show a Pi running idle (left) and after a few minutes of **Step 1:** Attach the fan to the 3D printed fan mount using the screws and nuts provided with the fan. Make sure that the fan is oriented such that it blows air towards the mount (when installed, it will blow air on the heatsink, rather than sucking air from the heatsink). -![Attach-Fan-to-Mount](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/Attach-Fan-to-Mount.jpg) +![Attach-Fan-to-Mount](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/Attach-Fan-to-Mount.jpg) **Step 2:** Attach the fan mount to the Pi circuit board using the two M2.5 x 12 machine screws and nuts. @@ -34,17 +35,17 @@ These infra-red images show a Pi running idle (left) and after a few minutes of ![Pi with Fan](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/Pi-with-Fan.jpg) -That's it - your Pi is ready to rock and roll. +That's it - your Pi is ready to rock and roll. #### Mounting on the Raspberry Pi 7" Touchscreen Display -Most other active cooling solutions for the Raspberry Pi come in the form of an active cooling enclosure or case. A variety of different active cooling cases can be purchased online and some of them work well. However, enclosing the Pi in a case isn't always desireable. For example, in many of our own projects, we like to mount our Pis onto the Raspberry Pi 7" Touchscreen Display. Therefore, we specifically designed our active cooling solution to be compatible with the 7" Touchscreen Display. The only difference in the assembly instructions is that we don't need nuts for the M2.5 x 12 screws. Instead, they screw directly into the M2.5 standoffs that come with the 7" Touchscreen Display. +Most other active cooling solutions for the Raspberry Pi come in the form of an active cooling enclosure or case. A variety of different active cooling cases can be purchased online and some of them work well. However, enclosing the Pi in a case isn't always desirable. For example, in many of our own projects, we like to mount our Pis onto the Raspberry Pi 7" Touchscreen Display. Therefore, we specifically designed our active cooling solution to be compatible with the 7" Touchscreen Display. The only difference in the assembly instructions is that we don't need nuts for the M2.5 x 12 screws. Instead, they screw directly into the M2.5 standoffs that come with the 7" Touchscreen Display. ![Pi with Fan on Display](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/Pi-with-Fan-on-Display.jpg) #### Printing without standoffs -You may have noticed that our *Pi 3 Fan Mount* comes in two versions, with or without standoffs. If your 3D printer can print support material, we recommend printing the version with standoffs. On the other hand, if your printer only supports a single filament, you may have more luck printing the version without standoffs. In that case, we recomment using appropriate spacers or standoffs. +You may have noticed that our *Pi 3 Fan Mount* comes in two versions, with or without standoffs. If your 3D printer can print support material, we recommend printing the version with standoffs. On the other hand, if your printer only supports a single filament, you may have more luck printing the version without standoffs. In that case, we recommend using appropriate spacers or standoffs. ![Two Mount Designs](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/Two-Mount-Designs.jpg) @@ -55,10 +56,10 @@ We've seen quite a few recommendations to cool the Pi using only a heatsink, wit * `none` - no cooling at all, just a Raspberry Pi as-is * `heat sink` - heat sink on the processor, fan turned off * `fan` - fan mounted on the Rapsberry Pi using our fan mount, blowing air on the bare processor, no heat sink -* `both` - our full cooling solution, as described above, with both fan and heat sink +* `both` - our full cooling solution, as described above, with both fan and heat sink Here are the results we observed: - + @@ -78,10 +79,10 @@ svg { multi_line_plot("/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/Pi-Heating-Data.tsv", "plot1", "Processor Temperature ºC", null, [35,90], 60); -The x-axis represents time in seconds and the y-axis represents the processor temperature in Celsius. The measurement starts with the idle processor tmeperature, and the stress test begins after 20 seconds. +The x-axis represents time in seconds and the y-axis represents the processor temperature in Celsius. The measurement starts with the idle processor temperature, and the stress test begins after 20 seconds. -The `none` configuration quickly overheats. Within a few minutes, the processor temperature hits 85 degrees, the temperature at which the processor starts protecting itself by throttling down its frequency. The passive cooling configuration, `heat sink`, isn't much better. At first, the heat sink absorbes some of the heat, causing the processor temperature to rise more slowly. However, the heat sink struggles to dissipate that heat and the processor temperature gradually climbs into the 70's. The passive heat sink prevented the processor from reaching the critical temperature of 85 degrees, but came too close for comfort. Processor temperature depends on many factors, such as ambient temperature, processor load, and processor frequency. Moreover, different Rapsberry Pi units behave differently. Our experiment was conducted in an air conditioned office (room temperature was about 26 degrees Celsius) and we can imagine getting into trouble under different circumstances. +The `none` configuration quickly overheats. Within a few minutes, the processor temperature hits 85 degrees, the temperature at which the processor starts protecting itself by throttling down its frequency. The passive cooling configuration, `heat sink`, isn't much better. At first, the heat sink absorbs some of the heat, causing the processor temperature to rise more slowly. However, the heat sink struggles to dissipate that heat and the processor temperature gradually climbs into the 70's. The passive heat sink prevented the processor from reaching the critical temperature of 85 degrees, but came too close for comfort. Processor temperature depends on many factors, such as ambient temperature, processor load, and processor frequency. Moreover, different Raspberry Pi units behave differently. Our experiment was conducted in an air conditioned office (room temperature was about 26 degrees Celsius) and we can imagine getting into trouble under different circumstances. -We were surprised to see the effectiveness of the fan in the `fan` configuration, where the processor temperature quickly stabilizes around 63 degrees. The clear winner is the `both` configutation, where the combination of fan and heat sink works like a champion, and the processor temperature remains below 50 degrees. +We were surprised to see the effectiveness of the fan in the `fan` configuration, where the processor temperature quickly stabilizes around 63 degrees. The clear winner is the `both` configuration, where the combination of fan and heat sink works like a champion, and the processor temperature remains below 50 degrees. -Try to repeat our experiment with different AI workloads and different cooling configurations. To measure processor temperature, use the command `watch /opt/vc/bin/vcgencmd measure_temp`. \ No newline at end of file +Try to repeat our experiment with different AI workloads and different cooling configurations. To measure processor temperature, use the command `watch /opt/vc/bin/vcgencmd measure_temp`. diff --git a/docs/tutorials/Fun-with-Dogs-and-Cats/catlabels.txt b/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/cats.txt similarity index 100% rename from docs/tutorials/Fun-with-Dogs-and-Cats/catlabels.txt rename to docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/cats.txt diff --git a/docs/tutorials/Fun-with-Dogs-and-Cats/doglabels.txt b/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/dogs.txt similarity index 100% rename from docs/tutorials/Fun-with-Dogs-and-Cats/doglabels.txt rename to docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/dogs.txt diff --git a/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/index.md b/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/index.md new file mode 100644 index 000000000..6b472223c --- /dev/null +++ b/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/index.md @@ -0,0 +1,268 @@ +--- +layout: default +title: Fun with Dogs and Cats +permalink: /tutorials/Boosting-classifier-accuracy-by-grouping-categories/ +--- +# Boosting classifier accuracy by grouping categories + +In this tutorial, we will split the 1000 image-categories, which our model was trained to classify, into three disjoint sets: *dogs*, *cats*, and *other* (anything that isn't a dog or a cat). We will demonstrate how a classifier with low accuracy on the original 1000-class problem can have a sufficiently high accuracy on the simpler 3-class problem. We will write a Python script that reads images from the camera, barks when it sees a dog, and meows when it sees a cat. + +--- + +[![screenshot](/ELL/tutorials/Boosting-classifier-accuracy-by-grouping-categories/thumbnail.png)](https://youtu.be/SOmV8tzg_DU) + +#### Materials + +* Laptop or desktop computer +* Raspberry Pi +* Headphones or speakers for your Raspberry Pi +* Raspberry Pi camera or USB webcam +* *optional* - Active cooling attachment (see our [tutorial on cooling your Pi](/ELL/tutorials/Active-cooling-your-Raspberry-Pi-3/)) + +#### Prerequisites + +* Install ELL on your computer ([Windows](https://github.com/Microsoft/ELL/blob/master/INSTALL-Windows.md), [Ubuntu Linux](https://github.com/Microsoft/ELL/blob/master/INSTALL-Ubuntu.md), [Mac](https://github.com/Microsoft/ELL/blob/master/INSTALL-Mac.md)). Specifically, this tutorial requires ELL, CMake, SWIG, and Python 3.6. +* Follow the instructions for [setting up your Raspberry Pi](/ELL/tutorials/Setting-up-your-Raspberry-Pi). +* Complete the basic tutorial, [Getting started with image classification on Raspberry Pi](/ELL/tutorials/Getting-started-with-image-classification-on-the-Raspberry-Pi/), to learn how to produce a Python wrapper for an ELL model. + +## Overview + +The pre-trained models in the [ELL gallery](/ELL/gallery/) are trained to identify 1000 different image categories (see the category names [here](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/categories.txt)). Often times, we are only interested in a subset of these categories and we don't require the fine-grained categorization that the model was trained to provide. For example, we may want to classify images of dogs versus images of cats, whereas the model is actually trained to distinguish between 6 different varieties of cat and 106 different varieties of dog. + +The dogs versus cats classification problem is easier than the original 1000 class problem, so a model that isn't very accurate on the original problem may be perfectly adequate on the simpler problem. Specifically, we will use a model that has an error rate of 64% on the 1000-class problem, but only 5% on the 3-class problem. We will build an application that grabs a frame from a camera, plays a barking sound when it recognizes one of the dog varieties, and plays a meow sound when it recognizes one of the cat varieties. + +## Step 1: Deploy a pre-trained model on a Raspberry Pi + +Start by repeating the steps of the basic tutorial, [Getting Started with Image Classification on Raspberry Pi](/ELL/tutorials/Getting-started-with-image-classification-on-the-Raspberry-Pi/), but replace the model suggested in that tutorial with [this faster and less accurate model](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I160x160x3NCMNCMNBMNBMNBMNBMNC1A/d_I160x160x3NCMNCMNBMNBMNBMNBMNC1A.ell.zip). Namely, download the model, use the `wrap` tool to compile it for the Raspberry Pi, copy the CMake project to the Pi, and build it. After completing these steps, you should have a Python module on your Pi named `model`. + +Copy the following files to your Pi. +- [dogs.txt](/ELL/tutorials/Boosting-classifier-accuracy-by-grouping-categories/dogs.txt) +- [cats.txt](/ELL/tutorials/Boosting-classifier-accuracy-by-grouping-categories/cats.txt) +- [tutorialHelpers.py](/ELL/tutorials/shared/tutorialHelpers.py) + +Additionally, download sound files (.wav) of a dog bark and a cat meow (for example, try this [bark](http://freesound.org/people/davidmenke/sounds/231762/) and this [meow](http://freesound.org/people/blimp66/sounds/397661/) ). Alternatively, record yourself barking and meowing. Rename the bark sound file `woof.wav` and the meow sound file `meow.wav`. + +## Step 2: Write a script + +We will write a Python script that invokes the model on a Raspberry Pi, groups the categories as described above, and takes action if a dog or cat is recognized. If you just want the code, copy the complete script from [here](/ELL/tutorials/Boosting-classifier-accuracy-by-grouping-categories/pets.py). Otherwise, create an empty text file named `pets.py` and copy in the code snippets below. + +First, import the modules we need: + +```python +import sys +import os +import numpy as np +import cv2 +import time +import subprocess +if (os.name == "nt"): + import winsound +import tutorialHelpers as helpers +``` + +Also, import the Python module for the compiled ELL model. + +```python +import model +``` + +As in previous tutorials, define some helper functions to get images from the camera and read a list of categories from file. + +```python +def get_image_from_camera(camera): + if camera is not None: + # if predictor is too slow frames get buffered, this is designed to flush that buffer + ret, frame = camera.read() + if (not ret): + raise Exception('your capture device is not returning images') + return frame + return None + +def get_categories_from_file(fileName): + labels = [] + with open(fileName) as f: + labels = f.read().splitlines() + return labels +``` + +Next, define helper functions that check whether a category is contained in a category list. Since categories can sometimes have more than one text description, each category label may be several strings, separated by commas. Checking whether a category label matches means checking whether any one of those elements is contained in the label, and whether any match occurs in the set. + +```python +def labels_match(a, b): + x = [s.strip().lower() for s in a.split(',')] + y = [s.strip().lower() for s in b.split(',')] + for w in x: + if (w in y): + return True + return False + +def label_in_set(label, label_set): + for x in label_set: + if labels_match(label, x): + return True + return False +``` + +When a prediction belonging to the dog group or the cat group is detected, we want to play the appropriate sound file. Define helper functions that play a bark or a meow. + +```python +# Declare variables that define where to find the sounds files we will play +script_path = os.path.dirname(os.path.abspath(__file__)) +woofSound = os.path.join(script_path, "woof.wav") +meowSound = os.path.join(script_path, "meow.wav") + +def play(filename): + if (os.name == "nt"): + winsound.PlaySound(filename, winsound.SND_FILENAME | winsound.SND_ASYNC) + else: + command = ["aplay", filename] + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, universal_newlines = True) + proc.wait() + +def take_action(group): + if group == "Dog": + # A prediction in the dog category group was detected, play a `woof` sound + play(woofSound) + elif group == "Cat": + # A prediction in the cat category group was detected, play a `meow` sound + play(meowSound) +``` + +Define the main entry point and start the camera. + +```python + if (len(args) < 1): + print("usage: python pets.py categories.txt") + exit() + + # Open the video camera. To use a different camera, change the camera index. + camera = cv2.VideoCapture(0) +``` + +Read the file of category names. Grouping uses subsets of the total categories, where each group is a unique subset. The `doglabels.txt` and `catlabels.txt` files are simply subsets of the larger file containing 1000 classes of objects. For example `doglabels.txt` contains a list of all the dog breeds that the model was trained to recognize. Read the overall categories, and the categories for the dogs and cats groups. + +```python + categories = get_categories_from_file(args[0]) + dogs = get_categories_from_file("dogs.txt") + cats = get_categories_from_file("cats.txt") +``` + +Get the model input and output shapes and allocate an array to hold the model output. + +```python + inputShape = model1.get_default_input_shape() + + outputShape = model1.get_default_output_shape() + predictions = model1.FloatVector(outputShape.Size()) +``` + +For this tutorial, we'll keep some state to ensure we don't keep taking the same action over and over for the same image. Initialize the state as follows. + +```python + lastHist = None + significantDiff = 5000 + lastPredictionTime = 0 + headerText = "" +``` + +Declare a loop where we get an image from the camera and prepare it to be used as input to the model. + +```python + while (cv2.waitKey(1) == 0xFF): + # Get an image from the camera. If you'd like to use a different image, load the image from some other source. + image = get_image_from_camera(camera) + + # Prepare the image to pass to the model. This helper: + # - crops and resizes the image maintaining proper aspect ratio + # - reorders the image channels if needed + # - returns the data as a ravelled numpy array of floats so it can be handed to the model + input = helpers.prepare_image_for_model(image, inputShape.columns, inputShape.rows) +``` + +We'll use OpenCV to get a histogram using OpenCV as a quick way to detect whether the image has changed significantly. This is to create a better experience than having the same action be taken on the same prediction over and over. We'll also ensure that enough time has passed for the sound file to have fully played out. + +```python + hist = np.histogram(input,16,[0,256])[0] + diff = 1 + if lastHist is None: + lastHist = hist + else: + diff = max(lastHist - hist) + + # Check whether the image has changed significantly and that enough time has passed + # since our last prediction to decide whether to predict again + now = time.time() + if diff >= significantDiff and now - lastPredictionTime > 2: +``` + +It's time to call the model to get predictions. + +```python + model1.predict(input, predictions) +``` + +Use the helpers to get the top predictions, which is returned a list of tuples. + +```python + topN = helpers.get_top_n_predictions(predictions, 1) +``` + +Check whether the prediction is part of a group. + +```python + group = "" + label = "" + if len(topN) > 0: + top = topN[0] + label = categories[top[0]] + if label_in_set(label, dogs): + group = "Dog" + elif label_in_set(label, cats): + group = "Cat"``` +``` + +If the prediction is in one of the define category groups, take the appropriate action. + +```python + if not group == "": + # A group was detected, so take action + top = topN[0] + take_action(group) + headerText = "(" + str(int(top[1]*100)) + "%) " + group + lastPredictionTime = now + lastHist = hist + else: + # No group was detected + headerText = "" +``` + +Finally, update the state if enough time has passed and display the image and header text. + +```python + if now - lastPredictionTime > 2: + # Reset the header text + headerText = "" + + helpers.draw_header(image, headerText) + # Display the image using opencv + cv2.imshow('Grouping', image) + +if __name__ == "__main__": + args = sys.argv + args.pop(0) + main(args) +``` + +## Step 3: Classify live video on the Raspberry Pi + +If you followed the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-up-your-Raspberry-Pi), you should have an anaconda environment named `py34`. Activate the environment and run the script. + +``` +source activate py34 +python pets.py categories.txt +``` + +Point your camera at different objects and see how the model classifies them. Look at `dogs.txt` and `cats.txt` to see which categories the model is trained to recognize and try to show those objects to the model. For quick experimentation, point the camera to your computer screen, have your computer display images of different animals, and see when it barks or meows. If you copied the full source for [pets.py](/ELL/tutorials/Boosting-classifier-accuracy-by-grouping-categories/pets.py), you will also see the average time it takes for the model to process a single frame. + +## Troubleshooting +If you run into trouble, you can find some troubleshooting instructions at the bottom of the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-Up-your-Raspberry-Pi). diff --git a/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/pets.py b/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/pets.py new file mode 100644 index 000000000..322ed00c5 --- /dev/null +++ b/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/pets.py @@ -0,0 +1,174 @@ +#################################################################################################### +## +## Project: Embedded Learning Library (ELL) +## File: pets.py +## Authors: Chris Lovett +## Byron Changuion +## +## Requires: Python 3.x +## +#################################################################################################### + +import sys +import os +import numpy as np +import cv2 +import time +import tutorialHelpers as helpers +import subprocess +if (os.name == "nt"): + import winsound + +# import the ELL model's Python module +import model1 + +# Function to return an image from our camera using OpenCV +def get_image_from_camera(camera): + if camera is not None: + # if predictor is too slow frames get buffered, this is designed to flush that buffer + ret, frame = camera.read() + if (not ret): + raise Exception('your capture device is not returning images') + return frame + return None + +# Return an array of strings corresponding to the model's recognized categories or classes. +# The order of the strings in this file are expected to match the order of the +# model's output predictions. +def get_categories_from_file(fileName): + labels = [] + with open(fileName) as f: + labels = f.read().splitlines() + return labels + +# Returns True if an element of the comma separated label `a` is an element of the comma separated label `b` +def labels_match(a, b): + x = [s.strip().lower() for s in a.split(',')] + y = [s.strip().lower() for s in b.split(',')] + for w in x: + if (w in y): + return True + return False + +# Returns True if the label is in the set of labels +def label_in_set(label, label_set): + for x in label_set: + if labels_match(label, x): + return True + return False + +# Declare variables that define where to find the sounds files we will play +script_path = os.path.dirname(os.path.abspath(__file__)) +woofSound = os.path.join(script_path, "woof.wav") +meowSound = os.path.join(script_path, "meow.wav") + +# Helper function to play a sound +def play(filename): + if (os.name == "nt"): + winsound.PlaySound(filename, winsound.SND_FILENAME | winsound.SND_ASYNC) + else: + command = ["aplay", filename] + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, universal_newlines = True) + proc.wait() + +# Helper function to decide what action to take when we detect a group +def take_action(group): + if group == "Dog": + # A prediction in the dog category group was detected, play a `woof` sound + play(woofSound) + elif group == "Cat": + # A prediction in the cat category group was detected, play a `meow` sound + play(meowSound) + +def main(args): + if (len(args) < 1): + print("usage: python pets.py categories.txt") + exit() + + # Open the video camera. To use a different camera, change the camera index. + camera = cv2.VideoCapture(0) + + # Read the category labels + categories = get_categories_from_file(args[0]) + dogs = get_categories_from_file("dogLabels.txt") + cats = get_categories_from_file("catLabels.txt") + + # Get the model's input dimensions. We'll use this information later to resize images appropriately. + inputShape = model1.get_default_input_shape() + + # Create a vector to hold the model's output predictions + outputShape = model1.get_default_output_shape() + predictions = model1.FloatVector(outputShape.Size()) + + # Declare variables to hold state that we use to ensure we play sounds on new recognition + lastHist = None + significantDiff = 5000 + lastPredictionTime = 0 + headerText = "" + + while (cv2.waitKey(1) == 0xFF): + # Get an image from the camera. If you'd like to use a different image, load the image from some other source. + image = get_image_from_camera(camera) + + # Prepare the image to pass to the model. This helper: + # - crops and resizes the image maintaining proper aspect ratio + # - reorders the image channels if needed + # - returns the data as a ravelled numpy array of floats so it can be handed to the model + input = helpers.prepare_image_for_model(image, inputShape.columns, inputShape.rows) + + # Get a histogram using OpenCV as a quick way to detect whether the image has changed significantly + hist = np.histogram(input,16,[0,256])[0] + diff = 1 + if lastHist is None: + lastHist = hist + else: + diff = max(lastHist - hist) + + # Check whether the image has changed significantly and that enough time has passed + # since our last prediction to decide whether to predict again + now = time.time() + if diff >= significantDiff and now - lastPredictionTime > 2: + # Get the predicted classes using the model's predict function on the image input data. + # The predictions are returned as a vector with the probability that the image + # contains the class represented by that index. + model1.predict(input, predictions) + + # Let's grab the value of the top prediction and its index, which represents the top most + # confident match and the class or category it belongs to. + topN = helpers.get_top_n_predictions(predictions, 1) + + # See whether the prediction is in one of our groups + group = "" + caption = "" + label = "" + if len(topN) > 0: + top = topN[0] + label = categories[top[0]] + if label_in_set(label, dogs): + group = "Dog" + elif label_in_set(label, cats): + group = "Cat" + + if not group == "": + # A group was detected, so take action + top = topN[0] + take_action(group) + headerText = "(" + str(int(top[1]*100)) + "%) " + group + lastPredictionTime = now + lastHist = hist + else: + # No group was detected + headerText = "" + + if now - lastPredictionTime > 2: + # Reset the header text + headerText = "" + + helpers.draw_header(image, headerText) + # Display the image using opencv + cv2.imshow('Grouping', image) + +if __name__ == "__main__": + args = sys.argv + args.pop(0) + main(args) diff --git a/docs/tutorials/Fun-with-Dogs-and-Cats/thumbnail.png b/docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/thumbnail.png similarity index 100% rename from docs/tutorials/Fun-with-Dogs-and-Cats/thumbnail.png rename to docs/tutorials/Boosting-classifier-accuracy-by-grouping-categories/thumbnail.png diff --git a/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/Screenshot.jpg b/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/Screenshot.jpg index deeeb55b4..f66fa20f2 100644 Binary files a/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/Screenshot.jpg and b/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/Screenshot.jpg differ diff --git a/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/index.md b/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/index.md index ec6ec1ec1..3321bbac7 100644 --- a/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/index.md +++ b/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/index.md @@ -1,312 +1,299 @@ --- layout: default -title: Comparing Image Classification models side by side on the Raspberry Pi -permalink: /tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/ +title: Comparing image classification models side by side on the Raspberry Pi +permalink: /tutorials/Comparing-image-classification-models-side-by-side-on-the-Raspberry-Pi/ --- -# Comparing Image Classification models side by side on the Raspberry Pi +# Comparing image classification models side by side on the Raspberry Pi -![screenshot](/ELL/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/Screenshot.jpg) +In this tutorial, we will download two models from the [ELL gallery](/ELL/gallery/) and run them side-by-side on a Raspberry Pi. Some of the models on the gallery are slower abd accurate, while others are faster but less accurate. Running two models as once allows us to get a sense of their relative speeds and accuracies. Specifically, we will compare a standard (real valued) Convolutional Neural Network to a Neural Network that contains binarized layers. The binarized model is smaller and faster, but less accurate. -### Materials +--- -* A Raspberry Pi -* Cooling fan attachment (see our [instructions on cooling your Pi](/ELL/gallery/Raspberry-Pi-3-Fan-Mount)) (optional) -* A USB webcam (optional) -* A display (optional) +![screenshot](Screenshot.jpg) -### Overview +#### Materials -In this tutorial, you will download two pretrained image classifiers from the gallery, compile the classifiers for the Raspberry Pi, and write a Python script that invokes the classifiers in a round-robin fashion and displays results side by side. When the Python script runs on the Pi, you will be able to point the camera at a variety of objects and compare both result and evaluation time per frame of the the classifiers. +* Laptop or desktop computer (Windows, Linux, or Mac) +* Raspberry Pi +* Raspberry Pi camera or USB webcam +* *optional* - Active cooling attachment (see our [tutorial on cooling your Pi](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/)) -For the example in this tutorial, we will download a real-valued model and a binarized version of that model to compare side by side. You'll see that the binarized model is much smaller, but less accurate. Runtime characteristics of the models differ too: different models have different resource and power requirements. Play around with other models in the gallery after this tutorial to find the one that best suits your particular scenario. +#### Prerequisites -### Prerequisites -We recommend that you are familiar with the concepts in [Getting Started with Image Classification on the Raspberry Pi](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/). +* Install ELL on your computer ([Windows](https://github.com/Microsoft/ELL/blob/master/INSTALL-Windows.md), [Ubuntu Linux](https://github.com/Microsoft/ELL/blob/master/INSTALL-Ubuntu.md), [Mac](https://github.com/Microsoft/ELL/blob/master/INSTALL-Mac.md)). Specifically, this tutorial requires ELL, CMake, SWIG, and Python 3.6. +* Follow the instructions for [setting up your Raspberry Pi](/ELL/tutorials/Setting-up-your-Raspberry-Pi). +* Complete the basic tutorial, [Getting started with image classification on Raspberry Pi](/ELL/tutorials/Getting-started-with-image-classification-on-the-Raspberry-Pi/), to learn how to produce a Python wrapper for an ELL model. -1. We will use `Python 3.6` for this tutorial on your dev box. -We highly recommend using the miniconda or full anaconda python environment because it comes with many -handy tools like `curl` which we will use later on. +## Step 1: Activate your environment and create a tutorial directory -2. You will also need a simple web cam or a pi cam. If you don't have one handy, we will show you how to load -static images or .mp4 videos and process those instead. +If you followed the setup instructions, you should have an environment named `py36`. Open a terminal window and activate your anaconda environment. -3. Additional Software is needed on your Raspberry Pi - See [Setup Raspberry Pi](/ELL/tutorials/Setting-Up-your-Raspberry-Pi). +``` +[Unix] source activate py36 +[Windows] activate py36 +``` -4. You will need to be sure you built ELL as per the ELL INSTALL-xxx.md instructions at the root of this git repo. You will need to build ELL after you install Python from your activate your conda environment, so that the `CMake` step picks up on the fact that you have Python 3.6 installed. +Then, cd into the directory where you built ELL and create a `sideBySide` directory -### Download pre-trained models -Make a new directory named `sideBySide` in the `build/tutorials` folder which is where we will download a pre-trained model. ``` +cd ELL/build mkdir sideBySide cd sideBySide ``` -ELL has a [gallery of pre-trained models](/ELL/gallery). For this tutorial, we'll use models trained on the ILSVRC2012 data set. Along with the models, you'll want to download a labels file that has friendly text names for each of the 1000 classes that the models are trained to recognize. -We'll use the (ILSVRC2012_labels.txt)(https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/ILSVRC2012_labels.txt) labels file and the following two model files: -* A real-valued model at https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.ell.zip -* A binarized version at https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3NCMNBMNBMNBMNBMNBMNC1A/d_I224x224x3NCMNBMNBMNBMNBMNBMNC1A.ell.zip -Note that the model files are zipped, and have long named indicating their architecture. For convenience, we'll just want to save it locally as `ell1.zip` and `ell2.zip`: +## Step 2: Download two pre-trained models + +Download this [real-valued ELL model](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I160x160x3CMCMCMCMCMCMC1A/d_I160x160x3CMCMCMCMCMCMC1A.ell.zip) and this [binarized ELL model](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I160x160x3NCMNCMNBMNBMNBMNBMNC1A/d_I160x160x3NCMNCMNBMNBMNBMNBMNC1A.ell.zip) +into the `sideBySide` directory. + +``` +curl --location -o model1.ell.zip https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I160x160x3CMCMCMCMCMCMC1A/d_I160x160x3CMCMCMCMCMCMC1A.ell.zip +curl --location -o model2.ell.zip https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I160x160x3NCMNCMNBMNBMNBMNBMNC1A/d_I160x160x3NCMNCMNBMNBMNBMNBMNC1A.ell.zip +``` + +Unzip the compressed files. + ``` -curl --location -o labels.txt https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/ILSVRC2012_labels.txt -curl --location -o ell1.zip https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.ell.zip -curl --location -o ell2.zip https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3NCMNBMNBMNBMNBMNBMNC1A/d_I224x224x3NCMNBMNBMNBMNBMNBMNC1A.ell.zip +unzip model1.ell.zip +unzip model2.ell.zip ``` -Inside `ell1.zip` is the ell model named `d_I224x224x3CMCMCMCMCMCMC1A.ell`, so unzip the archive to the current directory (`sideBySide`). -Inside `ell2.zip` is the ell model named `d_I224x224x3NCMNBMNBMNBMNBMNBMNC1A.ell`, so unzip the archive to the current directory (`sideBySide`). -Recent versions of git come with the `unzip` tool: +Rename them to `model1.ell` and `model2.ell` respectively. + ``` -unzip ell1.zip -unzip ell2.zip +[Unix] mv d_I160x160x3CMCMCMCMCMCMC1A.ell model1.ell && mv d_I160x160x3NCMNCMNBMNBMNBMNBMNC1A.ell model2.ell +[Windows] ren d_I160x160x3CMCMCMCMCMCMC1A.ell model1.ell && ren d_I160x160x3NCMNCMNBMNBMNBMNBMNC1A.ell model2.ell ``` - Rename the `d_I224x224x3CMCMCMCMCMCMC1A.ell` model file to `model1.ell` and rename the `d_I224x224x3NCMNBMNBMNBMNBMNBMNC1A.ell` model file to `model2.ell`: -| Unix | `mv d_I224x224x3CMCMCMCMCMCMC1A.ell model1.ell`
`mv d_I224x224x3NCMNBMNBMNBMNBMNBMNC1A.ell model2.ell` | -| Windows | `ren d_I224x224x3CMCMCMCMCMCMC1A.ell model1.ell`
`ren d_I224x224x3NCMNBMNBMNBMNBMNBMNC1A.ell model2.ell` | +(One Windows, unzip is part of the Git distribution, for example, in `\Program Files\Git\usr\bin`.) +Next, download the file of [category names](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/categories.txt) that correspond to these models. -You should now have a `labels.txt` file, a `model1.ell` file and a `model2.ell` file in the `sideBySide` folder. -### Wrap the models in Python callable modules -For this tutorial we want to call the model from Python. ELL provides a compiler that takes a model and compiles it into code that will run on a target platform - in this case the Raspberry Pi running Linux, so it generates code for armv7-linux-gnueabihf, and for the cortex-a53 CPU. -Similar to the [Getting Started with Image Classification on the Raspberry Pi](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/) tutorial, we'll use the `wrap.py` utility, this time with the `--oudir` option to put the models into different directories: +``` +curl --location -o categories.txt https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/categories.txt +``` + +There should now be `model1.ell` and `model2.ell` files as well as a `categories.txt` file in the `sideBySide` directory. + +## Step 3: Wrap the models in Python callable modules + +Use the `wrap.py` tool to compile the models and create Python wrappers. We'll use the `--outdir` option to put the models into different directories. ```` -python ../../tools/wrap/wrap.py labels.txt model1.ell -lang python -target pi3 -outdir model1 -python ../../tools/wrap/wrap.py labels.txt model2.ell -lang python -target pi3 -outdir model2 +python ../../tools/wrap/wrap.py categories.txt model1.ell -lang python -target pi3 -outdir model1 +python ../../tools/wrap/wrap.py categories.txt model2.ell -lang python -target pi3 -outdir model2 ```` You should see output similar to the following: ```` compiling model... generating python interfaces for model1 in model1 +running opt... running llc... success, now you can build the 'model1' folder ... compiling model... generating python interfaces for model2 in model2 +running opt... running llc... success, now you can build the 'model2' folder ```` -We also want to copy some additional python code to your Raspberry Pi for the purpose of running this tutorial. You can also copy a static image over for testing: +We also want to copy some additional python code to your Raspberry Pi for the purpose of running this tutorial. + +``` +[Unix] cp ../../../docs/tutorials/shared/tutorialHelpers.py pi3 +[Windows] copy ..\..\..\docs\tutorials\shared\tutorialHelpers.py pi3 +``` -| Unix | `cp ../../tools/utilities/pythonlibs/*.py .`
`cp ../../tools/utilities/pitest/coffeemug.jpg .` | -| Windows | `copy ..\..\tools\utilities\pythonlibs\*.py .`
`copy ..\..\tools\utilities\pitest\coffeemug.jpg .` | +You should now have a `sideBySide` folder containing `model1` and `model2` directories as well as some helpful python utilities which we'll use later in this tutorial. -You should now have a `sideBySide` folder containing `model1` and `model2` directories as well as some helpful python utilities which we'll use in the next section. +We are ready to move to the Raspberry Pi. You can copy the `sideBySide` folder to the Pi using the Unix `scp` tool or the Windows [WinSCP](https://winscp.net/eng/index.php) tool. -## Call your model from a Python app -Create a new text file called `sideBySideDemo.py` in your `sideBySide` folder. We'll add Python code to: -* Parse the list of models -* Load the compiled models -* Get an image -* Run the image through each model in turn -* Compose an image made up of the results from each model's predictions -* Show the tiled image result +## Step 4: Call your models from a Python script -If you don't want to type it out, the script can be found [here](/ELL/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/sideBySideDemo.py), otherwise follow along below. +We will write a Python script that grabs images from the camera, invokes the models one at a time, and displays the two frames side-by-side. If you just want the full script, copy it from [here](/ELL/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/sideBySide.py). Otherwise, create an empty text file named `sideBySide.py` and copy in the code snippets below. -First, we need to import the libraries we'll be using in this app, which include system ultilities, numpy and demoHelper that we copied over from ELL utilities: +First, import a few dependencies, including system utilities, opencv, and numpy. ```python import sys import os +import time import numpy as np import cv2 -import demoHelper as d +import tutorialHelpers as helpers ``` -Next, we're going to define a helper function that will split the argument string specifying the models to run side by side, and instantiate a model helper wrapper object for each, returning an array of all the wrapped models. We used the model helper class before in [Getting Started with Image Classification on the Raspberry Pi](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/), it provides handy functions to load and call ELL models. + +Next, we need to import the models. Since they are contained in different directories, add the relative paths so Python can find them: ```python -def get_model_helpers(demoArgs): - """ - Returns a list of model helpers, initialized from the commandline arguments in demoArgs - """ - models = [] - numModels = 0 - - # Split the labels string on commas. - demoArgs.labelsList = [x.strip() for x in demoArgs.labels.split(',')] - - if demoArgs.models: - # Split the models string on commas - demoArgs.modelsList = [x.strip() for x in demoArgs.models.split(',')] - numModels = len(demoArgs.modelsList) - demoArgs.compiledList = [None] * numModels - else: - # Split the compiled string on commas - demoArgs.compiledList = [x.strip() for x in demoArgs.compiledModels.split(',')] - numModels = len(demoArgs.compiledList) - demoArgs.modelsList = [None] * numModels - # If the number of elements in the labelsList is 1, then use the same labels file - # for all models - if (len(demoArgs.labelsList) == 1): - demoArgs.labelsList = demoArgs.labelsList * numModels - - # Iterate over the model list and instantiate a helper for each. - # Interactions with each model will be done via this wrapper class. - helperArgs = demoArgs - for i in range(numModels): - # Instantiate a new helper for the model and initialize it - helper = d.DemoHelper() - helperArgs.labels = demoArgs.labelsList[i] - helperArgs.model = demoArgs.modelsList[i] - helperArgs.compiledModel = demoArgs.compiledList[i] - helper.initialize(helperArgs) - helper.init_image_source() - # Add it to the list - models.append(helper) - - return models -``` -Next, we'll define the main function, which contains the primary application logic for this tutorial. We need to parse the commandline arguments, and call our helper function we just defined to get a list of models we will use. +sys.path.append("model1") +sys.path.append("model1/build/Release") +sys.path.append("model2") +sys.path.append("model2/build/Release") +import model1 +import model2 +``` + +The following functions help us get an image from the camera and read in the categories file. + +```python +def get_image_from_camera(camera): + if camera is not None: + ret, frame = camera.read() + if (not ret): + raise Exception('your capture device is not returning images') + return frame + return None + +# Return an array of strings corresponding to the model's recognized categories or classes. +# The order of the strings in this file are expected to match the order of the +# model's output predictions. +def get_categories_from_file(fileName): + labels = [] + with open(fileName) as f: + labels = f.read().splitlines() + return labels +``` + +Define our main entry point and use the camera as an image source. + ```python def main(args): - """Main function for the Side By Side tutorial""" - demoArgs = d.get_common_commandline_args(args, - "Runs a number of ELL models that predict the same categories, passing images from camera or static image file\n" - "in a round-robin fashion. The output is a tiled image, where each tile is the result of one model." - "Either the ELL model files, or the compiled models' Python modules must be given,\n" - "using the --models or --compiledModels options respectively.\n" - "Example:\n" - " python sideBySideDemo.py categories1.txt,categories2.txt --compiledModels models/pi3/model1,models/pi3/model2\n" - " python sideBySideDemo.py sameCategories.txt --models model3.ell,model4.ell\n" - "This shows opencv window with image classified by the models using given labels") - models = get_model_helpers(demoArgs) - if (len(models) < 1): - print('Found no models to run') - sys.exit() - -``` -We'll use a helper class called TiledImage to crete the output for this tutorial. The TiledImage class composes a set of images into one which can be displayed by OpenCV. Each tile in the output image is the result of passing a frame to a model instance. + camera = cv2.VideoCapture(0) +``` + +Use the function we defined above to read the category names from the file provided on the command line. + ```python - tiledImage = d.TiledImage(len(models)) + categories = get_categories_from_file("categories.txt") +``` + +Define an array to hold our models. + +```python + models = [model1, model2] +``` + +The models expect input in a certain shape. For each model, get this shape and store it for use later on. + +```python + inputShapes = [] + inputShapes.append(models[0].get_default_input_shape()) + inputShapes.append(models[1].get_default_input_shape()) ``` -Declare a loop so we can keep grabbing frames to push through the models. For simplicity, just ask the first model to grab a frame. The same frame will be passed to each model. + +Allocate arrays to store each model's output. + ```python - done = False - while (not done): - # Grab next frame - frame = models[0].get_next_frame() + predictionArrays = [] + outputShape = models[0].get_default_output_shape() + predictionArrays.append(models[0].FloatVector(outputShape.Size())) + outputShape = models[1].get_default_output_shape() + predictionArrays.append(models[1].FloatVector(outputShape.Size())) ``` -We want to pass the frame to each model in turn. For fairness, we'll randomize the order so that no one model benefits from cache locality more than another. + +Declare a tiled image used to compose our results. We get this from the helper module we imported earlier. + ```python - # Run through models in random order to get a fairer average of evaluation time - modelIndexes = np.arange(len(models)) - np.random.shuffle(modelIndexes) + tiledImage = helpers.TiledImage(len(models)) ``` -Iterate over the models. `frame` now holds image data for the model. However, it often cannot be used as-is, because models are typically trained with: -* specific image sizes e.g. 224 x 224 x 3 -* specific ordering of color channels e.g. RGB. Our helper uses OpenCV to grab images from the image source (file or webcam). Their size is dependent on the source, and the ordering is always BGR from OpenCV. Therefore, we need to crop and or resize the image while maintaining the same aspect ratio, and reorder the color channels from BGR to RGB. Since this is such a common operation, the helper implements this in a method called prepare_image_for_predictor: + +Next, set up a loop that keeps going until OpenCV indicates it is done, which is when the user hits any key. At the start of every loop iteration, read an image from the camera. + ```python + while (cv2.waitKey(1) == 0xFF): + image = get_image_from_camera(camera) +``` + +Iterate over the models. In this case, we'll randomize the order so that, on average, no model has an advantage over another due to caching and so on. + +```python + modelIndexes = np.arange(len(models)) + np.random.shuffle(modelIndexes) + for modelIndex in modelIndexes: model = models[modelIndex] - # Prepare the image to send to the model. - # This involves scaling to the required input dimension and re-ordering from BGR to RGB - data = model.prepare_image_for_predictor(frame) ``` -We are now ready to get a classify the image in the frame. The model has a 'predict' method, which will return a list of probabilities for each of the 1000 classes it can detect: +For each model, prepare the image as input to the model's predict function. + +```python + input = helpers.prepare_image_for_model(image, inputShapes[modelIndex].columns, inputShapes[modelIndex].rows) +``` + +With the processed image input handy, call the `predict` method to invoke the model. + ```python - # Get the compiled model to classify the image, by returning a list of probabilities for the classes it can detect - model.predict(data) + model.predict(input, predictionArrays[modelIndex]) ``` -Note that this is just an array of values, where each element is a probability between 0 and 1. It is typical to reject any that do not meet a particular threshold, since that represents low confidence results. Re-ordering so that we get only the Top 5 predictions is also useful. The index of the prediction represents the class, the value represents the score. We can use the labels file to match the index of the prediction to its text label, and then construct a string with the label and score. -We'll use this string as header text for each tile. + +As before, the `predict` method fills the `predictionsArray[modelIndex]` array with the model output. Each element of this array corresponds to one of the 1000 image classes recognized by the model. Extract the top 5 predicted categories by calling the helper function `get_top_n_predictions`. + ```python - # Get the (at most) top 5 predictions that meet our threshold. - top5 = model.get_top_n(model.results, 5) + top5 = helpers.get_top_n_predictions(predictionArrays[modelIndex], 5) +``` - # Turn the top5 into a text string to display - header_text = "".join([model.get_label(element[0]) + "(" + str(int(100 * element[1])) + "%) " for element in top5]) - # Draw the prediction text as a header - modelFrame = np.copy(frame) - model.draw_header(modelFrame, header_text) +`top5` is an array of tuples, where the first element is the category index and the second element is the probability of that category. Match the category indices in `top5` with the category names in `categories`. + +```python + headerText = "".join(["(" + str(int(element[1]*100)) + "%) " + categories[element[0]] + " " for element in top5]) ``` -Each model wrapper is keeping track of how long the `predict` function took to return a result. This is the model's evaluation time, and we will set it as part of the footer text: + +Use the `draw_header` helper function to write the predicted category on the image. Since each model will write its own result, we make a copy of the input image. + ```python - # Draw the evaluation time as the footer - evaluationTime = model.get_times() * 1000 # time in ms - if (evaluationTime is not None): - footerText = '{:.0f}'.format(evaluationTime) + 'ms/frame, ' + model.model_name - model.draw_footer(modelFrame, footerText) + modelFrame = np.copy(image) + helpers.draw_header(modelFrame, headerText) ``` -Lastly, update the tiled image with this frame. Each output frame composed from results of the model will be one tile in the output: + +The model has now produced a frame which has the input image and the model's prediction results. Set this as one of the tiles in the tiledImage and show the result. + ```python - # Set the image with the header and footer text as one of the tiles tiledImage.set_image_at(modelIndex, modelFrame) - done = tiledImage.show() - if done: - break + tiledImage.show() +``` + +Finally, write the code that invokes the `main` function and runs your script. +```python if __name__ == "__main__": args = sys.argv args.pop(0) # when an args list is passed to parse_args, the first argument (program name) needs to be dropped main(sys.argv) ``` -Your `sideBySide` folder is ready to copy to your Raspberry Pi. You can do that using the ‘scp’ tool. On Windows you can use WinSCP. -### SSH into Raspberry Pi +We are ready to move to the Raspberry Pi. You can copy the `sideBySide` folder to the Pi using the Unix `scp` tool or the Windows [WinSCP](https://winscp.net/eng/index.php) tool. -Now log into your Raspberry Pi, either remotely using SSH or directly if you have keyboard and screen attached. +## Step 5: Build the Python wrappers on the Raspberry Pi -Find the `model1` folder you just copied over using scp or winscp and run the following: +Log into your Raspberry Pi, either remotely using SSH or directly if you have a keyboard and screen connected. Find the `sideBySide` folder you just copied over and build the two CMake projects. -```` +``` cd model1 mkdir build cd build cmake .. make -cd .. -cd .. -```` -This builds the Python Module that is then loadable by the demo Python scripts. Do the same for the `model2` folder: - -```` +cd ../.. cd model2 mkdir build cd build cmake .. make -cd .. -cd .. -```` - -### Process a static image -Now if you followed the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-Up-your-Raspberry-Pi) you should have a miniconda -environment named py34. So to run the tutorial do this: - -```` -source activate py34 -python sideBySideDemo.py labels.txt --compiledModels model1/model1,model2/model2 --image coffeemug.jpg -```` - -If you have a display connected you should see the screen shot at the top of this page. - -### Process Video -If you have a USB camera attached to your Pi then you can also use ELL to process video frames: - -```` -python sideBySideDemo.py labels.txt --compiledModels model1/model1,model2/model2 -```` - -You will see the same kind of window appear only this time it is showing the video stream. -Then when your camera is pointed at an object that the model recognizes you will see the label and -confidence % at the top together with an estimated frame rate. +cd ../.. +``` -`Tip`: for quick image recognition results you can point the video camera at a web image of a dog -on your PC screen. ImageNet models can usually do a good job recognizing different dog breeds and -many types of African animals. +## Step 8: Classify live video on the Raspberry Pi -## Next steps -Different models have different characteristics. For example, some are slow but accurate, while others are faster and less accurate. Some have different power draw than others. +If you followed the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-up-your-Raspberry-Pi), you should have an anaconda environment named `py34`. Activate it and run the script that we wrote above. -Experiment with which model works best for you by downloading other models in the [ELL gallery](/ELL/gallery/). +``` +source activate py34 +python sideBySide.py +``` -Try these related tutorials: -* [Fun with Dogs and Cats](/ELL/tutorials/Fun-with-Dogs-and-Cats/) -* [Comparing Image Classification models side by side on the Raspberry Pi](/ELL/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/) +If you have a camera and display connected to your Pi, you should see a window similar to the screenshot at the top of this page. Point your camera at different objects and see how the model classifies them. If you downloaded the full source for [tutorial.py](/ELL/tutorials/shared/tutorial.py), you will also see the average time in milliseconds it takes each model to process a frame. Try to get a sense of the relative accuracy and speed of each model. -### Toubleshooting +## Troubleshooting -If you run into trouble there's some troubleshooting instructions at the bottom of the -[Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-Up-your-Raspberry-Pi). \ No newline at end of file +If you run into trouble, you can find some troubleshooting instructions at the bottom of the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-up-your-Raspberry-Pi). \ No newline at end of file diff --git a/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/sideBySide.py b/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/sideBySide.py new file mode 100644 index 000000000..1d1186e78 --- /dev/null +++ b/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/sideBySide.py @@ -0,0 +1,123 @@ +#################################################################################################### +## +## Project: Embedded Learning Library (ELL) +## File: sideBySide.py +## Authors: Byron Changuion +## +## Requires: Python 3.x +## +#################################################################################################### + +import sys +import os +import time +import numpy as np +import cv2 +import tutorialHelpers as helpers + +# Import models. Since they are contained in different directories, add the relative paths so Python can find them +sys.path.append("model1") +sys.path.append("model1/build/Release") +sys.path.append("model2") +sys.path.append("model2/build/Release") +import model1 +import model2 + +# Function to return an image from our camera using OpenCV +def get_image_from_camera(camera): + if camera is not None: + # if predictor is too slow frames get buffered, this is designed to flush that buffer + ret, frame = camera.read() + if (not ret): + raise Exception('your capture device is not returning images') + return frame + return None + +# Return an array of strings corresponding to the model's recognized categories or classes. +# The order of the strings in this file are expected to match the order of the +# model's output predictions. +def get_categories_from_file(fileName): + labels = [] + with open(fileName) as f: + labels = f.read().splitlines() + return labels + +def main(args): + + # Open the video camera. To use a different camera, change the camera index. + camera = cv2.VideoCapture(0) + + # Read the category labels + categories = get_categories_from_file("categories.txt") + + # Define the models we'll be using + models = [model1, model2] + + # Get the models' input dimensions. We'll use this information later to resize images appropriately. + inputShapes = [] + inputShapes.append(models[0].get_default_input_shape()) + inputShapes.append(models[1].get_default_input_shape()) + + # Create vectors to hold the models' output predictions + predictionArrays = [] + outputShape = models[0].get_default_output_shape() + predictionArrays.append(models[0].FloatVector(outputShape.Size())) + outputShape = models[1].get_default_output_shape() + predictionArrays.append(models[1].FloatVector(outputShape.Size())) + + # Declare a value to hold the prediction times + predictionTimes = [] + predictionTimes.append([]) + predictionTimes.append([]) + meanTimeToPredict = [0.0, 0.0] + + # Declare a tiled image used to compose our results + tiledImage = helpers.TiledImage(len(models)) + + while (cv2.waitKey(1) == 0xFF): + # Get an image from the camera. If you'd like to use a different image, load the image from some other source. + image = get_image_from_camera(camera) + + # Run through models in random order to get a fairer average of evaluation time + modelIndexes = np.arange(len(models)) + np.random.shuffle(modelIndexes) + + for modelIndex in modelIndexes: + model = models[modelIndex] + + # Prepare the image to pass to the model. This helper: + # - crops and resizes the image maintaining proper aspect ratio + # - reorders the image channels if needed + # - returns the data as a ravelled numpy array of floats so it can be handed to the model + input = helpers.prepare_image_for_model(image, inputShapes[modelIndex].columns, inputShapes[modelIndex].rows) + + # Get the predicted classes using the model's predict function on the image input data. + # The predictions are returned as a vector with the probability that the image + # contains the class represented by that index. + start = time.time() + model.predict(input, predictionArrays[modelIndex]) + end = time.time() + + # Let's grab the value of the top 5 predictions and their index, which represents the top five most + # confident matches and the class or category they belong to. + top5 = helpers.get_top_n_predictions(predictionArrays[modelIndex], 5) + + # Draw header text that represents the top5 predictions + modelFrame = np.copy(image) + headerText = "".join(["(" + str(int(element[1]*100)) + "%) " + categories[element[0]] + " " for element in top5]) + helpers.draw_header(modelFrame, headerText) + + # Draw footer text representing the mean evaluation time + meanTimeToPredict[modelIndex] = helpers.get_mean_duration(predictionTimes[modelIndex], end - start) + footerText = '{:.0f}'.format(meanTimeToPredict[modelIndex] * 1000) + 'ms/frame' + helpers.draw_footer(modelFrame, footerText) + + # Set the image with the header and footer text as one of the tiles + tiledImage.set_image_at(modelIndex, modelFrame) + tiledImage.show() + + +if __name__ == "__main__": + args = sys.argv + args.pop(0) # when an args list is passed to parse_args, the first argument (program name) needs to be dropped + main(sys.argv) diff --git a/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/sideBySideDemo.py b/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/sideBySideDemo.py deleted file mode 100644 index 8eb001e8f..000000000 --- a/docs/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/sideBySideDemo.py +++ /dev/null @@ -1,129 +0,0 @@ -#################################################################################################### -## -## Project: Embedded Learning Library (ELL) -## File: sideBySideDemo.py -## Authors: Byron Changuion -## -## Requires: Python 3.x -## -#################################################################################################### - -import sys -import os -import argparse -import numpy as np -import cv2 -script_path = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(script_path, "../../../tools/utilities/pythonlibs")) -import demoHelper as d - -# note: to run this in headless mode on a Linux machine run the following from your terminal window -# export DISPLAY=:0 -# then add the '-save' argument to get tagged frames to be saved to disk. - -def get_model_helpers(demoArgs): - """ - Returns a list of model helpers, initialized from the commandline arguments in demoArgs - """ - models = [] - numModels = 0 - - # Split the labels string on commas. - demoArgs.labelsList = [x.strip() for x in demoArgs.labels.split(',')] - - if demoArgs.models: - # Split the models string on commas - demoArgs.modelsList = [x.strip() for x in demoArgs.models.split(',')] - numModels = len(demoArgs.modelsList) - demoArgs.compiledList = [None] * numModels - else: - # Split the compiled string on commas - demoArgs.compiledList = [x.strip() for x in demoArgs.compiledModels.split(',')] - numModels = len(demoArgs.compiledList) - demoArgs.modelsList = [None] * numModels - # If the number of elements in the labelsList is 1, then use the same labels file - # for all models - if (len(demoArgs.labelsList) == 1): - demoArgs.labelsList = demoArgs.labelsList * numModels - - # Iterate over the model list and instantiate a helper for each. - # Interactions with each model will be done via this wrapper class. - helperArgs = demoArgs - for i in range(numModels): - # Instantiate a new helper for the model and initialize it - helper = d.DemoHelper() - helperArgs.labels = demoArgs.labelsList[i] - helperArgs.model = demoArgs.modelsList[i] - helperArgs.compiledModel = demoArgs.compiledList[i] - helper.initialize(helperArgs) - helper.init_image_source() - # Add it to the list - models.append(helper) - - return models - -def main(args): - """Main function for the Side By Side tutorial""" - - helper = d.DemoHelper() - arg_parser = argparse.ArgumentParser( - "Runs a number of ELL models that predict the same categories, passing images from camera or static image file\n" - "in a round-robin fashion. The output is a tiled image, where each tile is the result of one model." - "Either the ELL model files, or the compiled models' Python modules must be given,\n" - "using the --models or --compiledModels options respectively.\n" - "Example:\n" - " python sideBySideDemo.py categories1.txt,categories2.txt --compiledModels models/pi3/model1,models/pi3/model2\n" - " python sideBySideDemo.py sameCategories.txt --models model3.ell,model4.ell\n" - "This shows opencv window with image classified by the models using given labels") - helper.add_arguments(arg_parser) - demoArgs = arg_parser.parse_args(args) - models = get_model_helpers(demoArgs) - if (len(models) < 1): - print('Found no models to run') - sys.exit() - - tiledImage = d.TiledImage(len(models)) - done = False - while (not done): - # Grab next frame - frame = models[0].get_next_frame() - - # Run through models in random order to get a fairer average of evaluation time - modelIndexes = np.arange(len(models)) - np.random.shuffle(modelIndexes) - - for modelIndex in modelIndexes: - model = models[modelIndex] - # Prepare the image to send to the model. - # This involves scaling to the required input dimension and re-ordering from BGR to RGB - data = model.prepare_image_for_predictor(frame) - - # Get the compiled model to classify the image, by returning a list of probabilities for the classes it can detect - model.predict(data) - - # Get the (at most) top 5 predictions that meet our threshold. - top5 = model.get_top_n(model.results, 5) - - # Turn the top5 into a text string to display - header_text = ", ".join(["(" + str(int(100 * element[1])) + "%)" + model.get_label(element[0]) for element in top5]) - - # Draw the prediction text as a header - modelFrame = np.copy(frame) - model.draw_header(modelFrame, header_text) - # Draw the evaluation time as the footer - evaluationTime = model.get_times() * 1000 # time in ms - if (evaluationTime is not None): - footerText = '{:.0f}'.format(evaluationTime) + 'ms/frame, ' + model.model_name - model.draw_footer(modelFrame, footerText) - - # Set the image with the header and footer text as one of the tiles - tiledImage.set_image_at(modelIndex, modelFrame) - tiledImage.show() - - done = models[0].done() - if done: break - -if __name__ == "__main__": - args = sys.argv - args.pop(0) # when an args list is passed to parse_args, the first argument (program name) needs to be dropped - main(sys.argv) diff --git a/docs/tutorials/Fun-with-Dogs-and-Cats/index.md b/docs/tutorials/Fun-with-Dogs-and-Cats/index.md deleted file mode 100644 index fa00d5e11..000000000 --- a/docs/tutorials/Fun-with-Dogs-and-Cats/index.md +++ /dev/null @@ -1,190 +0,0 @@ ---- -layout: default -title: Fun with Dogs and Cats -permalink: /tutorials/Fun-with-Dogs-and-Cats/ ---- -# Fun with Dogs and Cats - - -[![screenshot](/ELL/tutorials/Fun-with-Dogs-and-Cats/thumbnail.png)](https://youtu.be/SOmV8tzg_DU) - -### Materials - -* A Raspberry Pi (or you can run it on your PC too if you want) -* Headphones or speakers for your Raspberry Pi -* A USB webcam - -### Overview - -This tutorial shows how to group the the predictions returned from an ELL model so you can -respond to any prediction in that group. To demonstrate this we create a grouping for dogs -and cats from the ImageNet model which contains many different breeds of dogs and cats. - -This technique can be useful if you don't care about the detailed breed information. Even if -your optimized ELL model is not accurate enough for individual breeds, you may still get value out of the model by grouping. - -This tutorial then creates a fun response to the recognition of a dog or a cat by playing a sound. -As you probably already guessed, it will bark or meow in response to it seeing a dog or a cat. -This also works in headless mode (no display). -So imagine your pi is watching your lawn, and when a dog arrives, it barks at the dog. -Your kids will have endless fun with this I'm sure. - -### Prerequisites - -This tutorial has the same prerequisites as [tutorial 1](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/). - -### Download pre-trained model - -Be sure to download the same pre-trained ImageNet model used in [tutorial 1](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/). - -### Wrap the model in a Python callable module - -Do the same steps here as shown in [tutorial 1](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/). - -Now you should now have a pi3 folder that is ready to copy to your Raspberry Pi. - -Add the following files to the pi3 folder: -- [doglabels.txt](/ELL/tutorials/Fun-with-Dogs-and-Cats/doglabels.txt) -- [catlabels.txt](/ELL/tutorials/Fun-with-Dogs-and-Cats/catlabels.txt) -- [woof.wav](/ELL/tutorials/Fun-with-Dogs-and-Cats/woof.wav) -- [meow.wav](/ELL/tutorials/Fun-with-Dogs-and-Cats/meow.wav) -- [pets.py](/ELL/tutorials/Fun-with-Dogs-and-Cats/pets.py) - - cp ~/git/ELL/tutorials/Fun-with-Dogs-and-Cats/* pi3 - -You can now copy this folder using the 'scp' tool. From Windows you can use [WinSCP](https://winscp.net/eng/index.php). - -### SSH into Raspberry Pi - -The build instructions are the same as [tutorial 1](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/). - -Log into your Raspberry Pi, either remotely using SSH or directly if you have keyboard and screen attached. - -Find the 'pi3' folder you just copied over using scp or winscp and run the following: - -```` -cd pi3 -mkdir build && cd build -cmake .. -make -cd .. -```` - -This builds the Python Module that is then loadable by the pets.py script. - -### Process a static image - -Now if you followed the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-Up-your-Raspberry-Pi) you should have a miniconda -environment named py34, so to run the tutorial do this: - -```` -source activate py34 -python pets.py categories.txt --compiledModel model --image cat.png --bgr true -```` -And it will classify the image, you should see output like this and you should hear the meow sound: -```` -Cat(90%) -```` - -### Process a folder - -If you have a folder full of your favorite animal pictures including some dogs and cats, you can run -the pets.py script on all those images using this command line: - -```` -python pets.py categories.txt --compiledModel model --folder myFavoriteImages --bgr true -```` - -It will show the first image, then press the `space bar` to advance to the next image -and `escape` to close the app. - -### Process video - -If you have a USB camera attached to your Pi then you can also use ELL to process video frames: - -```` -python pets.py categories.txt --compiledModel model --bgr true -```` -(just press `escape` on your keyboard to close the app) - -You will see the same kind of window appear only this time it is showing the video stream. -Then when your camera is pointed at an object that the model recognizes you will see the label and -confidence % at the top together with an estimated frame rate. - -When a dog or a cat appears you will also hear the woof.wav file or the meow.wav file play. - -`Tip`: for quick image recognition results you can point the video camera at a web image of a dog -on your PC screen. ImageNet models can usually do a good job recognizing different dog breeds and -many types of African animals. - -### How grouping works - -Grouping is a very simple concept. The `doglabels.txt` and `catlabels.txt` files are simply -subsets of the larger file containing 1000 classes of objects. For example `doglabels.txt` -contains a list of all the dog breeds that the model was trained to recognize. - -```` -Norwich terrier -Brittany spaniel -Ibizan hound -flat-coated retriever -toy terrier -Norfolk terrier -Tibetan terrier -Greater Swiss Mountain dog -... -```` - -The `pets.py` script then simply -takes the prediction from the ELL model and checks to see if the result is in one of these lists: -```` - top5 = self.helper.get_top_n(predictions, 5) - text = "" - if len(top5) > 0: - winner = top5[0] - label = winner[0] - if label in self.dogs: - text = "Dog" - elif label in self.cats: - text = "Cat" -```` - -So now we know if text comes back with "Dog" then one of the many dog breeds recognized by the -ImageNet model has been returned in the top confidence spot, namely, top5[0]. - -This just looks at the top prediction. You could easiliy change this code to look for a dog -or cat in any of the top 5 predictions. - -To play sounds we use the built-in Raspberry Pi "aplay" command. You can also run this on -your Windows PC and it will use the windows `winsound` library which is built into the windows -Python runtime. - -`pets.py` also contains some code to make sure it doesn't keep barking at you when it is just -looking at another frame of the same dog or cat. It uses a [numpy histogram](https://docs.scipy.org/doc/numpy/reference/generated/numpy.histogram.html) of the image to detect when each frame of the picture has significantly changed, which is done as follows: - -```` - hist = np.histogram(data,16,[0,256])[0] - diff = 1 - if lastHist is None: - lastHist = hist - else: - diff = max(lastHist - hist) - lastHist = hist -```` - -So if this `diff` is greater than some amount we know the user is looking at something new and then we go on to check to see if a new group is detected. - -### Toubleshooting - -If you run into trouble there's some troubleshooting instructions at the bottom of the -[Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-Up-your-Raspberry-Pi). - - -### Licenses - -The dog barking sound was recorded by davidmenke and published by freesounds.org under -[Creative Commons License Zero](https://creativecommons.org/publicdomain/zero/1.0/). - -The cat meow was recorded by blimp66 and published by freesounds.org under -[Creative Commons License Attribution](https://creativecommons.org/licenses/by/3.0/) -then the sound was edited, downsampled and converted to wav format to shrink the file size. \ No newline at end of file diff --git a/docs/tutorials/Fun-with-Dogs-and-Cats/meow.wav b/docs/tutorials/Fun-with-Dogs-and-Cats/meow.wav deleted file mode 100644 index 471f6e544..000000000 Binary files a/docs/tutorials/Fun-with-Dogs-and-Cats/meow.wav and /dev/null differ diff --git a/docs/tutorials/Fun-with-Dogs-and-Cats/pets.py b/docs/tutorials/Fun-with-Dogs-and-Cats/pets.py deleted file mode 100644 index 7e0420901..000000000 --- a/docs/tutorials/Fun-with-Dogs-and-Cats/pets.py +++ /dev/null @@ -1,150 +0,0 @@ -#################################################################################################### -## -## Project: Embedded Learning Library (ELL) -## File: pets.py -## Authors: Chris Lovett -## -## Requires: Python 3.x -## -#################################################################################################### - -import sys -import os -import numpy as np -import cv2 -import time -import demoHelper as d -import subprocess -if (os.name == "nt"): - import winsound - -# note: to run this in headless mode on a Linux machine run the following from your terminal window -# export DISPLAY=:0 -# then add the '-save' argument to get tagged frames to be saved to disk. - -class Demo: - def __init__(self, helper): - self.script_path = os.path.dirname(os.path.abspath(__file__)) - self.helper = helper - self.dogs = helper.load_labels("doglabels.txt") - self.cats = helper.load_labels("catlabels.txt") - self.woof = os.path.join(self.script_path, "woof.wav") - self.meow = os.path.join(self.script_path, "meow.wav") - self.last = time.time() - - def play(self, filename): - if (os.name == "nt"): - winsound.PlaySound(filename, winsound.SND_FILENAME | winsound.SND_ASYNC) - else: - command = ["aplay", filename] - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, universal_newlines = True) - proc.wait() - - - def labels_match(self, a, b): - x = [s.strip().lower() for s in a.split(',')] - y = [s.strip().lower() for s in b.split(',')] - for w in x: - if (w in y): - return True - return False - - def label_in_set(self, label, label_set): - for x in label_set: - if self.labels_match(label, x): - return True - return False - - - def run(self): - # Initialize image source - self.helper.init_image_source() - lastPrediction = "" - lastCaption = "" - lastHist = None - lastFrame = None - lastLabel = None - significantDiff = 5000 - while not self.helper.done(): - - # Grab next frame - frame = self.helper.get_next_frame() - - # Prepare the image to send to the model. - # This involves scaling to the required input dimension and re-ordering from BGR to RGB - data = self.helper.prepare_image_for_predictor(frame) - - hist = np.histogram(data,16,[0,256])[0] - diff = 1 - if lastHist is None: - lastHist = hist - else: - diff = max(lastHist - hist) - lastHist = hist - - # Get the model to classify the image, by returning a list of probabilities for the classes it can detect - predictions = self.helper.predict(data) - - # Get the (at most) top 5 predictions that meet our threshold. This is returned as a list of tuples, - # each with the text label and the prediction score. - top5 = self.helper.get_top_n(predictions, 5) - text = "" - caption = "" - label = "" - if len(top5) > 0: - winner = top5[0] - label = self.helper.get_label(winner[0]) - if self.label_in_set(label, self.dogs): - text = "Dog" - elif self.label_in_set(label, self.cats): - text = "Cat" - - if not text == "": - caption = text + "(" + str(int(100*winner[1])) + "%)" - else: - caption = "" - - if caption != lastCaption: - lastCaption = caption - - if lastLabel != label: - print(label) - lastLabel = label - - now = time.time() - if diff >= significantDiff and now - self.last > 2: - lastPrediction = "" - - save = False - if text != lastPrediction and text != "": - save = True - self.last = now - lastPrediction = text - if text == "Dog": - self.play(self.woof) - elif text == "Cat": - self.play(self.meow) - - # Draw the text on the frame - frameToShow = frame - helper.draw_label(frameToShow, caption) - helper.draw_fps(frameToShow) - - # Show the new frame - helper.show_image(frameToShow, save) - -if __name__ == "__main__": - args = sys.argv - args.pop(0) # when an args list is passed to parse_args, the first argument (program name) needs to be dropped - helper = d.DemoHelper() - helper.parse_arguments(args, - "Runs the given ELL model passing images from camera or static image file\n" - "Either the ELL model file, or the compiled model's Python module must be given,\n" - "using the --model or --compiledModel options respectively.\n" - "Example:\n" - " python pets.py categories.txt --compiledModel tutorial1/pi3/model1\n" - " python pets.py categories.txt --model model1.ell\n" - "This groups predictions from an image classifier by cats and dogs and plays an appropriate ") - "sound for one of these groups is recognized" - demo = Demo(helper) - demo.run() diff --git a/docs/tutorials/Fun-with-Dogs-and-Cats/woof.wav b/docs/tutorials/Fun-with-Dogs-and-Cats/woof.wav deleted file mode 100644 index 626b31f8f..000000000 Binary files a/docs/tutorials/Fun-with-Dogs-and-Cats/woof.wav and /dev/null differ diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/CMakeLists.txt b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/CMakeLists.txt new file mode 100644 index 000000000..a9e198006 --- /dev/null +++ b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 2.8) +project( tutorial ) +set(CMAKE_CXX_STANDARD 11) + +# Change this to set OpenCV_DIR to your opencv/build location +set(OpenCV_DIR C:/Install/OpenCV3/opencv/build) + +add_subdirectory(model) + +find_package(OpenCV REQUIRED) +add_executable(tutorial tutorial.cpp) +target_link_libraries(tutorial ${OpenCV_LIBS} model) diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/Screenshot.png b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/Screenshot.png new file mode 100644 index 000000000..0f629fa14 Binary files /dev/null and b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/Screenshot.png differ diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/index.md b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/index.md new file mode 100644 index 000000000..d9fc39596 --- /dev/null +++ b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/index.md @@ -0,0 +1,304 @@ +--- +layout: default +title: Getting started with image classification on the Raspberry Pi in C++ +permalink: /tutorials/Getting-started-with-image-classification-in-cpp/ +--- +# Getting started with image classification on Raspberry Pi in C++ + +In this tutorial, we will download a pretrained image classifier from the [ELL gallery](/ELL/gallery/) to a laptop or desktop computer. We will then compile the classifier as a C++ object file and create a CMake project that makes it easy to use the classifier in a C++ program. Finally, we will write a simple C++ application that captures images from the Raspberry Pi's camera and classifies them. + +--- + +![screenshot](/ELL/tutorials/Getting-Started-with-Image-Classification-in-Cpp/Screenshot.png) + +#### Materials + +* Laptop or desktop computer +* Raspberry Pi +* Raspberry Pi camera or USB webcam +* *optional* - Active cooling attachment (see our [tutorial on cooling your Pi](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/)) + +#### Prerequisites + +* Install ELL on your computer ([Windows](https://github.com/Microsoft/ELL/blob/master/INSTALL-Windows.md), [Ubuntu Linux](https://github.com/Microsoft/ELL/blob/master/INSTALL-Ubuntu.md), [Mac](https://github.com/Microsoft/ELL/blob/master/INSTALL-Mac.md)). Specifically, this tutorial requires ELL, CMake, and Python 3.6. Note that Python is required to run a tool named `wrap.py`, which makes compilation easy. If you prefer not to use `wrap.py`, you can perform the compilation steps manually, as described in the [wrap tool documentation](https://github.com/Microsoft/ELL/blob/master/tools/wrap/README.md). +* Follow the instructions for [setting up your Raspberry Pi](/ELL/tutorials/Setting-up-your-Raspberry-Pi). +* *optional* - Read through the instructions in [Getting Started with Image Classification on the Raspberry Pi](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/). + +## Step 1: Create a tutorial directory + +Change to the directory where you built ELL and create a `tutorials` directory. In that directory, create another directory named `cppTutorial`. + +``` +cd ELL/build +mkdir tutorials +cd tutorials +mkdir cppTutorial +cd cppTutorial +``` + +## Step 2: Download pre-trained model + +Download this [compressed ELL model file](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.ell.zip) into the `tutorial1` directory. The model file contains a pre-trained Deep Neural Network for image classification, and is one of the models available from the [ELL gallery](/ELL/gallery). The file's long name indicates the Neural Network's architecture, but don't worry about that for now and save it locally as `model.ell.zip`. + +``` +curl --location -o model.ell.zip https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.ell.zip +``` + +Unzip the compressed file and rename the `d_I224x224x3CMCMCMCMCMCMC1A.ell` model file to `model.ell`: + +``` +[Unix] unzip model.ell.zip`
`mv d_I224x224x3CMCMCMCMCMCMC1A.ell model.ell +[Windows] unzip model.ell.zip`
`ren d_I224x224x3CMCMCMCMCMCMC1A.ell model.ell +``` + +(One Windows, unzip is part of the Git distribution, for example, in `\Program Files\Git\usr\bin`.) +Next, download the file of [category names](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/categories.txt) that correspond to this model into the `cppTutorial` directory. + +``` +curl --location -o categories.txt https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/categories.txt +``` + +There should now be a `model.ell` file and a `categories.txt` file in the `cppTutorial` directory. + +## Step 3: Compile the model for static linking into a C++ program + +Deploying an ELL model to the Raspberry Pi using C++ requires two steps. First, the ELL compiler compiles `model.ell` into machine code. Next, we make all the preparations needed to wrap the compiled model in a CMake project. The result of this step is a special CMake project that contains all of the configurations and settings needed to link the compiled ELL model into a C++ project. + +These steps are performed by a handy tool named `wrap`. Run `wrap` as follows. + +``` +python "../../tools/wrap/wrap.py" categories.txt model.ell -lang cpp -target pi3 -outdir model +``` + +Note that we gave `wrap` the command line option `-target pi3`, which tells it to generate machine code for execution on the Raspberry Pi. We also used the `-outdir model` option to tell `wrap` to put the all output files into a directory named `model`. If all goes well, you should see the following output. + +``` + compiling model... + running llc... + success, now you can build the 'model' folder +``` + +The `wrap` tool creates a `cmake` project in a new directory named `model`. The `model` directory contains an object file with the compiled model and a header file that defines the following static functions. + * model_predict - The model's main function for classifying images + * model_GetInputShape - gets the shape of the input expected by the model + * model_GetOutputShape - get the shape of the output generated by the model + * A CMakeLists.txt file that defines a CMake project, which make it easy to link the model with other CMake projects. + +Copy over some additional C++ helper code that will make it easier to send images to our model. + +``` +[Unix] cp ../../../docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/*.h . +[Windows] copy ..\..\..\docs\tutorials\Getting-Started-with-Image-Classification-in-Cpp\*.h . +``` + +A this point, you should now have a `cppTutorial` folder that contains helper C++ code, a `model` subfolder with the compiled ELL model and a CMake project. + +## Step 4: Call your model from a C++ application + +We will write a C++ application that invokes the model and run the demo on a Raspberry Pi. The application will read images from the camera, pass them to the model, and display the results. Either copy the complete code from [here](/ELL/tutorials/Getting-Started-with-Image-Classification-in-Cpp/tutorial.cpp) or create an empty text file named `tutorial.cpp` in the `cppTutorial` directory and copy in the code snippets below. + +First, add the required include statements. Our code depends on some STL libraries and on the OpenCV library. + +```cpp +#include +#include +#include +#include +#include +``` + +Also include the header file that was generated by ELL when compiling the model. + +```cpp +#include "model.i.h" +``` + +Finally, include the helper functions that were copied over earlier. + +```cpp +#include "tutorialHelpers.h" +``` + +Define the following functions, which will help us get images from the camera and read in the categories file. + +```cpp +// Helper function to get an image from the camera using OpenCV +static cv::Mat getImageFromCamera(cv::VideoCapture& camera) +{ + cv::Mat frame; + camera >> frame; + return frame; +} + +// Return a vector of strings corresponding to the model's recognized categories or classes. +// The order of the strings in this file are expected to match the order of the +// model's output predictions. +static std::vector getCategoriesFromFile(const std::string& filename) +{ + std::vector categories; + + std::ifstream file(filename); + std::string line; + + while (std::getline(file, line)) + { + if (line.length() > 0) categories.emplace_back(line); + } + + return categories; +} +``` + +Define the `main` function and start the camera. + +```cpp +int main(int argc, char** argv ) +{ + if ( argc < 2 ) + { + printf("usage: tutorial categories.txt\n"); + return -1; + } + // Open the video camera. To use a different camera, change the camera index. + cv::VideoCapture camera(0); +``` + +Use the function we defined above to read the category names from the file provided on the command line. + +```cpp + auto categories = getCategoriesFromFile(argv[1]); +``` + +The model expects its input in a certain shape. Get this shape and store it for use later on. + +```cpp + TensorShape inputShape; + model_GetInputShape(0, &inputShape); +``` + +Allocate a vector to store the model's output. + +```cpp + std::vector predictions(model_GetOutputSize()); +``` + +Next, set up a loop that keeps going until OpenCV indicates it is done, which is when the user hits any key. At the start of every loop iteration, read an image from the camera: + +```cpp: + while (cv::waitKey(1) == 0xFF) + { + cv::Mat image = getImageFromCamera(camera); +``` + +The image stored in the `image` variable cannot be sent to the model as-is, because the model takes its input as an vector of `float` values. Moreover, the model expects the input image to have a certain shape and a specific ordering of the color channels (which, in this case, is Blue-Green-Red). Since preparing images for the model is such a common operation, we created a helper function for it named `prepareImageForModel`. + +```cpp + auto input = tutorialHelpers::prepareImageForModel(image, inputShape.columns, inputShape.rows); +``` + +With the processed image input handy, call the `predict` method to invoke the model. + +```cpp + model_predict(input, predictions); +``` + +The `predict` method fills the `predictions` vector with the model output. Each element of this array corresponds to one of the 1000 image classes recognized by the model. Extract the top 5 predicted categories by calling the helper function `getTopNPredictions`. + +```cpp + auto top5 = tutorialHelpers::getTopNPredictions(predictions, 5); +``` + +Match the category indices in `top5` with the category names in `categories`. + +```cpp + std::stringstream headerText; + for (auto element : top5) + { + headerText << categories[element.first] << " (" << std::floor(element.second * 100.0) << "%) "; + } +``` + +Use the `drawHeader` helper function to display the predicted category on the Raspberry Pi's display. Also display the camera image. + +```cpp + tutorialHelpers::drawHeader(image, headerText.str()); + cv::imshow("ELL model", image); + }; + return 0; +} +``` + +## Step 5: Create a CMake project for the application + +We will use CMake to build the application. Either download the complete `CMakeLists.txt` from [here](/ELL/tutorials/Getting-Started-with-Image-Classification-in-Cpp/CMakeLists.txt), or create an empty text file named `CMakeLists.txt` and copy in the project definitions below. + +```cmake +project( tutorial ) +``` + +Set the `OpenCV_DIR` variable to your `OpenCV/build` location. For example, if the full path to that directory is `C:/Install/OpenCV3/opencv/build` then add the following. + +```cmake +set(OpenCV_DIR C:/Install/OpenCV3/opencv/build) +``` + +Tell CMake to find OpenCV. + +```cmake +find_package(OpenCV REQUIRED) +``` + +Next, tell CMake to add your compiled ELL model. + +```cmake +add_subdirectory(model) +``` + +Define the target executable `tutorial` and tell CMake to build `tutorial.cpp` as part of it. + +```cmake +add_executable(tutorial tutorial.cpp) +``` + +Finally, tell CMake to link the application with OpenCV and the ELL model. + +```cmake +target_link_libraries(tutorial ${OpenCV_LIBS} model) +``` + +Your `cppTutorial` folder is ready to copy to your Raspberry Pi. You can do that using the 'scp' tool. On Windows you can use [WinSCP](https://winscp.net/eng/index.php). Be sure to copy the `cppTutorial/model` sub-directory too. + +## Step 6: Build the application on the Raspberry Pi + +Log into your Raspberry Pi, either remotely using SSH or directly if you have a keyboard and screen attached. Find the `cppTutorial` folder you just copied over and build the CMake project. + +``` +cd cppTutorial +mkdir build +cd build +cmake .. +make +cd .. +``` + + +## Step 7: Classify live video on the Raspberry Pi + +Make sure that a camera is connected to your Pi and run the application. + +```` +tutorial categories.txt +```` + +You should see a window similar to the screenshot at the top of this page. Point your camera at different objects and see how the model classifies them. Look at `categories.txt` to see which categories the model is trained to recognize and try to show those objects to the model. For quick experimentation, point the camera to your computer screen and have your computer display images of different objects. For example, experiment with different dog breeds and different types of African animals. + +If you copied the full source for [tutorial.cpp](/ELL/tutorials/Getting-Started-with-Image-Classification-in-Cpp/tutorial.cpp), +you will also see the average time in milliseconds it takes the model to process a frame. + +## Next steps + +The [ELL gallery](/ELL/gallery/) offers different models for image classification. Some are slow but accurate, while others are faster and less accurate. Different models can even lead to different power draw on the Raspberry Pi. Repeat the steps above with different models. + +## Troubleshooting + +If you run into trouble, you can find some troubleshooting instructions at the bottom of the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-up-your-Raspberry-Pi). \ No newline at end of file diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/tutorial.cpp b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/tutorial.cpp new file mode 100644 index 000000000..73480c368 --- /dev/null +++ b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/tutorial.cpp @@ -0,0 +1,121 @@ +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// Project: Embedded Learning Library (ELL) +// File: tutorial.cpp +// Authors: Byron Changuion +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include + +// Include the model interface file for the ELL model we compiled +#include "model.i.h" + +// Include a helper that has useful functions such as preparing data for the model +#include "tutorialHelpers.h" + +// Helper function to get an image from the camera using OpenCV +static cv::Mat getImageFromCamera(cv::VideoCapture& camera) +{ + cv::Mat frame; + camera >> frame; + return frame; +} + +// Helper function to get an image from file using OpenCV +static cv::Mat getImageFromFile(const std::string& filename) +{ + return cv::imread(filename); +} + +// Return a vector of strings corresponding to the model's recognized categories or classes. +// The order of the strings in this file are expected to match the order of the +// model's output predictions. +static std::vector getCategoriesFromFile(const std::string& filename) +{ + std::vector categories; + + std::ifstream file(filename); + std::string line; + + while (std::getline(file, line)) + { + if (line.length() > 0) categories.emplace_back(line); + } + + return categories; +} + +int main(int argc, char** argv ) +{ + if ( argc < 2 ) + { + printf("usage: tutorial categories.txt \n"); + return -1; + } + // Open the video camera. To use a different camera, change the camera index. + cv::VideoCapture camera(0); + + // Read the category labels + auto categories = getCategoriesFromFile(argv[1]); + + // Get the model's input dimensions. We'll use this information later to resize images appropriately. + TensorShape inputShape; + model_GetInputShape(0, &inputShape); + + // Create a vector to hold the model's output predictions + std::vector predictions(model_GetOutputSize()); + + // Declare a value to hold the prediction times + std::vector predictionTimes; + double meanTimeToPredict = 0.0; + + while (cv::waitKey(1) == 0xFF) + { + // Get an image from the camera. If you'd like to use a different image, change this to call getImageFromFile, + // or load the image from some other source. + cv::Mat image = getImageFromCamera(camera); + + // Prepare the image to pass to the model. This helper: + // - crops and resizes the image maintaining proper aspect ratio + // - reorders the image channels if needed + // - returns the data as a std::vector so it can be handed to the model + auto input = tutorialHelpers::prepareImageForModel(image, inputShape.columns, inputShape.rows); + + auto start = std::chrono::steady_clock::now(); + // Get the predicted classes using the model's predict function on the image input data. + // The predictions are returned as a vector with the probability that the image + // contains the class represented by that index. + model_predict(input, predictions); + auto end = std::chrono::steady_clock::now(); + + // Let's grab the value of the top 5 predictions and their index, which represents the top five most + // confident matches and the class or category they belong to. + auto top5 = tutorialHelpers::getTopNPredictions(predictions, 5); + + // Draw header text that represents the top5 predictions + std::stringstream headerText; + for (auto element : top5) + { + headerText << "(" << std::floor(element.second * 100.0) << "%) " << categories[element.first] << " "; + } + tutorialHelpers::drawHeader(image, headerText.str()); + + // Draw footer text representing the mean evaluation time + std::stringstream footerText; + meanTimeToPredict = std::floor(tutorialHelpers::getMeanDuration(predictionTimes, std::chrono::duration(end - start).count()) * 1000); + footerText << meanTimeToPredict << "ms/frame"; + tutorialHelpers::drawFooter(image, footerText.str()); + + // Display the image using opencv + cv::imshow("ELL model", image); + }; + + std::cout << "Mean time to predict: " << meanTimeToPredict << "ms/frame" << std::endl; + + return 0; +} \ No newline at end of file diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/tutorialHelpers.h b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/tutorialHelpers.h new file mode 100644 index 000000000..8d7599624 --- /dev/null +++ b/docs/tutorials/Getting-Started-with-Image-Classification-in-Cpp/tutorialHelpers.h @@ -0,0 +1,127 @@ +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// Project: Embedded Learning Library (ELL) +// File: tutorialHelpers.h +// Authors: Byron Changuion +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include + +class tutorialHelpers +{ +public: + // Prepare an image for use with a model. Typically, this involves: + // - Resize and center crop to the required width and height while preserving the image's aspect ratio. + // Simple resize may result in a stretched or squashed image which will affect the model's ability + // to classify images. + // - OpenCV gives the image in BGR order, so we may need to re-order the channels to RGB. + // - Convert the OpenCV result to a std::vector for use with ELL model + static std::vector prepareImageForModel(cv::Mat& image, int requiredWidth, int requiredHeight, bool reorderToRGB = false) + { + cv::Mat resultImage = cv::Mat::zeros(requiredWidth, requiredHeight, image.type() ); + std::vector result(resultImage.total() * resultImage.channels()); + + // Create a region of interest that defines a center crop, keeping the initial aspect ratio + // but whose destination dimensions are compatible with the requiredWidth and requiredHeight + cv::Rect roi; + if (image.rows >= image.cols) + { + float scale = ( (float) requiredWidth ) / image.cols; + + roi.width = image.cols; + roi.height = (int)(image.rows * scale); + roi.x = 0; + roi.y = (image.rows - roi.height) / 2; + } + else + { + float scale = ( (float) requiredHeight ) / image.rows; + + roi.width = int(image.cols * scale); + roi.height = image.rows; + roi.x = (image.cols - roi.width) / 2; + roi.y = 0; + } + // Crop the image to the region of interest + cv::Mat centerCropped = image(roi); + // Resize to the required dimensions + cv::resize(centerCropped, resultImage, resultImage.size()); + // Re-order if needed + if (reorderToRGB) + { + cv::cvtColor(resultImage, resultImage, cv::COLOR_BGR2RGB); + } + + // Convert the cv::Mat into a vector of floats + result.assign(resultImage.datastart, resultImage.dataend); + + return result; + } + + // Returns up to the top N predictions (that exceed our threshold) as a vector of std::pair, + // where the first represents the index or class of the prediction and the second + // represents that probability or confidence value. + static std::vector> getTopNPredictions(const std::vector& predictions, size_t topN = 5, double threshold = 0.20) + { + std::vector> result; + + // initialize original index locations + std::vector indexes(predictions.size()); + std::iota(indexes.begin(), indexes.end(), 0); + + // sort indexes based on comparing prediction values + std::sort(indexes.begin(), indexes.end(), [&predictions](size_t index1, size_t index2) + { + return predictions[index1] > predictions[index2]; + }); + + // Make a std::pair and append it to the result for N predictions + for (size_t i = 0; i < topN; i++) + { + if (predictions[indexes[i]] > threshold) + { + result.emplace_back(std::pair({ indexes[i], predictions[indexes[i]] })); + } + } + + return result; + } + + // Add a duration to a vector and calculate the mean duration. + static double getMeanDuration(std::vector& accumulated, double duration, size_t maxAccumulatedExntries = 30) + { + accumulated.emplace_back(duration); + if (accumulated.size() > maxAccumulatedExntries) accumulated.erase(accumulated.begin()); + auto meanFunction = [&accumulated](double a, double b) {return a + b / accumulated.size(); }; + + return std::accumulate(accumulated.begin(), accumulated.end(), 0.0, meanFunction); + } + + // Helper to draw a header text block on an image + static void drawHeader(cv::Mat& image, const std::string& text) + { + int blockHeight = 40; + drawTextBlock(image, text, cv::Point(0, 0), cv::Scalar(50, 200, 50), blockHeight); + } + + // Helper to draw a footer text block on an image + static void drawFooter(cv::Mat& image, const std::string& text) + { + int blockHeight = 40; + drawTextBlock(image, text, cv::Point(0, image.rows - blockHeight), cv::Scalar(200, 100, 100), blockHeight); + } + + // Helper to draw a colored text block with black text inside + static void drawTextBlock(cv::Mat& image, const std::string& text, cv::Point topLeft, cv::Scalar color, int height = 40) + { + double fontScale = 0.7; + cv::rectangle(image, topLeft, cv::Point(image.cols, topLeft.y + height), color, cv::FILLED); + cv::putText(image, text, cv::Point(topLeft.x + height / 4, topLeft.y + (int)(height * 0.667)), + cv::FONT_HERSHEY_COMPLEX_SMALL, fontScale, cv::Scalar(0, 0, 0), 1, cv::LINE_AA); + } +}; \ No newline at end of file diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/Screenshot.png b/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/Screenshot.png index dd8909b8e..9cf3bdcb9 100644 Binary files a/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/Screenshot.png and b/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/Screenshot.png differ diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/callModel.py b/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/callModel.py index e2edd4f2b..7da9dca5c 100644 --- a/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/callModel.py +++ b/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/callModel.py @@ -10,6 +10,7 @@ import sys import os +import cv2 import numpy as np # Add the appropriate paths so the wrapped model can be loaded. @@ -23,6 +24,35 @@ # Load the wrapped model's Python module import model +def prepare_image_for_model(image, requiredWidth, requiredHeight, reorderToRGB = False): + """ Prepare an image for use with a model. Typically, this involves: + - Resize and center crop to the required width and height while preserving the image's aspect ratio. + Simple resize may result in a stretched or squashed image which will affect the model's ability + to classify images. + - OpenCV gives the image in BGR order, so we may need to re-order the channels to RGB. + - Convert the OpenCV result to a std::vector for use with ELL model + """ + if image.shape[0] > image.shape[1]: # Tall (more rows than cols) + rowStart = int((image.shape[0] - image.shape[1]) / 2) + rowEnd = rowStart + image.shape[1] + colStart = 0 + colEnd = image.shape[1] + else: # Wide (more cols than rows) + rowStart = 0 + rowEnd = image.shape[0] + colStart = int((image.shape[1] - image.shape[0]) / 2) + colEnd = colStart + image.shape[0] + # Center crop the image maintaining aspect ratio + cropped = image[rowStart:rowEnd, colStart:colEnd] + # Resize to model's requirements + resized = cv2.resize(cropped, (requiredHeight, requiredWidth)) + # Re-order if needed + if not reorderToRGB: + resized = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) + # Return as a vector of floats + result = resized.astype(np.float).ravel() + return result + # Get the input and output shapes input_shape = model.get_default_input_shape() output_shape = model.get_default_output_shape() @@ -30,15 +60,20 @@ print("Model input shape: " + str([input_shape.rows,input_shape.columns,input_shape.channels])) print("Model output shape: " + str([output_shape.rows,output_shape.columns,output_shape.channels])) -# Create a blank input of the appropriate size -input = model.FloatVector(input_shape.Size()) - # Create a blank output of the appropriate size to hold the prediction results predictions = model.FloatVector(output_shape.Size()) +# Read in the sample image +image = cv2.imread("coffeemug.jpg") + +# Prepare the image to send to the model +input = prepare_image_for_model(image, input_shape.columns, input_shape.rows) + # Send the input to the predict function and get the prediction result model.predict(input, predictions) -# Print the indix of the highest confidence prediction -print(np.argmax(predictions)) +# Print the index of the highest confidence prediction +predictionIndex = int(np.argmax(predictions)) +print("Category index: " + str(predictionIndex)) +print("Confidence: " + str(predictions[predictionIndex])) diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/imageHelper.py b/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/imageHelper.py new file mode 100644 index 000000000..53acb0fb0 --- /dev/null +++ b/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/imageHelper.py @@ -0,0 +1,28 @@ +def prepare_image_for_model(image, requiredWidth, requiredHeight, reorderToRGB = False): + """ Prepare an image for use with a model. Typically, this involves: + - Resize and center crop to the required width and height while preserving the image's aspect ratio. + Simple resize may result in a stretched or squashed image which will affect the model's ability + to classify images. + - OpenCV gives the image in BGR order, so we may need to re-order the channels to RGB. + - Convert the OpenCV result to a std::vector for use with ELL model + """ + if image.shape[0] > image.shape[1]: # Tall (more rows than cols) + rowStart = int((image.shape[0] - image.shape[1]) / 2) + rowEnd = rowStart + image.shape[1] + colStart = 0 + colEnd = image.shape[1] + else: # Wide (more cols than rows) + rowStart = 0 + rowEnd = image.shape[0] + colStart = int((image.shape[1] - image.shape[0]) / 2) + colEnd = colStart + image.shape[0] + # Center crop the image maintaining aspect ratio + cropped = image[rowStart:rowEnd, colStart:colEnd] + # Resize to model's requirements + resized = cv2.resize(cropped, (requiredHeight, requiredWidth)) + # Re-order if needed + if not reorderToRGB: + resized = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) + # Return as a vector of floats + result = resized.astype(np.float).ravel() + return result diff --git a/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/index.md b/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/index.md index f88c03eee..ff085bb1d 100644 --- a/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/index.md +++ b/docs/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/index.md @@ -1,91 +1,119 @@ --- layout: default -title: Getting Started with Image Classification on the Raspberry Pi -permalink: /tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/ +title: Getting started with image classification on the Raspberry Pi +permalink: /tutorials/Getting-started-with-image-classification-on-the-Raspberry-Pi/ --- -# Getting Started with Image Classification on Raspberry Pi -![screenshot](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/Screenshot.png) +# Getting started with image classification on Raspberry Pi + +In this tutorial, we will download a pre-trained image classifier from the [ELL gallery](/ELL/gallery/) to a laptop or desktop computer. We will then compile the classifier and wrap it in a Python module. Finally, we will write a simple Python script that captures images from the Raspberry Pi's camera and sends them to the Python module for classification. + +--- -In this tutorial, you will download a pretrained image classifier from the [ELL gallery](/ELL/gallery/) to your laptop or desktop computer and compile it for the Raspberry Pi. Then, you will copy the compiled classifier to your Pi and write a Python script that captures images from the Pi's camera and attempts to classify them. +![screenshot](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/Screenshot.png) -## Materials +#### Materials -* Laptop or desktop computer (Windows, Linux, or Mac) +* Laptop or desktop computer * Raspberry Pi -* Raspberry Pi Camera or USB webcam (optional) -* Display (optional) -* Active cooling attachment (see our [tutorial on cooling your Pi](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/)) (optional) +* Raspberry Pi camera or USB webcam +* *optional* - Active cooling attachment (see our [tutorial on cooling your Pi](/ELL/tutorials/Active-cooling-your-Raspberry-Pi-3/)) + +#### Prerequisites -## Prerequisites +* Install ELL on your computer ([Windows](https://github.com/Microsoft/ELL/blob/master/INSTALL-Windows.md), [Ubuntu Linux](https://github.com/Microsoft/ELL/blob/master/INSTALL-Ubuntu.md), [Mac](https://github.com/Microsoft/ELL/blob/master/INSTALL-Mac.md)). Specifically, this tutorial requires ELL, CMake, SWIG, and Python 3.6. +* Follow the instructions for [setting up your Raspberry Pi](/ELL/tutorials/Setting-up-your-Raspberry-Pi). -* Install ELL on your computer ([Windows](https://github.com/Microsoft/ELL/blob/master/INSTALL-Windows.md), [Ubuntu Linux](https://github.com/Microsoft/ELL/blob/master/INSTALL-Ubuntu.md), [Mac](https://github.com/Microsoft/ELL/blob/master/INSTALL-Mac.md)). Specifically, this tutorial relies on ELL, CMake, SWIG, and Python 3.6. -* Follow our instructions for [setting up your Raspberry Pi](/ELL/tutorials/Setting-Up-your-Raspberry-Pi). +## Step 1: Activate your environment and create a tutorial directory + +If you followed the setup instructions, you should have an environment named `py36`. Open a terminal window and activate your Anaconda environment. -## Activate your environment, create tutorials directory -Open a terminal window and activate your anaconda environment. If you followed our setup instructions you will have -an environment named `py36` so you would do this to activate that: ``` -source activate py36 +[Unix] source activate py36 +[Windows] activate py36 ``` -Then cd into your ELL git repo where you did the build already, and create a `tutorials` folder to group all the tutorials under: + +Then, cd into the directory where you built ELL and create a `tutorials` directory. In that directory, create another directory named `tutorial1`. + ``` cd ELL/build mkdir tutorials cd tutorials +mkdir tutorial1 +cd tutorial1 +``` + +## Step 2: Download pre-trained model + +Download this [compressed ELL model file](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.ell.zip) into the `tutorial1` directory. The model file contains a pre-trained Deep Neural Network for image classification, and is one of the models available from the [ELL gallery](/ELL/gallery). The file's long name indicates the Neural Network's architecture, but don't worry about that for now and save it locally as `model.ell.zip`. + ``` +curl --location -o model.ell.zip https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.ell.zip +``` + +Unzip the compressed file. -## Download pre-trained model -Make a new directory named `tutorial1` in the `build/tutorials` folder which is where we will download a pre-trained model. ``` -mkdir tutorial1 -cd tutorial1 +unzip model.ell.zip +``` + +Rename the `d_I224x224x3CMCMCMCMCMCMC1A.ell` model file to `model.ell`: + +``` +[Unix] mv d_I224x224x3CMCMCMCMCMCMC1A.ell model.ell +[Windows] ren d_I224x224x3CMCMCMCMCMCMC1A.ell model.ell ``` -ELL has a [gallery of pre-trained models](/ELL/gallery). For this tutorial, we'll use a model trained on the ILSVRC2012 data set. Along with the model, you'll want to download a labels file that has friendly text names for each of the 1000 classes that the model is trained to recognize. -We'll use the (ILSVRC2012_labels.txt)(https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/ILSVRC2012_labels.txt) labels file and [this model file](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.ell.zip) from the gallery. Note that the model file is zipped, and has a long name indicating its architecture. For convenience, we'll just want to save it locally as `ell.zip`: + +(One Windows, unzip is part of the Git distribution, for example, in `\Program Files\Git\usr\bin`.) +Next, download the file with the [category names](https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/categories.txt) for this model into the `tutorial1` directory. The 1000 categories in this file are the types of objects that the model is trained to recognize. + ``` -curl --location -o labels.txt https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/ILSVRC2012_labels.txt -curl --location -o ell.zip https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.ell.zip +curl --location -o categories.txt https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/categories.txt ``` -Inside `ell.zip` is the ell model named `d_I224x224x3CMCMCMCMCMCMC1A.ell`, so unzip the archive to the current directory (`tutorial1`). Recent versions of git come with the `unzip` tool: + +There should now be a `model.ell` file and a `categories.txt` file in the `tutorial1` directory. + +## Step 3: Compile and run the model on your laptop or desktop computer + +Before deploying the model to the Raspberry Pi, we will practice deploying it to the laptop or desktop computer. Deploying an ELL model requires two steps. First, we run a tool called `wrap`, which both compiles `model.ell` into a library and generates a CMake project to build a Python wrapper for that library. Second, we call CMake to build the Python library. + +Run `wrap` as follows. + ``` -unzip ell.zip +python "../../tools/wrap/wrap.py" categories.txt model.ell -lang python -target host ``` - Rename the `d_I224x224x3CMCMCMCMCMCMC1A.ell` model file to `model.ell`: -| Unix | `mv d_I224x224x3CMCMCMCMCMCMC1A.ell model.ell` | -| Windows | `ren d_I224x224x3CMCMCMCMCMCMC1A.ell model.ell` | +Note that we gave `wrap` the command line option `-target host`, which tells it to generate machine code for execution on our computer, rather than machine code for the Raspberry Pi. If all goes well, you should see the following output. -You should now have a `labels.txt` file and a `model.ell` file in the `tutorial1` folder. +``` +compiling model... +generating python interfaces for model in host +running llc... +success, now you can build the 'host' folder +``` -Note: this particular model was trained to expect images in the format BGR (as opposed to RGB), so we will -use the "--bgr true" on the demo script below to tell it to use this format for the input to the model. +The `wrap` tool creates a `cmake` project in a new directory named `host`. Create a `build` directory inside the `host` directory and change to that directory -## Wrap the model in a Python module -ELL provides a compiler that takes a model and compiles it into code that will run on a target platform. Let's take a look at how we'd wrap the model for Python to run on the host platform. First, let's use the `wrap` tool to compile the model and generate a `cmake` project for a Python callable module: -```` -python "../../tools/wrap/wrap.py" labels.txt model.ell -lang python -target host -```` -You should see output similar to the following: +``` +cd host +mkdir build +cd build +``` - compiling model... - generating python interfaces for model in host - running llc... - success, now you can build the 'host' folder +To finish creating the Python wrapper, build the `cmake` project. -Next, we build the project in `host` folder using `cmake`. For example: +``` +[Unix] cmake .. && make +[Windows] cmake -G "Visual Studio 14 2015 Win64" -DPROCESSOR_HINT=haswell .. && cmake --build . --config release +``` -| Unix | `cd host`
`mkdir build`
`cd build`
`cmake ..`
`make`
`cd ..` | -| Windows | `cd host`
`mkdir build`
`cd build`
`cmake -G "Visual Studio 14 2015 Win64" -DPROCESSOR_HINT=haswell ..`
`cmake --build . --config release`
`cd ..` | +We have just created a Python module named `model`, which includes functions that report the model's input and output dimensions and enables us to pass images to the image classifier. -Let's take a look at how we might call the model's predict function using Python. Create a new text file called `callModel.py` in your `tutorial1` folder. We'll add Python code to: -* Load the model -* Create blank input and output data -* Run the input through the predictor and get the results +## Step 4: Invoke the model on your computer -If you don't want to type it out, the script can be found [here](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/callModel.py), otherwise follow along below. +The next step is to create a Python script that loads the model, sends images to it, and interprets its output. If you just want the full script, copy it from [here](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/callModel.py) into the `host` directory. Otherwise, create an empty text file named `callModel.py` in the `host` directory and copy in the code snippets below. -First, let's import some modules we'll need, and add some directories to our path so that Python can properly load the model's Python module. Typically, that would be the path of the wrapped model.py file, and the corresponding native module, in build/release: +First, import a few dependencies and add directories to our path, to allow Python to find the module that we created above. ```python import sys @@ -98,16 +126,14 @@ sys.path.append(os.path.join(scriptPath, 'build')) sys.path.append(os.path.join(scriptPath, 'build/Release')) ``` -Now, we can load the model's Python module with a simple `import`: +Next, load the Python module representing the wrapped ELL model. + ```python import model ``` -The model contains functions to: -* Get the input and output dimensions -* Produce predictions from input +Print the model's input and output shapes. -Let's print out the input and output dimensions for this model: ```python input_shape = model.get_default_input_shape() output_shape = model.get_default_output_shape() @@ -115,215 +141,225 @@ print("Model input shape: " + str([input_shape.rows,input_shape.columns,input_sh print("Model output shape: " + str([output_shape.rows,output_shape.columns,output_shape.channels])) ``` -When run on this model, you should see output something like: +You should see output similar to this. - Model input shape: [224, 224, 3] - Model output shape: [1, 1, 1000] +``` +Model input shape: [224, 224, 3] +Model output shape: [1, 1, 1000] +``` -So the input is a 3-channel image of height 224 and width 224 (which can also be represented as a vector of size 224 * 224 * 3), and the output is a tensor of 1x1x1000 which can be represented as a vector of 1000 elements. +The input to the model is a 3-channel image of height 224 and width 224 (which can also be represented as an array of size 224 * 224 * 3 = 150528). The shape of the output is 1 x 1 x 1000, which can be represented as an array of 1000 elements. Allocate an array to store the model's output. -Next, let's create data structures to hold the input and output data: ```python -# Create a blank input of the appropriate size -input = model.FloatVector(input_shape.Size()) -# Create a blank output of the appropriate size to hold the prediction results predictions = model.FloatVector(output_shape.Size()) ``` -The most useful thing this model does is to produce predictions given some input. Let's call the model's predict function with the input: + +Choose an image file to send to the model. For example, use our coffee mug image, [coffeemug.jpg](/ELL/tutorials/shared/coffeemug.jpg). Use OpenCV to read the image: + ```python -model.predict(input, predictions) +image = cv2.imread("coffeemug.jpg") ``` -The predictions come back as a vector of probability values. The index of the value represents the class, and the probability values represents the confidence that the model has that the input is that particular class. -Let's get the index of the highest confidence score: + +The image stored in the `image` variable cannot be sent to the model as-is, because the model takes its input as an array of `float` values. Moreover, the model expects the input image to have a certain shape and a specific ordering of the color channels (which, in this case, is Blue-Green-Red). Since preparing images for the model is such a common operation, we created a helper function for it called `prepare_image_for_model`. Copy and paste the code for [prepare_image_for_model](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/imageHelper.py) and put it at the top of the file, right after the `import model` statement. Then, call the helper function to prepare your image. + ```python -# Print the index of the highest confidence prediction -print(np.argmax(predictions)) +input = prepare_image_for_model(image, input_shape.columns, input_shape.rows) ``` -The output would be something like: - 650 +Finally, invoke the model by calling its `predict` method. -In the above case, we'd lookup line 650 of the labels file to get the text name of the predicted class. +```python +model.predict(input, predictions) +``` -So far, we've called the model with fake input data that is all zeros. Next, let's wrap the model for running on the Pi, and see how we can pass real images through the model instead. +The `predict` method fills the `predictions` array with probability scores, which sum to 1. Each element of this array corresponds to one of the 1000 image classes recognized by the model. Print the index of the highest confidence category. -## Wrap the model in a Python module for the Raspberry Pi -For this tutorial we want to call the model from Python on our Raspberry Pi. ELL's compiler takes a model and compiles it into code that will run on a target platform - in this case the Raspberry Pi running Linux, so it generates code for armv7-linux-gnueabihf, and for the cortex-a53 CPU. We use the `wrap` tool again and this time tell it to target the `pi3` platform: +```python +categories = open('categories.txt', 'r').readlines() +predictionIndex = int(np.argmax(predictions)) +print("Category index: " + str(predictionIndex)) +print("Category text: " + categories[predictionIndex]) +print("Confidence: " + str(predictions[predictionIndex])) +``` + +This code also looks up the category name by reading the corresponding line in `categories.txt`. For example, if the highest confidence category is 504, line 504 of `categories.txt` is `coffee mug`. The value at `predictions[504]` is the model's confidence in this prediction. For example, a value of `0.514353` means that the model is 51% confident that the image contains a coffee mug. + +## Step 5: Compile the model for execution on the Raspberry Pi -```` -# from the tutorials1 folder -python "../../tools/wrap/wrap.py" labels.txt model.ell -lang python -target pi3 -```` -You should see output similar to the following: +We are ready to cross-compile the model for deployment on the Raspberry Pi. As before, run the `wrap` tool on your laptop or desktop computer, but this time specify the target platform as `pi3`. This tells the ELL compiler to generate machine code for the Raspberry Pi's ARM Cortex A53 processor. + +``` +python "../../tools/wrap/wrap.py" categories.txt model.ell -lang python -target pi3 +``` - compiling model... - generating python interfaces for model in pi3 - running llc... - success, now copy the pi3 to your target machine and build it there +The `wrap` tool creates a new directory named `pi3`, which contains a CMake project that can be used to build the desired Python module. This time, we need to build this project on the Raspberry Pi. Before moving to the Pi, we also want to copy over some Python helper code: -We also want to copy some additional python code to your Raspberry Pi for the purpose of running this tutorial. You can also copy a static image over for testing: +``` +[Unix] cp ../../../docs/tutorials/shared/tutorialHelpers.py pi3 +[Windows] copy ..\..\..\docs\tutorials\shared\tutorialHelpers.py pi3 +``` -| Unix | `cp ../../tools/utilities/pythonlibs/demo*.py pi3`
`cp ../../tools/utilities/pitest/coffeemug.jpg pi3` | -| Windows | `copy ..\..\tools\utilities\pythonlibs\demo*.py pi3`
`copy ..\..\tools\utilities\pitest\coffeemug.jpg pi3` | +At this point, you should now have a `pi3` folder that contains a `cmake` project that builds the Python wrapper and some helpful Python utilities. -You should now have a `pi3` folder containing a python module for your model, as well as some helpful python utilities which we'll -use in the next section. +## Step 6: Write code to invoke the model on the Raspberry Pi -## Call your model from a Python app -Create a new text file called `demo.py` in your `tutorial1` folder. We'll add Python code to: -* Load the compiled image classification model -* Get an image -* Run the image through the model -* Show the classification results +We will write a Python script that invokes the model and runs the demo on a Raspberry Pi. The script will load the Python wrapper that we created above, read images from the camera, pass these images to the model, and display the classification results. If you just want the full script, copy it from [here](/ELL/tutorials/shared/tutorial.py). Otherwise, create an empty text file named `tutorial.py` and copy in the code snippets below. -If you don't want to type it out, the script can found [here](/ELL/tutorials/shared/demo.py), otherwise follow along below. +First, import a few dependencies, including system utilities, opencv, numpy and the `modelHelper` code that we copied over. -First, we need to import the libraries we'll be using in this app, which include system ultilities, numpy and demoHelper that we copied over from ELL utilities: ```python import sys import os +import time import numpy as np import cv2 -import demoHelper as d ``` -Then we define our main function, which instantiates a helper class to load the model. The helper knows which model to load by parsing the commandline arguments: +For convenience, include the helper code that we copied earlier. + +```python +import tutorialHelpers as helpers +``` + +Import the module that contains the compiled ELL model. + +```python +import model +``` + +The following functions help us get an image from the camera and read in the categories file. + +```python +# Function to return an image from our camera using OpenCV +def get_image_from_camera(camera): + if camera is not None: + # if predictor is too slow frames get buffered, this is designed to flush that buffer + ret, frame = camera.read() + if (not ret): + raise Exception('your capture device is not returning images') + return frame + return None + +# Return an array of strings corresponding to the model's recognized categories or classes. +# The order of the strings in this file are expected to match the order of the +# model's output predictions. +def get_categories_from_file(fileName): + labels = [] + with open(fileName) as f: + labels = f.read().splitlines() + return labels +``` +Define our main entry point and use the camera as our image source: ```python def main(args): - helper = d.DemoHelper() - helper.parse_arguments(args, - "Runs the given ELL model passing images from camera or static image file\n" - "Either the ELL model file, or the compiled model's Python module must be given,\n" - "using the --model or --compiledModel options respectively.\n" - "Example:\n" - " python demo.py categories.txt --compiledModel tutorial1/pi3/model1\n" - " python demo.py categories.txt --model model1.ell\n" - "This shows opencv window with image classified by the model using given labels") -``` -The command-line arguments also specify whether our images come from an image file or an attached webcam. Either way, we need to initialize our image source. + if (len(args) < 1): + print("usage: python tutorial.py categories.txt") + exit() + + # Open the video camera. To use a different camera, change the camera index. + camera = cv2.VideoCapture(0) +``` + +Use the function we defined above to read the category names from the file provided on the command line. + ```python - # Initialize image source - helper.init_image_source() - # In order to minimize console output, we compare the model's current prediction with its previous. Store the last prediction in 'lastPrediction' - lastPrediction = "" + categories = get_categories_from_file(args[0]) ``` -Next, we set up a loop that keeps going until the helper indicates it is done. Typically, this is when the user hits the Esc key. -At the start of every loop iteration, we want to grab an image from the image source: + +The model expects its input in a certain shape. Get this shape and store it for use later on. + ```python - while (not helper.done()): - # Grab next frame - frame = helper.get_next_frame() + inputShape = model.get_default_input_shape() ``` -`frame` now holds image data for the model. However, it often cannot be used as-is, because models are typically trained with: -* specific image sizes e.g. 224 x 224 x 3 -* specific ordering of color channels e.g. RGB. -Our helper uses OpenCV to grab images from the image source (file or webcam). Their size is dependent on the source, and the ordering is always BGR from OpenCV. -Therefore, we need to crop and or resize the image while maintaining the same aspect ratio, and reorder the color channels from BGR to RGB. Since -this is such a common operation, the helper implements this in a method called `prepare_image_for_predictor`: + +Allocate an array to store the model's output. ```python - # Prepare the image to send to the model. - # This involves scaling to the required input dimension and re-ordering from BGR to RGB - data = helper.prepare_image_for_predictor(frame) + outputShape = model.get_default_output_shape() + predictions = model.FloatVector(outputShape.Size()) ``` -We are now ready to get a classify the image in the frame. The model has a `predict` method, which will return a list of probabilities for each of the 1000 classes it can detect: + +Next, set up a loop that keeps going until OpenCV indicates it is done, which is when the user hits any key. At the start of every loop iteration, read an image from the camera. ```python - # Get the model to classify the image, by returning a list of probabilities for the classes it can detect - predictions = helper.predict(data) + while (cv2.waitKey(1) == 0xFF): + image = get_image_from_camera(camera) ``` -Note that this is just an array of values, where each element is a probability between 0 and 1. It is typical to reject any that do not meet a particular threshold, since that represents low confidence results. Re-ordering so that we get only the Top 5 predictions is also useful. The index of the prediction represents the class, the value represents the score. We can use the labels file to match the index of the prediction to its text label, and then construct a string with the label and score. + +As mentioned above, the image stored in the `image` variable cannot be sent to the model as-is and needs to be processed using the [prepare_image_for_model](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/imageHelper.py) helper function. ```python - # Get the (at most) top 5 predictions that meet our threshold. This is returned as a list of tuples, - # each with the text label and the prediction score. - top5 = helper.get_top_n(predictions, 5) + input = helpers.prepare_image_for_model(image, inputShape.columns, inputShape.rows) +``` - # Turn the top5 into a text string to display - text = "".join([helper.get_label(element[0]) + "(" + str(int(element[1]*100)) + "%) " for element in top5]) +With the processed image input handy, call the `predict` method to invoke the model. +```python + model.predict(input, predictions) ``` -Lastly, let's display the results. We'll do this in 2 ways: -* to the console by calling `print` with the prediction text. In order to not spam the console, we'll only print if the current -prediction text is different from the previous one. -* to a GUI window. The helper has methods that use OpenCV to add text to an image and then show that image in a window + +As before, the `predict` method fills the `predictions` array with the model output. Each element of this array corresponds to one of the 1000 image classes recognized by the model. Extract the top 5 predicted categories by calling the helper function `get_top_n_predictions`. ```python - save = False - if (text != lastPrediction): - print(text) - save = True - lastPrediction = text + top5 = helpers.get_top_n_predictions(predictions, 5) +``` - # Draw the text on the frame - frameToShow = frame - helper.draw_label(frameToShow, text) - helper.draw_fps(frameToShow) +`top5` is an array of tuples, where the first element is the category index and the second element is the probability of that category. Match the category indices in `top5` with the category names in `categories`. - # Show the new frame - helper.show_image(frameToShow, save) +```python + headerText = "".join(["(" + str(int(element[1]*100)) + "%) " + categories[element[0]] for element in top5]) ``` -Your `pi3` folder is ready to copy to your Raspberry Pi. You can do that using the 'scp' tool. On Windows you can use [WinSCP](https://winscp.net/eng/index.php). +Use the `draw_header` helper function to display the predicted category on the Raspberry Pi's display. Also display the camera image. -## SSH into Raspberry Pi +```python + helpers.draw_header(image, headerText) + cv2.imshow('ELL model', image) +``` +Finally, write the code that invokes the `main` function and runs your script. -Now log into your Raspberry Pi, either remotely using SSH or directly if you have keyboard and screen attached. +```python +if __name__ == "__main__": + args = sys.argv + args.pop(0) + main(args) +``` + +We are ready to move to the Raspberry Pi. You can copy the `pi3` folder to the Pi using the Unix `scp` tool or the Windows [WinSCP](https://winscp.net/eng/index.php) tool. + +## Step 7: Build the Python module on the Raspberry Pi -Find the `pi3` folder you just copied over using scp or winscp and run the following: +Log into your Raspberry Pi, either remotely using SSH or directly if you have a keyboard and display connected. Find the `pi3` folder you just copied from your computer and build the python module that wraps the ELL model. -```` +``` cd pi3 -mkdir build && cd build +mkdir build +cd build cmake .. make cd .. -```` +``` -This builds the Python Module that is then loadable by the demo Python scripts. +We just created a Python module named `model`, which includes functions that report the model's input and output dimensions and enables us to pass images to the model for classification. -## Process a static image +## Step 8: Classify live video on the Raspberry Pi -Now if you followed the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-Up-your-Raspberry-Pi) you should have a miniconda -environment named py34. So to run the tutorial do this: +If you followed the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-up-your-Raspberry-Pi), you should have an anaconda environment named `py34`. Activate it and run the script that we wrote above. -```` +``` source activate py34 -python demo.py labels.txt --compiledModel model --image coffeemug.jpg --bgr true -```` -And it will classify the image, you should see output like this: -```` -coffee mug(85%) -```` - -And if you have a display connected you should see something like the screenshot at the top of this page. - -## Process Video - -If you have a USB camera attached to your Pi then you can also use ELL to process video frames: - -```` -python demo.py labels.txt --compiledModel model --bgr true -```` -(just press ESCAPE on your keyboard to close the app) +python tutorial.py categories.txt +``` -You will see the same kind of window appear only this time it is showing the video stream. -Then when your camera is pointed at an object that the model recognizes you will see the label and -confidence % at the top together with an estimated frame rate. +If you have a camera and display connected to your Pi, you should see a window similar to the screenshot at the top of this page. Point your camera at different objects and see how the model classifies them. Look at `categories.txt` to see which categories the model is trained to recognize and try to show those objects to the model. For quick experimentation, point the camera to your computer screen and have your computer display images of different objects. For example, experiment with different dog breeds and different types of African animals. -`Tip`: for quick image recognition results you can point the video camera at a web image of a dog -on your PC screen. ImageNet models can usually do a good job recognizing different dog breeds and -many types of African animals. +If you copied the full source for [tutorial.py](/ELL/tutorials/shared/tutorial.py), you will also see the average time in milliseconds it takes the model to process a frame. ## Next steps -Different models have different characteristics. For example, some are slow but accurate, while others are faster and less accurate. Some have different power draw than others. -Experiment with which model works best for your scenario by downloading other models in the [ELL gallery](/ELL/gallery/). +The [ELL gallery](/ELL/gallery/) offers different models for image classification. Some are slow but accurate, while others are faster and less accurate. Different models can even lead to different power draw on the Raspberry Pi. Repeat the steps above with different models. -Try these related tutorials: -* [Fun with Dogs and Cats](/ELL/tutorials/Fun-with-Dogs-and-Cats/) -* [Importing new models](/ELL/tutorials/Importing-new-models/) +We used the `wrap` tool as a convenient way to compile the model and prepare for building its Python wrapper. To understand what `wrap` does under the hood, read the [wrap documentation](https://github.com/Microsoft/ELL/blob/master/tools/wrap/README.md). -## Toubleshooting +## Troubleshooting -If you run into trouble there's some troubleshooting instructions at the bottom of the -[Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-Up-your-Raspberry-Pi). \ No newline at end of file +If you run into trouble, you can find some troubleshooting instructions at the bottom of the [Raspberry Pi Setup Instructions](/ELL/tutorials/Setting-up-your-Raspberry-Pi). \ No newline at end of file diff --git a/docs/tutorials/Importing-models/index.md b/docs/tutorials/Importing-models/index.md new file mode 100644 index 000000000..73913b1c2 --- /dev/null +++ b/docs/tutorials/Importing-models/index.md @@ -0,0 +1,156 @@ +--- +layout: default +title: Importing models +permalink: /tutorials/Importing-models/ +--- + +# Importing models + +The [ELL gallery](/ELL/gallery/) includes different pre-trained ELL models for you to download and use. However, you may also want to train your own models. In this tutorial, we will import Neural Network models that were trained with the [Microsoft Cognitive Toolkit (CNTK)](https://www.microsoft.com/en-us/cognitive-toolkit/) or with [Darknet](https://pjreddie.com/darknet/). + +--- + +#### Materials + +* Laptop or desktop computer + +#### Prerequisites + +* Install ELL on your computer ([Windows](https://github.com/Microsoft/ELL/blob/master/INSTALL-Windows.md), [Ubuntu Linux](https://github.com/Microsoft/ELL/blob/master/INSTALL-Ubuntu.md), [Mac](https://github.com/Microsoft/ELL/blob/master/INSTALL-Mac.md)). Specifically, this tutorial requires ELL, CMake, SWIG, and Python 3.6. +* To import a model from the CNTK file format, you must install the [CNTK python packages](https://docs.microsoft.com/en-us/cognitive-toolkit/setup-cntk-on-your-machine). + +## Importing models + +If you followed the setup instructions, you should have an environment named `py36`. Open a terminal window and activate your Anaconda environment. + +| Platform | To activate the Anaconda environment... +| --- | --- +| Unix | `source activate py36` +| Windows | `activate py36` + +We recommend using Curl to download files. If Curl isn't installed in your Anaconda environment, install it as follows. + +``` +conda install curl +``` + +Choose whether you want to import a model from the CNTK file format or the Darknet file format and proceed to the relevant section below. + +## Importing a model from the Microsoft Cognitive Toolkit (CNTK) + +A Neural Network model in the CNTK file format can be converted into an ELL model using the *cntk_import.py* Python script, found in the `ELL/tools/importers/cntk` directory. As an example, we will import one of the models available on the [CNTK website](https://www.microsoft.com/en-us/cognitive-toolkit/). + +``` +curl --location -o model.cntk https://www.cntk.ai/Models/Caffe_Converted/VGG16_ImageNet_Caffe.model +``` + +Next, run the importer script, making sure to provide the CNTK model as a command line parameter. + +``` +python "~/ELL/tools/importers/cntk/cntk_import.py" model.cntk +``` + +You should see some output like this as the model is being imported. + +```` +Loading... +Selected CPU as the process wide default device. + +Finished loading. +Pre-processing... +Will not process ReduceElements - skipping this layer as irrelevant. +Will not process ClassificationError - skipping this layer as irrelevant. +Will not process ReduceElements - skipping this layer as irrelevant. +Will not process CrossEntropyWithSoftmax - skipping this layer as irrelevant. +Will not process Reshape - skipping this layer as irrelevant. +Will not process Combine - skipping this layer as irrelevant. +Minus : 224x224x3 -> 226x226x3 | input padding 0 output padding 1 +Convolution(LeakyReLU) : 226x226x3 -> 224x224x16 | input padding 1 output padding 0 +BatchNormalization : 224x224x16 -> 224x224x16 | input padding 0 output padding 0 +MaxPooling : 224x224x16 -> 114x114x16 | input padding 0 output padding 1 +Convolution(LeakyReLU) : 114x114x16 -> 112x112x64 | input padding 1 output padding 0 +BatchNormalization : 112x112x64 -> 112x112x64 | input padding 0 output padding 0 +MaxPooling : 112x112x64 -> 58x58x64 | input padding 0 output padding 1 +Convolution(LeakyReLU) : 58x58x64 -> 56x56x64 | input padding 1 output padding 0 +BatchNormalization : 56x56x64 -> 56x56x64 | input padding 0 output padding 0 +MaxPooling : 56x56x64 -> 30x30x64 | input padding 0 output padding 1 +Convolution(LeakyReLU) : 30x30x64 -> 28x28x128 | input padding 1 output padding 0 +BatchNormalization : 28x28x128 -> 28x28x128 | input padding 0 output padding 0 +MaxPooling : 28x28x128 -> 16x16x128 | input padding 0 output padding 1 +Convolution(LeakyReLU) : 16x16x128 -> 14x14x256 | input padding 1 output padding 0 +BatchNormalization : 14x14x256 -> 14x14x256 | input padding 0 output padding 0 +MaxPooling : 14x14x256 -> 9x9x256 | input padding 0 output padding 1 +Convolution(LeakyReLU) : 9x9x256 -> 7x7x512 | input padding 1 output padding 0 +BatchNormalization : 7x7x512 -> 7x7x512 | input padding 0 output padding 0 +MaxPooling : 7x7x512 -> 6x6x512 | input padding 0 output padding 1 +Convolution(LeakyReLU) : 6x6x512 -> 4x4x1024 | input padding 1 output padding 0 +BatchNormalization : 4x4x1024 -> 4x4x1024 | input padding 0 output padding 0 +Convolution : 4x4x1024 -> 4x4x1000 | input padding 0 output padding 0 +AveragePooling : 4x4x1000 -> 1x1x1000 | input padding 0 output padding 0 +Softmax : 1x1x1000 -> 1x1x1 | input padding 0 output padding 0 + +Finished pre-processing. +Saving model file: 'model.ell' +```` + +When the script finishes running, you should see a new file named `model.ell` appear in your directory. This ELL model file is a JSON file that contains both the model configuration and the model weights. + +In addition to the model, also download the categories file, which contains the names of the categories predicted by the model, and which we require below. + +``` +curl --location -o categories.txt https://raw.githubusercontent.com/Microsoft/ELL-models/master/models/ILSVRC2012/ILSVRC2012_labels.txt +``` + +## Importing a model from Darknet + +A Neural Network model in the Darknet file format can be converted into an ELL model using the *darknet_import.py* Python script, found in the `ELL/tools/importers/darknet` directory. As an example, we will import the Darknet reference model, available from the [Darknet website](https://pjreddie.com/darknet/). + +``` +curl --location -o model.cfg https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/darknet.cfg +curl --location -o model.weights https://pjreddie.com/media/files/darknet.weights +``` + +Next, run the importer script, making sure to provide the model configuration file (.cfg) and model weights files (.weights) as command line parameters. + +``` +python "~/ELL/tools/importers/darknet/darknet_import.py" model.cfg model.weights +``` + +You should see some output like this as the model is being imported. + +``` +convolutional: 224 x 224 x 3 -> 224 x 224 x 16 , pad 1 +max_pool: 224 x 224 x 16 -> 112 x 112 x 16 , pad 0 +convolutional: 112 x 112 x 16 -> 112 x 112 x 32 , pad 1 +max_pool: 112 x 112 x 32 -> 56 x 56 x 32 , pad 0 +convolutional: 56 x 56 x 32 -> 56 x 56 x 64 , pad 1 +max_pool: 56 x 56 x 64 -> 28 x 28 x 64 , pad 0 +convolutional: 28 x 28 x 64 -> 28 x 28 x 128 , pad 1 +max_pool: 28 x 28 x 128 -> 14 x 14 x 128 , pad 0 +convolutional: 14 x 14 x 128 -> 14 x 14 x 256 , pad 1 +max_pool: 14 x 14 x 256 -> 7 x 7 x 256 , pad 0 +convolutional: 7 x 7 x 256 -> 7 x 7 x 512 , pad 1 +max_pool: 7 x 7 x 512 -> 4 x 4 x 512 , pad 1 +convolutional: 4 x 4 x 512 -> 4 x 4 x 1024 , pad 1 +convolutional: 4 x 4 x 1024 -> 4 x 4 x 1000 , pad 0 +avg_pool: 4 x 4 x 1000 -> 1 x 1 x 1000 , pad 0 +softmax: 1 x 1 x 1000 -> 1 x 1 x 1000 +Saving model file: 'model.ell' +``` + +When the script finishes running, you should see a new file named `model.ell` appear in your directory. This ELL model file is a JSON file that contains both the model configuration and the model weights. + +In addition to the model, also download the categories file, which contains the names of the categories predicted by the model, and which we require below. + +``` +curl --location -o categories.txt https://raw.githubusercontent.com/pjreddie/darknet/master/data/imagenet.shortnames.list +``` + +## Using the model + +Once the model is in the ELL format, it no longer matters whether it came from CNTK or Darknet, and the only difference is the categories file. To test the model from Python, follow the steps in [Getting started with image classification on the Raspberry Pi](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/), but replace the model suggested in that tutorial with the model you just imported. Alternatively, to test the model from C++, follow the steps in [Getting started with image classification in C++](/ELL/tutorials/Getting-Started-with-Image-Classification-in-Cpp/). + +## Troubleshooting + +ELL currently supports a subset of the layer types and activation functions that can appear in CNTK and Darknet models. If you try to import a model that contains an unsupported layer type, the importer will fail. With time, ELL will support more layer types. In the meantime, try changing the model to use the supported layer types. + diff --git a/docs/tutorials/Importing-new-models/cntkVgg16ImageNetLabels.txt b/docs/tutorials/Importing-new-models/cntkVgg16ImageNetLabels.txt deleted file mode 100644 index 722c98456..000000000 --- a/docs/tutorials/Importing-new-models/cntkVgg16ImageNetLabels.txt +++ /dev/null @@ -1,1000 +0,0 @@ -tench, Tinca tinca -goldfish, Carassius auratus -great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias -tiger shark, Galeocerdo cuvieri -hammerhead, hammerhead shark -electric ray, crampfish, numbfish, torpedo -stingray -cock -hen -ostrich, Struthio camelus -brambling, Fringilla montifringilla -goldfinch, Carduelis carduelis -house finch, linnet, Carpodacus mexicanus -junco, snowbird -indigo bunting, indigo finch, indigo bird, Passerina cyanea -robin, American robin, Turdus migratorius -bulbul -jay -magpie -chickadee -water ouzel, dipper -kite -bald eagle, American eagle, Haliaeetus leucocephalus -vulture -great grey owl, great gray owl, Strix nebulosa -European fire salamander, Salamandra salamandra -common newt, Triturus vulgaris -eft -spotted salamander, Ambystoma maculatum -axolotl, mud puppy, Ambystoma mexicanum -bullfrog, Rana catesbeiana -tree frog, tree-frog -tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui -loggerhead, loggerhead turtle, Caretta caretta -leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea -mud turtle -terrapin -box turtle, box tortoise -banded gecko -common iguana, iguana, Iguana iguana -American chameleon, anole, Anolis carolinensis -whiptail, whiptail lizard -agama -frilled lizard, Chlamydosaurus kingi -alligator lizard -Gila monster, Heloderma suspectum -green lizard, Lacerta viridis -African chameleon, Chamaeleo chamaeleon -Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis -African crocodile, Nile crocodile, Crocodylus niloticus -American alligator, Alligator mississipiensis -triceratops -thunder snake, worm snake, Carphophis amoenus -ringneck snake, ring-necked snake, ring snake -hognose snake, puff adder, sand viper -green snake, grass snake -king snake, kingsnake -garter snake, grass snake -water snake -vine snake -night snake, Hypsiglena torquata -boa constrictor, Constrictor constrictor -rock python, rock snake, Python sebae -Indian cobra, Naja naja -green mamba -sea snake -horned viper, cerastes, sand viper, horned asp, Cerastes cornutus -diamondback, diamondback rattlesnake, Crotalus adamanteus -sidewinder, horned rattlesnake, Crotalus cerastes -trilobite -harvestman, daddy longlegs, Phalangium opilio -scorpion -black and gold garden spider, Argiope aurantia -barn spider, Araneus cavaticus -garden spider, Aranea diademata -black widow, Latrodectus mactans -tarantula -wolf spider, hunting spider -tick -centipede -black grouse -ptarmigan -ruffed grouse, partridge, Bonasa umbellus -prairie chicken, prairie grouse, prairie fowl -peacock -quail -partridge -African grey, African gray, Psittacus erithacus -macaw -sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita -lorikeet -coucal -bee eater -hornbill -hummingbird -jacamar -toucan -drake -red-breasted merganser, Mergus serrator -goose -black swan, Cygnus atratus -tusker -echidna, spiny anteater, anteater -platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus -wallaby, brush kangaroo -koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus -wombat -jellyfish -sea anemone, anemone -brain coral -flatworm, platyhelminth -nematode, nematode worm, roundworm -conch -snail -slug -sea slug, nudibranch -chiton, coat-of-mail shell, sea cradle, polyplacophore -chambered nautilus, pearly nautilus, nautilus -Dungeness crab, Cancer magister -rock crab, Cancer irroratus -fiddler crab -king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica -American lobster, Northern lobster, Maine lobster, Homarus americanus -spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish -crayfish, crawfish, crawdad, crawdaddy -hermit crab -isopod -white stork, Ciconia ciconia -black stork, Ciconia nigra -spoonbill -flamingo -little blue heron, Egretta caerulea -American egret, great white heron, Egretta albus -bittern -crane -limpkin, Aramus pictus -European gallinule, Porphyrio porphyrio -American coot, marsh hen, mud hen, water hen, Fulica americana -bustard -ruddy turnstone, Arenaria interpres -red-backed sandpiper, dunlin, Erolia alpina -redshank, Tringa totanus -dowitcher -oystercatcher, oyster catcher -pelican -king penguin, Aptenodytes patagonica -albatross, mollymawk -grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus -killer whale, killer, orca, grampus, sea wolf, Orcinus orca -dugong, Dugong dugon -sea lion -Chihuahua -Japanese spaniel -Maltese dog, Maltese terrier, Maltese -Pekinese, Pekingese, Peke -Shih-Tzu -Blenheim spaniel -papillon -toy terrier -Rhodesian ridgeback -Afghan hound, Afghan -basset, basset hound -beagle -bloodhound, sleuthhound -bluetick -black-and-tan coonhound -Walker hound, Walker foxhound -English foxhound -redbone -borzoi, Russian wolfhound -Irish wolfhound -Italian greyhound -whippet -Ibizan hound, Ibizan Podenco -Norwegian elkhound, elkhound -otterhound, otter hound -Saluki, gazelle hound -Scottish deerhound, deerhound -Weimaraner -Staffordshire bullterrier, Staffordshire bull terrier -American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier -Bedlington terrier -Border terrier -Kerry blue terrier -Irish terrier -Norfolk terrier -Norwich terrier -Yorkshire terrier -wire-haired fox terrier -Lakeland terrier -Sealyham terrier, Sealyham -Airedale, Airedale terrier -cairn, cairn terrier -Australian terrier -Dandie Dinmont, Dandie Dinmont terrier -Boston bull, Boston terrier -miniature schnauzer -giant schnauzer -standard schnauzer -Scotch terrier, Scottish terrier, Scottie -Tibetan terrier, chrysanthemum dog -silky terrier, Sydney silky -soft-coated wheaten terrier -West Highland white terrier -Lhasa, Lhasa apso -flat-coated retriever -curly-coated retriever -golden retriever -Labrador retriever -Chesapeake Bay retriever -German short-haired pointer -vizsla, Hungarian pointer -English setter -Irish setter, red setter -Gordon setter -Brittany spaniel -clumber, clumber spaniel -English springer, English springer spaniel -Welsh springer spaniel -cocker spaniel, English cocker spaniel, cocker -Sussex spaniel -Irish water spaniel -kuvasz -schipperke -groenendael -malinois -briard -kelpie -komondor -Old English sheepdog, bobtail -Shetland sheepdog, Shetland sheep dog, Shetland -collie -Border collie -Bouvier des Flandres, Bouviers des Flandres -Rottweiler -German shepherd, German shepherd dog, German police dog, alsatian -Doberman, Doberman pinscher -miniature pinscher -Greater Swiss Mountain dog -Bernese mountain dog -Appenzeller -EntleBucher -boxer -bull mastiff -Tibetan mastiff -French bulldog -Great Dane -Saint Bernard, St Bernard -Eskimo dog, husky -malamute, malemute, Alaskan malamute -Siberian husky -dalmatian, coach dog, carriage dog -affenpinscher, monkey pinscher, monkey dog -basenji -pug, pug-dog -Leonberg -Newfoundland, Newfoundland dog -Great Pyrenees -Samoyed, Samoyede -Pomeranian -chow, chow chow -keeshond -Brabancon griffon -Pembroke, Pembroke Welsh corgi -Cardigan, Cardigan Welsh corgi -toy poodle -miniature poodle -standard poodle -Mexican hairless -timber wolf, grey wolf, gray wolf, Canis lupus -white wolf, Arctic wolf, Canis lupus tundrarum -red wolf, maned wolf, Canis rufus, Canis niger -coyote, prairie wolf, brush wolf, Canis latrans -dingo, warrigal, warragal, Canis dingo -dhole, Cuon alpinus -African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus -hyena, hyaena -red fox, Vulpes vulpes -kit fox, Vulpes macrotis -Arctic fox, white fox, Alopex lagopus -grey fox, gray fox, Urocyon cinereoargenteus -tabby, tabby cat -tiger cat -Persian cat -Siamese cat, Siamese -Egyptian cat -cougar, puma, catamount, mountain lion, painter, panther, Felis concolor -lynx, catamount -leopard, Panthera pardus -snow leopard, ounce, Panthera uncia -jaguar, panther, Panthera onca, Felis onca -lion, king of beasts, Panthera leo -tiger, Panthera tigris -cheetah, chetah, Acinonyx jubatus -brown bear, bruin, Ursus arctos -American black bear, black bear, Ursus americanus, Euarctos americanus -ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus -sloth bear, Melursus ursinus, Ursus ursinus -mongoose -meerkat, mierkat -tiger beetle -ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle -ground beetle, carabid beetle -long-horned beetle, longicorn, longicorn beetle -leaf beetle, chrysomelid -dung beetle -rhinoceros beetle -weevil -fly -bee -ant, emmet, pismire -grasshopper, hopper -cricket -walking stick, walkingstick, stick insect -cockroach, roach -mantis, mantid -cicada, cicala -leafhopper -lacewing, lacewing fly -dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk -damselfly -admiral -ringlet, ringlet butterfly -monarch, monarch butterfly, milkweed butterfly, Danaus plexippus -cabbage butterfly -sulphur butterfly, sulfur butterfly -lycaenid, lycaenid butterfly -starfish, sea star -sea urchin -sea cucumber, holothurian -wood rabbit, cottontail, cottontail rabbit -hare -Angora, Angora rabbit -hamster -porcupine, hedgehog -fox squirrel, eastern fox squirrel, Sciurus niger -marmot -beaver -guinea pig, Cavia cobaya -sorrel -zebra -hog, pig, grunter, squealer, Sus scrofa -wild boar, boar, Sus scrofa -warthog -hippopotamus, hippo, river horse, Hippopotamus amphibius -ox -water buffalo, water ox, Asiatic buffalo, Bubalus bubalis -bison -ram, tup -bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis -ibex, Capra ibex -hartebeest -impala, Aepyceros melampus -gazelle -Arabian camel, dromedary, Camelus dromedarius -llama -weasel -mink -polecat, fitch, foulmart, foumart, Mustela putorius -black-footed ferret, ferret, Mustela nigripes -otter -skunk, polecat, wood pussy -badger -armadillo -three-toed sloth, ai, Bradypus tridactylus -orangutan, orang, orangutang, Pongo pygmaeus -gorilla, Gorilla gorilla -chimpanzee, chimp, Pan troglodytes -gibbon, Hylobates lar -siamang, Hylobates syndactylus, Symphalangus syndactylus -guenon, guenon monkey -patas, hussar monkey, Erythrocebus patas -baboon -macaque -langur -colobus, colobus monkey -proboscis monkey, Nasalis larvatus -marmoset -capuchin, ringtail, Cebus capucinus -howler monkey, howler -titi, titi monkey -spider monkey, Ateles geoffroyi -squirrel monkey, Saimiri sciureus -Madagascar cat, ring-tailed lemur, Lemur catta -indri, indris, Indri indri, Indri brevicaudatus -Indian elephant, Elephas maximus -African elephant, Loxodonta africana -lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens -giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca -barracouta, snoek -eel -coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch -rock beauty, Holocanthus tricolor -anemone fish -sturgeon -gar, garfish, garpike, billfish, Lepisosteus osseus -lionfish -puffer, pufferfish, blowfish, globefish -abacus -abaya -academic gown, academic robe, judge's robe -accordion, piano accordion, squeeze box -acoustic guitar -aircraft carrier, carrier, flattop, attack aircraft carrier -airliner -airship, dirigible -altar -ambulance -amphibian, amphibious vehicle -analog clock -apiary, bee house -apron -ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin -assault rifle, assault gun -backpack, back pack, knapsack, packsack, rucksack, haversack -bakery, bakeshop, bakehouse -balance beam, beam -balloon -ballpoint, ballpoint pen, ballpen, Biro -Band Aid -banjo -bannister, banister, balustrade, balusters, handrail -barbell -barber chair -barbershop -barn -barometer -barrel, cask -barrow, garden cart, lawn cart, wheelbarrow -baseball -basketball -bassinet -bassoon -bathing cap, swimming cap -bath towel -bathtub, bathing tub, bath, tub -beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon -beacon, lighthouse, beacon light, pharos -beaker -bearskin, busby, shako -beer bottle -beer glass -bell cote, bell cot -bib -bicycle-built-for-two, tandem bicycle, tandem -bikini, two-piece -binder, ring-binder -binoculars, field glasses, opera glasses -birdhouse -boathouse -bobsled, bobsleigh, bob -bolo tie, bolo, bola tie, bola -bonnet, poke bonnet -bookcase -bookshop, bookstore, bookstall -bottlecap -bow -bow tie, bow-tie, bowtie -brass, memorial tablet, plaque -brassiere, bra, bandeau -breakwater, groin, groyne, mole, bulwark, seawall, jetty -breastplate, aegis, egis -broom -bucket, pail -buckle -bulletproof vest -bullet train, bullet -butcher shop, meat market -cab, hack, taxi, taxicab -caldron, cauldron -candle, taper, wax light -cannon -canoe -can opener, tin opener -cardigan -car mirror -carousel, carrousel, merry-go-round, roundabout, whirligig -carpenter's kit, tool kit -carton -car wheel -cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM -cassette -cassette player -castle -catamaran -CD player -cello, violoncello -cellular telephone, cellular phone, cellphone, cell, mobile phone -chain -chainlink fence -chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour -chain saw, chainsaw -chest -chiffonier, commode -chime, bell, gong -china cabinet, china closet -Christmas stocking -church, church building -cinema, movie theater, movie theatre, movie house, picture palace -cleaver, meat cleaver, chopper -cliff dwelling -cloak -clog, geta, patten, sabot -cocktail shaker -coffee mug -coffeepot -coil, spiral, volute, whorl, helix -combination lock -computer keyboard, keypad -confectionery, confectionary, candy store -container ship, containership, container vessel -convertible -corkscrew, bottle screw -cornet, horn, trumpet, trump -cowboy boot -cowboy hat, ten-gallon hat -cradle -crane -crash helmet -crate -crib, cot -Crock Pot -croquet ball -crutch -cuirass -dam, dike, dyke -desk -desktop computer -dial telephone, dial phone -diaper, nappy, napkin -digital clock -digital watch -dining table, board -dishrag, dishcloth -dishwasher, dish washer, dishwashing machine -disk brake, disc brake -dock, dockage, docking facility -dogsled, dog sled, dog sleigh -dome -doormat, welcome mat -drilling platform, offshore rig -drum, membranophone, tympan -drumstick -dumbbell -Dutch oven -electric fan, blower -electric guitar -electric locomotive -entertainment center -envelope -espresso maker -face powder -feather boa, boa -file, file cabinet, filing cabinet -fireboat -fire engine, fire truck -fire screen, fireguard -flagpole, flagstaff -flute, transverse flute -folding chair -football helmet -forklift -fountain -fountain pen -four-poster -freight car -French horn, horn -frying pan, frypan, skillet -fur coat -garbage truck, dustcart -gasmask, respirator, gas helmet -gas pump, gasoline pump, petrol pump, island dispenser -goblet -go-kart -golf ball -golfcart, golf cart -gondola -gong, tam-tam -gown -grand piano, grand -greenhouse, nursery, glasshouse -grille, radiator grille -grocery store, grocery, food market, market -guillotine -hair slide -hair spray -half track -hammer -hamper -hand blower, blow dryer, blow drier, hair dryer, hair drier -hand-held computer, hand-held microcomputer -handkerchief, hankie, hanky, hankey -hard disc, hard disk, fixed disk -harmonica, mouth organ, harp, mouth harp -harp -harvester, reaper -hatchet -holster -home theater, home theatre -honeycomb -hook, claw -hoopskirt, crinoline -horizontal bar, high bar -horse cart, horse-cart -hourglass -iPod -iron, smoothing iron -jack-o'-lantern -jean, blue jean, denim -jeep, landrover -jersey, T-shirt, tee shirt -jigsaw puzzle -jinrikisha, ricksha, rickshaw -joystick -kimono -knee pad -knot -lab coat, laboratory coat -ladle -lampshade, lamp shade -laptop, laptop computer -lawn mower, mower -lens cap, lens cover -letter opener, paper knife, paperknife -library -lifeboat -lighter, light, igniter, ignitor -limousine, limo -liner, ocean liner -lipstick, lip rouge -Loafer -lotion -loudspeaker, speaker, speaker unit, loudspeaker system, speaker system -loupe, jeweler's loupe -lumbermill, sawmill -magnetic compass -mailbag, postbag -mailbox, letter box -maillot -maillot, tank suit -manhole cover -maraca -marimba, xylophone -mask -matchstick -maypole -maze, labyrinth -measuring cup -medicine chest, medicine cabinet -megalith, megalithic structure -microphone, mike -microwave, microwave oven -military uniform -milk can -minibus -miniskirt, mini -minivan -missile -mitten -mixing bowl -mobile home, manufactured home -Model T -modem -monastery -monitor -moped -mortar -mortarboard -mosque -mosquito net -motor scooter, scooter -mountain bike, all-terrain bike, off-roader -mountain tent -mouse, computer mouse -mousetrap -moving van -muzzle -nail -neck brace -necklace -nipple -notebook, notebook computer -obelisk -oboe, hautboy, hautbois -ocarina, sweet potato -odometer, hodometer, mileometer, milometer -oil filter -organ, pipe organ -oscilloscope, scope, cathode-ray oscilloscope, CRO -overskirt -oxcart -oxygen mask -packet -paddle, boat paddle -paddlewheel, paddle wheel -padlock -paintbrush -pajama, pyjama, pj's, jammies -palace -panpipe, pandean pipe, syrinx -paper towel -parachute, chute -parallel bars, bars -park bench -parking meter -passenger car, coach, carriage -patio, terrace -pay-phone, pay-station -pedestal, plinth, footstall -pencil box, pencil case -pencil sharpener -perfume, essence -Petri dish -photocopier -pick, plectrum, plectron -pickelhaube -picket fence, paling -pickup, pickup truck -pier -piggy bank, penny bank -pill bottle -pillow -ping-pong ball -pinwheel -pirate, pirate ship -pitcher, ewer -plane, carpenter's plane, woodworking plane -planetarium -plastic bag -plate rack -plow, plough -plunger, plumber's helper -Polaroid camera, Polaroid Land camera -pole -police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria -poncho -pool table, billiard table, snooker table -pop bottle, soda bottle -pot, flowerpot -potter's wheel -power drill -prayer rug, prayer mat -printer -prison, prison house -projectile, missile -projector -puck, hockey puck -punching bag, punch bag, punching ball, punchball -purse -quill, quill pen -quilt, comforter, comfort, puff -racer, race car, racing car -racket, racquet -radiator -radio, wireless -radio telescope, radio reflector -rain barrel -recreational vehicle, RV, R.V. -reel -reflex camera -refrigerator, icebox -remote control, remote -restaurant, eating house, eating place, eatery -revolver, six-gun, six-shooter -rifle -rocking chair, rocker -rotisserie -rubber eraser, rubber, pencil eraser -rugby ball -rule, ruler -running shoe -safe -safety pin -saltshaker, salt shaker -sandal -sarong -sax, saxophone -scabbard -scale, weighing machine -school bus -schooner -scoreboard -screen, CRT screen -screw -screwdriver -seat belt, seatbelt -sewing machine -shield, buckler -shoe shop, shoe-shop, shoe store -shoji -shopping basket -shopping cart -shovel -shower cap -shower curtain -ski -ski mask -sleeping bag -slide rule, slipstick -sliding door -slot, one-armed bandit -snorkel -snowmobile -snowplow, snowplough -soap dispenser -soccer ball -sock -solar dish, solar collector, solar furnace -sombrero -soup bowl -space bar -space heater -space shuttle -spatula -speedboat -spider web, spider's web -spindle -sports car, sport car -spotlight, spot -stage -steam locomotive -steel arch bridge -steel drum -stethoscope -stole -stone wall -stopwatch, stop watch -stove -strainer -streetcar, tram, tramcar, trolley, trolley car -stretcher -studio couch, day bed -stupa, tope -submarine, pigboat, sub, U-boat -suit, suit of clothes -sundial -sunglass -sunglasses, dark glasses, shades -sunscreen, sunblock, sun blocker -suspension bridge -swab, swob, mop -sweatshirt -swimming trunks, bathing trunks -swing -switch, electric switch, electrical switch -syringe -table lamp -tank, army tank, armored combat vehicle, armoured combat vehicle -tape player -teapot -teddy, teddy bear -television, television system -tennis ball -thatch, thatched roof -theater curtain, theatre curtain -thimble -thresher, thrasher, threshing machine -throne -tile roof -toaster -tobacco shop, tobacconist shop, tobacconist -toilet seat -torch -totem pole -tow truck, tow car, wrecker -toyshop -tractor -trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi -tray -trench coat -tricycle, trike, velocipede -trimaran -tripod -triumphal arch -trolleybus, trolley coach, trackless trolley -trombone -tub, vat -turnstile -typewriter keyboard -umbrella -unicycle, monocycle -upright, upright piano -vacuum, vacuum cleaner -vase -vault -velvet -vending machine -vestment -viaduct -violin, fiddle -volleyball -waffle iron -wall clock -wallet, billfold, notecase, pocketbook -wardrobe, closet, press -warplane, military plane -washbasin, handbasin, washbowl, lavabo, wash-hand basin -washer, automatic washer, washing machine -water bottle -water jug -water tower -whiskey jug -whistle -wig -window screen -window shade -Windsor tie -wine bottle -wing -wok -wooden spoon -wool, woolen, woollen -worm fence, snake fence, snake-rail fence, Virginia fence -wreck -yawl -yurt -web site, website, internet site, site -comic book -crossword puzzle, crossword -street sign -traffic light, traffic signal, stoplight -book jacket, dust cover, dust jacket, dust wrapper -menu -plate -guacamole -consomme -hot pot, hotpot -trifle -ice cream, icecream -ice lolly, lolly, lollipop, popsicle -French loaf -bagel, beigel -pretzel -cheeseburger -hotdog, hot dog, red hot -mashed potato -head cabbage -broccoli -cauliflower -zucchini, courgette -spaghetti squash -acorn squash -butternut squash -cucumber, cuke -artichoke, globe artichoke -bell pepper -cardoon -mushroom -Granny Smith -strawberry -orange -lemon -fig -pineapple, ananas -banana -jackfruit, jak, jack -custard apple -pomegranate -hay -carbonara -chocolate sauce, chocolate syrup -dough -meat loaf, meatloaf -pizza, pizza pie -potpie -burrito -red wine -espresso -cup -eggnog -alp -bubble -cliff, drop, drop-off -coral reef -geyser -lakeside, lakeshore -promontory, headland, head, foreland -sandbar, sand bar -seashore, coast, seacoast, sea-coast -valley, vale -volcano -ballplayer, baseball player -groom, bridegroom -scuba diver -rapeseed -daisy -yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum -corn -acorn -hip, rose hip, rosehip -buckeye, horse chestnut, conker -coral fungus -agaric -gyromitra -stinkhorn, carrion fungus -earthstar -hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa -bolete -ear, spike, capitulum -toilet tissue, toilet paper, bathroom tissue \ No newline at end of file diff --git a/docs/tutorials/Importing-new-models/darknetImageNetLabels.txt b/docs/tutorials/Importing-new-models/darknetImageNetLabels.txt deleted file mode 100644 index 3f8a89c57..000000000 --- a/docs/tutorials/Importing-new-models/darknetImageNetLabels.txt +++ /dev/null @@ -1,1000 +0,0 @@ -kit fox -English setter -Siberian husky -Australian terrier -English springer -grey whale -lesser panda -Egyptian cat -ibex -Persian cat -cougar -gazelle -porcupine -sea lion -malamute -badger -Great Dane -Walker hound -Welsh springer spaniel -whippet -Scottish deerhound -killer whale -mink -African elephant -Weimaraner -soft-coated wheaten terrier -Dandie Dinmont -red wolf -Old English sheepdog -jaguar -otterhound -bloodhound -Airedale -hyena -meerkat -giant schnauzer -titi -three-toed sloth -sorrel -black-footed ferret -dalmatian -black-and-tan coonhound -papillon -skunk -Staffordshire bullterrier -Mexican hairless -Bouvier des Flandres -weasel -miniature poodle -Cardigan -malinois -bighorn -fox squirrel -colobus -tiger cat -Lhasa -impala -coyote -Yorkshire terrier -Newfoundland -brown bear -red fox -Norwegian elkhound -Rottweiler -hartebeest -Saluki -grey fox -schipperke -Pekinese -Brabancon griffon -West Highland white terrier -Sealyham terrier -guenon -mongoose -indri -tiger -Irish wolfhound -wild boar -EntleBucher -zebra -ram -French bulldog -orangutan -basenji -leopard -Bernese mountain dog -Maltese dog -Norfolk terrier -toy terrier -vizsla -cairn -squirrel monkey -groenendael -clumber -Siamese cat -chimpanzee -komondor -Afghan hound -Japanese spaniel -proboscis monkey -guinea pig -white wolf -ice bear -gorilla -borzoi -toy poodle -Kerry blue terrier -ox -Scotch terrier -Tibetan mastiff -spider monkey -Doberman -Boston bull -Greater Swiss Mountain dog -Appenzeller -Shih-Tzu -Irish water spaniel -Pomeranian -Bedlington terrier -warthog -Arabian camel -siamang -miniature schnauzer -collie -golden retriever -Irish terrier -affenpinscher -Border collie -hare -boxer -silky terrier -beagle -Leonberg -German short-haired pointer -patas -dhole -baboon -macaque -Chesapeake Bay retriever -bull mastiff -kuvasz -capuchin -pug -curly-coated retriever -Norwich terrier -flat-coated retriever -hog -keeshond -Eskimo dog -Brittany spaniel -standard poodle -Lakeland terrier -snow leopard -Gordon setter -dingo -standard schnauzer -hamster -Tibetan terrier -Arctic fox -wire-haired fox terrier -basset -water buffalo -American black bear -Angora -bison -howler monkey -hippopotamus -chow -giant panda -American Staffordshire terrier -Shetland sheepdog -Great Pyrenees -Chihuahua -tabby -marmoset -Labrador retriever -Saint Bernard -armadillo -Samoyed -bluetick -redbone -polecat -marmot -kelpie -gibbon -llama -miniature pinscher -wood rabbit -Italian greyhound -lion -cocker spaniel -Irish setter -dugong -Indian elephant -beaver -Sussex spaniel -Pembroke -Blenheim spaniel -Madagascar cat -Rhodesian ridgeback -lynx -African hunting dog -langur -Ibizan hound -timber wolf -cheetah -English foxhound -briard -sloth bear -Border terrier -German shepherd -otter -koala -tusker -echidna -wallaby -platypus -wombat -revolver -umbrella -schooner -soccer ball -accordion -ant -starfish -chambered nautilus -grand piano -laptop -strawberry -airliner -warplane -airship -balloon -space shuttle -fireboat -gondola -speedboat -lifeboat -canoe -yawl -catamaran -trimaran -container ship -liner -pirate -aircraft carrier -submarine -wreck -half track -tank -missile -bobsled -dogsled -bicycle-built-for-two -mountain bike -freight car -passenger car -barrow -shopping cart -motor scooter -forklift -electric locomotive -steam locomotive -amphibian -ambulance -beach wagon -cab -convertible -jeep -limousine -minivan -Model T -racer -sports car -go-kart -golfcart -moped -snowplow -fire engine -garbage truck -pickup -tow truck -trailer truck -moving van -police van -recreational vehicle -streetcar -snowmobile -tractor -mobile home -tricycle -unicycle -horse cart -jinrikisha -oxcart -bassinet -cradle -crib -four-poster -bookcase -china cabinet -medicine chest -chiffonier -table lamp -file -park bench -barber chair -throne -folding chair -rocking chair -studio couch -toilet seat -desk -pool table -dining table -entertainment center -wardrobe -Granny Smith -orange -lemon -fig -pineapple -banana -jackfruit -custard apple -pomegranate -acorn -hip -ear -rapeseed -corn -buckeye -organ -upright -chime -drum -gong -maraca -marimba -steel drum -banjo -cello -violin -harp -acoustic guitar -electric guitar -cornet -French horn -trombone -harmonica -ocarina -panpipe -bassoon -oboe -sax -flute -daisy -yellow lady's slipper -cliff -valley -alp -volcano -promontory -sandbar -coral reef -lakeside -seashore -geyser -hatchet -cleaver -letter opener -plane -power drill -lawn mower -hammer -corkscrew -can opener -plunger -screwdriver -shovel -plow -chain saw -cock -hen -ostrich -brambling -goldfinch -house finch -junco -indigo bunting -robin -bulbul -jay -magpie -chickadee -water ouzel -kite -bald eagle -vulture -great grey owl -black grouse -ptarmigan -ruffed grouse -prairie chicken -peacock -quail -partridge -African grey -macaw -sulphur-crested cockatoo -lorikeet -coucal -bee eater -hornbill -hummingbird -jacamar -toucan -drake -red-breasted merganser -goose -black swan -white stork -black stork -spoonbill -flamingo -American egret -little blue heron -bittern -crane -limpkin -American coot -bustard -ruddy turnstone -red-backed sandpiper -redshank -dowitcher -oystercatcher -European gallinule -pelican -king penguin -albatross -great white shark -tiger shark -hammerhead -electric ray -stingray -barracouta -coho -tench -goldfish -eel -rock beauty -anemone fish -lionfish -puffer -sturgeon -gar -loggerhead -leatherback turtle -mud turtle -terrapin -box turtle -banded gecko -common iguana -American chameleon -whiptail -agama -frilled lizard -alligator lizard -Gila monster -green lizard -African chameleon -Komodo dragon -triceratops -African crocodile -American alligator -thunder snake -ringneck snake -hognose snake -green snake -king snake -garter snake -water snake -vine snake -night snake -boa constrictor -rock python -Indian cobra -green mamba -sea snake -horned viper -diamondback -sidewinder -European fire salamander -common newt -eft -spotted salamander -axolotl -bullfrog -tree frog -tailed frog -whistle -wing -paintbrush -hand blower -oxygen mask -snorkel -loudspeaker -microphone -screen -mouse -electric fan -oil filter -strainer -space heater -stove -guillotine -barometer -rule -odometer -scale -analog clock -digital clock -wall clock -hourglass -sundial -parking meter -stopwatch -digital watch -stethoscope -syringe -magnetic compass -binoculars -projector -sunglasses -loupe -radio telescope -bow -cannon -assault rifle -rifle -projectile -computer keyboard -typewriter keyboard -crane -lighter -abacus -cash machine -slide rule -desktop computer -hand-held computer -notebook -web site -harvester -thresher -printer -slot -vending machine -sewing machine -joystick -switch -hook -car wheel -paddlewheel -pinwheel -potter's wheel -gas pump -carousel -swing -reel -radiator -puck -hard disc -sunglass -pick -car mirror -solar dish -remote control -disk brake -buckle -hair slide -knot -combination lock -padlock -nail -safety pin -screw -muzzle -seat belt -ski -candle -jack-o'-lantern -spotlight -torch -neck brace -pier -tripod -maypole -mousetrap -spider web -trilobite -harvestman -scorpion -black and gold garden spider -barn spider -garden spider -black widow -tarantula -wolf spider -tick -centipede -isopod -Dungeness crab -rock crab -fiddler crab -king crab -American lobster -spiny lobster -crayfish -hermit crab -tiger beetle -ladybug -ground beetle -long-horned beetle -leaf beetle -dung beetle -rhinoceros beetle -weevil -fly -bee -grasshopper -cricket -walking stick -cockroach -mantis -cicada -leafhopper -lacewing -dragonfly -damselfly -admiral -ringlet -monarch -cabbage butterfly -sulphur butterfly -lycaenid -jellyfish -sea anemone -brain coral -flatworm -nematode -conch -snail -slug -sea slug -chiton -sea urchin -sea cucumber -iron -espresso maker -microwave -Dutch oven -rotisserie -toaster -waffle iron -vacuum -dishwasher -refrigerator -washer -Crock Pot -frying pan -wok -caldron -coffeepot -teapot -spatula -altar -triumphal arch -patio -steel arch bridge -suspension bridge -viaduct -barn -greenhouse -palace -monastery -library -apiary -boathouse -church -mosque -stupa -planetarium -restaurant -cinema -home theater -lumbermill -coil -obelisk -totem pole -castle -prison -grocery store -bakery -barbershop -bookshop -butcher shop -confectionery -shoe shop -tobacco shop -toyshop -fountain -cliff dwelling -yurt -dock -brass -megalith -bannister -breakwater -dam -chainlink fence -picket fence -worm fence -stone wall -grille -sliding door -turnstile -mountain tent -scoreboard -honeycomb -plate rack -pedestal -beacon -mashed potato -bell pepper -head cabbage -broccoli -cauliflower -zucchini -spaghetti squash -acorn squash -butternut squash -cucumber -artichoke -cardoon -mushroom -shower curtain -jean -carton -handkerchief -sandal -ashcan -safe -plate -necklace -croquet ball -fur coat -thimble -pajama -running shoe -cocktail shaker -chest -manhole cover -modem -tub -tray -balance beam -bagel -prayer rug -kimono -hot pot -whiskey jug -knee pad -book jacket -spindle -ski mask -beer bottle -crash helmet -bottlecap -tile roof -mask -maillot -Petri dish -football helmet -bathing cap -teddy -holster -pop bottle -photocopier -vestment -crossword puzzle -golf ball -trifle -suit -water tower -feather boa -cloak -red wine -drumstick -shield -Christmas stocking -hoopskirt -menu -stage -bonnet -meat loaf -baseball -face powder -scabbard -sunscreen -beer glass -hen-of-the-woods -guacamole -lampshade -wool -hay -bow tie -mailbag -water jug -bucket -dishrag -soup bowl -eggnog -mortar -trench coat -paddle -chain -swab -mixing bowl -potpie -wine bottle -shoji -bulletproof vest -drilling platform -binder -cardigan -sweatshirt -pot -birdhouse -hamper -ping-pong ball -pencil box -pay-phone -consomme -apron -punching bag -backpack -groom -bearskin -pencil sharpener -broom -mosquito net -abaya -mortarboard -poncho -crutch -Polaroid camera -space bar -cup -racket -traffic light -quill -radio -dough -cuirass -military uniform -lipstick -shower cap -monitor -oscilloscope -mitten -brassiere -French loaf -vase -milk can -rugby ball -paper towel -earthstar -envelope -miniskirt -cowboy hat -trolleybus -perfume -bathtub -hotdog -coral fungus -bullet train -pillow -toilet tissue -cassette -carpenter's kit -ladle -stinkhorn -lotion -hair spray -academic gown -dome -crate -wig -burrito -pill bottle -chain mail -theater curtain -window shade -barrel -washbasin -ballpoint -basketball -bath towel -cowboy boot -gown -window screen -agaric -cellular telephone -nipple -barbell -mailbox -lab coat -fire screen -minibus -packet -maze -pole -horizontal bar -sombrero -pickelhaube -rain barrel -wallet -cassette player -comic book -piggy bank -street sign -bell cote -fountain pen -Windsor tie -volleyball -overskirt -sarong -purse -bolo tie -bib -parachute -sleeping bag -television -swimming trunks -measuring cup -espresso -pizza -breastplate -shopping basket -wooden spoon -saltshaker -chocolate sauce -ballplayer -goblet -gyromitra -stretcher -water bottle -dial telephone -soap dispenser -jersey -school bus -jigsaw puzzle -plastic bag -reflex camera -diaper -Band Aid -ice lolly -velvet -tennis ball -gasmask -doormat -Loafer -ice cream -pretzel -quilt -maillot -tape player -clog -iPod -bolete -scuba diver -pitcher -matchstick -bikini -sock -CD player -lens cap -thatch -vault -beaker -bubble -cheeseburger -parallel bars -flagpole -coffee mug -rubber eraser -stole -carbonara -dumbbel \ No newline at end of file diff --git a/docs/tutorials/Importing-new-models/index.md b/docs/tutorials/Importing-new-models/index.md deleted file mode 100644 index 83d99b0df..000000000 --- a/docs/tutorials/Importing-new-models/index.md +++ /dev/null @@ -1,247 +0,0 @@ ---- -layout: default -title: Importing Models -permalink: /tutorials/Importing-new-models/ ---- - -# Importing New Models - -### Materials - -- Just your desktop PC (Windows, Mac, Linux) -- USB web cam (optional) - -### Prerequisites - -1. We will use `Python 3.6` for this tutorial on your desktop PC, -including the [setup for OpenCV](/ELL/Installing-OpenCV/). - -2. You will need to be sure you built ELL as per [Installing and building ELL](/ELL/Installing-and-building-ELL/). -You will need to build ELL after you install Python from your activate your conda environment, -so that the `CMake` step picks up on the fact that you have Python 3.6 installed. - -3. This tutorial assumes your ELL bits live in the location "~/ELL". On Linux and Mac -this may well be the case. On Windows you may need to replace this syntax with a real Windows -drive location like this: "d:/ELL". **Note:** We will also use "cp" to copy files, on windows change that to "copy". - -4. CMake 3.5 or newer, which you needed to build ELL. On Linux you will need GCC 4.9 ore later or CLang 3.9. -On Windows you will also need Visual Studio 2015 ot 2017 with C++ language feature installed. - -### Overview - -So far in our tutorials we have used pre-trained ELL models from the ELL model gallery. -But eventually you will want to use a model that you trained or one that the ELL team has -not converted to the ELL model format. - -In this tutorial you will learn how to import Darknet and Microsoft Cognitive Toolkit (CNTK) models and convert them -to the ELL model format, then we will test them to make sure they work. - -## Darknet - -First we start with Darknet. Feel free to skip to the CNTK section below if that's what -you are more interested in. - -[Joseph Redmon](https://pjreddie.com/darknet/) has published some CNN models on his website. -Joseph calls his project *Darknet*, which sounds spooky, but his models are harmless, and really quite fast. - -We have provided a Python script that can load those Darknet models. -See the *darknet_import.py* Python module (found in the `tools/importers/darknet` directory) - -So let's download a Darknet ImageNet reference model using `curl` to a new temporary directory -on your PC using your `anaconda environment`: - - curl --location -O https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/darknet.cfg - curl --location -O https://pjreddie.com/media/files/darknet.weights - -If your conda environment doesn't have curl just run this: - - conda install curl - -So, to convert this Darknet model to the ELL format simply create a new directory - - python "~/ELL/tools/importers/darknet/darknet_import.py" darknet.cfg darknet.weights - -You should see some output like this while it is loading the model: - - convolutional: 224 x 224 x 3 -> 224 x 224 x 16 , pad 1 - max_pool: 224 x 224 x 16 -> 112 x 112 x 16 , pad 0 - convolutional: 112 x 112 x 16 -> 112 x 112 x 32 , pad 1 - max_pool: 112 x 112 x 32 -> 56 x 56 x 32 , pad 0 - convolutional: 56 x 56 x 32 -> 56 x 56 x 64 , pad 1 - max_pool: 56 x 56 x 64 -> 28 x 28 x 64 , pad 0 - convolutional: 28 x 28 x 64 -> 28 x 28 x 128 , pad 1 - max_pool: 28 x 28 x 128 -> 14 x 14 x 128 , pad 0 - convolutional: 14 x 14 x 128 -> 14 x 14 x 256 , pad 1 - max_pool: 14 x 14 x 256 -> 7 x 7 x 256 , pad 0 - convolutional: 7 x 7 x 256 -> 7 x 7 x 512 , pad 1 - max_pool: 7 x 7 x 512 -> 4 x 4 x 512 , pad 1 - convolutional: 4 x 4 x 512 -> 4 x 4 x 1024 , pad 1 - convolutional: 4 x 4 x 1024 -> 4 x 4 x 1000 , pad 0 - avg_pool: 4 x 4 x 1000 -> 1 x 1 x 1000 , pad 0 - softmax: 1 x 1 x 1000 -> 1 x 1 x 1000 - Saving model file: 'darknet.ell' - -Wait for a few seconds (depending on your computer performance) and you should see the -following new file in your directory: - - darknet.ell - -This is a JSON file that contains all the information needed for ELL to be able to -compile this model to code. - -### Testing Darknet - -Now let's test that this model works on your PC. For this you will need to copy the -`darknetImageNetLabels.txt` from this tutorial folder: - - cp ~/ELL/docs/tutorials/Importing-new-models/darknetImageNetLabels.txt . - - Then to build an ELL project that will run on your PC: - - python ~/ELL/tools/wrap/wrap.py darknetImageNetLabels.txt darknet.ell -target host - cd host - mkdir build - cd build - -Run CMake: - -|= -| Linux | cmake .. -|- -| Windows with Visual Studio 2015 | cmake -G "Visual Studio 14 2015 Win64" .. -|- -| Windows with Visual Studio 2017 | cmake -G "Visual Studio 15 2017 Win64" .. -|= - - -Then to build it: - - cmake --build . --config Release - cd .. - cp ~/ELL/tools/utilities/pythonlibs/demoHelper.py . - cp ~/ELL/docs/tutorials/shared/demo.py . - -And finally to run the test on video input: - - python demo.py darknetImageNetLabels.txt --compiledModel darknet - -And to test with a static image: - - python demo.py darknetImageNetLabels.txt --compiledModel darknet --image coffeemug.jpg - -## CNTK - -To use the CNTK you will need to install the [CNTK python packages](https://docs.microsoft.com/en-us/cognitive-toolkit/setup-linux-python). -From your conda environment it will be something like this, but you need to find -the version that matches your Python environment version exactly, this is the -Python 3.6 version: - - pip install https://cntk.ai/PythonWheel/CPU-Only/cntk-2.0-cp36-cp36m-win_amd64.whl - -We have provided a Python script that can load CNTK models. -See the *cntk_import.py* Python module (found in tools/importers/cntk) - -After building ELL and the ELL Python Language Bindings create a new temporary directory -where we will put the cntk model and open a new conda environment terminal window in -that location. - -Then download the CNTK ImageNet reference model as follows: - - curl --location -o cntkmodel.zip https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I224x224x3CMCMCMCMCMCMC1A/d_I224x224x3CMCMCMCMCMCMC1A.cntk.zip - unzip cntkmodel.zip - -If your conda environment doesn't have curl just run this: - - conda install curl - -Now rename the file `d_I224x224x3CMCMCMCMCMCMC1A.cntk` to something simpler like `model2.cntk` and import this model into the ELL format: - - python "~/ELL/tools/importers/cntk/cntk_import.py" model2.cntk - -Now you should see some output like this: -```` -Loading... -Selected CPU as the process wide default device. - -Finished loading. -Pre-processing... -Will not process ReduceElements - skipping this layer as irrelevant. -Will not process ClassificationError - skipping this layer as irrelevant. -Will not process ReduceElements - skipping this layer as irrelevant. -Will not process CrossEntropyWithSoftmax - skipping this layer as irrelevant. -Will not process Reshape - skipping this layer as irrelevant. -Will not process Combine - skipping this layer as irrelevant. -Minus : 224x224x3 -> 226x226x3 | input padding 0 output padding 1 -Convolution(LeakyReLU) : 226x226x3 -> 224x224x16 | input padding 1 output padding 0 -BatchNormalization : 224x224x16 -> 224x224x16 | input padding 0 output padding 0 -MaxPooling : 224x224x16 -> 114x114x16 | input padding 0 output padding 1 -Convolution(LeakyReLU) : 114x114x16 -> 112x112x64 | input padding 1 output padding 0 -BatchNormalization : 112x112x64 -> 112x112x64 | input padding 0 output padding 0 -MaxPooling : 112x112x64 -> 58x58x64 | input padding 0 output padding 1 -Convolution(LeakyReLU) : 58x58x64 -> 56x56x64 | input padding 1 output padding 0 -BatchNormalization : 56x56x64 -> 56x56x64 | input padding 0 output padding 0 -MaxPooling : 56x56x64 -> 30x30x64 | input padding 0 output padding 1 -Convolution(LeakyReLU) : 30x30x64 -> 28x28x128 | input padding 1 output padding 0 -BatchNormalization : 28x28x128 -> 28x28x128 | input padding 0 output padding 0 -MaxPooling : 28x28x128 -> 16x16x128 | input padding 0 output padding 1 -Convolution(LeakyReLU) : 16x16x128 -> 14x14x256 | input padding 1 output padding 0 -BatchNormalization : 14x14x256 -> 14x14x256 | input padding 0 output padding 0 -MaxPooling : 14x14x256 -> 9x9x256 | input padding 0 output padding 1 -Convolution(LeakyReLU) : 9x9x256 -> 7x7x512 | input padding 1 output padding 0 -BatchNormalization : 7x7x512 -> 7x7x512 | input padding 0 output padding 0 -MaxPooling : 7x7x512 -> 6x6x512 | input padding 0 output padding 1 -Convolution(LeakyReLU) : 6x6x512 -> 4x4x1024 | input padding 1 output padding 0 -BatchNormalization : 4x4x1024 -> 4x4x1024 | input padding 0 output padding 0 -Convolution : 4x4x1024 -> 4x4x1000 | input padding 0 output padding 0 -AveragePooling : 4x4x1000 -> 1x1x1000 | input padding 0 output padding 0 -Softmax : 1x1x1000 -> 1x1x1 | input padding 0 output padding 0 - -Finished pre-processing. -Saving model file: 'model2.ell' -```` - -### Testing CNTK Model - -Now let's test that this model works on your PC. For this you will need to download the -labels file: - - curl --location -o categories.txt https://raw.githubusercontent.com/Microsoft/ELL-models/master/models/ILSVRC2012/ILSVRC2012_labels.txt - - Then to build an ELL project that will run on your PC: - - python "~/ELL/tools/wrap/wrap.py categories.txt model2.cntk -target host - cd host - mkdir build - cd build - -Run CMake: - -|= -| Linux | cmake .. -|- -| Windows with Visual Studio 2015 | cmake -G "Visual Studio 14 2015 Win64" .. -|- -| Windows with Visual Studio 2017 | cmake -G "Visual Studio 15 2017 Win64" .. -|= - -Then to build it: - - cmake --build . --config Release - cd .. - cp ~/ELL/tools/utilities/pythonlibs/demoHelper.py . - cp ~/ELL/docs/tutorials/shared/demo.py . --bgr true - -And finally to run the test on video input: - - python demo.py categories.txt --compiledModel model2 --bgr true - -And to test with a static image: - - python demo.py categories.txt --compiledModel model2 --image coffeemug.jpg --bgr true - -### Summary - -Notice that the steps for testing an ELL model are virtually identical. Once the model -is converted to .ell format it doesn't matter any more whether it came from Darknet -or CNTK. The only difference is the label file where the labels can be in a different order -because of how the model was trained. \ No newline at end of file diff --git a/docs/tutorials/Setting-Up-your-Raspberry-Pi.md b/docs/tutorials/Setting-Up-your-Raspberry-Pi.md index 27fcc6d23..3dfa66ec8 100644 --- a/docs/tutorials/Setting-Up-your-Raspberry-Pi.md +++ b/docs/tutorials/Setting-Up-your-Raspberry-Pi.md @@ -1,60 +1,60 @@ --- layout: default -title: Setting Up your Raspberry Pi -permalink: /tutorials/Setting-Up-your-Raspberry-Pi +title: Setting up your Raspberry Pi for tutorials +permalink: /tutorials/Setting-up-your-Raspberry-Pi --- -# Setting Up your Raspberry Pi for Tutorials +# Setting up your Raspberry Pi for tutorials -Most of our tutorials follow a common workflow. The first steps involve authoring and designing an ELL model - these steps are typically done on a laptop or desktop computer (Windows, Linux, or Mac). The next steps involve deploying the ELL model and invoking it on the Raspberry Pi - these steps are typically done on the Pi itself. Therefore, you need to setup both your computer and your Pi with the necessary prerequisites. The step-by-step instructions below describe how to setup your Pi. Installation steps for your computer can be found here: +Most of our tutorials follow a common workflow. The first steps involve authoring and designing an ELL model - these steps are typically done on a laptop or desktop computer (Windows, Linux, or Mac). The next steps involve deploying the ELL model and invoking it on the Raspberry Pi - these steps are typically done on the Pi itself. Therefore, you need to setup both your computer and your Pi with the necessary prerequisites. The step-by-step instructions below describe how to setup your Pi. Installation steps for your computer can be found here: * [Windows](https://github.com/Microsoft/ELL/blob/master/INSTALL-Windows.md) * [Ubuntu Linux](https://github.com/Microsoft/ELL/blob/master/INSTALL-Ubuntu.md) * [Mac](https://github.com/Microsoft/ELL/blob/master/INSTALL-Mac.md) -## Basic Pi Setup +## Basic Pi Setup ### Power Adapter and Power Cable -AI workloads guzzle power, so we recommend using a high quality USB power adapter and micro USB cable. An adapter rated for 12 Watts (2.4 Amps) per USB port works best. We've had a good experience with 12W-per-port USB power adapters from Apple, Anker, and Amazon Basics. Long and thin cables will often result in a noticeable voltage drop and fail to provide sufficient power to your Raspberry Pi. Generic unbranded cables are hit-and-miss. For a few extra dollars you can get a nice name-brand cable, like the Anker PowerLine, and save yourself a lot of frustration. +AI workloads guzzle power, so we recommend using a high quality USB power adapter and micro USB cable. An adapter rated for 12 Watts (2.4 Amps) per USB port works best. We've had a good experience with 12W-per-port USB power adapters from Apple, Anker, and Amazon Basics. Long and thin cables will often result in a noticeable voltage drop and fail to provide sufficient power to your Raspberry Pi. Generic unbranded cables are hit-and-miss. For a few extra dollars you can get a nice name-brand cable, like the Anker PowerLine, and save yourself a lot of frustration. ### Operating System -Our tutorials assume that the operating system running on your Pi is [*Raspbian Jessie*](https://downloads.raspberrypi.org/raspbian/images/raspbian-2017-07-05/2017-07-05-raspbian-jessie.zip), not the more recent *Raspian Stretch*. +Our tutorials assume that the operating system running on your Pi is [*Raspbian Jessie*](https://downloads.raspberrypi.org/raspbian/images/raspbian-2017-07-05/2017-07-05-raspbian-jessie.zip), not the more recent *Raspian Stretch*. ### CMake We use `CMake` on the Raspberry Pi to create Python modules that can be called from our tutorial code. To install `CMake` on your Pi, connect to the network, open a terminal window, and type sudo apt-get update - sudo apt-get install -y cmake + sudo apt-get install -y cmake -### OpenBLAS +### OpenBLAS `OpenBLAS` is a library for fast linear algebra operations, which can significantly increase the speed of your models. It is optional, but highly recommended. To install `OpenBLAS`, sudo apt-get install -y libopenblas-dev ### Python 3.4 via Miniconda -Our tutorials require Python 3.4 on the Pi (and Python 3.6 on your computer). We recommend using the [Miniconda](https://conda.io/miniconda.html) distribution of Python, which makes it easy to install any required Python modules. To install Miniconda, +Our tutorials require Python 3.4 on the Pi (and Python 3.6 on your computer). We recommend using the [Miniconda](https://conda.io/miniconda.html) distribution of Python, which makes it easy to install any required Python modules. To install Miniconda, wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-armv7l.sh chmod +x Miniconda3-latest-Linux-armv7l.sh ./Miniconda3-latest-Linux-armv7l.sh -When prompted to install, reply [yes] to add Miniconda3 to your PATH. Next create an environment, as follows +When prompted to install, reply [yes] to add Miniconda3 to your PATH. Next create an environment, as follows - source ~/.bashrc + source ~/.bashrc conda create --name py34 python=3 source activate py34 -Remember to run `source activate py34` each time you start a new terminal window. +Remember to run `source activate py34` each time you start a new terminal window. ### OpenCV -`OpenCV` is a computer vision library that enables us to read images from a camera, resize them, and convert them to NumPy arrays for processing by ELL. To install `OpenCV`, +`OpenCV` is a computer vision library that enables us to read images from a camera, resize them, and convert them to NumPy arrays for processing by ELL. To install `OpenCV`, conda install -c microsoft-ell opencv -## Tips and Tweaks +## Tips and Tweaks -Raspberry Pis weren't designed to run AI workloads. Many AI workloads, like visual object tracking and audio keyword detection, run continously for long periods of time and require near realtime responsiveness. You probably don't want the operating system to go into energy saving mode, turning off the screen and dynamically changing the processor speed. +Raspberry Pis weren't designed to run AI workloads. Many AI workloads, like visual object tracking and audio keyword detection, run continuously for long periods of time and require near realtime responsiveness. You probably don't want the operating system to go into energy saving mode, turning off the screen and dynamically changing the processor speed. ### Disabling Energy Star and Screensaver @@ -65,14 +65,14 @@ Edit the file `~/.config/lxsession/LXDE-pi/autostart`, for example, by typing Add the following lines to this file @xset -dpms - @xset s noblank - @xset s off - + @xset s noblank + @xset s off + The first line disables energy star and the next two lines disable the screensaver. These changes take effect after rebooting. ### Disable Dynamic Clocking -The Raspberry Pi supports dynamic clocking, which means that it can change the processor frequency according to processor load. You can check the range of processor frequencies that your Raspberry Pi is configured for by typing `lscpu`. To disable dynamic clocking, edit `/boot/config.txt`, for example, by typing +The Raspberry Pi supports dynamic clocking, which means that it can change the processor frequency according to processor load. You can check the range of processor frequencies that your Raspberry Pi is configured for by typing `lscpu`. To disable dynamic clocking, edit `/boot/config.txt`, for example, by typing sudo leafpad /boot/config.txt @@ -84,15 +84,15 @@ This change takes effect after rebooting. ### Underclocking and Overclocking -Computation produces heat. Even at the default processor frequency of 700MHz, a Raspberry Pi running a serious AI workload at room temperature can overheat unless fitted with a physical cooling device. We recommend following our instructions for [active cooling your Pi](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/). If you don't want to physically cool your Pi, you can cool it by reducing the processor frequency (underclocking). +Computation produces heat. Even at the default processor frequency of 700MHz, a Raspberry Pi running a serious AI workload at room temperature can overheat unless fitted with a physical cooling device. We recommend following our instructions for [active cooling your Pi](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3/). If you don't want to physically cool your Pi, you can cool it by reducing the processor frequency (underclocking). -If you do fit your Pi with active cooling, you can aslo increase the processor frequency (overclocking) without overheating. **Be warned** that overclocking your Raspberry Pi can void your warranty. Also, we've experienced some wierd behavior when overclocking our Pis: some Pi units freeze up while other units lose their USB peripherals. You won't know how your specific Pi will react to overclocking until you try it. Overclocking isn't for the faint of heart - **proceed at your own risk**. +If you do fit your Pi with active cooling, you can also increase the processor frequency (overclocking) without overheating. **Be warned** that overclocking your Raspberry Pi can void your warranty. Also, we've experienced some weird behavior when overclocking our Pis: some Pi units freeze up while other units lose their USB peripherals. You won't know how your specific Pi will react to overclocking until you try it. Overclocking isn't for the faint of heart - **proceed at your own risk**. -To change your processor frequency, edit `/boot/config.txt`, for example, by typing +To change your processor frequency, edit `/boot/config.txt`, for example, by typing sudo leafpad /boot/config.txt -The default processor speed is 700 MHz. To underclock your processor, add the setting +The default processor speed is 700 MHz. To underclock your processor, add the setting arm_freq=600 @@ -118,4 +118,4 @@ If you do not have the built in libavcodec.so.56, it probably means that you are **ImportError: No module named 'numpy'** -You probably forgot to activate your Miniconda environment using `source activate py34`. See `Miniconda` instructions above. \ No newline at end of file +You probably forgot to activate your Miniconda environment using `source activate py34`. See `Miniconda` instructions above. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 08ce1850c..44a9c6699 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -8,12 +8,16 @@ permalink: /tutorials/ ## Setting Up -* [Setting Up your Raspberry Pi](/ELL/tutorials/Setting-Up-your-Raspberry-Pi) -* [Active Cooling your Raspberry Pi 3](/ELL/tutorials/Active-Cooling-your-Raspberry-Pi-3) +* [Setting up your Raspberry Pi for tutorials](/ELL/tutorials/Setting-up-your-Raspberry-Pi) +* [Active cooling your Raspberry Pi 3](/ELL/tutorials/Active-cooling-your-Raspberry-Pi-3) -# Computer Vision +## Image Classification Tutorials in Python -* [Getting Started with Image Classification on the Raspberry Pi](/ELL/tutorials/Getting-Started-with-Image-Classification-on-the-Raspberry-Pi/) -* [Fun with Dogs and Cats](/ELL/tutorials/Fun-with-Dogs-and-Cats/) -* [Importing new models](/ELL/tutorials/Importing-new-models/) -* [Comparing Image Classification models side by side on the Raspberry Pi](/ELL/tutorials/Comparing-Image-Classification-models-side-by-side-on-the-Raspberry-Pi/) +* [Getting started with image classification on the Raspberry Pi](/ELL/tutorials/Getting-started-with-image-classification-on-the-Raspberry-Pi/) +* [Boosting classifier accuracy by grouping categories](/ELL/tutorials/Boosting-classifier-accuracy-by-grouping-categories/) +* [Importing models](/ELL/tutorials/Importing-models/) +* [Comparing image classification models side by side on the Raspberry Pi](/ELL/tutorials/Comparing-image-classification-models-side-by-side-on-the-Raspberry-Pi/) + +## Image Classification Tutorials in C++ + +* [Getting started with image classification on the Raspberry Pi in C++](/ELL/tutorials/Getting-started-with-image-classification-in-cpp/) diff --git a/docs/tutorials/shared/coffeemug.jpg b/docs/tutorials/shared/coffeemug.jpg new file mode 100644 index 000000000..1cfc2a8a0 Binary files /dev/null and b/docs/tutorials/shared/coffeemug.jpg differ diff --git a/docs/tutorials/shared/demo.py b/docs/tutorials/shared/demo.py index 2a156e293..7adfd8428 100644 --- a/docs/tutorials/shared/demo.py +++ b/docs/tutorials/shared/demo.py @@ -47,7 +47,7 @@ def main(args): # Get the (at most) top 5 predictions that meet our threshold. This is returned as a list of tuples, # each with the text label and the prediction score. - top5 = helper.get_top_n(predictions, 5) + top5 = helper.get_top_n_predictions(predictions, 5) # Turn the top5 into a text string to display text = ", ".join(["(" + str(int(element[1]*100)) + "%) " + helper.get_label(element[0]) for element in top5]) diff --git a/docs/tutorials/shared/tutorial.py b/docs/tutorials/shared/tutorial.py new file mode 100644 index 000000000..a5d2ee43e --- /dev/null +++ b/docs/tutorials/shared/tutorial.py @@ -0,0 +1,102 @@ +#################################################################################################### +## +## Project: Embedded Learning Library (ELL) +## File: tutorial.py +## Authors: Byron Changuion +## +## Requires: Python 3.x +## +#################################################################################################### + +import sys +import os +import time +import numpy as np +import cv2 + +# Include a helper that has useful functions such as preparing data for the model +import tutorialHelpers as helpers + +# import the ELL model's Python module +import model + +# Function to return an image from our camera using OpenCV +def get_image_from_camera(camera): + if camera is not None: + # if predictor is too slow frames get buffered, this is designed to flush that buffer + ret, frame = camera.read() + if (not ret): + raise Exception('your capture device is not returning images') + return frame + return None + +# Return an array of strings corresponding to the model's recognized categories or classes. +# The order of the strings in this file are expected to match the order of the +# model's output predictions. +def get_categories_from_file(fileName): + labels = [] + with open(fileName) as f: + labels = f.read().splitlines() + return labels + +def main(args): + if (len(args) < 1): + print("usage: python tutorial.py categories.txt") + exit() + + # Open the video camera. To use a different camera, change the camera index. + camera = cv2.VideoCapture(0) + + # Read the category labels + categories = get_categories_from_file(args[0]) + + # Get the model's input dimensions. We'll use this information later to resize images appropriately. + inputShape = model.get_default_input_shape() + + # Create a vector to hold the model's output predictions + outputShape = model.get_default_output_shape() + predictions = model.FloatVector(outputShape.Size()) + + # Declare a value to hold the prediction times + predictionTimes = [] + meanTimeToPredict = 0.0 + + while (cv2.waitKey(1) == 0xFF): + # Get an image from the camera. If you'd like to use a different image, load the image from some other source. + image = get_image_from_camera(camera) + + # Prepare the image to pass to the model. This helper: + # - crops and resizes the image maintaining proper aspect ratio + # - reorders the image channels if needed + # - returns the data as a ravelled numpy array of floats so it can be handed to the model + input = helpers.prepare_image_for_model(image, inputShape.columns, inputShape.rows) + + # Get the predicted classes using the model's predict function on the image input data. + # The predictions are returned as a vector with the probability that the image + # contains the class represented by that index. + start = time.time() + model.predict(input, predictions) + end = time.time() + + # Let's grab the value of the top 5 predictions and their index, which represents the top five most + # confident matches and the class or category they belong to. + top5 = helpers.get_top_n_predictions(predictions, 5) + + # Draw header text that represents the top5 predictions + headerText = "".join(["(" + str(int(element[1]*100)) + "%) " + categories[element[0]] for element in top5]) + helpers.draw_header(image, headerText) + + # Draw footer text representing the mean evaluation time + meanTimeToPredict = helpers.get_mean_duration(predictionTimes, end - start) + footerText = '{:.0f}'.format(meanTimeToPredict * 1000) + 'ms/frame' + helpers.draw_footer(image, footerText) + + # Display the image using opencv + cv2.imshow('ELL model', image) + + print('Mean time to predict: ' + '{:.0f}'.format(meanTimeToPredict) + 'ms/frame') + +if __name__ == "__main__": + args = sys.argv + args.pop(0) + main(args) diff --git a/docs/tutorials/shared/tutorialHelpers.py b/docs/tutorials/shared/tutorialHelpers.py new file mode 100644 index 000000000..9744c0753 --- /dev/null +++ b/docs/tutorials/shared/tutorialHelpers.py @@ -0,0 +1,180 @@ +#################################################################################################### +## +## Project: Embedded Learning Library (ELL) +## File: tutorialHelpers.py +## Authors: Chris Lovett +## Byron Changuion +## +## Requires: Python 3.x +## +#################################################################################################### + +import os +import sys +import argparse +import cv2 +import numpy as np +import time +import math + +script_path = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(script_path) +sys.path.append(os.path.join(script_path, 'build')) +sys.path.append(os.path.join(script_path, 'build/Release')) + +def prepare_image_for_model(image, requiredWidth, requiredHeight, reorderToRGB = False): + """ Prepare an image for use with a model. Typically, this involves: + - Resize and center crop to the required width and height while preserving the image's aspect ratio. + Simple resize may result in a stretched or squashed image which will affect the model's ability + to classify images. + - OpenCV gives the image in BGR order, so we may need to re-order the channels to RGB. + - Convert the OpenCV result to a std::vector for use with ELL model + """ + if image.shape[0] > image.shape[1]: # Tall (more rows than cols) + rowStart = int((image.shape[0] - image.shape[1]) / 2) + rowEnd = rowStart + image.shape[1] + colStart = 0 + colEnd = image.shape[1] + else: # Wide (more cols than rows) + rowStart = 0 + rowEnd = image.shape[0] + colStart = int((image.shape[1] - image.shape[0]) / 2) + colEnd = colStart + image.shape[0] + # Center crop the image maintaining aspect ratio + cropped = image[rowStart:rowEnd, colStart:colEnd] + # Resize to model's requirements + resized = cv2.resize(cropped, (requiredHeight, requiredWidth)) + # Re-order if needed + if not reorderToRGB: + resized = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) + # Return as a vector of floats + result = resized.astype(np.float).ravel() + return result + +def get_top_n_predictions(predictions, N = 5, threshold = 0.20): + """Return at most the top N predictions as a list of tuples that meet the threshold. + The first of element of each tuple represents the index or class of the prediction and the second + element represents that probability or confidence value. + """ + map = [(i,predictions[i]) for i in range(len(predictions)) if predictions[i] >= threshold] + map.sort(key=lambda tup: tup[1], reverse=True) + result = map[:N] + return result + +def get_mean_duration(accumulated, duration, maxAccumulatedEntries = 30): + """ Add a duration to an array and calculate the mean duration. """ + + accumulated.append(duration) + if (len(accumulated) > maxAccumulatedEntries): + accumulated.pop(0) + durations = np.array(accumulated) + mean = np.mean(durations) + return mean + +def draw_header(image, text): + """Helper to draw header text block onto an image""" + draw_text_block(image, text, (0, 0), (50, 200, 50)) + return + +def draw_footer(image, text): + """Helper to draw footer text block onto an image""" + draw_text_block(image, text, (0, image.shape[0] - 40), (200, 100, 100)) + return + +def draw_text_block(image, text, blockTopLeft=(0,0), blockColor=(50, 200, 50), blockHeight=40): + """Helper to draw a filled rectangle with text onto an image""" + fontScale = 0.7 + cv2.rectangle( + image, blockTopLeft, (image.shape[1], blockTopLeft[1] + blockHeight), blockColor, cv2.FILLED) + cv2.putText(image, text, (blockTopLeft[0] + int(blockHeight / 4), blockTopLeft[1] + int(blockHeight * 0.667)), + cv2.FONT_HERSHEY_COMPLEX_SMALL, fontScale, (0, 0, 0), 1, cv2.LINE_AA) + +class TiledImage: + def __init__(self, numImages=2, outputHeightAndWidth=(600, 800)): + """ Helper class to create a tiled image out of many smaller images. + The class calculates how many horizontal and vertical blocks are needed to fit the requested number of images + and fills in unused blocks as blank. For example, to fit 4 images, the number of tiles is 2x2, to fit 5 images, + the number of tiles is 3x2, with the last tile being blank. + numImages - the maximum number of images that need to be composed into the tiled image. Note that the + actual number of tiles is equal to or larger than this number. + outputHeightAndWidth - a list of two values giving the rows and columns of the output image. The output tiled image + is a composition of sub images. + """ + self.composed_image_shape = self.get_composed_image_shape(numImages) + self.number_of_tiles = self.composed_image_shape[0] * self.composed_image_shape[1] + self.output_height_and_width = outputHeightAndWidth + self.images = None + self.window_name = 'ELL side by side' + cv2.namedWindow(self.window_name, cv2.WINDOW_NORMAL) # Ensure the window is resizable + # The aspect ratio of the composed image is now self.composed_image_shape[0] : self.composed_image_shape[1] + # Adjust the height of the window to account for this, else images will look distorted + cv2.resizeWindow(self.window_name, outputHeightAndWidth[1], int(outputHeightAndWidth[0] * (self.composed_image_shape[0] / self.composed_image_shape[1]))) + + def get_composed_image_shape(self, numImages): + """Returns a tuple indicating the (rows,cols) of the required number of tiles to hold numImages.""" + # Split the image horizontally + numHorizontal = math.ceil(math.sqrt(numImages)) + # Split the image vertically + numVertical = math.ceil(numImages / numHorizontal) + + return (numVertical, numHorizontal) + + def resize_to_same_height(self, images): + minHeight = min([i.shape[0] for i in images]) + for i in range(len(images)): + shape = images[i].shape + h = shape[0] + if h > minHeight: + scale = minHeight / h + newSize = (int(shape[1] * scale), int(shape[0] * scale)) + images[i] = cv2.resize(images[i], newSize) + return images + + def compose(self): + """Composes an image made by tiling all the sub-images set with `set_image_at`. """ + yElements = [] + for verticalIndex in range(self.composed_image_shape[0]): + xElements = [] + for horizontalIndex in range(self.composed_image_shape[1]): + currentIndex = verticalIndex * self.composed_image_shape[1] + horizontalIndex + xElements.append(self.images[currentIndex]) + # np.hstack only works if the images are the same height + xElements = self.resize_to_same_height(xElements) + horizontalImage = np.hstack(tuple(xElements)) + yElements.append(horizontalImage) + composedImage = np.vstack(tuple(yElements)) + + # Draw separation lines + yStep = int(composedImage.shape[0] / self.composed_image_shape[0]) + xStep = int(composedImage.shape[1] / self.composed_image_shape[1]) + y = yStep + x = xStep + for horizontalIndex in range(1, self.composed_image_shape[1]): + cv2.line(composedImage, (x, 0), (x, composedImage.shape[0]), (0, 0, 0), 3) + x = x + xStep + for verticalIndex in range(1, self.composed_image_shape[0]): + cv2.line(composedImage, (0, y), (composedImage.shape[1], y), (0, 0, 0), 3) + y = y + yStep + + return composedImage + + def set_image_at(self, imageIndex, frame): + """Sets the image at the specified index. Once all images have been set, the tiled image result can be retrieved with `compose`.""" + # Ensure self.images is initialized. + if self.images is None: + self.images = [None] * self.number_of_tiles + for i in range(self.number_of_tiles): + self.images[i] = np.zeros((frame.shape), np.uint8) + + # Update the image at the specified index + if (imageIndex < self.number_of_tiles): + self.images[imageIndex] = frame + return True + return False + + def show(self): + """Shows the final result of the tiled image. Returns True if the user indicates they are done viewing by pressing `Esc`. """ + # Compose the tiled image + imageToShow = self.compose() + # Show the tiled image + cv2.imshow(self.window_name, imageToShow) diff --git a/tools/importers/darknet/darknet_to_ell.py b/tools/importers/darknet/darknet_to_ell.py index 20c48bf01..c4686e457 100644 --- a/tools/importers/darknet/darknet_to_ell.py +++ b/tools/importers/darknet/darknet_to_ell.py @@ -80,7 +80,10 @@ def parse_cfg(filename): layer['out_h'] = int(convolutional_out_height(layer)) layer['out_w'] = int(convolutional_out_width(layer)) layer['out_c'] = int(layer['filters']) - print("convolutional: ", layer['h'], 'x', layer['w'], 'x', layer['c'], '-> ', layer['out_h'], 'x', layer['out_w'], 'x', layer['out_c'], ', pad ', layer['padding']) + if ('xnor' in layer) and (layer['xnor'] == '1'): + print("binary convolutional: ", layer['h'], 'x', layer['w'], 'x', layer['c'], '-> ', layer['out_h'], 'x', layer['out_w'], 'x', layer['out_c'], ', pad ', layer['padding']) + else: + print("convolutional: ", layer['h'], 'x', layer['w'], 'x', layer['c'], '-> ', layer['out_h'], 'x', layer['out_w'], 'x', layer['out_c'], ', pad ', layer['padding']) elif layer['type'] == 'connected': layer['h'] = network[i-1]['out_h'] layer['w'] = network[i-1]['out_w'] diff --git a/tools/utilities/pitest/drivetest.py b/tools/utilities/pitest/drivetest.py index 96e301d18..669eab328 100644 --- a/tools/utilities/pitest/drivetest.py +++ b/tools/utilities/pitest/drivetest.py @@ -20,14 +20,14 @@ import operator from shutil import copyfile from shutil import rmtree -from urllib import request import paramiko -import ziptools +import zipfile +import requests class DriveTest: def __init__(self): self.arg_parser = argparse.ArgumentParser( - "This script uses ELL to create a demo project for a model (default is darknet)\n" + "This script uses ELL to create a demo project for a model (default is d_I160x160x3CMCMCMCMCMCMC1A from the ELL gallery)\n" "on a target device (default is Raspberry Pi 3), pushes it to the given\n" "device's ip address using ssh and scp, then executes the test.\n" "The test also measures the accuracy and performance of evaluating the model.\n") @@ -37,8 +37,6 @@ def __init__(self): self.test_dir = os.path.join(self.build_root, "test", "pitest") self.output_dir = None self.target_dir = "/home/pi/pi3" - self.config_file = None - self.weights_file = None self.model_name = None self.labels_file = None self.ell_model = None @@ -92,10 +90,9 @@ def init(self, args): def extract_model_info(self, ell_model, labels_file): if (ell_model is None or labels_file is None): - self.model_name = "darknet" - self.labels_file = os.path.join(self.test_dir, "darknetImageNetLabels.txt") + self.model_name = "d_I160x160x3CMCMCMCMCMCMC1A" + self.labels_file = os.path.join(self.test_dir, "categories.txt") else: - # extract the model if it's in an archive unzip = ziptools.Extractor(ell_model) success, filename = unzip.extract_file(".ell") if success: @@ -111,8 +108,11 @@ def extract_model_info(self, ell_model, labels_file): self.labels_file = labels_file def download(self, url, localpath): - req = request.URLopener() - req.retrieve(url, localpath) + # Download the file + response = requests.get(url, stream=True) + # Write the file to disk + with open(localpath, "wb") as handle: + handle.write(response.content) def copy_files(self, list, folder): target_dir = os.path.join(self.test_dir, folder) @@ -156,31 +156,22 @@ def get_bash_files(self): for bitcode in bitcode_files: os.remove(os.path.join(self.output_dir, bitcode)) - def get_darknet(self): - self.config_file = os.path.join(self.test_dir, "darknet.cfg") - self.weights_file = os.path.join(self.test_dir, "darknet.weights") - if (not os.path.isfile(self.config_file) or not os.path.isfile(self.weights_file)): - print("downloading darknet model...") - self.download("https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/darknet.cfg", self.config_file) - self.download("https://pjreddie.com/media/files/darknet.weights", self.weights_file) - - self.copy_files( [ os.path.join(self.ell_root, "docs/tutorials/Importing-new-models/darknetImageNetLabels.txt") ], "") - - def import_darknet(self): - self.ell_model = os.path.join(self.test_dir, self.model_name + ".ell") - sys.path.append(os.path.join(current_path, '../../importers/darknet')) - darknet_importer = __import__("darknet_import") - args = {} - args['config_file'] = self.config_file - args['weights_file'] = self.weights_file - args['output_directory'] = None - importer = darknet_importer.DarknetImporter(args) - importer.run() + def get_default_model(self): + # Download the model + self.model_file = self.model_name + '.ell' + self.ell_model = 'd_I160x160x3CMCMCMCMCMCMC1A.ell' + if (not os.path.isfile(self.model_file)) or (not os.path.isfile(self.labels_file)) : + print("downloading default model...") + self.download("https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/d_I160x160x3CMCMCMCMCMCMC1A/d_I160x160x3CMCMCMCMCMCMC1A.ell.zip", "d_I160x160x3CMCMCMCMCMCMC1A.ell.zip") + # extract the model if it's in an archive + with zipfile.ZipFile("d_I160x160x3CMCMCMCMCMCMC1A.ell.zip") as myzip: + myzip.extractall() + print("downloading default categories.txt...") + self.download("https://github.com/Microsoft/ELL-models/raw/master/models/ILSVRC2012/categories.txt", self.labels_file) def get_model(self): - if self.model_name == "darknet": - self.get_darknet() - self.import_darknet() + if self.model_name == "d_I160x160x3CMCMCMCMCMCMC1A": + self.get_default_model() print("using ELL model: " + self.model_name) def make_project(self): @@ -190,7 +181,7 @@ def make_project(self): sys.path.append(os.path.join(current_path, '../../wrap')) mpp = __import__("wrap") builder = mpp.ModuleBuilder() - builder_args = [labels_file, self.ell_model, "-target", self.target, "-outdir", self.output_dir] + builder_args = [labels_file, self.ell_model, "-target", self.target, "-outdir", self.output_dir, "-v"] if self.profile: builder_args.append("-profile") builder.parse_command_line(builder_args) diff --git a/tools/utilities/pythonlibs/batchtest.py b/tools/utilities/pythonlibs/batchtest.py index d798205d3..488b03c5b 100644 --- a/tools/utilities/pythonlibs/batchtest.py +++ b/tools/utilities/pythonlibs/batchtest.py @@ -215,7 +215,7 @@ def labels_match(self, a, b): return False def get_current_label(self): - topN = self.get_top_n(self.results, 1) + topN = self.get_top_n_predictions(self.results, 1) if len(topN) > 0: return self.get_label(topN[0][0]) return "unknown" @@ -267,7 +267,7 @@ def record_result(self): expected = self.get_grouped_result(expected) expected_label = self.group_names[expected] - topN = self.get_top_n(self.results, self.test_top_n) + topN = self.get_top_n_predictions(self.results, self.test_top_n) winner = None actual = [] first = True diff --git a/tools/utilities/pythonlibs/demoHelper.py b/tools/utilities/pythonlibs/demoHelper.py index 3bec95b11..47aafe514 100644 --- a/tools/utilities/pythonlibs/demoHelper.py +++ b/tools/utilities/pythonlibs/demoHelper.py @@ -256,9 +256,12 @@ def report_times(self, node_level=True): if hasattr(self.compiled_module, self.model_name + "_PrintNodeProfilingInfo"): getattr(self.compiled_module, self.model_name + "_PrintNodeProfilingInfo")() - def get_top_n(self, predictions, N): - """Return at most the top N predictions as a list of tuples that meet the threshold.""" - map = [(i,predictions[i]) for i in range(len(predictions)) if predictions[i] >= self.threshold] + def get_top_n_predictions(self, predictions, N = 5, threshold = 0.20): + """Return at most the top N predictions as a list of tuples that meet the threshold. + The first of element of each tuple represents the index or class of the prediction and the second + element represents that probability or confidence value. + """ + map = [(i,predictions[i]) for i in range(len(predictions)) if predictions[i] >= threshold] map.sort(key=lambda tup: tup[1], reverse=True) result = map[:N] return result diff --git a/tools/utilities/pythonlibs/gallery/validate.py b/tools/utilities/pythonlibs/gallery/validate.py index 6612a6759..fc55e8553 100644 --- a/tools/utilities/pythonlibs/gallery/validate.py +++ b/tools/utilities/pythonlibs/gallery/validate.py @@ -85,7 +85,7 @@ def validate_image(args, filename): # Get the (at most) top 5 predictions that meet our threshold. This is returned as a list of tuples, # each with the text label and the prediction score. - top5 = helper.get_top_n(predictions, 5) + top5 = helper.get_top_n_predictions(predictions, 5) text = "".join( [str(element[0]) + "(" + str(int(100 * element[1])) + "%) " for element in top5]) diff --git a/tools/utilities/pythonlibs/modelHelpers.py b/tools/utilities/pythonlibs/modelHelpers.py new file mode 100644 index 000000000..8f811c6ff --- /dev/null +++ b/tools/utilities/pythonlibs/modelHelpers.py @@ -0,0 +1,90 @@ +#################################################################################################### +## +## Project: Embedded Learning Library (ELL) +## File: modelHelpers.py +## Authors: Chris Lovett +## Byron Changuion +## +## Requires: Python 3.x +## +#################################################################################################### + +import os +import sys +import argparse +import cv2 +import numpy as np +import time +import math + +script_path = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(script_path) +sys.path.append(os.path.join(script_path, 'build')) +sys.path.append(os.path.join(script_path, 'build/Release')) + +def prepare_image_for_model(image, requiredWidth, requiredHeight, reorderToRGB = False): + """ Prepare an image for use with a model. Typically, this involves: + - Resize and center crop to the required width and height while preserving the image's aspect ratio. + Simple resize may result in a stretched or squashed image which will affect the model's ability + to classify images. + - OpenCV gives the image in BGR order, so we may need to re-order the channels to RGB. + - Convert the OpenCV result to a std::vector for use with ELL model + """ + if image.shape[0] > image.shape[1]: # Tall (more rows than cols) + rowStart = int((image.shape[0] - image.shape[1]) / 2) + rowEnd = rowStart + image.shape[1] + colStart = 0 + colEnd = image.shape[1] + else: # Wide (more cols than rows) + rowStart = 0 + rowEnd = image.shape[0] + colStart = int((image.shape[1] - image.shape[0]) / 2) + colEnd = colStart + image.shape[0] + # Center crop the image maintaining aspect ratio + cropped = image[rowStart:rowEnd, colStart:colEnd] + # Resize to model's requirements + resized = cv2.resize(cropped, (requiredHeight, requiredWidth)) + # Re-order if needed + if not reorderToRGB: + resized = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) + # Return as a vector of floats + result = resized.astype(np.float).ravel() + return result + +def get_top_n_predictions(predictions, N = 5, threshold = 0.20): + """Return at most the top N predictions as a list of tuples that meet the threshold. + The first of element of each tuple represents the index or class of the prediction and the second + element represents that probability or confidence value. + """ + map = [(i,predictions[i]) for i in range(len(predictions)) if predictions[i] >= threshold] + map.sort(key=lambda tup: tup[1], reverse=True) + result = map[:N] + return result + +def get_mean_duration(accumulated, duration, maxAccumulatedEntries = 30): + """ Add a duration to an array and calculate the mean duration. """ + + accumulated.append(duration) + if (len(accumulated) > maxAccumulatedEntries): + accumulated.pop(0) + durations = np.array(accumulated) + mean = np.mean(durations) + return mean + +def draw_header(image, text): + """Helper to draw header text block onto an image""" + draw_text_block(image, text, (0, 0), (50, 200, 50)) + return + +def draw_footer(image, text): + """Helper to draw footer text block onto an image""" + draw_text_block(image, text, (0, image.shape[0] - 40), (200, 100, 100)) + return + +def draw_text_block(image, text, blockTopLeft=(0,0), blockColor=(50, 200, 50), blockHeight=40): + """Helper to draw a filled rectangle with text onto an image""" + fontScale = 0.7 + cv2.rectangle( + image, blockTopLeft, (image.shape[1], blockTopLeft[1] + blockHeight), blockColor, cv2.FILLED) + cv2.putText(image, text, (blockTopLeft[0] + int(blockHeight / 4), blockTopLeft[1] + int(blockHeight * 0.667)), + cv2.FONT_HERSHEY_COMPLEX_SMALL, fontScale, (0, 0, 0), 1, cv2.LINE_AA) diff --git a/tools/wrap/CMakeLists.txt b/tools/wrap/CMakeLists.txt index 7fcdaaf2b..b33b137a7 100644 --- a/tools/wrap/CMakeLists.txt +++ b/tools/wrap/CMakeLists.txt @@ -12,8 +12,9 @@ else() set(ADDITIONAL_DISTUTILS_FLAGS install) endif() -set(module_name wrap) -set(builder_src +set (module_name wrap) +set (builder_src + templates/CMakeLists.cpp.txt.in templates/CMakeLists.python.txt.in wrap.py README.md) diff --git a/tools/wrap/README.md b/tools/wrap/README.md index f69266e63..cdb04ccec 100644 --- a/tools/wrap/README.md +++ b/tools/wrap/README.md @@ -6,6 +6,7 @@ This tool wraps a given ELL model in a CMake C++ buildable project that builds a The supported languages are: - `python` (default) +- `cpp` The supported target platforms are: - `pi3` raspberry pi 3 @@ -31,7 +32,7 @@ copy "d:\git\ELL\libraries/emitters/include/ClockInterface.h" "pi3\include\Clock copy "d:\git\ELL\interfaces/common/tcc/CallbackInterface.tcc" "pi3\tcc\CallbackInterface.tcc" ```` -### Templated CMakeLists.txt +### Templated CMakeLists.txt for Python Included in the templates folder is a CMakeLists.python.txt.in template. The final CMakeLists.txt file is generated from this template by filling @@ -50,6 +51,26 @@ table, td { border:1px solid #A0A0D0;border-colapse:collapse;padding:5px; } These parameters are filled and the resulting CMakeLists.txt file is written to the output folder. +### Templated CMakeLists.txt for C++ + +Included in the templates folder is a CMakeLists.cpp.txt.in template. +The final CMakeLists.txt file is generated from this template by filling +in some text parameters: + + + +
+ + + + +
ELL_modelof the model
Archtarget architecture, e.g. host, pi3
OBJECT_EXTENSIONobj or o
ELL_ROOTlocation of ELL git repo
+ +These parameters are filled and the resulting CMakeLists.txt file is written to the output folder. +This describes the compiled model to CMake so that it can be referenced in C++ projects, such as a calling application. + #### Compile Model Next it invokes the ell compiler to compile the given model. The command line looks like this: @@ -57,15 +78,23 @@ Next it invokes the ell compiler to compile the given model. The command line l compile -imap ImageNet.ell -cfn predict -cmn ImageNet --bitcode --swig --blas true --target pi3 -od pi3 ```` This compiler is created by the ELL build and you will find it in `~/git/ELL/build/bin` and on Windows it will be in `d:\git\ELL\ELL\build\bin\release\`. + +#### Optimizing the code + +Next, wrap invokes the optimizing LLVM tool `opt` on the output from compile. The command looks something like: +```` +opt.exe pi3\ImageNet.bc -o pi3\ImageNet.opt.bc -O3 +```` + #### Generating SWIG interfaces -ELL uses SWIG to generate the stubs necessary to convert between programming languages: +For language targets other than C++, ELL uses SWIG to generate the stubs necessary to convert between programming languages: ```` swig -python -c++ -Fmicrosoft -py3 -outdir pi3 -ID:\git\ELL\interfaces/common -ID:\git\ELL\interfaces/common/include -ID:\git\ELL\libraries/emitters/include -o pi3\ImageNetPYTHON_wrap.cxx pi3\ImageNet.i ```` On Windows we include SWIG in a nuget package, so you will find it in `d:\git\ELL\external\swigwintools.3.0.12\tools\swigwin-3.0.12\` -#### Targetting the Model +#### Targeting the Model Finally it uses LLC to cross-compile the model so that code runs on your specified target platform. diff --git a/tools/wrap/templates/CMakeLists.cpp.txt.in b/tools/wrap/templates/CMakeLists.cpp.txt.in new file mode 100644 index 000000000..0c2e0c06f --- /dev/null +++ b/tools/wrap/templates/CMakeLists.cpp.txt.in @@ -0,0 +1,44 @@ +# +# Generated CMakeLists.txt for linking with the @ELL_model@ static library on @Arch@ +# WARNING: Any changes made to this file will be overwritten! +# +# Required variables for configuring this file from CMake: +# ELL_model - name of the model +# Arch - target architecture, e.g. host, pi3. +# ELL_ROOT - location of ELL git repo +# +# Usage: +# +# Linux / Mac: +# mkdir build && cd build +# cmake .. +# make +# +# Windows 64 bit: +# mkdir build && cd build +# VS 2015: cmake -G "Visual Studio 14 2015 Win64" .. +# VS 2017: cmake -G "Visual Studio 15 2017 Win64" .. +# Build the generated .sln in Release, or run cmake --build . --config=Release +# + +cmake_minimum_required(VERSION 3.3) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +project(@ELL_model@) + +# dependencies +if(WIN32) + # path to the OpenBLAS Nuget + set(PACKAGE_ROOT "@ELL_ROOT@") +endif() +include(OpenBLASSetup.cmake) + +add_library(@ELL_model@ STATIC IMPORTED GLOBAL) + +set_property(TARGET model APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR};${CMAKE_CURRENT_SOURCE_DIR}/include") +set_target_properties(@ELL_model@ PROPERTIES IMPORTED_LOCATION ${CMAKE_CURRENT_SOURCE_DIR}/@ELL_model@.@OBJECT_EXTENSION@) +if(BLAS_LIBS) + target_link_libraries(@ELL_model@ INTERFACE ${BLAS_LIBS}) +endif() + diff --git a/tools/wrap/wrap.py b/tools/wrap/wrap.py index 2a6f61c13..4595512c0 100644 --- a/tools/wrap/wrap.py +++ b/tools/wrap/wrap.py @@ -54,7 +54,7 @@ def parse_command_line(self, argv): arg_parser.add_argument("model_file", help="path to the ELL model file") # optional arguments - arg_parser.add_argument("--language", "-lang", help="the language for the ELL module", choices=["python"], default=self.language) + arg_parser.add_argument("--language", "-lang", help="the language for the ELL module", choices=["python", "cpp"], default=self.language) arg_parser.add_argument("--target", "-target", help="the target platform", choices=["pi3", "pi3_64", "aarch64", "host"], default=self.target) arg_parser.add_argument("--outdir", "-outdir", help="the output directory") arg_parser.add_argument("--profile", "-profile", help="enable profiling functions in the ELL module", action="store_true") @@ -135,9 +135,9 @@ def run(self): self.copy_files(self.files, "") self.copy_files(self.includes, "include") self.copy_files(self.tcc, "tcc") - self.create_cmake_file() self.tools.compile(self.model_file, self.func_name, self.model_name, self.target, self.output_dir, self.profile) - self.tools.swig(self.output_dir, self.model_name, self.language) + if self.language != "cpp": + self.tools.swig(self.output_dir, self.model_name, self.language) self.tools.opt(self.output_dir, self.model_name) self.tools.llc(self.output_dir, self.model_name, self.target) self.create_cmake_file()