From ab8210686e338dd8e5cd902fa01116dc22324315 Mon Sep 17 00:00:00 2001 From: Mengqing Cao <52243582+MengqingCao@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:58:56 +0800 Subject: [PATCH] Merge pull request #3608 from MengqingCao:dvpp_support Add additional image processing operators for Ascend NPU by utilizing DVPP #3608 The user base for [Ascend NPU](https://www.hiascend.com/en/) and programming with CANN is increasing rapidly, with a growing number of users joining each day. To facilitate the use of these users, this PR provides more support for Ascend backend operators. All operators this PR offers are using use DVPP as the computational unit. Digital Vision Pre-Processing (DVPP) is an image processing unit built into the Ascend AI processor. Its main functions include image and video encoding/decoding, as well as image cropping and scaling. The high-frequency operators with NPU as the backend and basic data structure AscendMat has been provided in #3552, while it still lacks many image processing operators. Moreover, only two interpolation algorithms for the resize operator are supported in #3552. In this PR, the bilinear interpolation algorithm and nearest neighbour interpolation algorithm are implemented for the resize operator, as well as the Ascend implementation of the copyMakeBorder operator. In addition, the serialization of image processing operations is widely used in the preprocessing and post-processing stages of computer vision deep learning methods. Therefore, providing integrated operators is very meaningful for improving the convenience of use for OpenCV and deep learning crossover users. For example, torchvision also provides similar operators: [RESIZED_CROP](https://pytorch.org/vision/stable/generated/torchvision.transforms.functional.resized_crop.html?highlight=resizedcrop). Thus, this PR also provides two serialization processing operators: cropResize and cropResizeMakeBorder. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [N/A] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/cannops/include/opencv2/cann.hpp | 15 + .../include/opencv2/cann_interface.hpp | 165 ++++++++-- modules/cannops/include/opencv2/dvpp_call.hpp | 107 ++++++ .../cannops/misc/python/test/test_cannops.py | 45 +++ modules/cannops/perf/perf_core.cpp | 172 ++++++++++ modules/cannops/perf/perf_main.cpp | 14 +- modules/cannops/src/core.cpp | 209 ++++++++++-- modules/cannops/src/dvpp_call.cpp | 310 ++++++++++++++++++ modules/cannops/src/precomp.hpp | 1 + modules/cannops/test/test_core.cpp | 122 +++++++ modules/cannops/test/test_main.cpp | 14 +- .../ascend_npu_image_processing.markdown | 21 +- 12 files changed, 1126 insertions(+), 69 deletions(-) create mode 100644 modules/cannops/include/opencv2/dvpp_call.hpp create mode 100644 modules/cannops/src/dvpp_call.cpp diff --git a/modules/cannops/include/opencv2/cann.hpp b/modules/cannops/include/opencv2/cann.hpp index bd351481624..4f4f3a7d8d2 100644 --- a/modules/cannops/include/opencv2/cann.hpp +++ b/modules/cannops/include/opencv2/cann.hpp @@ -318,6 +318,21 @@ CV_EXPORTS_W void initAcl(); */ CV_EXPORTS_W void finalizeAcl(); +/** + * @brief init DVPP system. + * @note The DVPP interfaces used are all version V2. + * Supported devices: Atlas Inference Series products, Atlas 200/500 A2 Inference products and + * Atlas A2 Training Series products/Atlas 300I A2 Inference products + */ +CV_EXPORTS_W void initDvpp(); + +/** + * @brief finalize DVPP system. + * @note Supported devices: Atlas Inference Series products, Atlas 200/500 A2 Inference products and + * Atlas A2 Training Series products/Atlas 300I A2 Inference products + */ +CV_EXPORTS_W void finalizeDvpp(); + //! @} cann_init } // namespace cann diff --git a/modules/cannops/include/opencv2/cann_interface.hpp b/modules/cannops/include/opencv2/cann_interface.hpp index 6b13090f4f1..8d7c90a1cc1 100644 --- a/modules/cannops/include/opencv2/cann_interface.hpp +++ b/modules/cannops/include/opencv2/cann_interface.hpp @@ -111,6 +111,7 @@ CV_EXPORTS_W void subtract(const Scalar& src1, const AscendMat& src2, CV_OUT Asc * @param scale Optional scale factor. * @param dtype Optional depth of the output array. * @param stream AscendStream for the asynchronous version. + * @note when scale != 1, src must be one of the following types: float16, float32, int32 * @sa cv::multiply cuda::multiply */ CV_EXPORTS_W void multiply(const InputArray src1, const InputArray src2, OutputArray dst, @@ -145,6 +146,9 @@ CV_EXPORTS_W void multiply(const Scalar& src1, const AscendMat& src2, CV_OUT Asc * @param scale Optional scale factor. * @param dtype Optional depth of the output array. * @param stream AscendStream for the asynchronous version. + * @note when scale == 1, src must be one of the following types: float16, float32, double, uint16, + * int8, uint8, int16, int32, int64; when scale != 1, src must be one of the following types: + * int32, int16, float16, float32. * @sa cv::divide cuda::divide */ CV_EXPORTS_W void divide(const InputArray src1, const InputArray src2, OutputArray dst, @@ -178,6 +182,7 @@ CV_EXPORTS_W void divide(const Scalar& src1, const AscendMat& src2, CV_OUT Ascen * @param mask Optional operation mask, 8-bit single channel array, that specifies elements of the * destination array to be changed. The mask can be used only with single channel images. * @param stream AscendStream for the asynchronous version. + * @note src must be one of the following types: int32, int16, uint16 * @sa cv::bitwise_and cuda::bitwise_and */ CV_EXPORTS_W void bitwise_and(const InputArray src1, const InputArray src2, OutputArray dst, @@ -211,6 +216,7 @@ CV_EXPORTS_W void bitwise_and(const Scalar& src1, const AscendMat& src2, CV_OUT * @param mask Optional operation mask, 8-bit single channel array, that specifies elements of the * destination array to be changed. The mask can be used only with single channel images. * @param stream AscendStream for the asynchronous version. + * @note src must be one of the following types: int32, int16, uint16 * @sa cv::bitwise_or cuda::bitwise_or */ CV_EXPORTS_W void bitwise_or(const InputArray src1, const InputArray src2, OutputArray dst, @@ -245,6 +251,7 @@ CV_EXPORTS_W void bitwise_or(const Scalar& src1, const AscendMat& src2, CV_OUT A * @param mask Optional operation mask, 8-bit single channel array, that specifies elements of the * destination array to be changed. The mask can be used only with single channel images. * @param stream AscendStream for the asynchronous version. + * @note src must be one of the following types: int32, int16, uint16 * @sa cv::bitwise_xor cuda::bitwise_xor */ CV_EXPORTS_W void bitwise_xor(const InputArray src1, const InputArray src2, OutputArray dst, @@ -277,6 +284,7 @@ CV_EXPORTS_W void bitwise_xor(const Scalar& src1, const AscendMat& src2, CV_OUT * @param mask Optional operation mask, 8-bit single channel array, that specifies elements of the * destination array to be changed. The mask can be used only with single channel images. * @param stream AscendStream for the asynchronous version. + * @note src must be one of the following types: int32, int16, uint16 * @sa cv::bitwise_not cuda::bitwise_not */ CV_EXPORTS_W void bitwise_not(const InputArray src, OutputArray dst, @@ -306,6 +314,7 @@ The function addWeighted calculates the weighted sum of two arrays as follows: where I is a multi-dimensional index of array elements. In case of multi-channel arrays, each channel is processed independently. +@note src must be one of the following types: int32, int16, float16, float32. @sa cv::addWeighted cv::cuda::addWeighted */ @@ -326,6 +335,7 @@ CV_EXPORTS_W void addWeighted(const AscendMat& src1, double alpha, const AscendM @param type Threshold type. For details, see threshold . The THRESH_MASK, THRESH_OTSU and THRESH_TRIANGLE threshold types are not supported. @param stream AscendStream for the asynchronous version. +@note src must be one of the following types: float16, float32. @sa cv::threshold cv::cuda::threshold */ @@ -346,11 +356,13 @@ CV_EXPORTS_W double threshold(const AscendMat& src, CV_OUT AscendMat& dst, doubl @param n Number of source matrices. @param dst Destination matrix. @param stream AscendStream for the asynchronous version. +@note src must be one of the following types: float16, float32, double, int32, int16, int8, int64, +uint8, uint16, uint32, uint64. @sa cv::merge cv::cuda::merge */ CV_EXPORTS_W void merge(const AscendMat* src, size_t n, CV_OUT AscendMat& dst, - AscendStream& stream = AscendStream::Null()); + AscendStream& stream = AscendStream::Null()); /** @overload */ CV_EXPORTS_W void merge(const std::vector& src, CV_OUT AscendMat& dst, AscendStream& stream = AscendStream::Null()); @@ -366,11 +378,13 @@ CV_EXPORTS_W void merge(const std::vector& src, OutputArray& dst, @param src Source matrix. @param dst Destination array/vector of single-channel matrices. @param stream AscendStream for the asynchronous version. +@note src must be one of the types:float16, float32, double, int64, int32, uint8, uint16, uint32, + uint64, int8, int16, bool @sa cv::split cv::cuda::split */ CV_EXPORTS_W void split(const AscendMat& src, AscendMat* dst, - AscendStream& stream = AscendStream::Null()); + AscendStream& stream = AscendStream::Null()); /** @overload */ CV_EXPORTS_W void split(const AscendMat& src, CV_OUT std::vector& dst, AscendStream& stream = AscendStream::Null()); @@ -386,6 +400,8 @@ CV_EXPORTS_W void split(const InputArray src, CV_OUT std::vector& dst @param src Source matrix. @param dst Destination matrix. @param stream AscendStream for the asynchronous version. +@note src must be one of the following types: +float16,float,int8,int16,int32,int64,uint8,uint16,uint32,uint64,bool @sa cv::transpose cv::cuda::transpose */ @@ -403,6 +419,7 @@ CV_EXPORTS_W void transpose(const AscendMat& src, CV_OUT AscendMat& dst, - \> 0 Flips around y-axis. - \< 0 Flips around both axes. @param stream AscendStream for the asynchronous version. +@note src must be one of the following types: float16,float,int64,int32,int16,uint16 @sa cv::flip cv::cuda::flip */ @@ -421,6 +438,7 @@ The function cv::rotate rotates the array in one of three different ways: and the rows and cols are switched for ROTATE_90_CLOCKWISE and ROTATE_90_COUNTERCLOCKWISE. @param rotateCode an enum to specify how to rotate the array; see the enum #RotateFlags @param stream AscendStream for the asynchronous version. +@note src must be one of the following types: float16,float,int64,int32,int16,uint16 @sa cv::rotate */ @@ -445,21 +463,6 @@ CV_EXPORTS_W AscendMat crop(InputArray src, const Rect& rect, /** @overload */ CV_EXPORTS_W AscendMat crop(const AscendMat& src, const Rect& rect, AscendStream& stream = AscendStream::Null()); -/** @brief Resizes an image src down to or up to the specified size. -@param src input image -@param dst output image; it has the size dsize (when it is non-zero) or the size computed from -src.size(), fx, and fy; the type of dst is the same as of src. -@param dsize output image size; if it equals zero, it is computed as: - \f[𝚍𝚜𝚒𝚣𝚎 = 𝚂𝚒𝚣𝚎(𝚛𝚘𝚞𝚗𝚍(𝚏𝚡*𝚜𝚛𝚌.𝚌𝚘𝚕𝚜), 𝚛𝚘𝚞𝚗𝚍(𝚏𝚢*𝚜𝚛𝚌.𝚛𝚘𝚠𝚜))\f] - Either dsize or both fx and fy must be non-zero. -@param fx scale factor along the horizontal axis; when it equals 0, it is computed as -\f[(𝚍𝚘𝚞𝚋𝚕𝚎)𝚍𝚜𝚒𝚣𝚎.𝚠𝚒𝚍𝚝𝚑/𝚜𝚛𝚌.𝚌𝚘𝚕𝚜\f] - -@param fy scale factor along the vertical axis; when it equals 0, it is computed as -\f[(𝚍𝚘𝚞𝚋𝚕𝚎)𝚍𝚜𝚒𝚣𝚎.𝚑𝚎𝚒𝚐𝚑𝚝/𝚜𝚛𝚌.𝚛𝚘𝚠𝚜\f] -@param interpolation interpolation method(see **cv.cann.InterpolationFlags**) -@sa cv::resize -*/ //! interpolation algorithm enum InterpolationFlags @@ -478,14 +481,121 @@ enum InterpolationFlags INTER_MAX = 7, }; -CV_EXPORTS_W void resize(InputArray _src, OutputArray _dst, Size dsize, double inv_scale_x, - double inv_scale_y, int interpolation, - AscendStream& stream = AscendStream::Null()); +/** @brief Resizes an image src down to or up to the specified size. +@param src input image +@param dst output image; it has the size dsize (when it is non-zero) or the size computed from +src.size(), fx, and fy; the type of dst is the same as of src. +@param dsize output image size; if it equals zero, it is computed as: + \f[𝚍𝚜𝚒𝚣𝚎 = 𝚂𝚒𝚣𝚎(𝚛𝚘𝚞𝚗𝚍(𝚏𝚡*𝚜𝚛𝚌.𝚌𝚘𝚕𝚜), 𝚛𝚘𝚞𝚗𝚍(𝚏𝚢*𝚜𝚛𝚌.𝚛𝚘𝚠𝚜))\f] + Either dsize or both fx and fy must be non-zero. +@param fx scale factor along the horizontal axis; when it equals 0, it is computed as +\f[(𝚍𝚘𝚞𝚋𝚕𝚎)𝚍𝚜𝚒𝚣𝚎.𝚠𝚒𝚍𝚝𝚑/𝚜𝚛𝚌.𝚌𝚘𝚕𝚜\f] + +@param fy scale factor along the vertical axis; when it equals 0, it is computed as +\f[(𝚍𝚘𝚞𝚋𝚕𝚎)𝚍𝚜𝚒𝚣𝚎.𝚑𝚎𝚒𝚐𝚑𝚝/𝚜𝚛𝚌.𝚛𝚘𝚠𝚜\f] +@param interpolation interpolation method(see **cv.cann.InterpolationFlags**) +@param stream AscendStream for the asynchronous version. +* @note There are some constraints for the input datatype: + * when resampling using + * nearest neighbor or bilinear interpolation: Input images must be uint8, and only GRAY and BGR + images are supported. The resolution of input and output images must in range of [10*6, +4096*4096]. + * bicubic interpolation: Input images can be of different types, output images must be + float or uint8. + * pixel area interpolation: Input images can be of different types but output images + are always float.\n + * Only the following devices are supported when resampling using nearest neighbor or bilinear + interpolation: Atlas Inference Series products, Atlas 200/500 A2 Inference products and + Atlas A2 Training Series products/Atlas 300I A2 Inference products +@sa cv::resize +*/ +CV_EXPORTS_W void resize(InputArray src, OutputArray dst, Size dsize, double fx, double fy, + int interpolation, AscendStream& stream = AscendStream::Null()); /** @overload */ -CV_EXPORTS_W void resize(const AscendMat& src, CV_OUT AscendMat& dst, Size dsize, double inv_scale_x, - double inv_scale_y, int interpolation, - AscendStream& stream = AscendStream::Null()); +CV_EXPORTS_W void resize(const AscendMat& src, CV_OUT AscendMat& dst, Size dsize, double fx, + double fy, int interpolation, AscendStream& stream = AscendStream::Null()); + +/** @brief crop a sub image from a big one, and resize it to certain size. +@param src input array. +@param dst output array. it has the size dsize (when it is non-zero) or the size computed from +src.size(), fx, and fy; the type of dst is the same as of src. +@param rect a rect to crop a array to +@param dsize output image size; if it equals zero, it is computed as cv::resize do. +@param fx scale factor along the horizontal axis; when it equals 0, it is computed as +\f[(𝚍𝚘𝚞𝚋𝚕𝚎)𝚍𝚜𝚒𝚣𝚎.𝚠𝚒𝚍𝚝𝚑/𝚜𝚛𝚌.𝚌𝚘𝚕𝚜\f] +@param fy scale factor along the vertical axis; when it equals 0, it is computed as +\f[(𝚍𝚘𝚞𝚋𝚕𝚎)𝚍𝚜𝚒𝚣𝚎.𝚑𝚎𝚒𝚐𝚑𝚝/𝚜𝚛𝚌.𝚛𝚘𝚠𝚜\f] +@param interpolation interpolation method, only support INTER_NEAREST and INTER_LINEAR here. + (see **cv.cann.InterpolationFlags**) +@note The input images must be uint8, and only GRAY and BGR images are supported. The resolution of +input and output images must in range of [10*6, 4096*4096]. +@note Only the following devices are supported: Atlas Inference Series products, Atlas 200/500 A2 +Inference products and Atlas A2 Training Series products/Atlas 300I A2 Inference products. +@sa cv::gapi::crop, cv::resize, cv::cann::resize +*/ +CV_EXPORTS_W void cropResize(const InputArray src, OutputArray dst, const Rect& rect, Size dsize, + double fx, double fy, int interpolation); +/** @overload */ +CV_EXPORTS_W void cropResize(const AscendMat& src, CV_OUT AscendMat& dst, const Rect& rect, + Size dsize, double fx, double fy, int interpolation); +/** @brief crop a sub image from a big one, resize it to certain size, and form the top/left border +and fills it with specified bordertype. +@param src input array. +@param dst output array; it has the size Size(dsize.height + top, dsize.width + left). +@param rect a rect to crop a array to +@param dsize resize size; +@param fx scale factor along the horizontal axis; +@param fy scale factor along the vertical axis; +@param interpolation interpolation method, only INTER_NEAREST and INTER_LINEAR are supported. + (see **cv.cann.InterpolationFlags**) +@param borderType border extrapolate method, only cv::BorderTypes::BORDER_CONSTANT and +cv::BorderTypes::BORDER_REPLICATE are supported. +@param value Border BGR or YUV value if borderType==BORDER_CONSTANT. +@param top Number of pixels for top padding +@param left Number of pixels for left padding +@note The input images must be uint8, and only GRAY and BGR images are supported. The resolution of +input and output images must in range of [10*6, 4096*4096]. +@note Only the following devices are supported: Atlas Inference Series products, Atlas 200/500 A2 +Inference products and Atlas A2 Training Series products/Atlas 300I A2 Inference products. +@sa cv::gapi::crop, cv::resize, cv::cann::resize, cv::BorderTypes +*/ + +CV_EXPORTS_W void cropResizeMakeBorder(const InputArray src, OutputArray dst, const Rect& rect, + Size dsize, double fx, double fy, int interpolation, int top, + int left, const int borderType, Scalar value = Scalar()); +/** @overload */ +CV_EXPORTS_W void cropResizeMakeBorder(const AscendMat& src, CV_OUT AscendMat& dst, + const Rect& rect, Size dsize, double fx, double fy, + int interpolation, int top, int left, const int borderType, + Scalar value = Scalar()); +/** @brief Forms a border and fills it with specified bordertype around the copy of input image. +@param src Source image. +@param dst Destination image of the same type as src and the size Size(src.cols+left+right, +src.rows+top+bottom). +@param top Number of pixels for top padding +@param bottom Number of pixels for bottom padding +@param left Number of pixels for left padding +@param right Number of pixels for right padding +Parameter specifying how many pixels in each direction from the source image rectangle to +extrapolate. For example, top=1, bottom=1, left=1, right=1 mean that 1 pixel-wide border needs to be +built. +@param borderType Border type. only cv::BorderTypes::BORDER_CONSTANT and +cv::BorderTypes::BORDER_REPLICATE are supported. +@param value Border BGR or YUV value if borderType==BORDER_CONSTANT. +@note The input images must be uint8, and only GRAY and BGR images are supported. The resolution of +input and output images must in range of [10*6, 4096*4096]. +@note Only the following devices are supported: Atlas Inference Series products, Atlas 200/500 A2 +Inference products and Atlas A2 Training Series products/Atlas 300I A2 Inference products. +@sa cv::copyMakeBorder, cv::borderInterpolate +*/ +CV_EXPORTS_W void copyMakeBorder(const InputArray src, OutputArray dst, int top, int bottom, + int left, int right, int borderType, + const Scalar& value = Scalar()); +/** @overload */ +CV_EXPORTS_W void copyMakeBorder(const AscendMat& src, CV_OUT AscendMat& dst, int top, int bottom, + int left, int right, int borderType, + const Scalar& value = Scalar()); //! @} cannops_core //! @addtogroup cannimgproc @@ -495,10 +605,17 @@ CV_EXPORTS_W void resize(const AscendMat& src, CV_OUT AscendMat& dst, Size dsize @param src Source image with CV_8U , CV_16U , or CV_32F depth and 1, 3, or 4 channels. @param dst Destination image. -@param code Color space conversion code. For details, see cvtColor . +@param code Color space conversion code. For details, see cv::ColorConversionCodes . @param dstCn Number of channels in the destination image. If the parameter is 0, the number of the channels is derived automatically from src and the code . @param stream AscendStream for the asynchronous version. +@note The supported conversion types are as follows: + { CV_BGR2BGRA, CV_BGRA2BGR, CV_BGR2RGBA, CV_RGBA2BGR, + CV_BGR2RGB, CV_BGRA2RGBA, CV_BGR2GRAY, CV_RGB2GRAY, + CV_GRAY2BGR, CV_GRAY2BGRA, CV_BGRA2GRAY, CV_RGBA2GRAY, + CV_BGR2XYZ, CV_RGB2XYZ, CV_XYZ2BGR, CV_XYZ2RGB, + CV_BGR2YCrCb, CV_RGB2YCrCb, CV_YCrCb2BGR, CV_YCrCb2RGB, + CV_BGR2YUV, CV_RGB2YUV, CV_YUV2BGR, CV_YUV2RGB } @sa cv::cvtColor cv::cuda::cvtColor */ diff --git a/modules/cannops/include/opencv2/dvpp_call.hpp b/modules/cannops/include/opencv2/dvpp_call.hpp new file mode 100644 index 00000000000..e70d56ea801 --- /dev/null +++ b/modules/cannops/include/opencv2/dvpp_call.hpp @@ -0,0 +1,107 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef ENABLE_DVPP_INTERFACE + #define ENABLE_DVPP_INTERFACE +#endif // ENABLE_DVPP_INTERFACE + +#include +#include +#include +#include +#include +#include "acl/acl_op.h" +#include "cann_call.hpp" + +namespace cv +{ +namespace cann +{ +struct AscendPicDesc +{ + const char* name; + std::shared_ptr data; + std::vector batchNum; + + size_t widthAlignment = 16; + size_t heightAlignment = 1; + size_t sizeAlignment = 3; + size_t sizeNum = 3; + + hi_vpc_pic_info Pic; + AscendPicDesc& setMemAlign(); + AscendPicDesc& setPic(hi_pixel_format _picture_format); + std::shared_ptr allocate(); + AscendPicDesc(){}; + AscendPicDesc(const AscendMat& ascendMat, hi_pixel_format _picture_format); + AscendPicDesc(const Mat& mat, hi_pixel_format _picture_format); +}; + +/* + ***************************** hi_mpi_vpc warppers *************************** + The DVPP VPC interfaces here are all version v2. Only the following devices are supported: Atlas + Inference Series products, Atlas 200/500 A2 Inference products and Atlas A2 Training Series + products/Atlas 300I A2 Inference products. +*/ +inline void vpcResizeWarpper(hi_vpc_chn chnId, hi_vpc_pic_info& inPic, hi_vpc_pic_info& outPic, + int interpolation, uint32_t* taskID) +{ + uint32_t ret = hi_mpi_vpc_resize(chnId, &inPic, &outPic, 0, 0, interpolation, taskID, -1); + if (ret != HI_SUCCESS) + CV_Error(Error::StsBadFlag, "failed to resize image"); +} +void vpcCropResizeWarpper(hi_vpc_chn chnId, hi_vpc_pic_info& inPic, hi_vpc_pic_info& outPic, + int cnt, uint32_t* taskID, const Rect& rect, Size dsize, + int interpolation); + +void vpcCropResizeMakeBorderWarpper(hi_vpc_chn chnId, std::vector& inPicDesc, + std::vector& outPicDesc, int cnt, + uint32_t* taskID, const Rect& rect, Size dsize, + int interpolation, const int borderType, Scalar scalarV, + int top, int left); +void vpcCopyMakeBorderWarpper(hi_vpc_chn chnId, hi_vpc_pic_info& inPic, hi_vpc_pic_info& outPic, + uint32_t* taskID, int* offsets, int bordertype, Scalar value); +/*****************************************************************************/ + +/** + * @brief Interface for calling DVPP operator descriptors. + * The DVPP VPC interfaces here are all version v2. Supported devices: Atlas Inference Series + * products, Atlas 200/500 A2 Inference products and Atlas A2 Training Series products/Atlas 300I A2 + * Inference products. + */ +class DvppOperatorDesc +{ +private: + DvppOperatorDesc& addInput(AscendPicDesc& picDesc); + DvppOperatorDesc& addOutput(AscendPicDesc& picDesc); + std::set> holder; + +public: + DvppOperatorDesc() + { + chnId = 0; + stChnAttr = {}; + createChannel(); + } + virtual ~DvppOperatorDesc() { reset(); } + DvppOperatorDesc& addInput(const AscendMat& mat); + DvppOperatorDesc& addOutput(AscendMat& mat); + DvppOperatorDesc& addInput(const Mat& mat); + DvppOperatorDesc& addOutput(Mat& mat); + + DvppOperatorDesc& getResult(Mat& dst, uint32_t& taskIDResult); + DvppOperatorDesc& getResult(AscendMat& dst, uint32_t& taskIDResult); + + DvppOperatorDesc& reset(); + DvppOperatorDesc& createChannel(); + + std::vector inputDesc_; + std::vector outputDesc_; + + hi_vpc_chn chnId; + hi_vpc_chn_attr stChnAttr; +}; + +} // namespace cann +} // namespace cv \ No newline at end of file diff --git a/modules/cannops/misc/python/test/test_cannops.py b/modules/cannops/misc/python/test/test_cannops.py index f1b53bc192c..48d4ff18d11 100644 --- a/modules/cannops/misc/python/test/test_cannops.py +++ b/modules/cannops/misc/python/test/test_cannops.py @@ -24,6 +24,7 @@ def genMask(mask, listx, listy): class cannop_test(NewOpenCVTests): def test_ascend(self): cv.cann.initAcl() + cv.cann.initDvpp() cv.cann.getDevice() cv.cann.setDevice(0) stream = cv.cann.AscendStream_Null() @@ -275,6 +276,50 @@ def test_imgproc(self): aclMat, 127, 255, tType) self.assertTrue(np.allclose(cvThresh, cannThresh.download())) self.assertTrue(np.allclose(cvRet, cannRet)) + + npMat = (np.random.random((1280, 1024, 3)) * 255).astype(np.uint8) + w_off, h_off, crop_w, crop_h = 0, 0, 512, 384 + roi = [w_off, h_off, crop_w, crop_h] + aclMat = cv.cann.AscendMat() + aclMat.upload(npMat) + + # resize + dstSize = np.array([crop_w, crop_h]) + self.assertTrue(np.allclose(cv.cann.resize(npMat, dstSize, 0, 0, 1), + cv.resize(npMat, dstSize, 0, 0, 1))) + self.assertTrue(np.allclose(cv.cann.resize(aclMat, dstSize, 0, 0, 1).download(), + cv.resize(npMat, dstSize, 0, 0, 1))) + # cropResize + self.assertTrue(np.allclose(cv.cann.cropResize(npMat, roi, dstSize, 0, 0, 1), + cv.resize(npMat[h_off:crop_h, w_off:crop_w], dstSize, 0, 0, 1)), 0) + self.assertTrue(np.allclose(cv.cann.cropResize(aclMat, roi, dstSize, 0, 0, 1).download(), + cv.resize(npMat[h_off:crop_h, w_off:crop_w], dstSize, 0, 0, 1)), 0) + + # cropResizeMakeBorder + # TODO cv.copyMakeBorder ignores borderColorValue param; find the reason and fix it + borderColorValue = (100, 0, 255) + top, bottom, left, right = 32, 0, 10, 0 + borderTypes = [0, 1] + + for borderType in borderTypes: + self.assertTrue(np.allclose(cv.cann.cropResizeMakeBorder(npMat, roi, dstSize, + 0, 0, 1, top, left, borderType), + cv.copyMakeBorder(cv.resize(npMat[h_off:crop_h, w_off:crop_w], + dstSize, 0, 0, 1), top, bottom, left, right, borderType), 1)) + self.assertTrue(np.allclose(cv.cann.cropResizeMakeBorder(aclMat, roi, dstSize, + 0, 0, 1, top, left, borderType).download(), + cv.copyMakeBorder(cv.resize(npMat[h_off:crop_h, w_off:crop_w], + dstSize, 0, 0, 1), top, bottom, left, right, borderType), 1)) + + # copyMakeBorder + for borderType in borderTypes: + self.assertTrue(np.allclose(cv.cann.copyMakeBorder(npMat, top, bottom, left, right, + borderType), + cv.copyMakeBorder(npMat, top, bottom, left, right, borderType))) + self.assertTrue(np.allclose(cv.cann.copyMakeBorder(aclMat, top, bottom, left, right, + borderType).download(), + cv.copyMakeBorder(npMat, top, bottom, left, right, borderType))) + cv.cann.resetDevice() if __name__ == '__main__': diff --git a/modules/cannops/perf/perf_core.cpp b/modules/cannops/perf/perf_core.cpp index a9d86fca881..914a122d287 100644 --- a/modules/cannops/perf/perf_core.cpp +++ b/modules/cannops/perf/perf_core.cpp @@ -11,6 +11,7 @@ namespace { #define TYPICAL_ASCEND_MAT_SIZES \ Values(::perf::sz1080p, ::perf::sz2K, ::perf::sz2160p, ::perf::sz4320p) +#define DVPP_ASCEND_MAT_SIZES Values(::perf::sz1080p, ::perf::sz2K, ::perf::sz2160p, ::perf::sz5MP) #define DEF_PARAM_TEST(name, ...) \ typedef ::perf::TestBaseWithParam> name @@ -157,5 +158,176 @@ PERF_TEST_P(NPU, CROP_OVERLOAD, TYPICAL_ASCEND_MAT_SIZES) cv::cann::resetDevice(); SANITY_CHECK_NOTHING(); } + +PERF_TEST_P(CPU, RESIZE, DVPP_ASCEND_MAT_SIZES) +{ + Mat mat(GET_PARAM(0), CV_8UC3); + Mat dst; + declare.in(mat, WARMUP_RNG); + Size dsize = Size(256, 256); + TEST_CYCLE_N(10) { cv::resize(mat, dst, dsize, 0, 0, 1); } + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P(NPU, RESIZE, DVPP_ASCEND_MAT_SIZES) +{ + Mat mat(GET_PARAM(0), CV_32FC3); + AscendMat dst; + AscendMat src; + src.upload(mat); + declare.in(mat, WARMUP_RNG); + Size dsize = Size(256, 256); + TEST_CYCLE_N(10) { cv::cann::resize(src, dst, dsize, 0, 0, 3); } + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P(NPU, THRESHOLD, TYPICAL_ASCEND_MAT_SIZES) +{ + Mat mat(GET_PARAM(0), CV_32FC3); + AscendMat dst; + AscendMat src; + src.upload(mat); + declare.in(mat, WARMUP_RNG); + TEST_CYCLE_N(10) { cv::cann::threshold(src, dst, 100.0, 255.0, cv::THRESH_BINARY); } + SANITY_CHECK_NOTHING(); +} +PERF_TEST_P(CPU, THRESHOLD, TYPICAL_ASCEND_MAT_SIZES) +{ + Mat mat(GET_PARAM(0), CV_32FC3); + Mat dst; + declare.in(mat, WARMUP_RNG); + TEST_CYCLE_N(10) { cv::threshold(mat, dst, 100.0, 255.0, cv::THRESH_BINARY); } + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P(NPU, RESIZE_INTER_NEAREST, DVPP_ASCEND_MAT_SIZES) +{ + Mat mat(GET_PARAM(0), CV_8UC3); + Mat dst; + declare.in(mat, WARMUP_RNG); + Size dsize = Size(256, 256); + TEST_CYCLE_N(10) { cv::cann::resize(mat, dst, dsize, 0, 0, 0); } + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P(NPU, COPY_MAKE_BORDER, DVPP_ASCEND_MAT_SIZES) +{ + Mat resized_cv, checker, cpuOpRet, cpuMat(GET_PARAM(0), CV_8UC3); + declare.in(cpuMat, WARMUP_RNG); + int top, bottom, left, right; + top = (int)(20); + bottom = top; + left = (int)(20); + right = left; + int borderType = 1; + float scalarV[3] = {0, 0, 255}; + Scalar value = {scalarV[0], scalarV[1], scalarV[2]}; + + TEST_CYCLE_N(10) + { + cv::cann::copyMakeBorder(cpuMat, checker, top, bottom, left, right, borderType, value); + } + + SANITY_CHECK_NOTHING(); +} +PERF_TEST_P(CPU, COPY_MAKE_BORDER, DVPP_ASCEND_MAT_SIZES) +{ + Mat resized_cv, checker, cpuOpRet, cpuMat(GET_PARAM(0), CV_8UC3); + declare.in(cpuMat, WARMUP_RNG); + int top, bottom, left, right; + top = (int)(20); + bottom = top; + left = (int)(20); + right = left; + int borderType = 1; + float scalarV[3] = {0, 0, 255}; + Scalar value = {scalarV[0], scalarV[1], scalarV[2]}; + + TEST_CYCLE_N(10) + { + cv::copyMakeBorder(cpuMat, checker, top, bottom, left, right, borderType, value); + } + + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P(NPU, CROP_RESIZE_MAKE_BORDER, DVPP_ASCEND_MAT_SIZES) +{ + Size size = GET_PARAM(0); + Mat resized_cv, checker, cpuOpRet, cpuMat(size, CV_8UC3); + declare.in(cpuMat, WARMUP_RNG); + + const Rect b(1, 0, size.width / 2, size.height); + Size dsize = Size(size.width / 4, size.height / 2); + int top, left; + top = (int)(20); + left = (int)(20); + int borderType = 0; + float scalarV[3] = {1, 1, 1}; + Scalar value = {scalarV[0], scalarV[1], scalarV[2]}; + + TEST_CYCLE_N(10) + { + cv::cann::cropResizeMakeBorder(cpuMat, checker, b, dsize, 0, 0, 1, top, left, borderType, + value); + } + + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P(CPU, CROP_RESIZE_MAKE_BORDER, DVPP_ASCEND_MAT_SIZES) +{ + Size size = GET_PARAM(0); + Mat resized_cv, checker, cpuOpRet, cpuMat(size, CV_8UC3); + declare.in(cpuMat, WARMUP_RNG); + const Rect b(1, 0, size.width / 2, size.height); + Size dsize = Size(size.width / 4, size.height / 2); + int top, bottom, left, right; + top = (int)(20); + bottom = 0; + left = (int)(20); + right = 0; + int borderType = 0; + float scalarV[3] = {1, 1, 1}; + Scalar value = {scalarV[0], scalarV[1], scalarV[2]}; + + TEST_CYCLE_N(10) + { + Mat cropped_cv(cpuMat, b); + cv::resize(cropped_cv, resized_cv, dsize, 0, 0, 1); + cv::copyMakeBorder(resized_cv, cpuOpRet, top, bottom, left, right, borderType, value); + } + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P(NPU, CROP_RESIZE, DVPP_ASCEND_MAT_SIZES) +{ + Size size = GET_PARAM(0); + Mat resized_cv, checker, cpuOpRet, cpuMat(size, CV_8UC3); + declare.in(cpuMat, WARMUP_RNG); + const Rect b(1, 0, size.width / 2, size.height); + Size dsize = Size(size.width / 4, size.height / 2); + + TEST_CYCLE_N(10) { cv::cann::cropResize(cpuMat, checker, b, dsize, 0, 0, 1); } + + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P(CPU, CROP_RESIZE, DVPP_ASCEND_MAT_SIZES) +{ + Size size = GET_PARAM(0); + Mat resized_cv, checker, cpuOpRet, cpuMat(size, CV_8UC3); + declare.in(cpuMat, WARMUP_RNG); + const Rect b(1, 0, size.width / 2, size.height); + Size dsize = Size(size.width / 4, size.height / 2); + + TEST_CYCLE_N(10) + { + Mat cropped_cv(cpuMat, b); + cv::resize(cropped_cv, resized_cv, dsize, 0, 0, 1); + } + SANITY_CHECK_NOTHING(); +} + } // namespace } // namespace opencv_test diff --git a/modules/cannops/perf/perf_main.cpp b/modules/cannops/perf/perf_main.cpp index 33503ac4158..9e03d48904f 100644 --- a/modules/cannops/perf/perf_main.cpp +++ b/modules/cannops/perf/perf_main.cpp @@ -10,8 +10,18 @@ class CannEnvironment : public ::testing::Environment { public: virtual ~CannEnvironment() = default; - virtual void SetUp() CV_OVERRIDE { cv::cann::initAcl(); } - virtual void TearDown() CV_OVERRIDE { cv::cann::finalizeAcl(); } + virtual void SetUp() CV_OVERRIDE + { + initAcl(); + cv::cann::setDevice(DEVICE_ID); + initDvpp(); + } + virtual void TearDown() CV_OVERRIDE + { + finalizeAcl(); + cv::cann::resetDevice(); + finalizeDvpp(); + } }; static void initTests() diff --git a/modules/cannops/src/core.cpp b/modules/cannops/src/core.cpp index 7d328915ef9..027cd119f10 100644 --- a/modules/cannops/src/core.cpp +++ b/modules/cannops/src/core.cpp @@ -241,6 +241,56 @@ AscendMat crop(InputArray _src, const Rect& rect, AscendStream& stream) return crop(src, rect, stream); } +/************************** resize **************************/ +void checkResize(Size& ssize, Size& dsize, double inv_scale_x, double inv_scale_y, + int& interpolation) +{ + CV_Assert(!ssize.empty()); + float_t scaleX = (float_t)inv_scale_x; + float_t scaleY = (float_t)inv_scale_y; + // interpolation: resize mode, support bilinear/nearest neighbor/bicubic/pixel area relation. + CV_Assert(interpolation == INTER_LINEAR || interpolation == INTER_NEAREST || + interpolation == INTER_CUBIC || interpolation == INTER_AREA); + switch (interpolation) + { + case INTER_LINEAR: + interpolation = INTER_NEAREST; + break; + case INTER_NEAREST: + interpolation = INTER_LINEAR; + break; + default: + break; + } + + if (dsize.empty()) + { + CV_Assert(scaleX > 0); + CV_Assert(scaleY > 0); + dsize = Size(saturate_cast(ssize.width * inv_scale_x), + saturate_cast(ssize.height * inv_scale_y)); + CV_Assert(!dsize.empty()); + } + else + { + scaleX = (float_t)dsize.width / ssize.width; + scaleY = (float_t)dsize.height / ssize.height; + CV_Assert(scaleX > 0); + CV_Assert(scaleY > 0); + } +} + +template +void resize(const inMat& src, outMat& dst, int interpolation) +{ + DvppOperatorDesc op; + op.addInput(src).addOutput(dst); + uint32_t taskID = 0; + vpcResizeWarpper(op.chnId, op.inputDesc_[0].Pic, op.outputDesc_[0].Pic, interpolation, &taskID); + + uint32_t taskIDResult = taskID; + op.getResult(dst, taskIDResult); +} void resize(const AscendMat& src, AscendMat& dst, int32_t* dstSize, int interpolation, AscendStream& stream) { @@ -258,7 +308,6 @@ void resize(const AscendMat& src, AscendMat& dst, int32_t* dstSize, int interpol default: break; } - runner.setOp(mode) .addInput(src, "images") .addInput(dstSize, dims, 1, ACL_INT32, "size") @@ -271,30 +320,18 @@ void resize(const AscendMat& src, AscendMat& dst, Size dsize, double inv_scale_x double inv_scale_y, int interpolation, AscendStream& stream) { Size ssize = src.size(); - CV_Assert(!ssize.empty()); - float_t scaleX = (float_t)inv_scale_x; - float_t scaleY = (float_t)inv_scale_y; - CV_Assert(interpolation == INTER_CUBIC || interpolation == INTER_AREA); + checkResize(ssize, dsize, inv_scale_x, inv_scale_y, interpolation); + int32_t dstSize[] = {dsize.height, dsize.width}; + dst.create(dstSize[0], dstSize[1], src.type()); - if (dsize.empty()) + if (interpolation == INTER_CUBIC || interpolation == INTER_AREA) { - CV_Assert(scaleX > 0); - CV_Assert(scaleY > 0); - dsize = Size(saturate_cast(ssize.width * inv_scale_x), - saturate_cast(ssize.height * inv_scale_y)); - CV_Assert(!dsize.empty()); + resize(src, dst, dstSize, interpolation, stream); } else { - scaleX = (float_t)dsize.width / ssize.width; - scaleY = (float_t)dsize.height / ssize.height; - CV_Assert(scaleX > 0); - CV_Assert(scaleY > 0); + resize(src, dst, interpolation); } - - int32_t dstSize[] = {dsize.width, dsize.height}; - dst.create(dstSize[0], dstSize[1], src.type()); - resize(src, dst, dstSize, interpolation, stream); } void resize(InputArray _src, OutputArray _dst, Size dsize, double inv_scale_x, double inv_scale_y, @@ -302,8 +339,138 @@ void resize(InputArray _src, OutputArray _dst, Size dsize, double inv_scale_x, d { AscendMat src, dst; src.upload(_src, stream); - resize(src, dst, dsize, inv_scale_x, inv_scale_y, interpolation, stream); - dst.download(_dst, stream); + if (interpolation == INTER_CUBIC || interpolation == INTER_AREA) + { + resize(src, dst, dsize, inv_scale_x, inv_scale_y, interpolation, stream); + dst.download(_dst, stream); + } + else + { + Mat srcCV = _src.getMat(); + Size ssize = srcCV.size(); + checkResize(ssize, dsize, inv_scale_x, inv_scale_y, interpolation); + _dst.create(dsize, srcCV.type()); + Mat dstCV = _dst.getMat(); + resize(srcCV, dstCV, interpolation); + } +} + +/************************** CropResize **************************/ +template +void cropResize(const inMat& src, outMat& dst, const Rect& rect, Size dsize, int interpolation) +{ + DvppOperatorDesc op; + op.addInput(src).addOutput(dst); + uint32_t taskID = 0; + int cnt = 1; + + vpcCropResizeWarpper(op.chnId, op.inputDesc_[0].Pic, op.outputDesc_[0].Pic, cnt, &taskID, rect, + dsize, interpolation); + + uint32_t taskIDResult = taskID; + op.getResult(dst, taskIDResult); +} + +void cropResize(const AscendMat& src, AscendMat& dst, const Rect& rect, Size dsize, + double inv_scale_x, double inv_scale_y, int interpolation) +{ + Size ssize = src.size(); + checkResize(ssize, dsize, inv_scale_x, inv_scale_y, interpolation); + dst.create(dsize.height, dsize.width, src.type()); + cropResize(src, dst, rect, dsize, interpolation); +} + +void cropResize(const InputArray _src, OutputArray _dst, const Rect& rect, Size dsize, + double inv_scale_x, double inv_scale_y, int interpolation) +{ + Size ssize = _src.size(); + checkResize(ssize, dsize, inv_scale_x, inv_scale_y, interpolation); + + Mat src = _src.getMat(); + _dst.create(dsize.height, dsize.width, src.type()); + Mat dst = _dst.getMat(); + + cropResize(src, dst, rect, dsize, interpolation); +} + +/************************** CopyMakeBorder **************************/ +template +void copyMakeBorder(const inMat& src, outMat& dst, int* offsets, int borderType, + const Scalar& value) +{ + DvppOperatorDesc op; + op.addInput(src).addOutput(dst); + uint32_t taskID = 0; + vpcCopyMakeBorderWarpper(op.chnId, op.inputDesc_[0].Pic, op.outputDesc_[0].Pic, &taskID, + offsets, borderType, value); + + uint32_t taskIDResult = taskID; + op.getResult(dst, taskIDResult); +} + +void copyMakeBorder(const AscendMat& src, AscendMat& dst, int top, int bottom, int left, int right, + int borderType, const Scalar& value) +{ + dst.create(src.rows + top + bottom, src.cols + left + right, src.type()); + int offsets[] = {top, bottom, left, right}; + copyMakeBorder(src, dst, offsets, borderType, value); +} + +void copyMakeBorder(const InputArray _src, OutputArray _dst, int top, int bottom, int left, + int right, int borderType, const Scalar& value) +{ + CV_Assert(borderType < 2); + Mat src = _src.getMat(); + _dst.create(src.rows + top + bottom, src.cols + left + right, src.type()); + Mat dst = _dst.getMat(); + int offsets[] = {top, bottom, left, right}; + + copyMakeBorder(src, dst, offsets, borderType, value); +} + +/************************** CropResizeMakeBorder **************************/ + +template +void cropResizeMakeBorder(const inMat& src, outMat& dst, const Rect& rect, Size dsize, + int interpolation, int top, int left, const int borderType, + Scalar scalarV) +{ + DvppOperatorDesc op; + op.addInput(src).addOutput(dst); + uint32_t taskID = 0; + int cnt = 1; + vpcCropResizeMakeBorderWarpper(op.chnId, op.inputDesc_, op.outputDesc_, cnt, &taskID, rect, + dsize, interpolation, borderType, scalarV, top, left); + + uint32_t taskIDResult = taskID; + op.getResult(dst, taskIDResult); +} + +void cropResizeMakeBorder(const AscendMat& src, AscendMat& dst, const Rect& rect, Size dsize, + double inv_scale_x, double inv_scale_y, int interpolation, int top, + int left, const int borderType, Scalar scalarV) +{ + CV_Assert(borderType < 2); + Size ssize = src.size(); + checkResize(ssize, dsize, inv_scale_x, inv_scale_y, interpolation); + dst.create(dsize.height + top, dsize.width + left, src.type()); + + cropResizeMakeBorder(src, dst, rect, dsize, interpolation, top, left, borderType, scalarV); +} + +void cropResizeMakeBorder(const InputArray _src, OutputArray _dst, const Rect& rect, Size dsize, + double inv_scale_x, double inv_scale_y, int interpolation, int top, + int left, const int borderType, Scalar scalarV) +{ + CV_Assert(borderType < 2); + Size ssize = _src.size(); + checkResize(ssize, dsize, inv_scale_x, inv_scale_y, interpolation); + + Mat src = _src.getMat(); + _dst.create(dsize.height + top, dsize.width + left, src.type()); + Mat dst = _dst.getMat(); + + cropResizeMakeBorder(src, dst, rect, dsize, interpolation, top, left, borderType, scalarV); } } // namespace cann diff --git a/modules/cannops/src/dvpp_call.cpp b/modules/cannops/src/dvpp_call.cpp new file mode 100644 index 00000000000..f81604dc258 --- /dev/null +++ b/modules/cannops/src/dvpp_call.cpp @@ -0,0 +1,310 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include +#include +#include "opencv2/dvpp_call.hpp" +#include +#include +#include +#include + +#define unlikely(expr) __builtin_expect(!!(expr), 0) +#define likely(expr) __builtin_expect(!!(expr), 1) + +namespace cv +{ +namespace cann +{ + +/******************************AscendPicDesc****************************/ +AscendPicDesc& AscendPicDesc::setMemAlign() +{ + if (Pic.picture_format == HI_PIXEL_FORMAT_BGR_888 || + Pic.picture_format == HI_PIXEL_FORMAT_RGB_888 || + Pic.picture_format == HI_PIXEL_FORMAT_YUV_PACKED_444) + { + widthAlignment = 16; + heightAlignment = 1; + sizeAlignment = 3; + sizeNum = 3; + } + else if (Pic.picture_format == HI_PIXEL_FORMAT_YUV_400) + { + widthAlignment = 16; + heightAlignment = 1; + sizeAlignment = 1; + sizeNum = 1; + } + else if (Pic.picture_format == HI_PIXEL_FORMAT_ARGB_8888 || + Pic.picture_format == HI_PIXEL_FORMAT_ABGR_8888 || + Pic.picture_format == HI_PIXEL_FORMAT_RGBA_8888 || + Pic.picture_format == HI_PIXEL_FORMAT_BGRA_8888) + { + widthAlignment = 16; + heightAlignment = 1; + sizeAlignment = 4; + sizeNum = 4; + } + return *this; +} + +AscendPicDesc& AscendPicDesc::setPic(hi_pixel_format _picture_format) +{ + // set input + Pic.picture_format = _picture_format; + setMemAlign(); + Pic.picture_width_stride = ALIGN_UP(Pic.picture_width, widthAlignment) * sizeAlignment; + Pic.picture_height_stride = ALIGN_UP(Pic.picture_height, heightAlignment); + Pic.picture_buffer_size = + Pic.picture_width_stride * Pic.picture_height_stride * sizeAlignment / sizeNum; + return *this; +} + +std::shared_ptr AscendPicDesc::allocate() +{ + Pic.picture_address = nullptr; + uint32_t ret = hi_mpi_dvpp_malloc(0, &Pic.picture_address, Pic.picture_buffer_size); + if (ret != HI_SUCCESS) + CV_Error(Error::StsBadFlag, "failed to malloc mem on dvpp"); + + return std::shared_ptr(Pic.picture_address, [](void* ptr) { hi_mpi_dvpp_free(ptr); }); +} + +AscendPicDesc::AscendPicDesc(const AscendMat& ascendMat, hi_pixel_format _picture_format) +{ + Pic.picture_width = ascendMat.cols; + Pic.picture_height = ascendMat.rows; + setPic(_picture_format); + data = allocate(); +} + +AscendPicDesc::AscendPicDesc(const Mat& mat, hi_pixel_format _picture_format) +{ + Pic.picture_width = mat.cols; + Pic.picture_height = mat.rows; + setPic(_picture_format); + data = allocate(); +} + +/******************************hi_mpi_vpc warppers****************************/ +void vpcCropResizeWarpper(hi_vpc_chn chnId, hi_vpc_pic_info& inPic, hi_vpc_pic_info& outPic, + int cnt, uint32_t* taskID, const Rect& rect, Size dsize, + int interpolation) +{ + hi_vpc_crop_region cropRegion = {.top_offset = static_cast(rect.y), + .left_offset = static_cast(rect.x), + .crop_width = static_cast(rect.width), + .crop_height = static_cast(rect.height)}; + + hi_vpc_resize_info resize_info = {.resize_width = static_cast(dsize.width), + .resize_height = static_cast(dsize.height), + .interpolation = static_cast(interpolation)}; + hi_vpc_crop_resize_region crop_resize_info[1]; + crop_resize_info[0].dest_pic_info = outPic; + crop_resize_info[0].crop_region = cropRegion; + crop_resize_info[0].resize_info = resize_info; + uint32_t ret = hi_mpi_vpc_crop_resize(chnId, (const hi_vpc_pic_info*)&inPic, crop_resize_info, + cnt, taskID, -1); + if (ret != HI_SUCCESS) + CV_Error(Error::StsBadFlag, "failed to crop and resize image"); +} + +void vpcCopyMakeBorderWarpper(hi_vpc_chn chnId, hi_vpc_pic_info& inPic, hi_vpc_pic_info& outPic, + uint32_t* taskID, int* offsets, int bordertype, Scalar value) +{ + hi_vpc_make_border_info make_border_info; + make_border_info = {.top = static_cast(offsets[0]), + .bottom = static_cast(offsets[1]), + .left = static_cast(offsets[2]), + .right = static_cast(offsets[3]), + .border_type = saturate_cast(bordertype)}; + if (outPic.picture_format == HI_PIXEL_FORMAT_BGR_888) + { + make_border_info.scalar_value.val[0] = value[2]; + make_border_info.scalar_value.val[1] = value[1]; + make_border_info.scalar_value.val[2] = value[0]; + } + else if (outPic.picture_format == HI_PIXEL_FORMAT_YUV_400) + { + make_border_info.scalar_value.val[0] = value[0]; + make_border_info.scalar_value.val[1] = value[1]; + make_border_info.scalar_value.val[2] = value[2]; + } + make_border_info.scalar_value.val[3] = value[3]; + uint32_t ret = hi_mpi_vpc_copy_make_border(chnId, (const hi_vpc_pic_info*)&inPic, &outPic, + make_border_info, taskID, -1); + if (ret != HI_SUCCESS) + CV_Error(Error::StsBadFlag, "failed to crop and resize image"); +} + +void setBatchCropResizeMakeBorder(std::vector& outPicDesc, + hi_vpc_crop_resize_border_region crop_resize_make_border_info[], + const Rect& rect, Size dsize, int interpolation, + const int borderType, Scalar scalarV, int top, int left, + int batchSize) +{ + hi_vpc_crop_region cropRegion = {.top_offset = static_cast(rect.y), + .left_offset = static_cast(rect.x), + .crop_width = static_cast(rect.width), + .crop_height = static_cast(rect.height)}; + + hi_vpc_resize_info resize_info = {.resize_width = static_cast(dsize.width), + .resize_height = static_cast(dsize.height), + .interpolation = static_cast(interpolation)}; + for (int i = 0; i < batchSize; i++) + { + crop_resize_make_border_info[i].dest_pic_info = outPicDesc[i].Pic; + crop_resize_make_border_info[i].crop_region = cropRegion; + crop_resize_make_border_info[i].resize_info = resize_info; + crop_resize_make_border_info[i].dest_top_offset = top; + crop_resize_make_border_info[i].dest_left_offset = left; + crop_resize_make_border_info[i].border_type = static_cast(borderType); + if (crop_resize_make_border_info[i].dest_pic_info.picture_format == HI_PIXEL_FORMAT_BGR_888) + { + crop_resize_make_border_info[i].scalar_value.val[0] = scalarV[2]; + crop_resize_make_border_info[i].scalar_value.val[1] = scalarV[1]; + crop_resize_make_border_info[i].scalar_value.val[2] = scalarV[0]; + } + else if (crop_resize_make_border_info[i].dest_pic_info.picture_format == + HI_PIXEL_FORMAT_YUV_400) + { + crop_resize_make_border_info[i].scalar_value.val[0] = scalarV[0]; + crop_resize_make_border_info[i].scalar_value.val[1] = scalarV[1]; + crop_resize_make_border_info[i].scalar_value.val[2] = scalarV[2]; + } + crop_resize_make_border_info[i].scalar_value.val[3] = scalarV[3]; + } +} + +void vpcCropResizeMakeBorderWarpper(hi_vpc_chn chnId, std::vector& inPicDesc, + std::vector& outPicDesc, int cnt, + uint32_t* taskID, const Rect& rect, Size dsize, + int interpolation, const int borderType, Scalar scalarV, + int top, int left) +{ + hi_vpc_crop_resize_border_region crop_resize_make_border_info[1]; + + setBatchCropResizeMakeBorder(outPicDesc, crop_resize_make_border_info, rect, dsize, + interpolation, borderType, scalarV, top, left, 1); + uint32_t ret = + hi_mpi_vpc_crop_resize_make_border(chnId, (const hi_vpc_pic_info*)&inPicDesc[0].Pic, + crop_resize_make_border_info, cnt, taskID, -1); + if (ret != HI_SUCCESS) + CV_Error(Error::StsBadFlag, "failed to crop, resize and make border of image"); +} + +/******************************DvppOperatorDesc****************************/ +DvppOperatorDesc& DvppOperatorDesc::reset() +{ + uint32_t ret = hi_mpi_vpc_destroy_chn(chnId); + if (ret != HI_SUCCESS) + CV_Error(Error::StsBadFlag, "failed to destory DVPP vpc channel"); + inputDesc_.clear(); + outputDesc_.clear(); + holder.clear(); + return *this; +} +void initDvpp() { hi_mpi_sys_init(); } + +void finalizeDvpp() { hi_mpi_sys_exit(); } + +DvppOperatorDesc& DvppOperatorDesc::createChannel() +{ + uint32_t ret = hi_mpi_vpc_sys_create_chn(&chnId, &stChnAttr); + if (ret != HI_SUCCESS) + CV_Error(Error::StsBadFlag, "failed to create DVPP vpc channel"); + return *this; +} + +// copy input array to dvpp memory +DvppOperatorDesc& DvppOperatorDesc::addInput(AscendPicDesc& picDesc) +{ + inputDesc_.push_back(picDesc); + holder.insert(picDesc.data); + return *this; +} + +template +hi_pixel_format setPixelFormat(const inMat& mat) +{ + CV_Assert(mat.channels() == 3 || mat.channels() == 1); + hi_pixel_format _picture_format; + if (mat.channels() == 3) + { + _picture_format = HI_PIXEL_FORMAT_BGR_888; + } + else if (mat.channels() == 1) + { + _picture_format = HI_PIXEL_FORMAT_YUV_400; + } + return _picture_format; +} + +DvppOperatorDesc& DvppOperatorDesc::addInput(const AscendMat& mat) +{ + Mat matHost; + mat.download(matHost); + return addInput(matHost); +} + +DvppOperatorDesc& DvppOperatorDesc::addInput(const Mat& mat) +{ + hi_pixel_format _picture_format = setPixelFormat(mat); + + AscendPicDesc picDesc(mat, _picture_format); + aclrtMemcpy2d(picDesc.Pic.picture_address, picDesc.Pic.picture_width_stride, mat.data, + mat.step[0], mat.step[0], picDesc.Pic.picture_height, ACL_MEMCPY_HOST_TO_DEVICE); + + return addInput(picDesc); +} + +// malloc memory for output +DvppOperatorDesc& DvppOperatorDesc::addOutput(AscendPicDesc& picDesc) +{ + outputDesc_.push_back(picDesc); + holder.insert(picDesc.data); + return *this; +} + +DvppOperatorDesc& DvppOperatorDesc::addOutput(AscendMat& mat) +{ + hi_pixel_format _picture_format = setPixelFormat(mat); + AscendPicDesc picDesc(mat, _picture_format); + return addOutput(picDesc); +} + +DvppOperatorDesc& DvppOperatorDesc::addOutput(Mat& mat) +{ + hi_pixel_format _picture_format = setPixelFormat(mat); + AscendPicDesc picDesc(mat, _picture_format); + return addOutput(picDesc); +} + +// get process result and copy it to host/device +DvppOperatorDesc& DvppOperatorDesc::getResult(Mat& dst, uint32_t& taskIDResult) +{ + uint32_t ret = hi_mpi_vpc_get_process_result(chnId, taskIDResult, -1); + if (ret != HI_SUCCESS) + CV_Error(Error::StsBadFlag, "failed to get process result."); + const uint32_t esz = CV_ELEM_SIZE(dst.type()); + size_t step = esz * dst.cols; + + aclrtMemcpy2d(dst.data, dst.step[0], outputDesc_[0].Pic.picture_address, + outputDesc_[0].Pic.picture_width_stride, dst.step[0], + outputDesc_[0].Pic.picture_height, ACL_MEMCPY_DEVICE_TO_HOST); + return *this; +} + +DvppOperatorDesc& DvppOperatorDesc::getResult(AscendMat& dst, uint32_t& taskIDResult) +{ + Mat matHost; + matHost.create(dst.rows, dst.cols, dst.type()); + getResult(matHost, taskIDResult); + dst.upload(matHost); + return *this; +} + +} // namespace cann +} // namespace cv diff --git a/modules/cannops/src/precomp.hpp b/modules/cannops/src/precomp.hpp index 8aadaf4d8de..fe81c8a42cb 100644 --- a/modules/cannops/src/precomp.hpp +++ b/modules/cannops/src/precomp.hpp @@ -10,6 +10,7 @@ #include "opencv2/cann_call.hpp" #include "opencv2/cann_interface.hpp" #include "opencv2/cann_private.hpp" +#include "opencv2/dvpp_call.hpp" #include "opencv2/ascendc_kernels.hpp" #define ALIGN_UP(num, align) (((num) + (align) - 1) & ~((align) - 1)) diff --git a/modules/cannops/test/test_core.cpp b/modules/cannops/test/test_core.cpp index 6b63a8cf061..98d554335aa 100644 --- a/modules/cannops/test/test_core.cpp +++ b/modules/cannops/test/test_core.cpp @@ -212,6 +212,128 @@ TEST(CORE, RESIZE) cv::cann::resetDevice(); } +TEST(CORE, RESIZE_NEW) +{ + Mat resized_cv, checker; + Mat cpuMat = randomMat(1280, 1706, CV_8UC3, 100.0, 255.0); + Size dsize = Size(768, 832); + // add support for {0 INTER_NEAREST} and {1 INTER_LINEAR} + // only the resize result of INTER_LINEAR is close to CV's. + int interpolation = 1; + cv::resize(cpuMat, resized_cv, dsize, 0, 0, interpolation); + cv::cann::resize(cpuMat, checker, dsize, 0, 0, interpolation); + EXPECT_MAT_NEAR(resized_cv, checker, 1); + + cv::resize(cpuMat, resized_cv, Size(), 0.5, 0.5, interpolation); + cv::cann::resize(cpuMat, checker, Size(), 0.5, 0.5, interpolation); + EXPECT_MAT_NEAR(resized_cv, checker, 1); + + AscendMat npuMat, npuChecker; + npuMat.upload(cpuMat); + cv::resize(cpuMat, resized_cv, dsize, 0, 0, interpolation); + cv::cann::resize(npuMat, npuChecker, dsize, 0, 0, interpolation); + npuChecker.download(checker); + EXPECT_MAT_NEAR(resized_cv, checker, 1); + + cv::resize(cpuMat, resized_cv, Size(), 0.5, 0.5, interpolation); + cv::cann::resize(npuMat, npuChecker, Size(), 0.5, 0.5, interpolation); + npuChecker.download(checker); + EXPECT_MAT_NEAR(resized_cv, checker, 1); +} + +TEST(CORE, CROP_RESIZE) +{ + Mat cpuMat = randomMat(1280, 1706, CV_8UC1, 100.0, 255.0); + Mat resized_cv, checker, cpuOpRet; + Size dsize = Size(496, 512); + const Rect b(300, 500, 224, 256); + + cv::cann::cropResize(cpuMat, checker, b, dsize, 0, 0, 1); + Mat cropped_cv(cpuMat, b); + cv::resize(cropped_cv, cpuOpRet, dsize, 0, 0, 1); + EXPECT_MAT_NEAR(checker, cpuOpRet, 1); + + AscendMat npuMat, npuChecker; + npuMat.upload(cpuMat); + cv::cann::cropResize(npuMat, npuChecker, b, dsize, 0, 0, 1); + npuChecker.download(checker); + EXPECT_MAT_NEAR(cpuOpRet, checker, 1); +} +TEST(CORE, CROP_RESIZE_MAKE_BORDER) +{ + Mat cpuMat = randomMat(1024, 896, CV_8UC1, 100.0, 255.0); + + Mat resized_cv, checker, cpuOpRet; + Size dsize = Size(320, 256); + const Rect b(300, 500, 496, 512); + RNG rng(12345); + float scalarV[3] = {0, 0, 255}; + int top, bottom, left, right; + top = 54; + bottom = 0; + left = 32; + right = 0; + int interpolation = 1; + + Scalar value = {scalarV[0], scalarV[1], scalarV[2], 0}; + for (int borderType = 0; borderType < 2; borderType++) + { + cv::cann::cropResizeMakeBorder(cpuMat, checker, b, dsize, 0, 0, interpolation, top, left, + borderType, value); + Mat cropped_cv(cpuMat, b); + cv::resize(cropped_cv, resized_cv, dsize, 0, 0, interpolation); + cv::copyMakeBorder(resized_cv, cpuOpRet, top, bottom, left, right, borderType, value); + EXPECT_MAT_NEAR(checker, cpuOpRet, 1e-10); + } + AscendMat npuMat, npuChecker; + npuMat.upload(cpuMat); + for (int borderType = 0; borderType < 2; borderType++) + { + cv::cann::cropResizeMakeBorder(npuMat, npuChecker, b, dsize, 0, 0, interpolation, top, left, + borderType, value); + npuChecker.download(checker); + Mat cropped_cv(cpuMat, b); + cv::resize(cropped_cv, resized_cv, dsize, 0, 0, interpolation); + cv::copyMakeBorder(resized_cv, cpuOpRet, top, bottom, left, right, borderType, value); + EXPECT_MAT_NEAR(checker, cpuOpRet, 1e-10); + } +} + +TEST(CORE, COPY_MAKE_BORDER) +{ + Mat cpuMat = randomMat(1280, 1706, CV_8UC3, 100, 255); + + Mat cpuOpRet, checker; + RNG rng(12345); + Scalar value = {static_cast(rng.uniform(0, 255)), + static_cast(rng.uniform(0, 255)), + static_cast(rng.uniform(0, 255))}; + int top, bottom, left, right; + top = 20; + bottom = 30; + left = 30; + right = 20; + + int borderType = 0; + for (borderType = 0; borderType < 2; borderType++) + { + cv::cann::copyMakeBorder(cpuMat, checker, top, bottom, left, right, borderType, value); + + cv::copyMakeBorder(cpuMat, cpuOpRet, top, bottom, left, right, borderType, value); + EXPECT_MAT_NEAR(checker, cpuOpRet, 1e-10); + } + + AscendMat npuMat, npuChecker; + npuMat.upload(cpuMat); + for (borderType = 0; borderType < 2; borderType++) + { + cv::cann::copyMakeBorder(npuMat, npuChecker, top, bottom, left, right, borderType, value); + npuChecker.download(checker); + + cv::copyMakeBorder(cpuMat, cpuOpRet, top, bottom, left, right, borderType, value); + EXPECT_MAT_NEAR(checker, cpuOpRet, 1e-10); + } +} } // namespace } // namespace opencv_test diff --git a/modules/cannops/test/test_main.cpp b/modules/cannops/test/test_main.cpp index 202c6af27ee..d14f2a2869e 100644 --- a/modules/cannops/test/test_main.cpp +++ b/modules/cannops/test/test_main.cpp @@ -8,8 +8,18 @@ class CannEnvironment : public ::testing::Environment { public: virtual ~CannEnvironment() = default; - virtual void SetUp() CV_OVERRIDE { initAcl(); } - virtual void TearDown() CV_OVERRIDE { finalizeAcl(); } + virtual void SetUp() CV_OVERRIDE + { + initAcl(); + cv::cann::setDevice(DEVICE_ID); + initDvpp(); + } + virtual void TearDown() CV_OVERRIDE + { + finalizeAcl(); + cv::cann::resetDevice(); + finalizeDvpp(); + } }; static void initTests() diff --git a/modules/cannops/tutorials/ascend_npu_image_processing.markdown b/modules/cannops/tutorials/ascend_npu_image_processing.markdown index ed905831d31..80e54b4cc23 100644 --- a/modules/cannops/tutorials/ascend_npu_image_processing.markdown +++ b/modules/cannops/tutorials/ascend_npu_image_processing.markdown @@ -108,23 +108,4 @@ Results 4. Upon applying the flip operation with a flip code of 0 (flipping around the x-axis), we achieve the final result: - ![puppy_processed_normalized](./puppy_processed.jpg) - - - -## Usage Limitations - -While Ascend supports most commonly used operators, there are still some limitations that need to be addressed. - -- There is no strict limit on the size of the input image used for encoding; however, it depends on the available RAM size of your device. -- Please note that not all data types (dtypes) are supported by every operator. The current dtype limitations are outlined in the following table. We are actively working on addressing these limitations through automatic dtype conversion in an upcoming commit. - - -| Operator | Supported Dtype | -| ---------------------- | ------------------------------------------------------------ | -| multiply (with scale) | float16,float32,int32 | -| divide (with scale) | float16,float,int32,int8,uint8 | -| bitwise add/or/xor/not | int32,int16,uint16 | -| flip | float16,float,int64,int32,int16,uint16 | -| transpose | float16,float,int64,int32,int16,int8,uint64,uint32,uint16,uint8,bool | -| rotate | float16,float,int64,int32,int16,uint16 | + ![puppy_processed_normalized](./puppy_processed.jpg) \ No newline at end of file