Merge pull request #3608 from MengqingCao:dvpp_support

Add additional image processing operators for Ascend NPU by utilizing DVPP #3608

The user base for [Ascend NPU](https://www.hiascend.com/en/) and programming with CANN is increasing rapidly, with a growing number of users joining each day. To facilitate the use of these users, this PR provides more support for Ascend backend operators. All operators this PR offers are using use DVPP as the computational unit. Digital Vision Pre-Processing (DVPP) is an image processing unit built into the Ascend AI processor. Its main functions include image and video encoding/decoding, as well as image cropping and scaling. 

The high-frequency operators with NPU as the backend and basic data structure AscendMat has been provided in #3552, while it still lacks many image processing operators. Moreover, only two interpolation algorithms for the resize operator are supported in #3552. In this PR, the bilinear interpolation algorithm and nearest neighbour interpolation algorithm are implemented for the resize operator, as well as the Ascend implementation of the copyMakeBorder operator. 

In addition, the serialization of image processing operations is widely used in the preprocessing and post-processing stages of computer vision deep learning methods. Therefore, providing integrated operators is very meaningful for improving the convenience of use for OpenCV and deep learning crossover users. For example, torchvision also provides similar operators: [RESIZED_CROP](https://pytorch.org/vision/stable/generated/torchvision.transforms.functional.resized_crop.html?highlight=resizedcrop).
Thus, this PR also provides two serialization processing operators: cropResize and cropResizeMakeBorder. 

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [N/A] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

modules/cannops/include/opencv2/cann.hpp

@@ -318,6 +318,21 @@ CV_EXPORTS_W void initAcl();
  */
 CV_EXPORTS_W void finalizeAcl();
 
+/**
+ * @brief init DVPP system.
+ * @note The DVPP interfaces used are all version V2.
+ * Supported devices: Atlas Inference Series products, Atlas 200/500 A2 Inference products and
+ * Atlas A2 Training Series products/Atlas 300I A2 Inference products
+ */
+CV_EXPORTS_W void initDvpp();
+
+/**
+ * @brief finalize DVPP system.
+ * @note Supported devices: Atlas Inference Series products, Atlas 200/500 A2 Inference products and
+ * Atlas A2 Training Series products/Atlas 300I A2 Inference products
+ */
+CV_EXPORTS_W void finalizeDvpp();
+
 //! @} cann_init
 
 } // namespace cann

modules/cannops/include/opencv2/dvpp_call.hpp

@@ -0,0 +1,107 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef ENABLE_DVPP_INTERFACE
+    #define ENABLE_DVPP_INTERFACE
+#endif // ENABLE_DVPP_INTERFACE
+
+#include <vector>
+#include <string>
+#include <acl/acl.h>
+#include <acl/acl_op_compiler.h>
+#include <acl/dvpp/hi_dvpp.h>
+#include "acl/acl_op.h"
+#include "cann_call.hpp"
+
+namespace cv
+{
+namespace cann
+{
+struct AscendPicDesc
+{
+    const char* name;
+    std::shared_ptr<hi_void> data;
+    std::vector<int64_t> batchNum;
+
+    size_t widthAlignment = 16;
+    size_t heightAlignment = 1;
+    size_t sizeAlignment = 3;
+    size_t sizeNum = 3;
+
+    hi_vpc_pic_info Pic;
+    AscendPicDesc& setMemAlign();
+    AscendPicDesc& setPic(hi_pixel_format _picture_format);
+    std::shared_ptr<hi_void> allocate();
+    AscendPicDesc(){};
+    AscendPicDesc(const AscendMat& ascendMat, hi_pixel_format _picture_format);
+    AscendPicDesc(const Mat& mat, hi_pixel_format _picture_format);
+};
+
+/*
+ ***************************** hi_mpi_vpc warppers ***************************
+ The DVPP VPC interfaces here are all version v2. Only the following devices are supported: Atlas
+ Inference Series products, Atlas 200/500 A2 Inference products and Atlas A2 Training Series
+ products/Atlas 300I A2 Inference products.
+*/
+inline void vpcResizeWarpper(hi_vpc_chn chnId, hi_vpc_pic_info& inPic, hi_vpc_pic_info& outPic,
+                             int interpolation, uint32_t* taskID)
+{
+    uint32_t ret = hi_mpi_vpc_resize(chnId, &inPic, &outPic, 0, 0, interpolation, taskID, -1);
+    if (ret != HI_SUCCESS)
+        CV_Error(Error::StsBadFlag, "failed to resize image");
+}
+void vpcCropResizeWarpper(hi_vpc_chn chnId, hi_vpc_pic_info& inPic, hi_vpc_pic_info& outPic,
+                          int cnt, uint32_t* taskID, const Rect& rect, Size dsize,
+                          int interpolation);
+
+void vpcCropResizeMakeBorderWarpper(hi_vpc_chn chnId, std::vector<AscendPicDesc>& inPicDesc,
+                                    std::vector<AscendPicDesc>& outPicDesc, int cnt,
+                                    uint32_t* taskID, const Rect& rect, Size dsize,
+                                    int interpolation, const int borderType, Scalar scalarV,
+                                    int top, int left);
+void vpcCopyMakeBorderWarpper(hi_vpc_chn chnId, hi_vpc_pic_info& inPic, hi_vpc_pic_info& outPic,
+                              uint32_t* taskID, int* offsets, int bordertype, Scalar value);
+/*****************************************************************************/
+
+/**
+ * @brief Interface for calling DVPP operator descriptors.
+ * The DVPP VPC interfaces here are all version v2. Supported devices: Atlas Inference Series
+ * products, Atlas 200/500 A2 Inference products and Atlas A2 Training Series products/Atlas 300I A2
+ * Inference products.
+ */
+class DvppOperatorDesc
+{
+private:
+    DvppOperatorDesc& addInput(AscendPicDesc& picDesc);
+    DvppOperatorDesc& addOutput(AscendPicDesc& picDesc);
+    std::set<std::shared_ptr<hi_void>> holder;
+
+public:
+    DvppOperatorDesc()
+    {
+        chnId = 0;
+        stChnAttr = {};
+        createChannel();
+    }
+    virtual ~DvppOperatorDesc() { reset(); }
+    DvppOperatorDesc& addInput(const AscendMat& mat);
+    DvppOperatorDesc& addOutput(AscendMat& mat);
+    DvppOperatorDesc& addInput(const Mat& mat);
+    DvppOperatorDesc& addOutput(Mat& mat);
+
+    DvppOperatorDesc& getResult(Mat& dst, uint32_t& taskIDResult);
+    DvppOperatorDesc& getResult(AscendMat& dst, uint32_t& taskIDResult);
+
+    DvppOperatorDesc& reset();
+    DvppOperatorDesc& createChannel();
+
+    std::vector<AscendPicDesc> inputDesc_;
+    std::vector<AscendPicDesc> outputDesc_;
+
+    hi_vpc_chn chnId;
+    hi_vpc_chn_attr stChnAttr;
+};
+
+} // namespace cann
+} // namespace cv

modules/cannops/misc/python/test/test_cannops.py

@@ -24,6 +24,7 @@ def genMask(mask, listx, listy):
 class cannop_test(NewOpenCVTests):
     def test_ascend(self):
         cv.cann.initAcl()
+        cv.cann.initDvpp()
         cv.cann.getDevice()
         cv.cann.setDevice(0)
         stream = cv.cann.AscendStream_Null()
@@ -275,6 +276,50 @@ def test_imgproc(self):
                 aclMat, 127, 255, tType)
             self.assertTrue(np.allclose(cvThresh, cannThresh.download()))
             self.assertTrue(np.allclose(cvRet, cannRet))
+
+        npMat = (np.random.random((1280, 1024, 3)) * 255).astype(np.uint8)
+        w_off, h_off, crop_w, crop_h = 0, 0, 512, 384
+        roi = [w_off, h_off, crop_w, crop_h]
+        aclMat = cv.cann.AscendMat()
+        aclMat.upload(npMat)
+
+        # resize
+        dstSize = np.array([crop_w, crop_h])
+        self.assertTrue(np.allclose(cv.cann.resize(npMat, dstSize, 0, 0, 1),
+                        cv.resize(npMat, dstSize, 0, 0, 1)))
+        self.assertTrue(np.allclose(cv.cann.resize(aclMat, dstSize, 0, 0, 1).download(),
+                        cv.resize(npMat, dstSize, 0, 0, 1)))
+        # cropResize
+        self.assertTrue(np.allclose(cv.cann.cropResize(npMat, roi, dstSize, 0, 0, 1),
+                        cv.resize(npMat[h_off:crop_h, w_off:crop_w], dstSize, 0, 0, 1)), 0)
+        self.assertTrue(np.allclose(cv.cann.cropResize(aclMat, roi, dstSize, 0, 0, 1).download(),
+                        cv.resize(npMat[h_off:crop_h, w_off:crop_w], dstSize, 0, 0, 1)), 0)
+
+        # cropResizeMakeBorder
+        # TODO cv.copyMakeBorder ignores borderColorValue param; find the reason and fix it
+        borderColorValue = (100, 0, 255)
+        top, bottom, left, right = 32, 0, 10, 0
+        borderTypes = [0, 1]
+
+        for borderType in borderTypes:
+            self.assertTrue(np.allclose(cv.cann.cropResizeMakeBorder(npMat, roi, dstSize,
+                                0, 0, 1, top, left, borderType),
+                            cv.copyMakeBorder(cv.resize(npMat[h_off:crop_h, w_off:crop_w],
+                                dstSize, 0, 0, 1), top, bottom, left, right, borderType), 1))
+            self.assertTrue(np.allclose(cv.cann.cropResizeMakeBorder(aclMat, roi, dstSize,
+                                0, 0, 1, top, left, borderType).download(),
+                            cv.copyMakeBorder(cv.resize(npMat[h_off:crop_h, w_off:crop_w],
+                                dstSize, 0, 0, 1), top, bottom, left, right, borderType), 1))
+
+        # copyMakeBorder
+        for borderType in borderTypes:
+            self.assertTrue(np.allclose(cv.cann.copyMakeBorder(npMat, top, bottom, left, right,
+                                                               borderType),
+                            cv.copyMakeBorder(npMat, top, bottom, left, right, borderType)))
+            self.assertTrue(np.allclose(cv.cann.copyMakeBorder(aclMat, top, bottom, left, right,
+                                                               borderType).download(),
+                            cv.copyMakeBorder(npMat, top, bottom, left, right, borderType)))
+
         cv.cann.resetDevice()
 
 if __name__ == '__main__':

modules/cannops/perf/perf_core.cpp

@@ -11,6 +11,7 @@ namespace
 {
 #define TYPICAL_ASCEND_MAT_SIZES \
     Values(::perf::sz1080p, ::perf::sz2K, ::perf::sz2160p, ::perf::sz4320p)
+#define DVPP_ASCEND_MAT_SIZES Values(::perf::sz1080p, ::perf::sz2K, ::perf::sz2160p, ::perf::sz5MP)
 #define DEF_PARAM_TEST(name, ...) \
     typedef ::perf::TestBaseWithParam<testing::tuple<__VA_ARGS__>> name
 
@@ -157,5 +158,176 @@ PERF_TEST_P(NPU, CROP_OVERLOAD, TYPICAL_ASCEND_MAT_SIZES)
     cv::cann::resetDevice();
     SANITY_CHECK_NOTHING();
 }
+
+PERF_TEST_P(CPU, RESIZE, DVPP_ASCEND_MAT_SIZES)
+{
+    Mat mat(GET_PARAM(0), CV_8UC3);
+    Mat dst;
+    declare.in(mat, WARMUP_RNG);
+    Size dsize = Size(256, 256);
+    TEST_CYCLE_N(10) { cv::resize(mat, dst, dsize, 0, 0, 1); }
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST_P(NPU, RESIZE, DVPP_ASCEND_MAT_SIZES)
+{
+    Mat mat(GET_PARAM(0), CV_32FC3);
+    AscendMat dst;
+    AscendMat src;
+    src.upload(mat);
+    declare.in(mat, WARMUP_RNG);
+    Size dsize = Size(256, 256);
+    TEST_CYCLE_N(10) { cv::cann::resize(src, dst, dsize, 0, 0, 3); }
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST_P(NPU, THRESHOLD, TYPICAL_ASCEND_MAT_SIZES)
+{
+    Mat mat(GET_PARAM(0), CV_32FC3);
+    AscendMat dst;
+    AscendMat src;
+    src.upload(mat);
+    declare.in(mat, WARMUP_RNG);
+    TEST_CYCLE_N(10) { cv::cann::threshold(src, dst, 100.0, 255.0, cv::THRESH_BINARY); }
+    SANITY_CHECK_NOTHING();
+}
+PERF_TEST_P(CPU, THRESHOLD, TYPICAL_ASCEND_MAT_SIZES)
+{
+    Mat mat(GET_PARAM(0), CV_32FC3);
+    Mat dst;
+    declare.in(mat, WARMUP_RNG);
+    TEST_CYCLE_N(10) { cv::threshold(mat, dst, 100.0, 255.0, cv::THRESH_BINARY); }
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST_P(NPU, RESIZE_INTER_NEAREST, DVPP_ASCEND_MAT_SIZES)
+{
+    Mat mat(GET_PARAM(0), CV_8UC3);
+    Mat dst;
+    declare.in(mat, WARMUP_RNG);
+    Size dsize = Size(256, 256);
+    TEST_CYCLE_N(10) { cv::cann::resize(mat, dst, dsize, 0, 0, 0); }
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST_P(NPU, COPY_MAKE_BORDER, DVPP_ASCEND_MAT_SIZES)
+{
+    Mat resized_cv, checker, cpuOpRet, cpuMat(GET_PARAM(0), CV_8UC3);
+    declare.in(cpuMat, WARMUP_RNG);
+    int top, bottom, left, right;
+    top = (int)(20);
+    bottom = top;
+    left = (int)(20);
+    right = left;
+    int borderType = 1;
+    float scalarV[3] = {0, 0, 255};
+    Scalar value = {scalarV[0], scalarV[1], scalarV[2]};
+
+    TEST_CYCLE_N(10)
+    {
+        cv::cann::copyMakeBorder(cpuMat, checker, top, bottom, left, right, borderType, value);
+    }
+
+    SANITY_CHECK_NOTHING();
+}
+PERF_TEST_P(CPU, COPY_MAKE_BORDER, DVPP_ASCEND_MAT_SIZES)
+{
+    Mat resized_cv, checker, cpuOpRet, cpuMat(GET_PARAM(0), CV_8UC3);
+    declare.in(cpuMat, WARMUP_RNG);
+    int top, bottom, left, right;
+    top = (int)(20);
+    bottom = top;
+    left = (int)(20);
+    right = left;
+    int borderType = 1;
+    float scalarV[3] = {0, 0, 255};
+    Scalar value = {scalarV[0], scalarV[1], scalarV[2]};
+
+    TEST_CYCLE_N(10)
+    {
+        cv::copyMakeBorder(cpuMat, checker, top, bottom, left, right, borderType, value);
+    }
+
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST_P(NPU, CROP_RESIZE_MAKE_BORDER, DVPP_ASCEND_MAT_SIZES)
+{
+    Size size = GET_PARAM(0);
+    Mat resized_cv, checker, cpuOpRet, cpuMat(size, CV_8UC3);
+    declare.in(cpuMat, WARMUP_RNG);
+
+    const Rect b(1, 0, size.width / 2, size.height);
+    Size dsize = Size(size.width / 4, size.height / 2);
+    int top, left;
+    top = (int)(20);
+    left = (int)(20);
+    int borderType = 0;
+    float scalarV[3] = {1, 1, 1};
+    Scalar value = {scalarV[0], scalarV[1], scalarV[2]};
+
+    TEST_CYCLE_N(10)
+    {
+        cv::cann::cropResizeMakeBorder(cpuMat, checker, b, dsize, 0, 0, 1, top, left, borderType,
+                                       value);
+    }
+
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST_P(CPU, CROP_RESIZE_MAKE_BORDER, DVPP_ASCEND_MAT_SIZES)
+{
+    Size size = GET_PARAM(0);
+    Mat resized_cv, checker, cpuOpRet, cpuMat(size, CV_8UC3);
+    declare.in(cpuMat, WARMUP_RNG);
+    const Rect b(1, 0, size.width / 2, size.height);
+    Size dsize = Size(size.width / 4, size.height / 2);
+    int top, bottom, left, right;
+    top = (int)(20);
+    bottom = 0;
+    left = (int)(20);
+    right = 0;
+    int borderType = 0;
+    float scalarV[3] = {1, 1, 1};
+    Scalar value = {scalarV[0], scalarV[1], scalarV[2]};
+
+    TEST_CYCLE_N(10)
+    {
+        Mat cropped_cv(cpuMat, b);
+        cv::resize(cropped_cv, resized_cv, dsize, 0, 0, 1);
+        cv::copyMakeBorder(resized_cv, cpuOpRet, top, bottom, left, right, borderType, value);
+    }
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST_P(NPU, CROP_RESIZE, DVPP_ASCEND_MAT_SIZES)
+{
+    Size size = GET_PARAM(0);
+    Mat resized_cv, checker, cpuOpRet, cpuMat(size, CV_8UC3);
+    declare.in(cpuMat, WARMUP_RNG);
+    const Rect b(1, 0, size.width / 2, size.height);
+    Size dsize = Size(size.width / 4, size.height / 2);
+
+    TEST_CYCLE_N(10) { cv::cann::cropResize(cpuMat, checker, b, dsize, 0, 0, 1); }
+
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST_P(CPU, CROP_RESIZE, DVPP_ASCEND_MAT_SIZES)
+{
+    Size size = GET_PARAM(0);
+    Mat resized_cv, checker, cpuOpRet, cpuMat(size, CV_8UC3);
+    declare.in(cpuMat, WARMUP_RNG);
+    const Rect b(1, 0, size.width / 2, size.height);
+    Size dsize = Size(size.width / 4, size.height / 2);
+
+    TEST_CYCLE_N(10)
+    {
+        Mat cropped_cv(cpuMat, b);
+        cv::resize(cropped_cv, resized_cv, dsize, 0, 0, 1);
+    }
+    SANITY_CHECK_NOTHING();
+}
+
 } // namespace
 } // namespace opencv_test

modules/cannops/perf/perf_main.cpp

@@ -10,8 +10,18 @@ class CannEnvironment : public ::testing::Environment
 {
 public:
     virtual ~CannEnvironment() = default;
-    virtual void SetUp() CV_OVERRIDE { cv::cann::initAcl(); }
-    virtual void TearDown() CV_OVERRIDE { cv::cann::finalizeAcl(); }
+    virtual void SetUp() CV_OVERRIDE
+    {
+        initAcl();
+        cv::cann::setDevice(DEVICE_ID);
+        initDvpp();
+    }
+    virtual void TearDown() CV_OVERRIDE
+    {
+        finalizeAcl();
+        cv::cann::resetDevice();
+        finalizeDvpp();
+    }
 };
 
 static void initTests()