Support for ROI Random crop

rocAL/rocAL/include/api/rocal_api_meta_data.h

@@ -284,4 +284,23 @@ extern "C" void ROCAL_API_CALL rocalGetImageId(RocalContext p_context, int* buf)
  */
 extern "C" void ROCAL_API_CALL rocalGetJointsDataPtr(RocalContext p_context, RocalJointsData** joints_data);
 
+/*! \brief initialize the values required for ROI Random crop
+ * \ingroup group_rocal_meta_data
+ * \param [in] rocal_context rocal context
+ * \param [in] crop_shape_batch
+ * \param [in] roi_begin_batch
+ * \param [in] input_shape_batch
+ * \param [in] roi_end_batch
+ */
+extern "C" void ROCAL_API_CALL rocalROIRandomCrop(RocalContext p_context, RocalTensor p_input, int *crop_shape);
+
+/*! \brief get the ROI Random crop values
+ * \ingroup group_rocal_meta_data
+ * \param [in] rocal_context rocal context
+ * \param [in] crop_shape_batch
+ * \param [in] roi_begin_batch
+ * \param [in] input_shape_batch
+ * \param [in] roi_end_batch
+ */
+extern "C" RocalTensor ROCAL_API_CALL rocalGetROIRandomCropValues(RocalContext p_context);
 #endif  // MIVISIONX_ROCAL_API_META_DATA_H

rocAL/rocAL/include/pipeline/master_graph.h

@@ -131,6 +131,9 @@ class MasterGraph {
     void set_sequence_batch_size(size_t sequence_length) { _sequence_batch_size = _user_batch_size * sequence_length; }
     std::vector<rocalTensorList *> get_bbox_encoded_buffers(size_t num_encoded_boxes);
     size_t bounding_box_batch_count(pMetaDataBatch meta_data_batch);
+    void roi_random_crop(Tensor *input, int *crop_shape);
+    void update_roi_random_crop(int *crop_shape_batch, int *roi_begin_batch, int *roi_end_batch);
+    Tensor * get_roi_random_crop_values() { return _roi_random_crop_tensor; }
 #if ENABLE_OPENCL
     cl_command_queue get_ocl_cmd_q() { return _device.resources()->cmd_queue; }
 #endif
@@ -203,6 +206,12 @@ class MasterGraph {
     bool _offset;                                                                 // Returns normalized offsets ((encoded_bboxes*scale - anchors*scale) - mean) / stds in EncodedBBoxes that use std and the mean and scale arguments if offset="True"
     std::vector<float> _means, _stds;                                             //_means:  [x y w h] mean values for normalization _stds: [x y w h] standard deviations for offset normalization.
     bool _augmentation_metanode = false;
+    bool _is_roi_random_crop = false;
+    uint _input_dims = 1;
+    int *_crop_shape_batch = nullptr;
+    Tensor *_roi_random_crop_tensor = nullptr;
+    void *_roi_random_crop_buf = nullptr;
+    vx_tensor _roi_random_crop_vx_tensor = nullptr;
 #if ENABLE_HIP
     BoxEncoderGpu *_box_encoder_gpu = nullptr;
 #endif

rocAL/rocAL/include/pipeline/tensor.h

@@ -279,6 +279,7 @@ class Tensor : public rocalTensor {
     // create_from_handle() no internal memory allocation is done here since
     // tensor's handle should be swapped with external buffers before usage
     int create_from_handle(vx_context context);
+    int create_from_handle_new(vx_context context, void *ptr);
     int create_virtual(vx_context context, vx_graph graph);
     bool is_handle_set() { return (_vx_handle != 0); }
     void set_dims(std::vector<size_t> dims) { _info.set_dims(dims); }

rocAL/rocAL/source/api/rocal_api_meta_data.cpp

@@ -465,3 +465,24 @@ void
 
     *joints_data = (RocalJointsData*)(&(meta_data.second->get_joints_data_batch()));
 }
+
+void
+    ROCAL_API_CALL
+    rocalROIRandomCrop(RocalContext p_context, RocalTensor p_input, int *crop_shape) {
+     if ((p_context == nullptr) || (p_input == nullptr)) {
+        ERR("Invalid ROCAL context or invalid input tensor")
+    }
+    auto context = static_cast<Context*>(p_context);
+    auto input = static_cast<Tensor*>(p_input);
+    context->master_graph->roi_random_crop(input, crop_shape);
+}
+
+RocalTensor
+    ROCAL_API_CALL
+    rocalGetROIRandomCropValues(RocalContext p_context) {
+     if (p_context == nullptr) {
+        ERR("Invalid ROCAL context or invalid input tensor")
+    }
+    auto context = static_cast<Context*>(p_context);
+    return context->master_graph->get_roi_random_crop_values();
+}

rocAL/rocAL/source/pipeline/master_graph.cpp

@@ -928,6 +928,28 @@ void MasterGraph::output_routine() {
                     _meta_data_graph->process(_augmented_meta_data, output_meta_data);
                 }
             }
+
+            if(_is_roi_random_crop)
+            {
+                // get the roi_begin and roi_end values from random_object_bbox
+                int *roi_begin_batch = new int[_user_batch_size * _input_dims];
+                int *roi_end_batch = new int[_user_batch_size * _input_dims];
+                for(uint i = 0; i < _user_batch_size; i++)
+                {
+                    int sample_idx = i * _input_dims;
+                    int *roi_begin = &roi_begin_batch[sample_idx];
+                    int *roi_end = &roi_end_batch[sample_idx];
+                    for(uint j = 0; j < _input_dims; j++)
+                    {
+                        roi_begin[j] = std::rand() % (_input_dims * 5);
+                        roi_end[j] = roi_begin[j] + (std::rand() % (_input_dims * 5));
+                    }
+                }
+                update_roi_random_crop(_crop_shape_batch, roi_begin_batch, roi_end_batch);
+                delete[] roi_begin_batch;
+                delete[] roi_end_batch;
+            }
+
             _process_time.start();
             _graph->process();
             _process_time.end();
@@ -1373,6 +1395,115 @@ TensorList *MasterGraph::mask_meta_data() {
 
     return &_mask_tensor_list;
 }
+class BatchRNGUniform {
+ public:
+  /**
+   * @brief Used to keep batch of RNGs, so Operators can be immune to order of sample processing
+   * while using randomness
+   *
+   * @param seed Used to generate seed_seq to initialize batch of RNGs
+   * @param batch_size How many RNGs to store
+   * @param state_size How many seed are used to initialize one RNG. Used to lower probablity of
+   * collisions between seeds used to initialize RNGs in different operators.
+   */
+  BatchRNGUniform(int64_t seed, int batch_size, int state_size = 4)
+  : seed_(seed) {
+    std::seed_seq seq{seed_};
+    std::vector<uint32_t> seeds(batch_size * state_size);
+    seq.generate(seeds.begin(), seeds.end());
+    rngs_.reserve(batch_size);
+    for (int i = 0; i < batch_size * state_size; i += state_size) {
+      std::seed_seq s(seeds.begin() + i, seeds.begin() + i + state_size);
+      rngs_.emplace_back(s);
+    }
+  }
+
+
+  /**
+   * Returns engine corresponding to given sample ID
+   */
+  std::mt19937 &operator[](int sample) noexcept {
+    return rngs_[sample];
+  }
+
+ private:
+  int64_t seed_;
+  std::vector<std::mt19937> rngs_;
+};
+
+void MasterGraph::roi_random_crop(Tensor *input, int *crop_shape)
+{
+    _is_roi_random_crop = true;
+    if(input->info().is_image())
+        _input_dims = input->num_of_dims() - 2; // TO be changed later when generic roi changes are added
+    else
+        _input_dims = input->num_of_dims() - 1;
+
+    _crop_shape_batch = new int[_input_dims * _user_batch_size]; // TODO handle this case later when different crop_shape is given for each tensor
+
+    // replicate crop_shape values for all samples in a batch
+    for(uint i = 0; i < _user_batch_size; i++)
+    {
+        int sample_idx = i * _input_dims;
+        memcpy(&(_crop_shape_batch[sample_idx]), crop_shape, _input_dims * sizeof(int));
+    }
+
+    // create new instance of tensor class
+    std::vector<size_t> dims = {_user_batch_size, _input_dims};
+    auto info = TensorInfo(std::move(dims), RocalMemType::HOST, RocalTensorDataType::INT32);
+    _roi_random_crop_tensor = new Tensor(info);
+
+    // allocate memory for the raw buffer pointer in tensor object
+    _roi_random_crop_buf = new int[_user_batch_size * _input_dims];
+    _roi_random_crop_tensor->create_from_handle_new(_context, _roi_random_crop_buf);
+}
+
+void MasterGraph::update_roi_random_crop(int *crop_shape_batch, int *roi_begin_batch, int *roi_end_batch) {
+    void *roi_random_crop_buf = _roi_random_crop_tensor->buffer();
+    int *crop_begin_batch = static_cast<int *>(roi_random_crop_buf);
+    uint seed = std::time(0);
+    int *roi_batch = reinterpret_cast<int *>(_internal_tensor_list[0]->get_roi());
+
+    BatchRNGUniform _rng = {seed, static_cast<int>(_user_batch_size)};
+    for(uint i = 0; i < _user_batch_size; i++) {
+        int sample_idx = i * _input_dims;
+        int *crop_shape = &crop_shape_batch[sample_idx];
+        int *roi_begin = &roi_begin_batch[sample_idx];
+        int *input_shape = &roi_batch[sample_idx * 2 + _input_dims];
+        int *roi_end = &roi_end_batch[sample_idx];
+        int *crop_begin = &crop_begin_batch[sample_idx];
+
+        for(uint j = 0; j < _input_dims; j++) {
+            // check if crop_shape, roi_end is greater than input_shape
+            if(crop_shape[j] > input_shape[j])
+                THROW("crop shape cannot be greater than input shape");
+            if (roi_end[j] > input_shape[j])
+                THROW("ROI shape cannot be greater than input shape");
+
+            int roi_length = roi_end[j] - roi_begin[j];
+            int crop_length = crop_shape[j];
+            if (roi_length == crop_length) {
+                crop_begin[j] = roi_begin[j];
+            } else {
+                int64_t start_range[2] = {roi_begin[j], roi_end[j] - crop_length};
+
+                // swap range values if start_range[0] > start_range[1]
+                if (start_range[0] > start_range[1]) {
+                    int64_t temp = start_range[0];
+                    start_range[0] = start_range[1];
+                    start_range[1] = temp;
+                }
+
+                // check if range is within the bounds of input
+                start_range[0] = std::max<int64_t>(0, start_range[0]);
+                start_range[1] = std::min<int64_t>(input_shape[j] - crop_length, start_range[1]);
+
+                auto dist = std::uniform_int_distribution<int64_t>(start_range[0], start_range[1]);
+                crop_begin[j] = dist(_rng[i]);
+            }
+        }
+    }
+}
 
 void MasterGraph::notify_user_thread() {
     if (_output_routine_finished_processing)

rocAL/rocAL/source/pipeline/tensor.cpp

@@ -298,6 +298,33 @@ int Tensor::create_from_handle(vx_context context) {
     return 0;
 }
 
+int Tensor::create_from_handle_new(vx_context context, void *ptr) {
+    if (_vx_handle) {
+        WRN("Tensor object create method is already called ")
+        return -1;
+    }
+
+    _context = context;
+    vx_enum tensor_data_type = interpret_tensor_data_type(_info.data_type());
+    unsigned num_of_dims = _info.num_of_dims();
+    vx_size stride[num_of_dims];
+    // void *ptr[1] = {nullptr};
+
+    stride[0] = tensor_data_size(_info.data_type());
+    for (unsigned i = 1; i < num_of_dims; i++)
+        stride[i] = stride[i - 1] * _info.dims().at(i - 1);
+
+    _vx_handle = vxCreateTensorFromHandle(_context, _info.num_of_dims(), _info.dims().data(), tensor_data_type, 0, stride, ptr, vx_mem_type(_info._mem_type));
+    vx_status status;
+    if ((status = vxGetStatus((vx_reference)_vx_handle)) != VX_SUCCESS)
+        THROW("Error: vxCreateTensorFromHandle(input: failed " + TOSTR(status))
+    _info._type = TensorInfo::Type::HANDLE;
+    _mem_handle = ptr;
+    // void *roi_handle = reinterpret_cast<void *>(_info.get_roi());
+    // create_roi_tensor_from_handle(&roi_handle); // Create ROI tensor from handle
+    return 0;
+}
+
 int Tensor::create(vx_context context) {
     if (_vx_handle) {
         WRN("Tensor object create method is already called ")

rocAL/rocAL_pybind/amd/rocal/fn.py

@@ -1055,3 +1055,9 @@ def box_iou_matcher(*inputs, anchors, criteria=0.5, high_threshold=0.5,
         Pipeline._current_pipeline._handle, *(kwargs_pybind.values()))
     Pipeline._current_pipeline._box_iou_matcher = True
     return (box_iou_matcher, [])
+
+def roi_random_crop(*inputs, crop_shape=None):
+
+    # pybind call arguments
+    kwargs_pybind = {"crop_shape": crop_shape}
+    roi_random_crop = b.roiRandomCrop(Pipeline._current_pipeline._handle, *(kwargs_pybind.values()))

rocAL/rocAL_pybind/amd/rocal/pipeline.py

@@ -394,3 +394,6 @@ def create_pipeline(*args, **kwargs):
         return create_pipeline
 
     return actual_decorator(fn) if fn else actual_decorator
+
+def get_roi_random_crop_values(self):
+    return b.rocalGetROIRandomCropValues(self._handle)

rocAL/rocAL_pybind/rocal_pybind.cpp

@@ -362,6 +362,8 @@ PYBIND11_MODULE(rocal_pybind, m) {
         int *ptr = static_cast<int *>(buf.ptr);
         rocalGetImageSizes(context, ptr);
     });
+    m.def("roiRandomCrop", &rocalROIRandomCrop);
+    m.def("getROIRandomCropValues", &rocalGetROIRandomCropValues, py::return_value_policy::reference);
     // rocal_api_parameter.h
     m.def("setSeed", &rocalSetSeed);
     m.def("getSeed", &rocalGetSeed);

utilities/rocAL/rocAL_basic_test/rocal_basic_test.cpp

@@ -108,6 +108,10 @@ int main(int argc, const char **argv) {
         rocalCreateTextFileBasedLabelReader(handle, label_text_file_path);
     rocalCropResizeFixed(handle, decoded_output, 224, 224, true, 0.9, 1.1, 0.1, 0.1);
 
+    int crop_shape[] = {2, 2};
+    rocalROIRandomCrop(handle, decoded_output, crop_shape);
+    RocalTensor crop_output;
+
     if (rocalGetStatus(handle) != ROCAL_OK) {
         std::cout << "JPEG source could not initialize : " << rocalGetErrorMessage(handle) << std::endl;
         return -1;
@@ -163,6 +167,16 @@ int main(int argc, const char **argv) {
             if (rocalRun(handle) != 0)
                 break;
 
+            std::cout << "printing ROI Random crop outputs:" << std::endl;
+            crop_output = rocalGetROIRandomCropValues(handle);
+            int *crop_begin = static_cast<int *>(crop_output->buffer());
+            for(int i = 0; i < inputBatchSize; i++)
+            {
+                for(int j = 0; j < 2; j++)
+                    std::cout<<crop_begin[i * 2 + j] << " ";
+                std::cout<<std::endl;
+            }
+
             rocalCopyToOutput(handle, mat_input.data, h * w * p);
 
             counter += inputBatchSize;