diff --git a/rocAL/rocAL/include/api/rocal_api_meta_data.h b/rocAL/rocAL/include/api/rocal_api_meta_data.h index dfe961acd9..061000f116 100644 --- a/rocAL/rocAL/include/api/rocal_api_meta_data.h +++ b/rocAL/rocAL/include/api/rocal_api_meta_data.h @@ -284,4 +284,23 @@ extern "C" void ROCAL_API_CALL rocalGetImageId(RocalContext p_context, int* buf) */ extern "C" void ROCAL_API_CALL rocalGetJointsDataPtr(RocalContext p_context, RocalJointsData** joints_data); +/*! \brief initialize the values required for ROI Random crop + * \ingroup group_rocal_meta_data + * \param [in] rocal_context rocal context + * \param [in] crop_shape_batch + * \param [in] roi_begin_batch + * \param [in] input_shape_batch + * \param [in] roi_end_batch + */ +extern "C" void ROCAL_API_CALL rocalROIRandomCrop(RocalContext p_context, RocalTensor p_input, int *crop_shape); + +/*! \brief get the ROI Random crop values + * \ingroup group_rocal_meta_data + * \param [in] rocal_context rocal context + * \param [in] crop_shape_batch + * \param [in] roi_begin_batch + * \param [in] input_shape_batch + * \param [in] roi_end_batch + */ +extern "C" RocalTensor ROCAL_API_CALL rocalGetROIRandomCropValues(RocalContext p_context); #endif // MIVISIONX_ROCAL_API_META_DATA_H diff --git a/rocAL/rocAL/include/pipeline/master_graph.h b/rocAL/rocAL/include/pipeline/master_graph.h index 98349ae86c..e9b827bb32 100644 --- a/rocAL/rocAL/include/pipeline/master_graph.h +++ b/rocAL/rocAL/include/pipeline/master_graph.h @@ -131,6 +131,9 @@ class MasterGraph { void set_sequence_batch_size(size_t sequence_length) { _sequence_batch_size = _user_batch_size * sequence_length; } std::vector get_bbox_encoded_buffers(size_t num_encoded_boxes); size_t bounding_box_batch_count(pMetaDataBatch meta_data_batch); + void roi_random_crop(Tensor *input, int *crop_shape); + void update_roi_random_crop(int *crop_shape_batch, int *roi_begin_batch, int *roi_end_batch); + Tensor * get_roi_random_crop_values() { return _roi_random_crop_tensor; } #if ENABLE_OPENCL cl_command_queue get_ocl_cmd_q() { return _device.resources()->cmd_queue; } #endif @@ -203,6 +206,12 @@ class MasterGraph { bool _offset; // Returns normalized offsets ((encoded_bboxes*scale - anchors*scale) - mean) / stds in EncodedBBoxes that use std and the mean and scale arguments if offset="True" std::vector _means, _stds; //_means: [x y w h] mean values for normalization _stds: [x y w h] standard deviations for offset normalization. bool _augmentation_metanode = false; + bool _is_roi_random_crop = false; + uint _input_dims = 1; + int *_crop_shape_batch = nullptr; + Tensor *_roi_random_crop_tensor = nullptr; + void *_roi_random_crop_buf = nullptr; + vx_tensor _roi_random_crop_vx_tensor = nullptr; #if ENABLE_HIP BoxEncoderGpu *_box_encoder_gpu = nullptr; #endif diff --git a/rocAL/rocAL/include/pipeline/tensor.h b/rocAL/rocAL/include/pipeline/tensor.h index dd7868e882..08486adfe3 100644 --- a/rocAL/rocAL/include/pipeline/tensor.h +++ b/rocAL/rocAL/include/pipeline/tensor.h @@ -279,6 +279,7 @@ class Tensor : public rocalTensor { // create_from_handle() no internal memory allocation is done here since // tensor's handle should be swapped with external buffers before usage int create_from_handle(vx_context context); + int create_from_handle_new(vx_context context, void *ptr); int create_virtual(vx_context context, vx_graph graph); bool is_handle_set() { return (_vx_handle != 0); } void set_dims(std::vector dims) { _info.set_dims(dims); } diff --git a/rocAL/rocAL/source/api/rocal_api_meta_data.cpp b/rocAL/rocAL/source/api/rocal_api_meta_data.cpp index 1fa4768a78..fe4e79ed95 100644 --- a/rocAL/rocAL/source/api/rocal_api_meta_data.cpp +++ b/rocAL/rocAL/source/api/rocal_api_meta_data.cpp @@ -465,3 +465,24 @@ void *joints_data = (RocalJointsData*)(&(meta_data.second->get_joints_data_batch())); } + +void + ROCAL_API_CALL + rocalROIRandomCrop(RocalContext p_context, RocalTensor p_input, int *crop_shape) { + if ((p_context == nullptr) || (p_input == nullptr)) { + ERR("Invalid ROCAL context or invalid input tensor") + } + auto context = static_cast(p_context); + auto input = static_cast(p_input); + context->master_graph->roi_random_crop(input, crop_shape); +} + +RocalTensor + ROCAL_API_CALL + rocalGetROIRandomCropValues(RocalContext p_context) { + if (p_context == nullptr) { + ERR("Invalid ROCAL context or invalid input tensor") + } + auto context = static_cast(p_context); + return context->master_graph->get_roi_random_crop_values(); +} diff --git a/rocAL/rocAL/source/pipeline/master_graph.cpp b/rocAL/rocAL/source/pipeline/master_graph.cpp index f361391222..dd73e1907e 100644 --- a/rocAL/rocAL/source/pipeline/master_graph.cpp +++ b/rocAL/rocAL/source/pipeline/master_graph.cpp @@ -928,6 +928,28 @@ void MasterGraph::output_routine() { _meta_data_graph->process(_augmented_meta_data, output_meta_data); } } + + if(_is_roi_random_crop) + { + // get the roi_begin and roi_end values from random_object_bbox + int *roi_begin_batch = new int[_user_batch_size * _input_dims]; + int *roi_end_batch = new int[_user_batch_size * _input_dims]; + for(uint i = 0; i < _user_batch_size; i++) + { + int sample_idx = i * _input_dims; + int *roi_begin = &roi_begin_batch[sample_idx]; + int *roi_end = &roi_end_batch[sample_idx]; + for(uint j = 0; j < _input_dims; j++) + { + roi_begin[j] = std::rand() % (_input_dims * 5); + roi_end[j] = roi_begin[j] + (std::rand() % (_input_dims * 5)); + } + } + update_roi_random_crop(_crop_shape_batch, roi_begin_batch, roi_end_batch); + delete[] roi_begin_batch; + delete[] roi_end_batch; + } + _process_time.start(); _graph->process(); _process_time.end(); @@ -1373,6 +1395,115 @@ TensorList *MasterGraph::mask_meta_data() { return &_mask_tensor_list; } +class BatchRNGUniform { + public: + /** + * @brief Used to keep batch of RNGs, so Operators can be immune to order of sample processing + * while using randomness + * + * @param seed Used to generate seed_seq to initialize batch of RNGs + * @param batch_size How many RNGs to store + * @param state_size How many seed are used to initialize one RNG. Used to lower probablity of + * collisions between seeds used to initialize RNGs in different operators. + */ + BatchRNGUniform(int64_t seed, int batch_size, int state_size = 4) + : seed_(seed) { + std::seed_seq seq{seed_}; + std::vector seeds(batch_size * state_size); + seq.generate(seeds.begin(), seeds.end()); + rngs_.reserve(batch_size); + for (int i = 0; i < batch_size * state_size; i += state_size) { + std::seed_seq s(seeds.begin() + i, seeds.begin() + i + state_size); + rngs_.emplace_back(s); + } + } + + + /** + * Returns engine corresponding to given sample ID + */ + std::mt19937 &operator[](int sample) noexcept { + return rngs_[sample]; + } + + private: + int64_t seed_; + std::vector rngs_; +}; + +void MasterGraph::roi_random_crop(Tensor *input, int *crop_shape) +{ + _is_roi_random_crop = true; + if(input->info().is_image()) + _input_dims = input->num_of_dims() - 2; // TO be changed later when generic roi changes are added + else + _input_dims = input->num_of_dims() - 1; + + _crop_shape_batch = new int[_input_dims * _user_batch_size]; // TODO handle this case later when different crop_shape is given for each tensor + + // replicate crop_shape values for all samples in a batch + for(uint i = 0; i < _user_batch_size; i++) + { + int sample_idx = i * _input_dims; + memcpy(&(_crop_shape_batch[sample_idx]), crop_shape, _input_dims * sizeof(int)); + } + + // create new instance of tensor class + std::vector dims = {_user_batch_size, _input_dims}; + auto info = TensorInfo(std::move(dims), RocalMemType::HOST, RocalTensorDataType::INT32); + _roi_random_crop_tensor = new Tensor(info); + + // allocate memory for the raw buffer pointer in tensor object + _roi_random_crop_buf = new int[_user_batch_size * _input_dims]; + _roi_random_crop_tensor->create_from_handle_new(_context, _roi_random_crop_buf); +} + +void MasterGraph::update_roi_random_crop(int *crop_shape_batch, int *roi_begin_batch, int *roi_end_batch) { + void *roi_random_crop_buf = _roi_random_crop_tensor->buffer(); + int *crop_begin_batch = static_cast(roi_random_crop_buf); + uint seed = std::time(0); + int *roi_batch = reinterpret_cast(_internal_tensor_list[0]->get_roi()); + + BatchRNGUniform _rng = {seed, static_cast(_user_batch_size)}; + for(uint i = 0; i < _user_batch_size; i++) { + int sample_idx = i * _input_dims; + int *crop_shape = &crop_shape_batch[sample_idx]; + int *roi_begin = &roi_begin_batch[sample_idx]; + int *input_shape = &roi_batch[sample_idx * 2 + _input_dims]; + int *roi_end = &roi_end_batch[sample_idx]; + int *crop_begin = &crop_begin_batch[sample_idx]; + + for(uint j = 0; j < _input_dims; j++) { + // check if crop_shape, roi_end is greater than input_shape + if(crop_shape[j] > input_shape[j]) + THROW("crop shape cannot be greater than input shape"); + if (roi_end[j] > input_shape[j]) + THROW("ROI shape cannot be greater than input shape"); + + int roi_length = roi_end[j] - roi_begin[j]; + int crop_length = crop_shape[j]; + if (roi_length == crop_length) { + crop_begin[j] = roi_begin[j]; + } else { + int64_t start_range[2] = {roi_begin[j], roi_end[j] - crop_length}; + + // swap range values if start_range[0] > start_range[1] + if (start_range[0] > start_range[1]) { + int64_t temp = start_range[0]; + start_range[0] = start_range[1]; + start_range[1] = temp; + } + + // check if range is within the bounds of input + start_range[0] = std::max(0, start_range[0]); + start_range[1] = std::min(input_shape[j] - crop_length, start_range[1]); + + auto dist = std::uniform_int_distribution(start_range[0], start_range[1]); + crop_begin[j] = dist(_rng[i]); + } + } + } +} void MasterGraph::notify_user_thread() { if (_output_routine_finished_processing) diff --git a/rocAL/rocAL/source/pipeline/tensor.cpp b/rocAL/rocAL/source/pipeline/tensor.cpp index 2cebfd0e00..5cf2fcb383 100644 --- a/rocAL/rocAL/source/pipeline/tensor.cpp +++ b/rocAL/rocAL/source/pipeline/tensor.cpp @@ -298,6 +298,33 @@ int Tensor::create_from_handle(vx_context context) { return 0; } +int Tensor::create_from_handle_new(vx_context context, void *ptr) { + if (_vx_handle) { + WRN("Tensor object create method is already called ") + return -1; + } + + _context = context; + vx_enum tensor_data_type = interpret_tensor_data_type(_info.data_type()); + unsigned num_of_dims = _info.num_of_dims(); + vx_size stride[num_of_dims]; + // void *ptr[1] = {nullptr}; + + stride[0] = tensor_data_size(_info.data_type()); + for (unsigned i = 1; i < num_of_dims; i++) + stride[i] = stride[i - 1] * _info.dims().at(i - 1); + + _vx_handle = vxCreateTensorFromHandle(_context, _info.num_of_dims(), _info.dims().data(), tensor_data_type, 0, stride, ptr, vx_mem_type(_info._mem_type)); + vx_status status; + if ((status = vxGetStatus((vx_reference)_vx_handle)) != VX_SUCCESS) + THROW("Error: vxCreateTensorFromHandle(input: failed " + TOSTR(status)) + _info._type = TensorInfo::Type::HANDLE; + _mem_handle = ptr; + // void *roi_handle = reinterpret_cast(_info.get_roi()); + // create_roi_tensor_from_handle(&roi_handle); // Create ROI tensor from handle + return 0; +} + int Tensor::create(vx_context context) { if (_vx_handle) { WRN("Tensor object create method is already called ") diff --git a/rocAL/rocAL_pybind/amd/rocal/fn.py b/rocAL/rocAL_pybind/amd/rocal/fn.py index a5b60b62cd..ee60174aad 100644 --- a/rocAL/rocAL_pybind/amd/rocal/fn.py +++ b/rocAL/rocAL_pybind/amd/rocal/fn.py @@ -1055,3 +1055,9 @@ def box_iou_matcher(*inputs, anchors, criteria=0.5, high_threshold=0.5, Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) Pipeline._current_pipeline._box_iou_matcher = True return (box_iou_matcher, []) + +def roi_random_crop(*inputs, crop_shape=None): + + # pybind call arguments + kwargs_pybind = {"crop_shape": crop_shape} + roi_random_crop = b.roiRandomCrop(Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) \ No newline at end of file diff --git a/rocAL/rocAL_pybind/amd/rocal/pipeline.py b/rocAL/rocAL_pybind/amd/rocal/pipeline.py index dca380e09c..156d01412f 100644 --- a/rocAL/rocAL_pybind/amd/rocal/pipeline.py +++ b/rocAL/rocAL_pybind/amd/rocal/pipeline.py @@ -394,3 +394,6 @@ def create_pipeline(*args, **kwargs): return create_pipeline return actual_decorator(fn) if fn else actual_decorator + +def get_roi_random_crop_values(self): + return b.rocalGetROIRandomCropValues(self._handle) diff --git a/rocAL/rocAL_pybind/rocal_pybind.cpp b/rocAL/rocAL_pybind/rocal_pybind.cpp index 3c83772adc..20ebd28b14 100644 --- a/rocAL/rocAL_pybind/rocal_pybind.cpp +++ b/rocAL/rocAL_pybind/rocal_pybind.cpp @@ -362,6 +362,8 @@ PYBIND11_MODULE(rocal_pybind, m) { int *ptr = static_cast(buf.ptr); rocalGetImageSizes(context, ptr); }); + m.def("roiRandomCrop", &rocalROIRandomCrop); + m.def("getROIRandomCropValues", &rocalGetROIRandomCropValues, py::return_value_policy::reference); // rocal_api_parameter.h m.def("setSeed", &rocalSetSeed); m.def("getSeed", &rocalGetSeed); diff --git a/utilities/rocAL/rocAL_basic_test/rocal_basic_test.cpp b/utilities/rocAL/rocAL_basic_test/rocal_basic_test.cpp index a7a91e7ba8..832c76ef9f 100644 --- a/utilities/rocAL/rocAL_basic_test/rocal_basic_test.cpp +++ b/utilities/rocAL/rocAL_basic_test/rocal_basic_test.cpp @@ -108,6 +108,10 @@ int main(int argc, const char **argv) { rocalCreateTextFileBasedLabelReader(handle, label_text_file_path); rocalCropResizeFixed(handle, decoded_output, 224, 224, true, 0.9, 1.1, 0.1, 0.1); + int crop_shape[] = {2, 2}; + rocalROIRandomCrop(handle, decoded_output, crop_shape); + RocalTensor crop_output; + if (rocalGetStatus(handle) != ROCAL_OK) { std::cout << "JPEG source could not initialize : " << rocalGetErrorMessage(handle) << std::endl; return -1; @@ -163,6 +167,16 @@ int main(int argc, const char **argv) { if (rocalRun(handle) != 0) break; + std::cout << "printing ROI Random crop outputs:" << std::endl; + crop_output = rocalGetROIRandomCropValues(handle); + int *crop_begin = static_cast(crop_output->buffer()); + for(int i = 0; i < inputBatchSize; i++) + { + for(int j = 0; j < 2; j++) + std::cout<