Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add semantic segmentation support #38

Open
wants to merge 6 commits into
base: lk_rocalTensor_vxfix
Choose a base branch
from
Open

Add semantic segmentation support #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0825a03
Add semantic segmentation support
shobana-mcw Jul 23, 2023
326e253
Add thirdparty lib
shobana-mcw Jul 24, 2023
330f685
Fix review comments
shobana-mcw Jul 26, 2023
120b6ae
Fix review comments
shobana-mcw Jul 30, 2023
b854c6c
code cleanup
fiona-gladwin Aug 3, 2023
a6a7253
revert the changes made to thirdparty code
fiona-gladwin Aug 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion rocAL/rocAL/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ if(${BUILD_ROCAL})
./include/readers/image/
./include/readers/video/
./include/pipeline/
./../../thirdparty/coco_RLE/include/
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this a cloned repo or just a single cpp file? If its a repo, better to add it as a dependency in Dockerfile and figure out how to import the RLE structs from there since they removed all 3rd party libs from rocAL and added them as dependencies

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure Sundar. We can discuss on it

include
)

Expand All @@ -200,10 +201,11 @@ if(${BUILD_ROCAL})
protobuf_generate_cpp(TF_PROTO_SRCS TF_PROTO_HEADERS proto/example.proto proto/feature.proto)
protobuf_generate_cpp(CAFFE2_PROTO_SRCS CAFFE2_PROTO_HEADERS proto/caffe2_protos.proto)
protobuf_generate_cpp(CAFFE_PROTO_SRCS CAFFE_PROTO_HEADERS proto/caffe_protos.proto)
file(GLOB_RECURSE PYCOCOTOOLS_SOURCES "../../thirdparty/coco_RLE/source/*.cpp")
link_directories(${AMDRPP_LIBRARIES_DIRS} ${TurboJpeg_LIBRARIES_DIR} ${PROTOBUF_LIBRARY_DIRS} /usr/local/lib/)

file(GLOB_RECURSE SOURCES "./source/*.cpp")
add_library(${PROJECT_NAME} SHARED ${SOURCES} ${TF_PROTO_SRCS} ${TF_PROTO_HEADERS} ${CAFFE_PROTO_HEADERS} ${CAFFE_PROTO_SRCS} ${CAFFE2_PROTO_SRCS} ${CAFFE2_PROTO_HEADERS})
add_library(${PROJECT_NAME} SHARED ${SOURCES} ${TF_PROTO_SRCS} ${TF_PROTO_HEADERS} ${PYCOCOTOOLS_SOURCES} ${CAFFE_PROTO_HEADERS} ${CAFFE_PROTO_SRCS} ${CAFFE2_PROTO_SRCS} ${CAFFE2_PROTO_HEADERS})

if("${BACKEND}" STREQUAL "HIP" AND HIP_FOUND)
add_dependencies(${PROJECT_NAME} rocAL_hip)
Expand Down
7 changes: 6 additions & 1 deletion rocAL/rocAL/include/api/rocal_api_meta_data.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ extern "C" RocalMetaData ROCAL_API_CALL rocalCreateTFReaderDetection(RocalContex
/// \param rocal_context
/// \param source_path path to the coco json file
/// \return RocalMetaData object, can be used to inquire about the rocal's output (processed) tensors
extern "C" RocalMetaData ROCAL_API_CALL rocalCreateCOCOReader(RocalContext rocal_context, const char* source_path, bool is_output, bool mask = false, bool ltrb = true, bool is_box_encoder = false);
extern "C" RocalMetaData ROCAL_API_CALL rocalCreateCOCOReader(RocalContext rocal_context, const char* source_path, bool is_output, bool is_polygon_mask = false, bool is_pixelwise_mask = false, bool ltrb = true, bool is_box_encoder = false);

///
/// \param rocal_context
Expand Down Expand Up @@ -143,6 +143,11 @@ extern "C" unsigned ROCAL_API_CALL rocalGetMaskCount(RocalContext p_context, int
/// \return The tensorlist with the mask coordinates
extern "C" RocalTensorList ROCAL_API_CALL rocalGetMaskCoordinates(RocalContext p_context, int* bufcount);

///
/// \param rocal_context
/// \return The tensorlist with the pixelwise coordinates
extern "C" RocalTensorList ROCAL_API_CALL rocalGetPixelwiseMaskLabels(RocalContext p_context);

///
/// \param rocal_context
/// \param buf The user's buffer that will be filled with bounding box label info for the images in the output batch. It needs to be of size returned by a call to the rocalGetBoundingBoxCount
Expand Down
16 changes: 14 additions & 2 deletions rocAL/rocAL/include/meta_data/coco_meta_data_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ THE SOFTWARE.
#include "meta_data.h"
#include "meta_data_reader.h"
#include "timing_debug.h"
#include "maskApi.h"

class COCOMetaDataReader: public MetaDataReader
{
Expand All @@ -37,7 +38,7 @@ class COCOMetaDataReader: public MetaDataReader
void release() override;
void print_map_contents();
bool set_timestamp_mode() override { return false; }

std::pair<uint32_t, uint32_t> get_max_size() override { return std::make_pair(_max_height, _max_width); }
const std::map<std::string, std::shared_ptr<MetaData>> & get_map_content() override { return _map_content;}
COCOMetaDataReader();
private:
Expand All @@ -47,11 +48,22 @@ class COCOMetaDataReader: public MetaDataReader
void add(std::string image_name, BoundingBoxCords bbox, Labels labels, ImgSize image_size, uint image_id = 0);
void add(std::string image_name, BoundingBoxCords bbox, Labels labels, ImgSize image_size, MaskCords mask_cords, std::vector<int> polygon_count, std::vector<std::vector<int>> vertices_count); // To add Mask coordinates to Metadata struct
bool exists(const std::string &image_name) override;
void generate_pixelwise_mask(std::string filename, RLE* rle_in);
std::map<std::string, std::shared_ptr<MetaData>> _map_content;
std::map<std::string, std::shared_ptr<MetaData>>::iterator _itr;
std::map<std::string, ImgSize> _map_img_sizes;
std::map<std::string, ImgSize> ::iterator itr;
std::map<int, int> _label_info;
std::map<int, int> _label_info = {{1,1},{2,2},{3,3},{4,4},{5,5},{6,6},{7,7},{8,8}, \
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hardcoding the contiguous label mapping for coco is not preferred. Use a for loop to generate the mapping in meta_data_reader.cpp where its needed

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We already have this support right @SundarRajan28 ?

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This label generation can't be done on-fly like mask in semantic and it needs this information while storing RLE value

{9,9},{10,10},{11,11},{13,12},{14,13},{15,14},{16,15},{17,16}, \
{18,17},{19,18},{20,19},{21,20},{22,21},{23,22},{24,23},{25,24},{27,25},{28,26}, \
{31,27},{32,28},{33,29},{34,30},{35,31},{36,32},{37,33},{38,34},{39,35},{40,36}, \
{41,37},{42,38},{43,39},{44,40},{46,41},{47,42},{48,43},{49,44},{50,45},{51,46}, \
{52,47},{53,48},{54,49},{55,50},{56,51},{57,52},{58,53},{59,54},{60,55},{61,56}, \
{62,57},{63,58},{64,59},{65,60},{67,61},{70,62},{72,63},{73,64},{74,65},{75,66}, \
{76,67},{77,68},{78,69},{79,70},{80,71},{81,72},{82,73},{84,74},{85,75},{86,76}, \
{87,77},{88,78},{89,79},{90,80}};
uint32_t _max_width = 0;
uint32_t _max_height = 0;
std::map<int, int> ::iterator _it_label;
TimingDBG _coco_metadata_read_time;
};
Expand Down
108 changes: 108 additions & 0 deletions rocAL/rocAL/include/meta_data/meta_data.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ enum class MetaDataType
Label,
BoundingBox,
PolygonMask,
PixelwiseMask,
KeyPoints
};

Expand Down Expand Up @@ -117,6 +118,8 @@ class MetaData
virtual void set_mask_cords(MaskCords mask_cords) = 0;
virtual void set_polygon_counts(std::vector<int> polygon_count) = 0;
virtual void set_vertices_counts(std::vector<std::vector<int>> vertices_count) = 0;
virtual std::vector<int>& get_pixelwise_label() = 0;
virtual void set_pixelwise_label(std::vector<int>& pixelwise_label) = 0;
virtual JointsData& get_joints_data() = 0;
virtual void set_joints_data(JointsData *joints_data) = 0;
ImgSize& get_img_size() { return _info.img_size; }
Expand Down Expand Up @@ -145,6 +148,8 @@ class Label : public MetaData
void set_mask_cords(MaskCords mask_cords) override { THROW("Not Implemented") }
void set_polygon_counts(std::vector<int> polygon_count) override { THROW("Not Implemented") }
void set_vertices_counts(std::vector<std::vector<int>> vertices_count) override { THROW("Not Implemented") }
std::vector<int>& get_pixelwise_label() override { THROW("Not Implemented") };
void set_pixelwise_label(std::vector<int>& pixelwise_label) override { THROW("Not Implemented") }
JointsData& get_joints_data() override { THROW("Not Implemented") }
void set_joints_data(JointsData *joints_data) override { THROW("Not Implemented") }
protected:
Expand All @@ -170,6 +175,7 @@ class BoundingBox : public Label

struct PolygonMask : public BoundingBox {
public:
PolygonMask() = default;
PolygonMask(BoundingBoxCords bb_cords, Labels bb_label_ids, ImgSize img_size, MaskCords mask_cords, std::vector<int> polygon_count, std::vector<std::vector<int>> vertices_count)
{
_bb_cords = std::move(bb_cords);
Expand All @@ -191,6 +197,24 @@ struct PolygonMask : public BoundingBox {
std::vector<std::vector<int>> _vertices_count = {};
};

struct PixelwiseMask : public PolygonMask {
public:
PixelwiseMask() = default;
PixelwiseMask(BoundingBoxCords bb_cords, Labels bb_label_ids, ImgSize img_size, MaskCords mask_cords, std::vector<int> polygon_count, std::vector<std::vector<int>> vertices_count)
{
_bb_cords = std::move(bb_cords);
_label_ids = std::move(bb_label_ids);
_info.img_size = std::move(img_size);
_mask_cords = std::move(mask_cords);
_polygon_count = std::move(polygon_count);
_vertices_count = std::move(vertices_count);
}
std::vector<int>& get_pixelwise_label() override { return _pixelwise_label; }
void set_pixelwise_label(std::vector<int>& pixelwise_label) override { _pixelwise_label = std::move(pixelwise_label); }
protected:
std::vector<int> _pixelwise_label = {};
};

class KeyPoint : public BoundingBox
{
public:
Expand Down Expand Up @@ -251,6 +275,7 @@ class MetaDataBatch
virtual std::vector<MaskCords>& get_mask_cords_batch() = 0;
virtual std::vector<std::vector<int>>& get_mask_polygons_count_batch() = 0;
virtual std::vector<std::vector<std::vector<int>>>& get_mask_vertices_count_batch() = 0;
virtual std::vector<std::vector<int>>& get_pixelwise_labels_batch() = 0;
virtual JointsDataBatch & get_joints_data_batch() = 0;
std::vector<uint>& get_image_id_batch() { return _info_batch.img_ids; }
std::vector<std::string>& get_image_names_batch() {return _info_batch.img_names; }
Expand Down Expand Up @@ -327,6 +352,7 @@ class LabelBatch : public MetaDataBatch
std::vector<MaskCords>& get_mask_cords_batch() override { THROW("Not Implemented") }
std::vector<std::vector<int>>& get_mask_polygons_count_batch() override { THROW("Not Implemented") }
std::vector<std::vector<std::vector<int>>>& get_mask_vertices_count_batch() override { THROW("Not Implemented") }
std::vector<std::vector<int>>& get_pixelwise_labels_batch() override { THROW("Not Implemented") };
JointsDataBatch & get_joints_data_batch() override { THROW("Not Implemented") }
protected:
std::vector<Labels> _label_ids = {};
Expand Down Expand Up @@ -487,6 +513,87 @@ struct PolygonMaskBatch: public BoundingBoxBatch {
std::vector<std::vector<std::vector<int>>> _vertices_counts = {};
};

class PixelwiseMaskBatch : public PolygonMaskBatch {
public:
void clear() override
{
_bb_cords.clear();
_label_ids.clear();
_info_batch.clear();
_mask_cords.clear();
_polygon_counts.clear();
_vertices_counts.clear();
_buffer_size.clear();
_pixelwise_labels.clear();
}
MetaDataBatch& operator += (MetaDataBatch& other) override
{
_bb_cords.insert(_bb_cords.end(), other.get_bb_cords_batch().begin(), other.get_bb_cords_batch().end());
_label_ids.insert(_label_ids.end(), other.get_labels_batch().begin(), other.get_labels_batch().end());
_info_batch.insert(other.get_info_batch());
_mask_cords.insert(_mask_cords.end(), other.get_mask_cords_batch().begin(), other.get_mask_cords_batch().end());
_polygon_counts.insert(_polygon_counts.end(), other.get_mask_polygons_count_batch().begin(), other.get_mask_polygons_count_batch().end());
_vertices_counts.insert(_vertices_counts.end(), other.get_mask_vertices_count_batch().begin(), other.get_mask_vertices_count_batch().end());
_pixelwise_labels.insert(_pixelwise_labels.end(), other.get_pixelwise_labels_batch().begin(), other.get_pixelwise_labels_batch().end());
return *this;
}
void resize(int batch_size) override
{
_bb_cords.resize(batch_size);
_label_ids.resize(batch_size);
_info_batch.resize(batch_size);
_mask_cords.resize(batch_size);
_polygon_counts.resize(batch_size);
_vertices_counts.resize(batch_size);
_pixelwise_labels.resize(batch_size);
}
std::vector<std::vector<int>>& get_pixelwise_labels_batch() override { return _pixelwise_labels; }
std::shared_ptr<MetaDataBatch> clone(bool copy_contents) override
{
if(copy_contents) {
return std::make_shared<PixelwiseMaskBatch>(*this);
} else {
std::shared_ptr<MetaDataBatch> mask_batch_instance = std::make_shared<PixelwiseMaskBatch>();
mask_batch_instance->resize(this->size());
mask_batch_instance->get_info_batch() = this->get_info_batch();
return mask_batch_instance;
}
}
void copy_data(std::vector<void*> buffer) override
{
if(buffer.size() < 2)
THROW("The buffers are insufficient") // TODO -change
int *labels_buffer = (int *)buffer[0];
float *bbox_buffer = (float *)buffer[1];
int *mask_buffer = (int *)buffer[2];
for(unsigned i = 0; i < (unsigned int)_label_ids.size(); i++)
{
mempcpy(labels_buffer, _label_ids[i].data(), _label_ids[i].size() * sizeof(int));
if(_bbox_output_type == BoundingBoxType::XYWH) convert_ltrb_to_xywh(_bb_cords[i]);
memcpy(bbox_buffer, _bb_cords[i].data(), _label_ids[i].size() * 4 * sizeof(float));
memcpy(mask_buffer, _pixelwise_labels[i].data(), _pixelwise_labels[i].size() * sizeof(int));
labels_buffer += _label_ids[i].size();
bbox_buffer += (_label_ids[i].size() * 4);
mask_buffer += _pixelwise_labels[i].size();
}
}
std::vector<size_t>& get_buffer_size() override
{
size_t size = 0;
for (auto label : _label_ids)
size += label.size();
_buffer_size.emplace_back(size * sizeof(int));
_buffer_size.emplace_back(size * 4 * sizeof(float));
size = 0;
for (auto mask : _pixelwise_labels)
size += mask.size();
_buffer_size.emplace_back(size * sizeof(int));
return _buffer_size;
}
protected:
std::vector<std::vector<int>> _pixelwise_labels;
};

class KeyPointBatch : public BoundingBoxBatch
{
public:
Expand Down Expand Up @@ -546,5 +653,6 @@ using ImageNameBatch = std::vector<std::string>;
using pMetaData = std::shared_ptr<Label>;
using pMetaDataBox = std::shared_ptr<BoundingBox>;
using pMetaDataPolygonMask = std::shared_ptr<PolygonMask>;
using pMetaDataPixelwiseMask = std::shared_ptr<PixelwiseMask>;
using pMetaDataKeyPoint = std::shared_ptr<KeyPoint>;
using pMetaDataBatch = std::shared_ptr<MetaDataBatch>;
1 change: 1 addition & 0 deletions rocAL/rocAL/include/meta_data/meta_data_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,5 +92,6 @@ class MetaDataReader
virtual const std::map<std::string, std::shared_ptr<MetaData>> & get_map_content()=0;
virtual bool exists(const std::string &image_name) = 0;
virtual bool set_timestamp_mode() = 0;
virtual std::pair<uint32_t, uint32_t> get_max_size() { return std::pair<uint32_t, uint32_t>(); }
};

2 changes: 1 addition & 1 deletion rocAL/rocAL/include/pipeline/master_graph.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ class MasterGraph
const std::pair<ImageNameBatch,pMetaDataBatch>& meta_data();
TensorList * labels_meta_data();
TensorList * bbox_meta_data();
TensorList * mask_meta_data();
TensorList * mask_meta_data(bool is_polygon_mask);
void set_loop(bool val) { _loop = val; }
void set_output(Tensor* output_tensor);
size_t calculate_cpu_num_threads(size_t shard_count);
Expand Down
19 changes: 16 additions & 3 deletions rocAL/rocAL/source/api/rocal_api_meta_data.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,16 @@ ROCAL_API_CALL rocalCreateVideoLabelReader(RocalContext p_context, const char* s
}

RocalMetaData
ROCAL_API_CALL rocalCreateCOCOReader(RocalContext p_context, const char* source_path, bool is_output, bool mask, bool ltrb, bool is_box_encoder) {
ROCAL_API_CALL rocalCreateCOCOReader(RocalContext p_context, const char* source_path, bool is_output, bool is_polygon_mask, bool is_pixelwise_mask, bool ltrb, bool is_box_encoder) {
if (!p_context)
THROW("Invalid rocal context passed to rocalCreateCOCOReader")
if (is_polygon_mask && is_pixelwise_mask)
THROW("PixelwiseMask and PolygonMask are mutually exclusive")
auto context = static_cast<Context*>(p_context);
if(mask) {
if(is_polygon_mask) {
return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_META_DATA_READER, MetaDataType::PolygonMask, ltrb, is_box_encoder);
} else if (is_pixelwise_mask) {
return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_META_DATA_READER, MetaDataType::PixelwiseMask, ltrb, is_box_encoder);
}
return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_META_DATA_READER, MetaDataType::BoundingBox, ltrb, is_box_encoder);
}
Expand Down Expand Up @@ -362,7 +366,16 @@ ROCAL_API_CALL rocalGetMaskCoordinates(RocalContext p_context, int *bufcount)
}
}
}
return context->master_graph->mask_meta_data();
return context->master_graph->mask_meta_data(true);
}

RocalTensorList
ROCAL_API_CALL rocalGetPixelwiseMaskLabels(RocalContext p_context)
{
if (p_context == nullptr)
THROW("Invalid rocal context passed to rocalGetPixelwiseMaskLabels")
auto context = static_cast<Context *>(p_context);
return context->master_graph->mask_meta_data(false);
}

void
Expand Down
Loading