From 9f4e022db5766971b30a1dc938f542f5fcbd0d94 Mon Sep 17 00:00:00 2001 From: Pushkar-Bhuse Date: Thu, 30 Jun 2022 12:30:40 -0700 Subject: [PATCH 1/3] Image Payload changes --- forte/common/constants.py | 3 + forte/data/ontology/top.py | 69 ++++++-- tests/forte/data/data_store_test.py | 208 +++++++++++++++++----- tests/forte/image_annotation_test.py | 248 ++++++++++++++++++++++++++- 4 files changed, 470 insertions(+), 58 deletions(-) diff --git a/forte/common/constants.py b/forte/common/constants.py index db28078ff..d7762379d 100644 --- a/forte/common/constants.py +++ b/forte/common/constants.py @@ -11,6 +11,9 @@ # The index storing entry type in the internal entry data of DataStore. ENTRY_TYPE_INDEX = 3 +# The index storing the payload ID in internal entry data of DataStore +PAYLOAD_INDEX = 0 + # The index storing entry type (specific to Link and Group type). It is saved # in the `tid_idx_dict` in DataStore. ENTRY_DICT_TYPE_INDEX = 0 diff --git a/forte/data/ontology/top.py b/forte/data/ontology/top.py index 39688b906..b49aaf1e8 100644 --- a/forte/data/ontology/top.py +++ b/forte/data/ontology/top.py @@ -43,6 +43,7 @@ PARENT_TID_INDEX, CHILD_TID_INDEX, MEMBER_TID_INDEX, + PAYLOAD_INDEX, ) __all__ = [ @@ -72,8 +73,15 @@ make sure it available across the ontology system: 1. Create a new top level class that inherits from `Entry` or `MultiEntry` 2. Add the new class to `SinglePackEntries` or `MultiPackEntries` - 3. Register a new method in `DataStore`: `add__raw()` - 4. Insert a new conditional branch in `EntryConverter.save_entry_object()` + 3. Insert a new conditional branch in `EntryConverter.save_entry_object()` + 4. Decide two main attributes which will qualify as your `attribute_data` + parameters. These parameters will be passes in your branch of + `EntryConverter.save_entry_object()`. If there are no such parameters, + you can pass None + 5. add `getter` and `setter` functions to update `attribute_data` parameters + if you have any + 6. If additional attributes are required, make the class a `dataclass` and set + `dataclass` attributes. """ @@ -879,7 +887,9 @@ def image(self): "Cannot get image because image annotation is not " "attached to any data pack." ) - return self.pack.get_image_array(self._image_payload_idx) + return self.pack.get_payload_data_at( + Modality.Image, self._image_payload_idx + ) @property def max_x(self): @@ -1052,12 +1062,35 @@ def __init__(self, pack: PackType, image_payload_idx: int = 0): else: self._image_payload_idx = image_payload_idx + @property + def image_payload_idx(self): + r"""Getter function of ``image_payload_idx``. The function will first try to + retrieve the image_payload_idx index from ``DataStore`` in ``self.pack``. If + this attempt fails, it will directly return the value in ``_image_payload_idx``. + """ + try: + self._image_payload_idx = self.pack.get_entry_raw(self.tid)[ + PAYLOAD_INDEX + ] + except KeyError: + pass + return self._image_payload_idx + + @image_payload_idx.setter + def image_payload_idx(self, val: int): + r"""Setter function of ``image_payload_idx``. The update will also be populated + into ``DataStore`` in ``self.pack``. + """ + self._image_payload_idx = val + self.pack.get_entry_raw(self.tid)[PAYLOAD_INDEX] = val + def compute_iou(self, other) -> int: intersection = np.sum(np.logical_and(self.image, other.image)) union = np.sum(np.logical_or(self.image, other.image)) return intersection / union +@dataclass class Box(Region): """ A box class with a center position and a box configuration. @@ -1078,13 +1111,18 @@ class Box(Region): width: the width of the box, the unit is one image array entry. """ + _cy: int + _cx: int + _height: int + _width: int + def __init__( self, pack: PackType, - cy: int, - cx: int, - height: int, - width: int, + cy: int = 0, + cx: int = 0, + height: int = 1, + width: int = 1, image_payload_idx: int = 0, ): # assume Box is associated with Grids @@ -1180,6 +1218,7 @@ def compute_iou(self, other): return intersection / union +@dataclass class BoundingBox(Box): """ A bounding box class that associates with image payload and grids and @@ -1208,15 +1247,17 @@ class BoundingBox(Box): """ + _grid_id: int + def __init__( self, pack: PackType, - height: int, - width: int, - grid_height: int, - grid_width: int, - grid_cell_h_idx: int, - grid_cell_w_idx: int, + height: int = 1, + width: int = 1, + grid_height: int = 1, + grid_width: int = 1, + grid_cell_h_idx: int = 0, + grid_cell_w_idx: int = 0, image_payload_idx: int = 0, ): self.grids = Grids(pack, grid_height, grid_width, image_payload_idx) @@ -1228,6 +1269,8 @@ def __init__( image_payload_idx, ) + self._grid_id = self.grids.tid + class Payload(Entry): """ diff --git a/tests/forte/data/data_store_test.py b/tests/forte/data/data_store_test.py index 6dde1c92a..ed07e237d 100644 --- a/tests/forte/data/data_store_test.py +++ b/tests/forte/data/data_store_test.py @@ -27,6 +27,7 @@ Annotation, Generics, AudioAnnotation, + ImageAnnotation, Group, Link, MultiPackGeneric, @@ -142,42 +143,51 @@ def setUp(self) -> None: }, "parent_class": set(), }, - } - - self.base_type_attributes = { - 'forte.data.ontology.top.Generics': {'parent_class': {'Entry'}}, - 'forte.data.ontology.top.Annotation': {'parent_class': {'Entry'}}, - 'forte.data.ontology.top.Link': {'parent_class': {'BaseLink'}}, - 'forte.data.ontology.top.Group': {'parent_class': {'Entry', 'BaseGroup'}}, - 'forte.data.ontology.top.MultiPackGeneric': {'parent_class': {'Entry', 'MultiEntry'}}, - 'forte.data.ontology.top.MultiPackLink': {'parent_class': {'MultiEntry', 'BaseLink'}}, - 'forte.data.ontology.top.MultiPackGroup': {'parent_class': {'Entry', 'MultiEntry', 'BaseGroup'}}, - 'forte.data.ontology.top.Query': {'parent_class': {'Generics'}}, - 'forte.data.ontology.top.AudioAnnotation': {'parent_class': {'Entry'}} - } - - DataStore._type_attributes["ft.onto.base_ontology.Document"] = { + "forte.data.ontology.top.BoundingBox": { "attributes": { - "document_class": 4, - "sentiment": 5, - "classifications": 6, + "_cy": 4, + "_cx": 5, + "_height": 6, + "_width": 7, + "_grid_id": 8, }, "parent_class": set(), - } + }, + } + + DataStore._type_attributes["ft.onto.base_ontology.Document"] = { + "attributes": { + "document_class": 4, + "sentiment": 5, + "classifications": 6, + }, + "parent_class": set(), + } DataStore._type_attributes["ft.onto.base_ontology.Sentence"] = { - "attributes": { - "speaker": 4, - "part_id": 5, - "sentiment": 6, - "classification": 7, - "classifications": 8, - }, - "parent_class": set(), - } - # The order is [Document, Sentence]. Initialize 2 entries in each list. + "attributes": { + "speaker": 4, + "part_id": 5, + "sentiment": 6, + "classification": 7, + "classifications": 8, + }, + "parent_class": set(), + } + DataStore._type_attributes["forte.data.ontology.top.BoundingBox"] = { + "attributes": { + "_cy": 4, + "_cx": 5, + "_height": 6, + "_width": 7, + "_grid_id": 8, + }, + "parent_class": set(), + } + # The order is [Document, Sentence, BoundingBox]. Initialize 2 entries in each list. # Document entries have tid 1234, 3456. # Sentence entries have tid 9999, 1234567. + # BoundingBox entries have tid 1212, 3434. # The type id for Document is 0, Sentence is 1. ref1 = [ @@ -226,6 +236,31 @@ def setUp(self) -> None: 7654, "forte.data.ontology.top.Annotation", ] + ref6 = [ + 1, + None, + 1212, + "forte.data.ontology.top.BoundingBox", + 3, + 5, + 2, + 2, + 1000, + ] + ref7 = [ + 2, + None, + 3434, + "forte.data.ontology.top.BoundingBox", + 3, + 5, + 1, + 3, + 2000, + ] + ref8 = [0, None, 1000, "forte.data.ontology.top.Grids"] + ref9 = [0, None, 2000, "forte.data.ontology.top.Grids"] + ref10 = [0, None, 8888, "forte.data.ontology.top.ImageAnnotation"] sorting_fn = lambda s: ( s[constants.BEGIN_INDEX], @@ -271,6 +306,9 @@ def setUp(self) -> None: "forte.data.ontology.top.Link", ], ], + "forte.data.ontology.top.BoundingBox": [ref6, ref7], + "forte.data.ontology.top.Grids": [ref8, ref9], + "forte.data.ontology.top.ImageAnnotation": [ref10], } self.data_store._DataStore__tid_ref_dict = { 1234: ref1, @@ -284,6 +322,11 @@ def setUp(self) -> None: 23456: ["forte.data.ontology.top.Group", 1], 34567: ["forte.data.ontology.top.Group", 2], 88888: ["forte.data.ontology.top.Link", 0], + 1212: ["forte.data.ontology.top.BoundingBox", 0], + 3434: ["forte.data.ontology.top.BoundingBox", 1], + 1000: ["forte.data.ontology.top.Grids", 0], + 2000: ["forte.data.ontology.top.Grids", 1], + 8888: ["forte.data.ontology.top.ImageAnnotation", 0], } def test_get_type_info(self): @@ -304,6 +347,14 @@ def test_get_type_info(self): DataStore._type_attributes["ft.onto.base_ontology.Document"], self.reference_type_attributes["ft.onto.base_ontology.Document"], ) + + empty_data_store._get_type_info("forte.data.ontology.top.BoundingBox") + self.assertEqual( + DataStore._type_attributes["forte.data.ontology.top.BoundingBox"], + self.reference_type_attributes[ + "forte.data.ontology.top.BoundingBox" + ], + ) # test the return value self.assertEqual( doc_attr_dict, @@ -321,7 +372,7 @@ def test_get_type_info(self): doc_attr_dict = empty_data_store._get_type_info( "ft.onto.base_ontology.Document" ) - self.assertEqual(len(DataStore._type_attributes), 2) + self.assertEqual(len(DataStore._type_attributes), 3) self.assertEqual( doc_attr_dict, DataStore._type_attributes["ft.onto.base_ontology.Document"], @@ -344,8 +395,11 @@ def test_entry_methods(self): doc_type = "ft.onto.base_ontology.Document" ann_type = "forte.data.ontology.top.Annotation" group_type = "forte.data.ontology.top.Group" + box_type = "forte.data.ontology.top.BoundingBox" + sent_list = list(self.data_store._DataStore__elements[sent_type]) doc_list = list(self.data_store._DataStore__elements[doc_type]) + box_list = list(self.data_store._DataStore__elements[box_type]) ann_list = list( self.data_store.co_iterator_annotation_like( list(self.data_store._get_all_subclass(ann_type, True)) @@ -356,13 +410,16 @@ def test_entry_methods(self): sent_entries = list(self.data_store.all_entries(sent_type)) doc_entries = list(self.data_store.all_entries(doc_type)) ann_entries = list(self.data_store.all_entries(ann_type)) + box_entries = list(self.data_store.all_entries(box_type)) self.assertEqual(sent_list, sent_entries) self.assertEqual(doc_list, doc_entries) self.assertEqual(ann_list, ann_entries) + self.assertEqual(box_list, box_entries) self.assertEqual(self.data_store.num_entries(sent_type), len(sent_list)) self.assertEqual(self.data_store.num_entries(doc_type), len(doc_list)) + self.assertEqual(self.data_store.num_entries(box_type), len(box_list)) self.assertEqual( self.data_store.num_entries(ann_type), len(ann_entries) ) @@ -397,6 +454,12 @@ def test_entry_methods(self): num_group_entries = self.data_store.num_entries(group_type) self.assertEqual(num_group_entries, len(group_list) - 1) + # remove a boundingbox + self.data_store.delete_entry(1212) + self.assertEqual( + self.data_store.num_entries(box_type), len(box_list) - 1 + ) + def test_co_iterator_annotation_like(self): type_names = [ "ft.onto.base_ontology.Sentence", @@ -737,6 +800,63 @@ def test_add_audio_annotation_raw(self): [0, 1, tid, "ft.onto.base_ontology.Recording", []], ) + def test_add_image_annotation_raw(self): + tid_bounding_box: int = self.data_store.add_entry_raw( + type_name="forte.data.ontology.top.BoundingBox", + attribute_data=[25, None], + base_class=ImageAnnotation, + ) + + tid_box: int = self.data_store.add_entry_raw( + type_name="forte.data.ontology.top.Box", + attribute_data=[30, None], + base_class=ImageAnnotation, + ) + + tid_region: int = self.data_store.add_entry_raw( + type_name="forte.data.ontology.top.Region", + attribute_data=[35, None], + base_class=ImageAnnotation, + ) + self.assertEqual( + len( + self.data_store._DataStore__elements[ + "forte.data.ontology.top.Box" + ] + ), + 1, + ) + self.assertEqual( + len( + self.data_store._DataStore__elements[ + "forte.data.ontology.top.Region" + ] + ), + 1, + ) + + tid = 88 + self.data_store.add_entry_raw( + type_name="forte.data.ontology.top.BoundingBox", + attribute_data=[45, None], + base_class=ImageAnnotation, + tid=tid, + ) + self.assertEqual( + self.data_store.get_entry(tid=88)[0], + [ + 45, + None, + tid, + "forte.data.ontology.top.BoundingBox", + None, + None, + None, + None, + None, + ], + ) + def test_add_generics_raw(self): # test add Document entry tid_generics: int = self.data_store.add_entry_raw( @@ -910,6 +1030,7 @@ def test_set_attribute(self): # set attribute with originally none value self.data_store.set_attribute(1234, "document_class", "Class D") speaker = self.data_store.get_attribute(9999, "speaker") + doc_class = self.data_store.get_attribute(1234, "document_class") self.assertEqual(speaker, "student") @@ -1031,7 +1152,7 @@ def test_delete_entry(self): # delete group self.data_store.delete_entry(10123) - self.assertEqual(len(self.data_store._DataStore__tid_idx_dict), 3) + self.assertEqual(len(self.data_store._DataStore__tid_idx_dict), 8) self.data_store.delete_entry(23456) self.data_store.delete_entry(34567) self.assertTrue( @@ -1054,7 +1175,7 @@ def test_delete_entry(self): def test_delete_entry_nonexist(self): # Entry tid does not exist; should raise a KeyError with self.assertRaises(KeyError): - self.data_store.delete_entry(1000) + self.data_store.delete_entry(5000) def test_delete_entry_by_loc(self): self.data_store._delete_entry_by_loc( @@ -1276,19 +1397,28 @@ def test_check_onto_file(self): "umls_link": 5, }, "parent_class": {"ft.onto.test.EntityMention"}, - } + }, } - data_store_from_file = DataStore(onto_file_path=os.path.abspath(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "ontology/test_specs/test_check_onto_file.json" - ))) + data_store_from_file = DataStore( + onto_file_path=os.path.abspath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "ontology/test_specs/test_check_onto_file.json", + ) + ) + ) # Check whether `_type_attributes` contains all items in `expected_type_attributes` - union_dict: Dict = dict(data_store_from_file._type_attributes, **expected_type_attributes) + union_dict: Dict = dict( + data_store_from_file._type_attributes, **expected_type_attributes + ) self.assertDictEqual(data_store_from_file._type_attributes, union_dict) # DataStores share a static type_attribute dict. data_store_non_file = DataStore() - self.assertDictEqual(data_store_non_file._type_attributes, data_store_from_file._type_attributes) + self.assertDictEqual( + data_store_non_file._type_attributes, + data_store_from_file._type_attributes, + ) def test_entry_conversion(self): data_pack = DataPack() diff --git a/tests/forte/image_annotation_test.py b/tests/forte/image_annotation_test.py index 35f523c80..5305f75ea 100644 --- a/tests/forte/image_annotation_test.py +++ b/tests/forte/image_annotation_test.py @@ -19,12 +19,14 @@ import numpy as np from numpy import array_equal -from forte.data.ontology.top import ImageAnnotation - +from forte.common.exception import ProcessExecutionException from ft.onto.base_ontology import ImagePayload - from forte.data.data_pack import DataPack import unittest +from sortedcontainers import SortedList +from forte.data.ontology.top import ImageAnnotation, BoundingBox, Box, Region +from forte.data.data_pack import DataPack +from forte.common import constants class ImageAnnotationTest(unittest.TestCase): @@ -38,15 +40,141 @@ def setUp(self): self.line[2, 2] = 1 self.line[3, 3] = 1 self.line[4, 4] = 1 - ip = ImagePayload(self.datapack, 0) - ip.set_cache(self.line) + ip1 = ImagePayload(self.datapack, 0) + ip1.set_cache(self.line) ImageAnnotation(self.datapack) - def test_image_annotation(self): + self.datapack_1 = DataPack("image_1") + self.line1 = np.zeros((6, 12)) + self.line1[2, 2] = 1 + self.line1[3, 3] = 1 + self.line1[4, 4] = 1 + ip2 = ImagePayload(self.datapack_1, 0) + ip2.set_cache(self.line1) + ImageAnnotation(self.datapack_1) + + self.mark = np.zeros((6, 6)) + self.mark[3, 3:5] = 1 + self.mark[4, 4:5] = 1 + ip2 = ImagePayload(self.datapack_1, 1) + ip2.set_cache(self.mark) + ImageAnnotation(self.datapack_1) + + self.datapack_2 = DataPack("image_2") + self.chunk = np.zeros((12, 12)) + self.chunk[2, :] = 1 + self.chunk[3, :] = 1 + self.chunk[4, :] = 1 + ip3 = ImagePayload(self.datapack_2, 0) + ip3.set_cache(self.chunk) + ImageAnnotation(self.datapack_2) + + # Entries for first Data Pack + self.bb1 = BoundingBox( + pack=self.datapack_1, + height=2, + width=2, + grid_height=3, + grid_width=4, + grid_cell_h_idx=1, + grid_cell_w_idx=1, + ) + self.datapack_1.add_entry(self.bb1) + + self.bb2 = BoundingBox( + pack=self.datapack_1, + height=3, + width=4, + grid_height=5, + grid_width=5, + grid_cell_h_idx=3, + grid_cell_w_idx=3, + ) + self.datapack_1.add_entry(self.bb2) + + # Entries for another image in the same Data Pack + self.bb4 = BoundingBox( + pack=self.datapack_1, + height=3, + width=4, + grid_height=6, + grid_width=6, + grid_cell_h_idx=2, + grid_cell_w_idx=2, + image_payload_idx=1, + ) + self.datapack_1.add_entry(self.bb4) + + # Entries for second Data Pack + self.bb3 = BoundingBox( + pack=self.datapack_2, + height=5, + width=5, + grid_height=3, + grid_width=3, + grid_cell_h_idx=6, + grid_cell_w_idx=6, + image_payload_idx=0, + ) + self.datapack_2.add_entry(self.bb3) + + self.box1 = Box( + pack=self.datapack_2, + cy=7, + cx=7, + height=4, + width=2, + image_payload_idx=0, + ) + self.datapack_2.add_entry(self.box1) + + self.region1 = Region(pack=self.datapack_2, image_payload_idx=1) + self.datapack_2.add_entry(self.region1) + + def test_entry_methods(self): + bb_type = "forte.data.ontology.top.BoundingBox" + region_type = "forte.data.ontology.top.Region" + box_type = "forte.data.ontology.top.Box" + + # Analyzing entries in first Data Pack + bb_list = list( + self.datapack_1._data_store._DataStore__elements[bb_type] + ) + bb_entries = list(self.datapack_1._data_store.all_entries(bb_type)) + + img1_box_list = self.datapack_1.get_payload_data_at( + modality=Modality.Image, payload_index=0 + ) + + self.assertEqual(bb_list, bb_entries) + self.assertEqual( + self.datapack_1._data_store.num_entries(bb_type), len(bb_list) + ) + self.assertEqual( self.datapack.get_single(ImageAnnotation).image_payload_idx, 0 ) + self.assertEqual( + len(img1_box_list), 6 + ) # For each bounding box, there is a grid payload created as well + + with self.assertRaises(ProcessExecutionException): + impossible_box_list = self.datapack_1.get_payload_data_at( + modality=Modality.Image, payload_index=2 + ) + + # Analyzing entries in second Data Pack + bb_list = list( + self.datapack_2._data_store._DataStore__elements[bb_type] + ) + bb_entries = list(self.datapack_2._data_store.all_entries(bb_type)) + + box_list = list( + self.datapack_2._data_store._DataStore__elements[box_type] + ) + box_entries = list(self.datapack_2._data_store.all_entries(box_type)) + self.assertTrue( array_equal( self.datapack.get_payload_at(Modality.Image, 0).cache, self.line @@ -56,3 +184,111 @@ def test_image_annotation(self): self.assertEqual( new_pack.audio_annotations, self.datapack.audio_annotations ) + region_list = list( + self.datapack_2._data_store._DataStore__elements[region_type] + ) + region_entries = list( + self.datapack_2._data_store.all_entries(region_type) + ) + + # Box and BoundingBox are subclasses of Region + self.assertEqual( + len(region_list + box_list + bb_list), len(region_entries) + ) + self.assertEqual( + self.datapack_2._data_store.num_entries(region_type), + len(region_entries), + ) + + # BoundingBox is a subclass of Box + self.assertEqual(len(box_list + bb_list), len(box_entries)) + self.assertEqual( + self.datapack_2._data_store.num_entries(box_type), len(box_entries) + ) + + def test_delete_image_annotations(self): + + box_type = "forte.data.ontology.top.BoundingBox" + box_list = len( + list(self.datapack_1._data_store._DataStore__elements[box_type]) + ) + + self.datapack_1._data_store.delete_entry(self.bb1.tid) + self.assertEqual( + self.datapack_1._data_store.num_entries(box_type), box_list - 1 + ) + + def test_update_image_annotation(self): + # Check current value + self.assertEqual(self.bb1._height, 2) + + # Change a parameter of the entry object + self.bb1._height = 5 + + # Fetch attribute value from data store + bb1_height = self.datapack_1._data_store.get_attribute( + self.bb1.tid, "_height" + ) + # Check new value + self.assertEqual(bb1_height, 5) + + # Updating Non-Dataclass fields + + # Check current value + self.assertEqual(self.bb4.image_payload_idx, 1) + + # Change a parameter of the entry object + self.bb4.image_payload_idx = 2 + + # Fetch attribute value from data store + bb4_payload = self.datapack_1._data_store.get_entry(self.bb4.tid)[0][ + constants.PAYLOAD_INDEX + ] + # Check new value + self.assertEqual(bb4_payload, 2) + + def test_compute_iou(self): + box1 = self.bb1 + box2 = self.bb2 + box3 = self.bb3 + box4 = self.bb4 + + iou1 = box1.compute_iou(box4) + self.assertEqual(iou1, 0.14285714285714285) + + iou2 = box1.compute_iou(box2) + self.assertEqual(iou2, 0) + + iou3 = box1.compute_iou(box3) + self.assertEqual(iou3, 0) + + def test_compute_overlap_from_data_store(self): + bb1 = self.datapack_1.get_entry(tid=self.bb1.tid) + bb2 = self.datapack_1.get_entry(tid=self.bb2.tid) + + overlap = bb1.is_overlapped(bb2) + self.assertTrue(overlap) + + def test_add_image_annotation(self): + + new_box = Box( + pack=self.datapack_1, + cy=7, + cx=7, + height=4, + width=2, + image_payload_idx=0, + ) + + self.assertEqual( + len( + self.datapack_1._data_store._DataStore__elements[ + "forte.data.ontology.top.Box" + ] + ), + 1, + ) + + +if __name__ == "__main__": + unittest.main() From 05f883fab362900e02f16b1de71638ac7a6bb607 Mon Sep 17 00:00:00 2001 From: Pushkar-Bhuse Date: Thu, 7 Jul 2022 15:45:20 -0700 Subject: [PATCH 2/3] Improving interface for CV ontologies --- forte/data/entry_converter.py | 3 + forte/data/ontology/core.py | 204 ++++++++ forte/data/ontology/top.py | 699 ++++++++++++++++++++++----- tests/forte/data/data_store_test.py | 56 ++- tests/forte/image_annotation_test.py | 56 ++- 5 files changed, 843 insertions(+), 175 deletions(-) diff --git a/forte/data/entry_converter.py b/forte/data/entry_converter.py index 343d6693e..066df06ce 100644 --- a/forte/data/entry_converter.py +++ b/forte/data/entry_converter.py @@ -32,6 +32,7 @@ SinglePackEntries, MultiPackEntries, ) +from forte.data.ontology.top import BoundingBox, Box from forte.utils import get_class, get_full_module_name logger = logging.getLogger(__name__) @@ -205,6 +206,8 @@ def get_entry_object( # the value can be arbitrary since they will all be routed to DataStore. if data_store_ref._is_annotation(type_name): entry = entry_class(pack=pack, begin=0, end=0) + elif any(entry_class == box_class for box_class in [BoundingBox, Box]): + entry = entry_class(pack=pack, height=1, width=1) elif any( data_store_ref._is_subclass(type_name, type_class) for type_class in SinglePackEntries + MultiPackEntries diff --git a/forte/data/ontology/core.py b/forte/data/ontology/core.py index 10f3b2641..db780733d 100644 --- a/forte/data/ontology/core.py +++ b/forte/data/ontology/core.py @@ -15,6 +15,7 @@ Defines the basic data structures and interfaces for the Forte data representation system. """ +import math import uuid from abc import abstractmethod, ABC @@ -23,6 +24,7 @@ from typing import ( Iterable, Optional, + Tuple, Type, Hashable, TypeVar, @@ -635,5 +637,207 @@ def index_key(self) -> int: return self.tid +class Grid: + """ + Regular grid with a grid configuration dependent on the image size. + It is a data structure used to retrieve grid-related objects such as grid + cells from the image. Grid itself doesn't store any data. + Based the image size and the grid shape, + we compute the height and the width of grid cells. + For example, if the image size (image_height,image_width) is (640, 480) + and the grid shape (height, width) is (2, 3) + the size of grid cells (self.c_h, self.c_w) will be (320, 160). + However, when the image size is not divisible by the grid shape, we round + up the resulting size(floating number) to an integer. + In this way, as each grid + cell taking one more pixel, we make the last grid cell per column and row + size(height and width) to be the remainder of the image size divided by the + grid cell size which is smaller than other grid cell. + For example, if the image + size is (128, 128) and the grid shape is (13, 13), the first 11 grid cells + per column and row will have a size of (10, 10) since 128/13=9.85, so we + round up to 10. The last grid cell per column and row will have a size of + (8, 8) since 128%10=8. + We require each grid to be bounded/intialized with one image size since + the number of different image shapes are limited per computer vision task. + For example, we can only have one image size (640, 480) from a CV dataset, + and we could augment the dataset with few other image sizes + (320, 240), (480, 640). Then there are only three image sizes. + Therefore, it won't be troublesome to + have a grid for each image size, and we can check the image size during the + initialization of the grid. + By contrast, if the grid is totally "free-form" + that we don't initialize it with any + image size and pass the image size directly into the method/operation on + the fly, the API would be more complex and image size check would be + repeated everytime the method is called. + Args: + height: the number of grid cell per column, the unit is one grid cell. + width: the number of grid cell per row, the unit is one grid cell. + image_height: the number of pixels per column in the image. + image_width: the number of pixels per row in the image. + """ + + def __init__( + self, + height: int, + width: int, + image_height: int, + image_width: int, + ): + if image_height <= 0 or image_width <= 0: + raise ValueError( + "both image height and width must be positive" + f"but the image shape is {(image_height, image_width)}" + "please input a valid image shape" + ) + if height <= 0 or width <= 0: + raise ValueError( + f"height({height}) and " + f"width({width}) both must be larger than 0" + ) + if height >= image_height or width >= image_width: + raise ValueError( + "Grid height and width must be smaller than image height and width" + ) + + self._height = height + self._width = width + + self._image_height = image_height + self._image_width = image_width + + # if the resulting size of grid is not an integer, we round it up. + # The last grid cell per row and column might be out of the image size + # since we constrain the maximum pixel locations by the image size + self.c_h, self.c_w = ( + math.ceil(image_height / self._height), + math.ceil(image_width / self._width), + ) + + if self.c_h <= 0 or self.c_w <= 0: + raise ValueError( + "cell height and width must be positive" + f"but the cell shape is {(self.c_h, self.c_w)}" + "please adjust image shape or grid shape accordingly" + ) + + def get_grid_cell(self, img_arr: np.ndarray, h_idx: int, w_idx: int): + """ + Get the array data of a grid cell from image of the image payload index. + The array is a masked version of the original image, and it has + the same size as the original image. The array entries that are not + within the grid cell will masked as zeros. The image array entries that + are within the grid cell will kept. + Note: all indices are zero-based and counted from top left corner of + the image. + Args: + img_arr: image data represented as a numpy array. + h_idx: the zero-based height(row) index of the grid cell in the + grid, the unit is one grid cell. + w_idx: the zero-based width(column) index of the grid cell in the + grid, the unit is one grid cell. + Raises: + ValueError: ``h_idx`` is out of the range specified by ``height``. + ValueError: ``w_idx`` is out of the range specified by ``width``. + Returns: + numpy array that represents the grid cell. + """ + if not 0 <= h_idx < self._height: + raise ValueError( + f"input parameter h_idx ({h_idx}) is" + "out of scope of h_idx range" + f" {(0, self._height)}" + ) + if not 0 <= w_idx < self._width: + raise ValueError( + f"input parameter w_idx ({w_idx}) is" + "out of scope of w_idx range" + f" {(0, self._width)}" + ) + # initialize a numpy zeros array + array = np.zeros((self._image_height, self._image_width)) + # set grid cell entry values to the values of the original image array + # (entry values outside of grid cell remain zeros) + # An example of computing grid height index range is + # index * cell height : min((index + 1) * cell height, image_height). + # It's similar for computing cell width index range + # Plus, we constrain the maximum pixel locations by the image size as + # the last grid cell per row and column might be out of the image size + array[ + h_idx * self.c_h : min((h_idx + 1) * self.c_h, self._image_height), + w_idx * self.c_w : min((w_idx + 1) * self.c_w, self._image_width), + ] = img_arr[ + h_idx * self.c_h : min((h_idx + 1) * self.c_h, self._image_height), + w_idx * self.c_w : min((w_idx + 1) * self.c_w, self._image_width), + ] + return array + + def get_grid_cell_center(self, h_idx: int, w_idx: int) -> Tuple[int, int]: + """ + Get the center pixel position of the grid cell at the specific height + index and width index in the ``Grid``. + The computation of the center position of the grid cell is + dividing the grid cell height range and width range by 2 (round down) + Suppose an extreme case that a grid cell has a height range of (0, 3) + and a width range of (0, 3) the grid cell center would be (1, 1). + Since the grid cell size is usually very large, + the offset of the grid cell center is minor. + Note: all indices are zero-based and counted from top left corner of + the grid. + Args: + h_idx: the height(row) index of the grid cell in the grid, + the unit is one grid cell. + w_idx: the width(column) index of the grid cell in the + grid, the unit is one grid cell. + Returns: + A tuple of (y index, x index) + """ + + return ( + (h_idx * self.c_h + min((h_idx + 1) * self.c_h, self._image_height)) + // 2, + (w_idx * self.c_w + min((w_idx + 1) * self.c_w, self._image_width)) + // 2, + ) + + @property + def num_grid_cells(self): + return self._height * self._width + + @property + def height(self): + return self._height + + @property + def width(self): + return self._width + + def __repr__(self): + return str( + (self._height, self._width, self._image_height, self._image_width) + ) + + def __eq__(self, other): + if other is None: + return False + return ( + self._height, + self._width, + self._image_height, + self._image_width, + ) == ( + other._height, + other._width, + other.image_height, + other.image_width, + ) + + def __hash__(self): + return hash( + (self._height, self._width, self._image_height, self._image_width) + ) + + GroupType = TypeVar("GroupType", bound=BaseGroup) LinkType = TypeVar("LinkType", bound=BaseLink) diff --git a/forte/data/ontology/top.py b/forte/data/ontology/top.py index b49aaf1e8..fcd944da9 100644 --- a/forte/data/ontology/top.py +++ b/forte/data/ontology/top.py @@ -35,6 +35,7 @@ BaseGroup, MultiEntry, EntryType, + Grid, ) from forte.data.span import Span from forte.common.constants import ( @@ -864,7 +865,6 @@ def __init__(self, pack: PackType, image_payload_idx: int = 0): ImageAnnotation type entries, such as "edge" and "bounding box". Each ImageAnnotation has a ``image_payload_idx`` corresponding to its image representation in the payload array. - Args: pack: The container that this image annotation will be added to. @@ -880,6 +880,14 @@ def __init__(self, pack: PackType, image_payload_idx: int = 0): def image_payload_idx(self) -> int: return self._image_payload_idx + @image_payload_idx.setter + def image_payload_idx(self, val: int): + r"""Setter function of ``image_payload_idx``. The update will also be populated + into ``DataStore`` in ``self.pack``. + """ + self._image_payload_idx = val + self.pack.get_entry_raw(self.tid)[PAYLOAD_INDEX] = val + @property def image(self): if self.pack is None: @@ -887,9 +895,7 @@ def image(self): "Cannot get image because image annotation is not " "attached to any data pack." ) - return self.pack.get_payload_data_at( - Modality.Image, self._image_payload_idx - ) + return self.pack.get_image_array(self._image_payload_idx) @property def max_x(self): @@ -905,6 +911,37 @@ def __eq__(self, other): return self.image_payload_idx == other.image_payload_idx +class Region(ImageAnnotation): + """ + A region class associated with an image payload. + Args: + pack: the container that this ``Region`` will be added to. + image_payload_idx: the index of the image payload. If it's not set, + it defaults to 0 which meaning it will load the first image payload. + """ + + def __init__(self, pack: PackType, image_payload_idx: int = 0): + super().__init__(pack, image_payload_idx) + if image_payload_idx is None: + self._image_payload_idx = 0 + else: + self._image_payload_idx = image_payload_idx + + def compute_iou(self, other) -> float: + """ + Compute the IoU between this region and another region. + Args: + other: Another region object. + Returns: + the IoU between this region and another region as a float. + """ + if not isinstance(other, Region): + raise TypeError("other must be a Region object") + intersection = np.sum(np.logical_and(self.image, other.image)) + union = np.sum(np.logical_or(self.image, other.image)) + return intersection / union + + class Grids(Entry): """ Regular grids with a grid configuration. @@ -1043,146 +1080,496 @@ def __eq__(self, other): ) -class Region(ImageAnnotation): +@dataclass +class Box(Region): + # pylint: disable=too-many-public-methods """ - A region class associated with an image payload. - + A box class with a reference point which is the box center and a box + configuration. + Given a box with shape parameters (height, width), we want to locate its + four corners (top-left, top-right, bottom-left, bottom-right). We need + to know the locaton of the center of the box. + Generally, we can either use box standalone or as a bounding box in object + detection tasks. In the later case, we need to consider its association with + a grid as the task is performed on each grid cell. + There are several use cases for a box: + 1. When we use a box standalone, we need the box center to be set. + The offset between the box center and the grid cell center is not used. + 2. When we represent a ground truth box, the box center and shape are + given. If we want to compute loss, its grid cell center is required to + compute the offset between the box center and the + grid cell center. + 3. When we predict a box, we will have the predicted box shape (height, + width) and the offset between the box center and the grid cell center, + then we can compute the box center. + .. code-block:: python + pack = DataPack("box examples") + # create a box + simple_box = Box(pack, 640, 480, 320, 240) + # create a bounding box at with its center at (320, 240) + gt_bbx = Box(pack, 640, 480, 320, 240) + # create a predicted bounding box without known its center + # and compute its center from the offset and the grid center + predicted_bbx = Box(pack, 640, 480) + predicted_bbx.set_offset(305, 225) + b.set_grid_cell_center(Grid(64, 48, 640, 480), 1, 1) + print(b.cy, b.cx) # it prints (320, 240) + For example, in the object detection task, dataset label contains a ground + truth box (box shape and box center). + The inference pipeline is that given a grid cell, we make a prediction of a + bounding box (box shape and box offset from the grid cell center). + 1. If we want to locate the predicted box, we compute the box center + based on the second use case. + 2. If we want to compute the loss, we need values for the center, shape + and offset for both boxes. And we need to compute the offset between + the box center and the grid cell center based on the third use case. + A more detailed explanation can be found in the following blog: + https://towardsdatascience.com/yolo2-walkthrough-with-examples-e40452ca265f + Based on the use cases, there are two important class conditions: + 1. Whether the box center is set. + 2. Whether it's associated with a grid. (It might or might not + depending on the box use cases) + Note: all indices are zero-based and counted from top left corner of + image. But the unit could be a pixel or a grid cell. Args: - pack: the container that this ``Region`` will be added to. - image_payload_idx: the index of the image payload in the DataPack's - image payload list. - If it's not set, + pack: the container that this ``Box`` will be added to. + height: the height of the box, the unit is one pixel. + width: the width of the box, the unit is one pixel. + cy: the row index of the box center in the image array, + the unit is one pixel. If not set, it defaults to None. + cx: the column index of the box center in the image array, + the unit is one pixel. If not set, it defaults to None. + image_payload_idx: the index of the image payload. If it's not set, it defaults to 0 which meaning it will load the first image payload. """ + cy: Optional[int] + cx: Optional[int] + height: int + width: int + cy_offset: Optional[int] + cx_offset: Optional[int] + grid_cy: Optional[int] + grid_cx: Optional[int] + is_grid_associated: bool - def __init__(self, pack: PackType, image_payload_idx: int = 0): + def __init__( + self, + pack: PackType, + height: int, + width: int, + cy: Optional[int] = None, + cx: Optional[int] = None, + image_payload_idx: int = 0, + ): super().__init__(pack, image_payload_idx) - if image_payload_idx is None: - self._image_payload_idx = 0 - else: - self._image_payload_idx = image_payload_idx - @property - def image_payload_idx(self): - r"""Getter function of ``image_payload_idx``. The function will first try to - retrieve the image_payload_idx index from ``DataStore`` in ``self.pack``. If - this attempt fails, it will directly return the value in ``_image_payload_idx``. + self._image_height, self._image_width = self.pack.get_payload_data_at( + Modality.Image, image_payload_idx + ).shape[-2:] + # We don't initialize the grid cell center/offset during the class + # initialization because we cannot pass the grid cell to the constructor. + # Instead, we initialize the grid cell center when we set the grid cell. + if not (height > 0 and width > 0): + raise ValueError( + f"Box height({height}) and width({width}) must be positive." + ) + if not (height < self._image_height and width < self._image_width): + raise ValueError( + f"Box height({height}) and width({width}) must be smaller " + f"than image height({self._image_height}) and width({self._image_width})." + ) + self.height = height + self.width = width + if cy is not None and cx is not None: + self._check_center_validity(cy, cx) + self.cy = cy + self.cx = cx + # intialize the grid cell center/offset to None + # they must be set later if the ``Box`` use case is associated with Grid + self.is_grid_associated = False + self.cy_offset: Optional[int] = None + self.cx_offset: Optional[int] = None + self.grid_cy: Optional[int] = None + self.grid_cx: Optional[int] = None + + def _check_center_validity(self, cy: Optional[int], cx: Optional[int]): + """ + Check whether the box center is valid. + Args: + cy: the row index of the box center in the image array. + cx: the column index of the box center in the image array. + Returns: + True if the box center is valid, False otherwise. """ - try: - self._image_payload_idx = self.pack.get_entry_raw(self.tid)[ - PAYLOAD_INDEX - ] - except KeyError: - pass - return self._image_payload_idx + if cy is None or cx is None: + raise ValueError( + "Box center cy, cx must be set to check" + " their numerical validaty." + "Currently they are None." + ) - @image_payload_idx.setter - def image_payload_idx(self, val: int): - r"""Setter function of ``image_payload_idx``. The update will also be populated - into ``DataStore`` in ``self.pack``. + if cy < self.height / 2 or cx < self.width / 2: + raise ValueError( + f"Box center({cy}, {cx}) must be greater than half " + f"height({self.height/2}) and half width({self._width/2})" + "respectively." + ) + + if ( + cy >= self._image_height - self.height / 2 + or cx >= self._image_width - self.width / 2 + ): + raise ValueError( + f"Box center({cy}, {cx}) must be less than half " + f"height({self._image_height - self.height/2}) and" + f" half width({self._image_width - self.width/2})" + "respectively." + ) + + def _check_offset_validity(self, cy_offset: int, cx_offset: int): """ - self._image_payload_idx = val - self.pack.get_entry_raw(self.tid)[PAYLOAD_INDEX] = val + Check the validaty of y coordinate offset and x coordinate offset. + Args: + cy_offset: the offset between the box center and the grid cell + center. + cx_offset: the offset between the box center and the grid cell + center. + Returns: + True if the offset is valid, False otherwise. + """ + if self.grid_cy is not None and self.grid_cx is not None: + # the computed cell center + computed_cy = self.grid_cy + cy_offset + computed_cx = self.grid_cx + cx_offset + if computed_cy < 0 or computed_cx < 0: + return False - def compute_iou(self, other) -> int: - intersection = np.sum(np.logical_and(self.image, other.image)) - union = np.sum(np.logical_or(self.image, other.image)) - return intersection / union + if ( + computed_cy > self._image_height + or computed_cx > self._image_width + ): + return False + def set_center(self, cy: int, cx: int): + """ + Set the y coordinate and x coordinate of the center(a pixel) of the Box. + Args: + cy: the row index of the box center in the image array, + the unit is one pixel. + cx: the column index of the box center in the image array, + the unit is one pixel. + """ + self._check_center_validity(cy, cx) + self.cy = cy + self.cx = cx -@dataclass -class Box(Region): - """ - A box class with a center position and a box configuration. + def set_grid_cell_center( + self, grid: Grid, grid_h_idx: int, grid_w_idx: int + ): + """ + Set the center of a grid cell that the Box is associated with. + Args: + grid: the grid that the box is associated with. + grid_h_idx: the row index of the grid cell center in the image + array, the unit is one grid cell. + grid_w_idx: the column index of the grid cell center in the image + array, the unit is one grid cell. + """ + self.is_grid_associated = True + # given a grid cell, compute its center + self.grid_cy, self.grid_cx = grid.get_grid_cell_center( + grid_h_idx, grid_w_idx + ) - Note: all indices are zero-based and counted from top left corner of - image. + @property + def grid_center_y(self) -> int: + """ + The row index(unit: pixel) of the grid cell center in the image array. + Raises: + ValueError: if the box is not associated with a grid. + Returns: + The row index of the grid cell center in the image array. + """ + if not self.is_grid_associated or self.grid_cy is None: + raise ValueError( + "The box is not associated with a grid." + "Therefore, there is no grid cell center." + ) + return self.grid_cy - Args: - pack: the container that this ``Box`` will be added to. - image_payload_idx: the index of the image payload in the DataPack's - image payload list. If it's not set, - it defaults to 0 which meaning it will load the first image payload. - cy: the row index of the box center in the image array, - the unit is one image array entry. - cx: the column index of the box center in the image array, - the unit is one image array entry. - height: the height of the box, the unit is one image array entry. - width: the width of the box, the unit is one image array entry. - """ + @grid_center_y.setter + def grid_center_y(self, val): + raise ValueError( + "Use in-built method set_grid_cell_center" + "to set center coordinates." + ) - _cy: int - _cx: int - _height: int - _width: int + @property + def grid_center_x(self) -> int: + """ + The column index(unit: pixel) of the grid cell center in the image + array. + Raises: + ValueError: if the box is not associated with a grid. + Returns: + The column index of the grid cell center in the image array. + """ - def __init__( - self, - pack: PackType, - cy: int = 0, - cx: int = 0, - height: int = 1, - width: int = 1, - image_payload_idx: int = 0, + if not self.is_grid_associated or self.grid_cx is None: + raise ValueError( + "The box is not associated with a grid." + "Therefore, there is no grid cell center." + ) + return self.grid_cx + + @grid_center_x.setter + def grid_center_x(self, val): + raise ValueError( + "Use in-built method set_grid_cell_center" + "to set center coordinates" + ) + + @property + def grid_cell_center(self) -> Tuple[int, int]: + """ + The position of center(a pixel) of the grid cell that the Box is + associated with. + Raises: + ValueError: if the box is not associated with a grid. + Returns: + Tuple[int, int]: the center of the grid cell that the Box is associated with. + """ + if ( + not self.is_grid_associated + or self.grid_cy is None + or self.grid_cx is None + ): + raise ValueError( + "The box is not associated with a grid." + "Therefore, there is no grid cell center." + ) + return self.grid_cy, self.grid_cx + + def set_offset( + self, cy_offset: int, cx_offset: int, check_validity: bool = False ): - # assume Box is associated with Grids - super().__init__(pack, image_payload_idx) - # center location - self._cy = cy - self._cx = cx - self._height = height - self._width = width + """ + Set the offset(unit: pixel) of the box center from the grid cell center. + Args: + cy_offset: the row index of the box center offset from the grid cell + center in the image array, the unit is one pixel. + cx_offset: the column index of the box center offset from the grid + cell center in the image array, the unit is one pixel. + check_validity: a boolean indicating whether to check the validity + of the offset. + """ + if check_validity: + self._check_offset_validity(cy_offset, cx_offset) + self.cy_offset = cy_offset + self.cx_offset = cx_offset @property - def center(self): - return (self._cy, self._cx) + def offset(self): + """ + The offset(unit: pixel) of the box center from the grid cell center. + Returns: + the offset of the box center from the grid cell + """ + return self.cy_offset, self.cx_offset @property - def corners(self): + def center_y_offset(self) -> int: """ - Get corners of box. + The row index difference(unit: pixel) between the box center and the + grid cell. + Returns: + The row index difference between the box center and the grid cell. """ - return [ - (self._cy + h_offset, self._cx + w_offset) - for h_offset in [-0.5 * self._height, 0.5 * self._height] - for w_offset in [-0.5 * self._width, 0.5 * self._width] - ] + if self.cy_offset is not None: + return self.cy_offset + if self.grid_cy is not None and self.cy is not None: + self.cy_offset = self.cy - self.grid_cy + else: + self._offset_condition_check() + raise ValueError("cy_offset is not set.") + return self.cy_offset @property - def box_min_x(self): - return max(self._cx - round(0.5 * self._width), 0) + def center_x_offset(self) -> int: + """ + The column index difference(unit: pixel) between the box center and the + grid cell + Returns: + The column index difference between the box center and the grid cell + """ + if self.cx_offset is not None: + return self.cx_offset + if self.cx is not None and self.grid_cx is not None: + self.cx_offset = self.cx - self.grid_cx + else: + self._offset_condition_check() + raise ValueError("cx_offset is not set") + return self.cx_offset @property - def box_max_x(self): - return min(self._cx + round(0.5 * self._width), self.max_x) + def center_y(self) -> int: + """ + Compute and return row index(unit: pixel) of the box center + in the image array. + It returns the row index of the box center in the image array directly + if the box center is set. + Otherwise, if it computes and sets the box center y coordinate when the + box is both associated with a grid and the + offset is set. + Returns: + The row index of the box center in the image array. + """ + if self.cy is not None: + return self.cy + else: + # if cy computation condition is met, then cy is set + if self.grid_cy is not None and self.cy_offset is not None: + self.cy = self.grid_cy + self.cy_offset + else: + self._center_condition_check() + raise ValueError("cy is not set.") + return self.cy + + @center_y.setter + def center_y(self, val: int): + r"""Setter function of ``center_y``. The update will also be populated + into ``DataStore`` in ``self.pack``. + """ + self.cy = val @property - def box_min_y(self): - return max(self._cy - round(0.5 * self._height), 0) + def center_x(self) -> int: + """ + The column index(unit: pixel) of the box center in the image array. + It returns the column index of the box center in the image array + directly if the box center is set. + Otherwise, if it computes and sets the box center x coordinate when the + box is both associated with a grid and the + offset is set. + Returns: + The column index of the box center in the image array. + """ + if self.cx is not None: + return self.cx + else: + # if cx computation condition is met, then cx is set + if self.grid_cx is not None and self.cx_offset is not None: + self.cx = self.grid_cx + self.cx_offset + else: + self._center_condition_check() + raise ValueError("cx is not set.") + return self.cx + + @center_x.setter + def center_x(self, val): + r"""Setter function of ``center_x``. The update will also be populated + into ``DataStore`` in ``self.pack``. + """ + self.cx = val + + @property + def box_center(self) -> Tuple[int, int]: + """ + Get the box center y coordinate and x coordinate. + If box center is not set not computable, it raises a ``ValueError`` + Returns: + The box center in a ``Tuple`` format. + """ + return (self.center_y, self.center_x) @property - def box_max_y(self): - return min(self._cy + round(0.5 * self._height), self.max_y) + def corners(self) -> Tuple[Tuple[int, int], ...]: + """ + Compute and return the positions of corners of the box, (top left, + top right, bottom left, bottom right). + Raises: + ValueError: if the box center is not set. + Returns: + The corners of the box in a ``Tuple`` format. + """ + return tuple( + (self.center_y + h_offset, self.center_x + w_offset) + for h_offset in [-self.height // 2, self.height // 2] + for w_offset in [-self.width // 2, self.width // 2] + ) @property - def area(self): - return self._height * self._width + def box_min_x(self) -> int: + """ + Compute the minimum x coordinate(unit: pixel) of the box. + Raises: + ValueError: if the box center is not set. + Returns: + The minimum x coordinate of the box. + """ + return max(self.center_x - round(0.5 * self.width), 0) - def is_overlapped(self, other): + @property + def box_max_x(self) -> int: + """ + Compute the maximum x coordinate(unit: pixel) of the box. + Raises: + ValueError: if the box center is not set. + Returns: + The maximum x coordinate of the box. + """ + return self.center_x + self.width // 2 + + @property + def box_min_y(self) -> int: + """ + Compute the minimum y coordinate(unit: pixel) of the box. + Raises: + ValueError: if the box center is not set. + Returns: + The minimum y coordinate of the box. + """ + return max(self.center_y - round(0.5 * self.height), 0) + + @property + def box_max_y(self) -> int: + """ + Compute the maximum y coordinate(unit: pixel) of the box. + Raises: + ValueError: if the box center is not set. + Returns: + The maximum y coordinate of the box. + """ + return self.center_y + round(0.5 * self.height) + + @property + def area(self) -> int: + """ + Compute the area of the box(unit: pixel). + Returns: + The area of the box. + """ + return self.height * self.width + + def is_overlapped(self, other) -> bool: """ A function checks whether two boxes are overlapped(two box area have intersections). - Note: in edges cases where two bounding boxes' boundaries share the same line segment/corner in the image array, it won't be considered overlapped. - Args: other: the other ``Box`` object to compared to. - Returns: - A boolean value indicating whether there is overlapped. + True if the two boxes are overlapped, False otherwise. """ + if not isinstance(other, Box): + raise ValueError( + "The other object to check overlapping with is" + " not a Box object." + "You need to check the type of the other object." + ) + # If one box is on left side of other if self.box_min_x > other.box_max_x or other.box_min_x > self.box_max_x: return False @@ -1192,17 +1579,25 @@ def is_overlapped(self, other): return False return True - def compute_iou(self, other): + def compute_iou(self, other) -> float: """ - A function computes iou(intersection over union) between two boxes. - + A function computes iou(intersection over union) between two boxes + (unit: pixel). + It overwrites the ``compute_iou`` function in it's parent class + ``Region``. Args: - other: the other ``Box`` object to compared to. - + other: the other ``Box`` object to be computed with. Returns: A float value which is (intersection area/ union area) between two boxes. """ + if not isinstance(other, Box): + raise ValueError( + "The other object to compute iou with is" + " not a Box object." + "You need to check the type of the other object." + ) + if not self.is_overlapped(other): return 0 box_x_diff = min( @@ -1217,60 +1612,102 @@ def compute_iou(self, other): union = self.area + other.area - intersection return intersection / union + def _offset_condition_check(self): + """ + When the the offset is not set, this function checks the reason the + offset cannot be computed and raises the corresponding error. + Raises: + ValueError: if the grid cell is not associated with the box and + the box center is not set. + ValueError: if the grid cell is not associated with the box. + ValueError: if the box center is not set. + """ + result_msg = ( + "Hence, the offset of the box center from the grid cell" + + "center cannot be computed." + ) + if not self.is_grid_associated and not self._is_box_center_set(): + raise ValueError( + "The box center is not set and the grid cell center is not set." + + result_msg + ) + elif not self.is_grid_associated: + raise ValueError("The grid cell center is not set." + result_msg) + elif not self._is_box_center_set(): + raise ValueError("The box center is not set." + result_msg) + + def _center_condition_check(self): + """ + When the the center is not set, this function checks the reason the box + center cannot be computed and raises the corresponding error. + Raises: + ValueError: if the box center is not set and the grid cell center + is not set. + ValueError: if the grid cell center is not set. + ValueError: if the box center is not set. + """ + result_msg = "Hence, the position of the box center cannot be computed." + if not self.is_grid_associated and not self._is_offset_set(): + raise ValueError( + "The box center is not set and the grid cell center is not set." + + result_msg + ) + elif not self.is_grid_associated: + raise ValueError("The grid cell center is not set." + result_msg) + elif not self._is_offset_set(): + raise ValueError("The offset is not set." + result_msg) + + def _is_box_center_set(self) -> bool: + """ + A function checks whether the box center is set. + Returns: + True if the box center is set, False otherwise. + """ + return self.cy is not None and self.cx is not None + + def _is_offset_set(self) -> bool: + """ + A function checks whether the offset of the box center from the grid + is set. + Returns: + True if the offset is set, False otherwise. + """ + return self.cy_offset is not None and self.cx_offset is not None + @dataclass class BoundingBox(Box): """ - A bounding box class that associates with image payload and grids and + A bounding box class that associates with image payload and grid and has a configuration of height and width. - Note: all indices are zero-based and counted from top left corner of the image/grid. - Args: pack: The container that this BoundingBox will be added to. image_payload_idx: the index of the image payload in the DataPack's image payload list. If it's not set, it defaults to 0 which means it will load the first image payload. - height: the height of the bounding box, the unit is one image array - entry. - width: the width of the bounding box, the unit is one image array entry. - grid_height: the height of the associated grid, the unit is one grid - cell. - grid_width: the width of the associated grid, the unit is one grid - cell. - grid_cell_h_idx: the height index of the associated grid cell in - the grid, the unit is one grid cell. - grid_cell_w_idx: the width index of the associated grid cell in - the grid, the unit is one grid cell. - + height: the height of the bounding box, the unit is one pixel. + width: the width of the bounding box, the unit is one pixel. """ - _grid_id: int - def __init__( self, pack: PackType, - height: int = 1, - width: int = 1, - grid_height: int = 1, - grid_width: int = 1, - grid_cell_h_idx: int = 0, - grid_cell_w_idx: int = 0, + height: int, + width: int, image_payload_idx: int = 0, ): - self.grids = Grids(pack, grid_height, grid_width, image_payload_idx) + + # self._is_grid_associated = False super().__init__( pack, - *self.grids.get_grid_cell_center(grid_cell_h_idx, grid_cell_w_idx), height, width, image_payload_idx, ) - self._grid_id = self.grids.tid - class Payload(Entry): """ diff --git a/tests/forte/data/data_store_test.py b/tests/forte/data/data_store_test.py index ed07e237d..1b1c3b23b 100644 --- a/tests/forte/data/data_store_test.py +++ b/tests/forte/data/data_store_test.py @@ -145,11 +145,15 @@ def setUp(self) -> None: }, "forte.data.ontology.top.BoundingBox": { "attributes": { - "_cy": 4, - "_cx": 5, - "_height": 6, - "_width": 7, - "_grid_id": 8, + "cy": 4, + "cx": 5, + "height": 6, + "width": 7, + "cy_offset": 8, + "cx_offset": 9, + "grid_cy": 10, + "grid_cx": 11, + "is_grid_associated": 12 }, "parent_class": set(), }, @@ -176,11 +180,15 @@ def setUp(self) -> None: } DataStore._type_attributes["forte.data.ontology.top.BoundingBox"] = { "attributes": { - "_cy": 4, - "_cx": 5, - "_height": 6, - "_width": 7, - "_grid_id": 8, + "cy": 4, + "cx": 5, + "height": 6, + "width": 7, + "cy_offset": 8, + "cx_offset": 9, + "grid_cy": 10, + "grid_cx": 11, + "is_grid_associated": 12 }, "parent_class": set(), } @@ -241,22 +249,30 @@ def setUp(self) -> None: None, 1212, "forte.data.ontology.top.BoundingBox", - 3, + 1, + 1, 5, 2, - 2, - 1000, + None, + None, + None, + None, + False ] ref7 = [ - 2, + 1, None, - 3434, + 1212, "forte.data.ontology.top.BoundingBox", 3, - 5, - 1, 3, - 2000, + 3, + 4, + None, + None, + None, + None, + False ] ref8 = [0, None, 1000, "forte.data.ontology.top.Grids"] ref9 = [0, None, 2000, "forte.data.ontology.top.Grids"] @@ -854,6 +870,10 @@ def test_add_image_annotation_raw(self): None, None, None, + None, + None, + None, + None, ], ) diff --git a/tests/forte/image_annotation_test.py b/tests/forte/image_annotation_test.py index 5305f75ea..884c1c15e 100644 --- a/tests/forte/image_annotation_test.py +++ b/tests/forte/image_annotation_test.py @@ -74,22 +74,16 @@ def setUp(self): pack=self.datapack_1, height=2, width=2, - grid_height=3, - grid_width=4, - grid_cell_h_idx=1, - grid_cell_w_idx=1, ) + self.bb1.set_center(1,1) self.datapack_1.add_entry(self.bb1) self.bb2 = BoundingBox( pack=self.datapack_1, height=3, width=4, - grid_height=5, - grid_width=5, - grid_cell_h_idx=3, - grid_cell_w_idx=3, ) + self.bb2.set_center(3,3) self.datapack_1.add_entry(self.bb2) # Entries for another image in the same Data Pack @@ -97,12 +91,9 @@ def setUp(self): pack=self.datapack_1, height=3, width=4, - grid_height=6, - grid_width=6, - grid_cell_h_idx=2, - grid_cell_w_idx=2, image_payload_idx=1, ) + self.bb4.set_center(2.5,2.5) self.datapack_1.add_entry(self.bb4) # Entries for second Data Pack @@ -110,22 +101,18 @@ def setUp(self): pack=self.datapack_2, height=5, width=5, - grid_height=3, - grid_width=3, - grid_cell_h_idx=6, - grid_cell_w_idx=6, image_payload_idx=0, ) + self.bb3.set_center(4,5) self.datapack_2.add_entry(self.bb3) self.box1 = Box( pack=self.datapack_2, - cy=7, - cx=7, height=4, width=2, image_payload_idx=0, ) + self.box1.set_center(7,7) self.datapack_2.add_entry(self.box1) self.region1 = Region(pack=self.datapack_2, image_payload_idx=1) @@ -220,14 +207,14 @@ def test_delete_image_annotations(self): def test_update_image_annotation(self): # Check current value - self.assertEqual(self.bb1._height, 2) + self.assertEqual(self.bb1.height, 2) # Change a parameter of the entry object - self.bb1._height = 5 + self.bb1.height = 5 # Fetch attribute value from data store bb1_height = self.datapack_1._data_store.get_attribute( - self.bb1.tid, "_height" + self.bb1.tid, "height" ) # Check new value self.assertEqual(bb1_height, 5) @@ -235,7 +222,7 @@ def test_update_image_annotation(self): # Updating Non-Dataclass fields # Check current value - self.assertEqual(self.bb4.image_payload_idx, 1) + self.assertEqual(self.bb4.image_payload_idx, 0) # Change a parameter of the entry object self.bb4.image_payload_idx = 2 @@ -247,6 +234,20 @@ def test_update_image_annotation(self): # Check new value self.assertEqual(bb4_payload, 2) + + self.assertEqual( + self.bb2.center_x, 3 + ) + + self.bb2.center_x = 5 + + bb2_c_x = self.datapack_1._data_store.get_attribute( + self.bb2.tid, "cx" + ) + + self.assertEqual(bb2_c_x, 5) + + def test_compute_iou(self): box1 = self.bb1 box2 = self.bb2 @@ -254,15 +255,20 @@ def test_compute_iou(self): box4 = self.bb4 iou1 = box1.compute_iou(box4) - self.assertEqual(iou1, 0.14285714285714285) + self.assertEqual(iou1, 0.16363636363636364) iou2 = box1.compute_iou(box2) - self.assertEqual(iou2, 0) + self.assertEqual(iou2, 0.06666666666666667) iou3 = box1.compute_iou(box3) self.assertEqual(iou3, 0) def test_compute_overlap_from_data_store(self): + + # Clearing DataIndex and EntryConverter Cache + self.datapack_1._index._entry_index = {} + self.datapack_1._entry_converter._entry_dict = {} + bb1 = self.datapack_1.get_entry(tid=self.bb1.tid) bb2 = self.datapack_1.get_entry(tid=self.bb2.tid) @@ -273,8 +279,6 @@ def test_add_image_annotation(self): new_box = Box( pack=self.datapack_1, - cy=7, - cx=7, height=4, width=2, image_payload_idx=0, From db8a5d90bcc328aefcf5a4dfc10c1c596badc397 Mon Sep 17 00:00:00 2001 From: Pushkar-Bhuse Date: Thu, 7 Jul 2022 15:52:03 -0700 Subject: [PATCH 3/3] Bug Fixes --- forte/data/ontology/top.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forte/data/ontology/top.py b/forte/data/ontology/top.py index fcd944da9..28c653363 100644 --- a/forte/data/ontology/top.py +++ b/forte/data/ontology/top.py @@ -1211,7 +1211,7 @@ def _check_center_validity(self, cy: Optional[int], cx: Optional[int]): if cy < self.height / 2 or cx < self.width / 2: raise ValueError( f"Box center({cy}, {cx}) must be greater than half " - f"height({self.height/2}) and half width({self._width/2})" + f"height({self.height/2}) and half width({self.width/2})" "respectively." )