From bad3ee04c271c0e1ecd676980477bf624562a89f Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Fri, 5 Jul 2024 15:06:29 -0400 Subject: [PATCH 01/66] removed old unused code --- tasks/point_extraction/utils.py | 35 --------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 tasks/point_extraction/utils.py diff --git a/tasks/point_extraction/utils.py b/tasks/point_extraction/utils.py deleted file mode 100644 index c3d5eba8..00000000 --- a/tasks/point_extraction/utils.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import hashlib -from typing import Dict - -LOCAL_CACHE_PATH = "~/.points" - - -def filename_to_id(filename: str) -> int: - """ - Hash filename and modification time to create a unique id. - """ - mod_time = os.path.getmtime(filename) - unique_string = filename + str(mod_time) - hash_object = hashlib.md5(unique_string.encode()) - return int(hash_object.hexdigest()[:8], 16) - - -def filter_coco_images(coco_dict: Dict) -> Dict: - # Filter out images that have no annotations. - filtered_coco_dict = { - "images": [], - "annotations": [], - "categories": coco_dict["categories"], - } - image_ids = set(anno["image_id"] for anno in coco_dict["annotations"]) - for image in coco_dict["images"]: - if image["id"] in image_ids: - filtered_coco_dict["images"].append(image) - for anno in coco_dict["annotations"]: - if anno["image_id"] in image_ids: - filtered_coco_dict["annotations"].append(anno) - print( - f'Filtering {len(coco_dict["images"]) - len(filtered_coco_dict["images"])} out of {len(coco_dict["images"])} images without annotations' - ) - return filtered_coco_dict From c74368fb885ee731d20c9b310ee5be3e8494eb61 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Fri, 5 Jul 2024 15:13:37 -0400 Subject: [PATCH 02/66] adding code to handle CDR-based legend item annotations inside points pipeline (WIP) --- pipelines/point_extraction/run_pipeline.py | 33 ++- tasks/point_extraction/entities.py | 15 + tasks/point_extraction/legend_analyzer.py | 38 ++- tasks/point_extraction/legend_item_utils.py | 258 ++++++++++++++++++ .../point_extraction/point_extractor_utils.py | 82 +----- 5 files changed, 332 insertions(+), 94 deletions(-) create mode 100644 tasks/point_extraction/legend_item_utils.py diff --git a/pipelines/point_extraction/run_pipeline.py b/pipelines/point_extraction/run_pipeline.py index bc4113b6..b593ede4 100644 --- a/pipelines/point_extraction/run_pipeline.py +++ b/pipelines/point_extraction/run_pipeline.py @@ -6,7 +6,10 @@ from tasks.common.pipeline import PipelineInput, BaseModelOutput, ImageDictOutput from pipelines.point_extraction.point_extraction_pipeline import PointExtractionPipeline from tasks.common.io import ImageFileInputIterator, JSONFileWriter, ImageFileWriter -from tasks.point_extraction.point_extractor_utils import parse_legend_point_hints +from tasks.point_extraction.legend_item_utils import ( + parse_legend_annotations, + parse_legend_point_hints, +) from tasks.point_extraction.entities import ( LEGEND_ITEMS_OUTPUT_KEY, ) @@ -26,6 +29,7 @@ def main(): parser.add_argument("--model_segmenter", type=str, default=None) parser.add_argument("--cdr_schema", action="store_true") # False by default parser.add_argument("--bitmasks", action="store_true") # False by default + parser.add_argument("--legend_annotations_dir", type=str, default="") parser.add_argument("--legend_hints_dir", type=str, default="") p = parser.parse_args() @@ -61,15 +65,34 @@ def main(): logger.info(f"Processing {doc_id}") image_input = PipelineInput(image=image, raster_id=doc_id) - # load JSON legend hints file, if present, parse and add to PipelineInput - if p.legend_hints_dir: + if p.legend_annotations_dir: + # load JSON legend annotations file, if present, parse and add to PipelineInput + # expected format is LegendItemResponse CDR pydantic objects + try: + # check for legend annotations for this image + with open( + os.path.join(p.legend_annotations_dir, doc_id + ".json"), "r" + ) as fp: + legend_anns = json.load(fp) + legend_pt_items = parse_legend_annotations(legend_anns, doc_id) + # add legend item annotations as a pipeline input param + image_input.params[LEGEND_ITEMS_OUTPUT_KEY] = legend_pt_items + logger.info( + f"Number of legend point items loaded for this map: {len(legend_pt_items.items)}" + ) + + except Exception as e: + logger.error("EXCEPTION loading legend hints json: " + repr(e)) + + elif p.legend_hints_dir: + # load JSON legend hints file, if present, parse and add to PipelineInput try: - # check or legend hints for this image (JSON CMA contest data) + # check for legend hints for this image (JSON CMA contest data) with open( os.path.join(p.legend_hints_dir, doc_id + ".json"), "r" ) as fp: legend_hints = json.load(fp) - legend_pt_items = parse_legend_point_hints(legend_hints) + legend_pt_items = parse_legend_point_hints(legend_hints, doc_id) # add legend item hints as a pipeline input param image_input.params[LEGEND_ITEMS_OUTPUT_KEY] = legend_pt_items logger.info( diff --git a/tasks/point_extraction/entities.py b/tasks/point_extraction/entities.py index 5cd7c79c..82a8b0e5 100644 --- a/tasks/point_extraction/entities.py +++ b/tasks/point_extraction/entities.py @@ -171,6 +171,13 @@ class LegendPointItem(BaseModel): # TODO -- could be modified to use CDR PointLegendAndFeaturesResult class in the future name: str = Field(description="Label of the map unit in the legend") + class_name: str = Field( + default="", + description="Normalized label of the legend map unit, if available (based on point symbol ontology)", + ) + abbreviation: str = Field( + default="", description="Abbreviation of the map unit label." + ) description: str = Field( default="", description="Description of the map unit in the legend" ) @@ -186,6 +193,14 @@ class LegendPointItem(BaseModel): label. Format is expected to be [x,y] coordinate pairs where the top left is the origin (0,0).""", ) + system: str = Field( + default="", description="System that published this item" + ) # dgdg -- is this needed? (or get rid of provenance below?) + validated: bool = Field(default=False, description="Validated by human") + confidence: Optional[float] = Field( + default=None, + description="Confidence for this legend item (whether extracted by a model or human annotated)", + ) class LegendPointItems(BaseModel): diff --git a/tasks/point_extraction/legend_analyzer.py b/tasks/point_extraction/legend_analyzer.py index 7dc531d2..64d25d60 100644 --- a/tasks/point_extraction/legend_analyzer.py +++ b/tasks/point_extraction/legend_analyzer.py @@ -4,6 +4,11 @@ LegendPointItems, LEGEND_ITEMS_OUTPUT_KEY, ) +from tasks.point_extraction.legend_item_utils import ( + filter_labelme_annotations, + LEGEND_ANNOTATION_PROVENANCE, +) +from tasks.segmentation.entities import MapSegmentation, SEGMENTATION_OUTPUT_KEY logger = logging.getLogger(__name__) @@ -26,10 +31,12 @@ def run(self, task_input: TaskInput) -> TaskResult: run point symbol legend analysis """ + legend_pt_items = None if LEGEND_ITEMS_OUTPUT_KEY in task_input.data: # legend items for point symbols already exist - result = self._create_result(task_input) - return result + legend_pt_items = LegendPointItems.model_validate( + task_input.data[LEGEND_ITEMS_OUTPUT_KEY] + ) elif LEGEND_ITEMS_OUTPUT_KEY in task_input.request: # legend items for point symbols already exist as a request param # (ie, loaded from a JSON hints file) @@ -37,13 +44,28 @@ def run(self, task_input: TaskInput) -> TaskResult: legend_pt_items = LegendPointItems.model_validate( task_input.request[LEGEND_ITEMS_OUTPUT_KEY] ) + + if ( + legend_pt_items + and legend_pt_items.provenance == LEGEND_ANNOTATION_PROVENANCE.LABELME + ): + if SEGMENTATION_OUTPUT_KEY in task_input.data: + segmentation = MapSegmentation.model_validate( + task_input.data[SEGMENTATION_OUTPUT_KEY] + ) + # use segmentation results to filter noisy "labelme" legend annotations + # (needed because all labelme annotations are set to type "polygon" regardless of feature type: polygons, lines or points) + filter_labelme_annotations(legend_pt_items, segmentation) + logger.info( + f"Number of legend point annotations after filtering: {len(legend_pt_items.items)}" + ) + else: + logger.warning( + "No segmentation results available. Disregarding labelme legend annotations as noisy." + ) + legend_pt_items.items = [] return TaskResult( task_id=self._task_id, output={LEGEND_ITEMS_OUTPUT_KEY: legend_pt_items} ) - # TODO WIP - # Continue here... - # If no legend item hints available then could do our own naive version using segmentation, ocr, etc.? - - result = self._create_result(task_input) - return result + return self._create_result(task_input) diff --git a/tasks/point_extraction/legend_item_utils.py b/tasks/point_extraction/legend_item_utils.py new file mode 100644 index 00000000..c7d3a6c2 --- /dev/null +++ b/tasks/point_extraction/legend_item_utils.py @@ -0,0 +1,258 @@ +import logging +from enum import Enum +from typing import List +from collections import defaultdict +from shapely import Polygon, distance + +from tasks.point_extraction.entities import LegendPointItem, LegendPointItems +from tasks.point_extraction.label_map import LABEL_MAPPING +from schema.cdr_schemas.cdr_responses.legend_items import LegendItemResponse +from tasks.segmentation.entities import MapSegmentation + + +logger = logging.getLogger(__name__) + +SEGMENT_PT_LEGEND_CLASS = ( + "legend_points_lines" # class label for points legend area segmentation +) + + +# Legend item annotations "system" or provenance labels +class LEGEND_ANNOTATION_PROVENANCE(str, Enum): + GROUND_TRUTH = "ground_truth" + LABELME = "labelme" # aka STEPUP + POLYMER = "polymer" # Jatware's annotation system + + def __str__(self): + return self.value + + +def parse_legend_annotations( + legend_anns: list, + raster_id: str, + system_filter=[ + LEGEND_ANNOTATION_PROVENANCE.POLYMER, + LEGEND_ANNOTATION_PROVENANCE.LABELME, + ], + check_validated=False, +) -> LegendPointItems: + """ + parse legend annotations JSON data (CDR LegendItemResponse json format) + and convert to LegendPointItem objects + """ + + # parse legend annotations and group by system label + legend_item_resps = defaultdict(list) + count_leg_items = 0 + for leg_ann in legend_anns: + try: + leg_resp = LegendItemResponse(**leg_ann) + if leg_resp.system in system_filter or ( + check_validated and leg_resp.validated + ): + # only keep legend item responses from desired systems + # or with validated=True + legend_item_resps[leg_resp.system].append(leg_resp) + count_leg_items += 1 + except Exception as e: + # legend_pt_items = LegendPointItems(items=[], provenance="") + logger.error( + f"EXCEPTION parsing legend annotations json for raster {raster_id}: {repr(e)}" + ) + logger.info(f"Successfully loaded {count_leg_items} LegendItemResponse objects") + + # try to parse non-labelme annotations first + legend_point_items = [] + system_label = "" + for system, leg_anns in legend_item_resps.items(): + if system == LEGEND_ANNOTATION_PROVENANCE.LABELME: + continue + system_label = system + legend_point_items.extend(legend_ann_to_legend_items(leg_anns, raster_id)) + if legend_point_items: + return LegendPointItems(items=legend_point_items, provenance=system_label) + else: + # try to parse label annotations 2nd (since labelme anns have noisy data for point/line features) + for system, leg_anns in legend_item_resps.items(): + if not system == LEGEND_ANNOTATION_PROVENANCE.LABELME: + continue + legend_point_items.extend(legend_ann_to_legend_items(leg_anns, raster_id)) + if legend_point_items: + return LegendPointItems( + items=legend_point_items, + provenance=LEGEND_ANNOTATION_PROVENANCE.LABELME, + ) + return LegendPointItems(items=[], provenance="") + + +def parse_legend_point_hints(legend_hints: dict, raster_id: str) -> LegendPointItems: + """ + parse legend hints JSON data (from the CMA contest) + and convert to LegendPointItem objects + + legend_hints -- input hints dict + """ + + legend_point_items = [] + for shape in legend_hints["shapes"]: + label = shape["label"] + if not label.endswith("_pt") and not label.endswith("_point"): + continue # not a point symbol, skip + + # contour coords for the legend item's thumbnail swatch + xy_pts = shape.get("points", []) + if xy_pts: + x_min = xy_pts[0][0] + x_max = xy_pts[0][0] + y_min = xy_pts[0][1] + y_max = xy_pts[0][1] + for x, y in xy_pts: + x_min = int(min(x, x_min)) + x_max = int(max(x, x_max)) + y_min = int(min(y, y_min)) + y_max = int(max(y, y_max)) + else: + x_min = 0 + x_max = 0 + y_min = 0 + y_max = 0 + xy_pts = [ + [x_min, y_min], + [x_max, y_min], + [x_max, y_max], + [x_min, y_max], + ] + class_name = find_legend_keyword_match(label, raster_id) + legend_point_items.append( + LegendPointItem( + name=label, + class_name=class_name, + legend_bbox=[x_min, y_min, x_max, y_max], + legend_contour=xy_pts, + confidence=1.0, + system=LEGEND_ANNOTATION_PROVENANCE.GROUND_TRUTH, + validated=True, + ) + ) + return LegendPointItems( + items=legend_point_items, provenance=LEGEND_ANNOTATION_PROVENANCE.GROUND_TRUTH + ) + + +def find_legend_keyword_match(legend_item_name: str, raster_id: str) -> str: + """ + Use keyword matching to map legend item label to point extractor ontology class names + """ + leg_label_norm = raster_id + "_" + legend_item_name.strip().lower() + matches = [] + for symbol_class, suffixs in LABEL_MAPPING.items(): + for s in suffixs: + if s in leg_label_norm: + # match found + matches.append((s, symbol_class)) + if matches: + # sort to get longest suffix match + matches.sort(key=lambda a: len(a[0]), reverse=True) + symbol_class = matches[0][1] + logger.info( + f"Legend label: {legend_item_name} matches point class: {symbol_class}" + ) + return symbol_class + + logger.info(f"No point class match found for legend label: {legend_item_name}") + return "" + + +def legend_ann_to_legend_items( + legend_anns: List[LegendItemResponse], raster_id: str +) -> List[LegendPointItems]: + """ + convert LegendItemResponse (CDR schema format) + to internal LegendPointItem objects + """ + legend_point_items = [] + prev_label = "" + for leg_ann in legend_anns: + label = leg_ann.label if leg_ann.label else leg_ann.abbreviation + if ( + leg_ann.system == LEGEND_ANNOTATION_PROVENANCE.LABELME + and prev_label + and prev_label == label + ): + # special base to handle labelme (STEPUP) annotations... + # skip the 2nd labelme annotation in each pair + # (this 2nd entry is just the bbox for the legend item text; TODO -- could extract and include this text too?) + continue + class_name = find_legend_keyword_match(label, raster_id) + xy_pts = ( + leg_ann.px_geojson.coordinates[0] + if leg_ann.px_geojson + else [ + [leg_ann.px_bbox[0], leg_ann.px_bbox[1]], + [leg_ann.px_bbox[2], leg_ann.px_bbox[1]], + [leg_ann.px_bbox[2], leg_ann.px_bbox[3]], + [leg_ann.px_bbox[0], leg_ann.px_bbox[3]], + ] + ) + + legend_point_items.append( + LegendPointItem( + name=label, + class_name=class_name, + abbreviation=leg_ann.abbreviation, + description=leg_ann.description, + legend_bbox=leg_ann.px_bbox, + legend_contour=xy_pts, + system=leg_ann.system, + confidence=leg_ann.confidence, + validated=leg_ann.validated, + ) + ) + prev_label = label + + return legend_point_items + + +def filter_labelme_annotations( + leg_point_items: LegendPointItems, + segmentation: MapSegmentation, + width_thres=120, + shape_thres=2.0, +): + """ + labelme (aka STEPUP) legend annotations are noisy, with all items for polygons, points and lines grouped together. + These are filtered using segmentation info and shape heuristics to estimate which items, if any, correspond to point features + """ + + segs_point_legend = list( + filter( + lambda s: (s.class_label == SEGMENT_PT_LEGEND_CLASS), + segmentation.segments, + ) + ) + if not segs_point_legend: + logger.warning( + "No Points-Legend segment found. Disregarding labelme legend annotations as noisy." + ) + leg_point_items.items = [] + return + + filtered_leg_items = [] + for seg in segs_point_legend: + p_seg = Polygon(seg.poly_bounds) + for leg in leg_point_items.items: + p_leg = Polygon(leg.legend_contour) + if not p_seg.intersects(p_leg.centroid): + # this legend swatch is not within the points legend area; disregard + continue + # legend swatch intersects the points legend area + # check other properties to determine if swatch is line vs point symbol + w = leg.legend_bbox[2] - leg.legend_bbox[0] + h = leg.legend_bbox[3] - leg.legend_bbox[1] + if leg.class_name: + # legend item label is in points ontology + filtered_leg_items.append(leg) + elif w < width_thres and w < shape_thres * h: + # legend item swatch bbox is close to square + filtered_leg_items.append(leg) + leg_point_items.items = filtered_leg_items diff --git a/tasks/point_extraction/point_extractor_utils.py b/tasks/point_extraction/point_extractor_utils.py index 75c29e49..725b8107 100644 --- a/tasks/point_extraction/point_extractor_utils.py +++ b/tasks/point_extraction/point_extractor_utils.py @@ -5,9 +5,8 @@ from PIL import Image from collections import defaultdict -from tasks.point_extraction.entities import LegendPointItem, LegendPointItems, MapImage +from tasks.point_extraction.entities import MapImage from tasks.text_extraction.entities import TextExtraction -from tasks.point_extraction.label_map import LABEL_MAPPING from shapely.geometry import Polygon from shapely.strtree import STRtree from typing import List, Tuple, Dict @@ -314,85 +313,6 @@ def mask_ocr_blocks( return im -def parse_legend_point_hints(legend_hints: dict) -> LegendPointItems: - """ - parse legend hints JSON data (from the CMA contest) - and convert to LegendPointItem objects - - legend_hints -- input hints dict - only_keep_points -- if True, will discard any hints about line or polygon features - """ - - legend_point_items = [] - for shape in legend_hints["shapes"]: - label = shape["label"] - if not label.endswith("_pt") and not label.endswith("_point"): - continue # not a point symbol, skip - - # contour coords for the legend item's thumbnail swatch - xy_pts = shape.get("points", []) - if xy_pts: - x_min = xy_pts[0][0] - x_max = xy_pts[0][0] - y_min = xy_pts[0][1] - y_max = xy_pts[0][1] - for x, y in xy_pts: - x_min = int(min(x, x_min)) - x_max = int(max(x, x_max)) - y_min = int(min(y, y_min)) - y_max = int(max(y, y_max)) - else: - x_min = 0 - x_max = 0 - y_min = 0 - y_max = 0 - legend_point_items.append( - LegendPointItem( - name=label, - legend_bbox=[x_min, y_min, x_max, y_max], - legend_contour=xy_pts, - ) - ) - return LegendPointItems(items=legend_point_items, provenance="ground_truth") - - -def find_legend_label_matches( - legend_items: LegendPointItems, - raster_id: str, -) -> Dict[str, LegendPointItem]: - """ - Use keyword matching to map point extractor YOLO classes to legend item labels - Output is dict: point extractor model class -> legend label - """ - - def find_label_match(legend_item: LegendPointItem, raster_id: str) -> str: - leg_label_norm = raster_id + "_" + legend_item.name.strip().lower() - matches = [] - for symbol_class, suffixs in LABEL_MAPPING.items(): - for s in suffixs: - if s in leg_label_norm: - # match found - matches.append((s, symbol_class)) - if matches: - # sort to get longest suffix match - matches.sort(key=lambda a: len(a[0]), reverse=True) - symbol_class = matches[0][1] - logger.info( - f"Legend label: {legend_item.name} matches point class: {symbol_class}" - ) - return symbol_class - - logger.info(f"No point class match found for legend label: {legend_item.name}") - return "" - - label_mappings = {} - for legend_item in legend_items.items: - symbol_class = find_label_match(legend_item, raster_id) - if symbol_class: - label_mappings[symbol_class] = legend_item - return label_mappings - - def convert_preds_to_bitmasks( map_image: MapImage, legend_pt_labels: List[str], From 544284b78cfbd4dd451a33efece09fe025636dba Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Fri, 5 Jul 2024 16:52:00 -0400 Subject: [PATCH 03/66] re-factoring point pipeline entities, for integration with legend item annotations (WIP) --- cdr/result_subscriber.py | 2 +- pipelines/point_extraction/README.md | 2 +- .../point_extraction_pipeline.py | 6 +- schema/mappers/cdr.py | 5 +- tasks/README.md | 6 +- tasks/point_extraction/entities.py | 69 +++++++++--------- tasks/point_extraction/point_extractor.py | 54 +++++++------- .../point_extraction/point_extractor_utils.py | 6 +- .../point_orientation_extractor.py | 14 ++-- tasks/point_extraction/pytorch/utils.py | 4 +- .../template_match_point_extractor.py | 24 +++---- tasks/point_extraction/tiling.py | 71 +++++++++++-------- 12 files changed, 139 insertions(+), 124 deletions(-) diff --git a/cdr/result_subscriber.py b/cdr/result_subscriber.py index 00228cf4..ef4fe4b8 100644 --- a/cdr/result_subscriber.py +++ b/cdr/result_subscriber.py @@ -34,7 +34,7 @@ from schema.mappers.cdr import get_mapper from tasks.geo_referencing.entities import GeoreferenceResult as LARAGeoreferenceResult from tasks.metadata_extraction.entities import MetadataExtraction as LARAMetadata -from tasks.point_extraction.entities import MapImage as LARAPoints +from tasks.point_extraction.entities import PointLabels as LARAPoints from tasks.segmentation.entities import MapSegmentation as LARASegmentation import datetime diff --git a/pipelines/point_extraction/README.md b/pipelines/point_extraction/README.md index c9bd52b6..683c7809 100644 --- a/pipelines/point_extraction/README.md +++ b/pipelines/point_extraction/README.md @@ -53,7 +53,7 @@ The segmentation dependencies are required due to the use of the map segmentatio * Pipeline is defined in `point_extraction_pipeline.py` and is suitable for integration into other systems * Input is a image (ie binary image file buffer) * Ouput is the set of extracted points materialized as a: - * `MapImage` JSON object, which contains a list of `MapPointLabel` capturing the point information + * `PointLabels` JSON object, which contains a list of `PointLabel` capturing the point information * List of `FeatureResults` JSON objects as defined in the CMA TA1 CDR schema ### Command Line Execution ### diff --git a/pipelines/point_extraction/point_extraction_pipeline.py b/pipelines/point_extraction/point_extraction_pipeline.py index a0570ca0..6c4a6b1b 100644 --- a/pipelines/point_extraction/point_extraction_pipeline.py +++ b/pipelines/point_extraction/point_extraction_pipeline.py @@ -12,7 +12,7 @@ ) from tasks.point_extraction.tiling import Tiler, Untiler from tasks.point_extraction.entities import ( - MapImage, + PointLabels, LegendPointItems, LEGEND_ITEMS_OUTPUT_KEY, ) @@ -117,13 +117,13 @@ def __init__(self, id: str): def create_output(self, pipeline_result: PipelineResult) -> Output: """ - Creates a MapPointLabel object from the pipeline result. + Creates a PointLabel object from the pipeline result. Args: pipeline_result (PipelineResult): The pipeline result. Returns: - MapPointLabel: The map point label extraction object. + PointLabel: The map point label extraction object. """ map_image = MapImage.model_validate(pipeline_result.data["map_image"]) return BaseModelOutput( diff --git a/schema/mappers/cdr.py b/schema/mappers/cdr.py index 746552de..d728e927 100644 --- a/schema/mappers/cdr.py +++ b/schema/mappers/cdr.py @@ -31,7 +31,7 @@ MapShape, MetadataExtraction as LARAMetadata, ) -from tasks.point_extraction.entities import MapImage as LARAPoints +from tasks.point_extraction.entities import PointLabels as LARAPoints from tasks.point_extraction.label_map import YOLO_TO_CDR_LABEL from tasks.segmentation.entities import MapSegmentation as LARASegmentation @@ -256,6 +256,7 @@ def map_to_cdr(self, model: LARAPoints) -> FeatureResults: if pt_label not in point_features_by_class: # init result object for this point type... # TODO -- fill in legend item info if available in future + # ( we should use legend item annotations here, if possible, to fill in legend fields and then append point extractions afterwards) point_features_by_class[pt_label] = [] point_features_result = PointLegendAndFeaturesResult( id="id", @@ -263,7 +264,7 @@ def map_to_cdr(self, model: LARAPoints) -> FeatureResults: name=pt_label, abbreviation=pt_label, description=pt_label.replace("_", " ").strip().lower(), - legend_bbox=map_pt_label.legend_bbox, + # legend_bbox=map_pt_label.legend_bbox, point_features=None, # points are filled in below ) point_features.append(point_features_result) diff --git a/tasks/README.md b/tasks/README.md index 1a74f398..e94fc2df 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -117,9 +117,9 @@ Point orientation (ie "strike" direction) and the "dip" magnitude are also extra #### Using the Point Extraction Tasks #### * The main point extraction is available in the `YOLOPointDetector` task -* Ouput is a`MapImage` JSON object, which contains a list of `MapPointLabel` capturing the point information. -* Both dectector tasks take `MapTiles` objects as inputs - `MapTiles` are produced by the `Tiler` task -* `MapTiles` can be re-assembled into a `MapImage` using the `Untiler` task +* Ouput is a`PointLabels` JSON object, which contains a list of `PointLabel` capturing the point information. +* Both dectector tasks take `ImageTiles` objects as inputs - `ImageTiles` are produced by the `Tiler` task +* `ImageTiles` can be re-assembled into a `PointLabels` using the `Untiler` task A pipeline using these task, along with a CLI and sever wrapper are available at [../pipelines/point_extraction](../pipelines/point_extraction) diff --git a/tasks/point_extraction/entities.py b/tasks/point_extraction/entities.py index 82a8b0e5..0b57dc31 100644 --- a/tasks/point_extraction/entities.py +++ b/tasks/point_extraction/entities.py @@ -11,14 +11,14 @@ LEGEND_ITEMS_OUTPUT_KEY = "legend_point_items" -class MapPointLabel(BaseModel): +class PointLabel(BaseModel): """ - Represents a label on a map image. - Class ID should correspond to the ID encoded in the underlying model. + Represents a point feature extraction. + Can be used for either ML Object Detection or One-shot template matching results """ - classifier_name: str - classifier_version: str + model_name: str + model_version: str class_id: int class_name: str x1: int @@ -28,27 +28,22 @@ class MapPointLabel(BaseModel): score: float direction: Optional[float] = None # [deg] orientation of point symbol dip: Optional[float] = None # [deg] dip angle associated with symbol - legend_name: str - legend_bbox: List[Union[float, int]] + legend_name: str = Field( + default="", + description="Label for the legend item associated with this extraction", + ) + # legend_bbox: List[Union[float, int]] # TODO -- is this needed here? -class MapImage(BaseModel): +class PointLabels(BaseModel): """ - Represents a map image with point symbol prediction results + Represents a collection of PointLabel objects for an image or region-of-interest """ path: str raster_id: str - labels: Optional[List[MapPointLabel]] = None - map_bounds: Optional[List[int]] = ( - None # [x1, y1, h, w] location of map. TODO: Accept polygonal seg mask. - ) - point_legend_bounds: Optional[List[int]] = ( - None # [x1, y1, h, w] location of point legend. - ) - polygon_legend_bounds: Optional[List[int]] = ( - None # [x1, y1, h, w] location of polygon legend. - ) + roi_label: str = "" # roi (segment) area label for these tiles + labels: Optional[List[PointLabel]] = None _cached_image = None @@ -66,11 +61,10 @@ def image(self): if img.size[0] == 0 or img.size[1] == 0: raise ValueError("Image cannot have 0 height or width") self._cached_image = img - # TODO: Use polygonal segmask stored in self.map_bounds to filter the image and crop out the non-map regions. return img -class MapTile(BaseModel): +class ImageTile(BaseModel): """ Represents a tile of a map image in (x, y, width, height) format. x and y are coordinates on the original map image. @@ -82,10 +76,9 @@ class MapTile(BaseModel): y_offset: int # y offset of the tile in the original image. width: int height: int - map_bounds: tuple # map global bounds (x_min, y_min, x_max, y_max) image: Any # torch.Tensor or PIL.Image - map_path: str # Path to the original map image. - predictions: Optional[List[MapPointLabel]] = None + image_path: str # Path to the original map image. + predictions: Optional[List[PointLabel]] = None @validator("image", pre=True, always=True) def validate_image(cls, value): @@ -102,11 +95,13 @@ class Config: arbitrary_types_allowed = True -class MapTiles(BaseModel): +class ImageTiles(BaseModel): raster_id: str - tiles: List[MapTile] + tiles: List[ImageTile] + roi_bounds: tuple # roi global bounds (x_min, y_min, x_max, y_max) + roi_label: str = "" # roi (segment) area label for these tiles - def format_for_caching(self) -> MapTiles: + def format_for_caching(self) -> ImageTiles: """ Reformat point extraction tiles prior to caching - tile image raster is discarded @@ -114,29 +109,33 @@ def format_for_caching(self) -> MapTiles: tiles_cache = [] for t in self.tiles: - t_cache = MapTile( + t_cache = ImageTile( x_offset=t.x_offset, y_offset=t.y_offset, width=t.width, height=t.height, - map_bounds=t.map_bounds, image=None, - map_path=t.map_path, + image_path=t.image_path, predictions=t.predictions, ) tiles_cache.append(t_cache) - return MapTiles(raster_id=self.raster_id, tiles=tiles_cache) + return ImageTiles( + raster_id=self.raster_id, + roi_bounds=self.roi_bounds, + roi_label=self.roi_label, + tiles=tiles_cache, + ) def join_with_cached_predictions( - self, cached_preds: MapTiles, point_legend_mapping: Dict[str, LegendPointItem] + self, cached_preds: ImageTiles, point_legend_mapping: Dict[str, LegendPointItem] ) -> bool: """ - Append cached point predictions to MapTiles + Append cached point predictions to ImageTiles """ try: # re-format cached predictions with key as (x_offset, y_offset) - cached_dict: Dict[Any, MapTile] = {} + cached_dict: Dict[Any, ImageTile] = {} for p in cached_preds.tiles: cached_dict[(p.x_offset, p.y_offset)] = p for t in self.tiles: @@ -144,7 +143,7 @@ def join_with_cached_predictions( if key not in cached_dict: # cached predictions not found for this tile! return False - t_cached: MapTile = cached_dict[key] + t_cached: ImageTile = cached_dict[key] t.predictions = t_cached.predictions if t.predictions is not None: diff --git a/tasks/point_extraction/point_extractor.py b/tasks/point_extraction/point_extractor.py index cd0143f0..33b136ce 100644 --- a/tasks/point_extraction/point_extractor.py +++ b/tasks/point_extraction/point_extractor.py @@ -1,12 +1,13 @@ from tasks.point_extraction.entities import ( - MapTile, - MapTiles, - MapPointLabel, + ImageTile, + ImageTiles, + PointLabel, LegendPointItem, LegendPointItems, LEGEND_ITEMS_OUTPUT_KEY, ) -from tasks.point_extraction.point_extractor_utils import find_legend_label_matches + +from tasks.point_extraction.legend_item_utils import find_legend_label_matches from tasks.common.s3_data_cache import S3DataCache from tasks.common.task import Task, TaskInput, TaskResult @@ -26,6 +27,8 @@ CONF_THRES = 0.20 # (0.25) minimum confidence threshold for detections IOU_THRES = 0.7 # IoU threshold for NMS +MODEL_NAME = "uncharted_ml_point_extractor" + logger = logging.getLogger(__name__) @@ -109,10 +112,10 @@ def _prep_model_data(self, model_data_path: str, data_cache_path: str) -> Path: def process_output( self, predictions: Results, point_legend_mapping: Dict[str, LegendPointItem] - ) -> List[MapPointLabel]: + ) -> List[PointLabel]: """ Convert point detection inference results from YOLO model format - to a list of MapPointLabel objects + to a list of PointLabel objects """ pt_labels = [] for pred in predictions: @@ -128,15 +131,15 @@ def process_output( # map YOLO class name to legend item name, if available class_name = self.model.names[int(class_id)] legend_name = class_name - legend_bbox = [] + # legend_bbox = [] if class_name in point_legend_mapping: legend_name = point_legend_mapping[class_name].name - legend_bbox = point_legend_mapping[class_name].legend_bbox + # legend_bbox = point_legend_mapping[class_name].legend_bbox pt_labels.append( - MapPointLabel( - classifier_name="unchartNet_point_extractor", - classifier_version=self._model_id, + PointLabel( + model_name=MODEL_NAME, + model_version=self._model_id, class_id=int(class_id), class_name=self.model.names[int(class_id)], x1=int(x1), @@ -145,7 +148,7 @@ def process_output( y2=int(y2), score=score, legend_name=legend_name, - legend_bbox=legend_bbox, + # legend_bbox=legend_bbox, ) ) return pt_labels @@ -154,7 +157,7 @@ def run(self, task_input: TaskInput) -> TaskResult: """ run YOLO model inference for point symbol detection """ - map_tiles = MapTiles.model_validate(task_input.data["map_tiles"]) + image_tiles = ImageTiles.model_validate(task_input.data["map_tiles"]) point_legend_mapping: Dict[str, LegendPointItem] = {} if LEGEND_ITEMS_OUTPUT_KEY in task_input.data: @@ -174,8 +177,8 @@ def run(self, task_input: TaskInput) -> TaskResult: doc_key = f"{task_input.raster_id}_points-{self._model_id}" # check cache and re-use existing file if present json_data = self.fetch_cached_result(doc_key) - if json_data and map_tiles.join_with_cached_predictions( - MapTiles(**json_data), point_legend_mapping + if json_data and image_tiles.join_with_cached_predictions( + ImageTiles(**json_data), point_legend_mapping ): # cached point predictions loaded successfully logger.info( @@ -183,14 +186,14 @@ def run(self, task_input: TaskInput) -> TaskResult: ) return TaskResult( task_id=self._task_id, - output={"map_tiles": map_tiles}, + output={"map_tiles": image_tiles}, ) - output: List[MapTile] = [] + tiles_out: List[ImageTile] = [] # run batch model inference... - for i in tqdm(range(0, len(map_tiles.tiles), self.bsz)): + for i in tqdm(range(0, len(image_tiles.tiles), self.bsz)): logger.info(f"Processing batch {i} to {i + self.bsz}") - batch = map_tiles.tiles[i : i + self.bsz] + batch = image_tiles.tiles[i : i + self.bsz] images = [tile.image for tile in batch] # note: ideally tile sizes used should be the same size as used during model training # tiles can be resized during inference pre-processing, if needed using 'imgsz' param @@ -203,16 +206,17 @@ def run(self, task_input: TaskInput) -> TaskResult: ) for tile, preds in zip(batch, batch_preds): tile.predictions = self.process_output(preds, point_legend_mapping) - output.append(tile) - result_map_tiles = MapTiles(raster_id=map_tiles.raster_id, tiles=output) + tiles_out.append(tile) + # save tile results with point extraction predictions + image_tiles.tiles = tiles_out # write to cache self.write_result_to_cache( - result_map_tiles.format_for_caching().model_dump(), doc_key + image_tiles.format_for_caching().model_dump(), doc_key ) return TaskResult( - task_id=self._task_id, output={"map_tiles": result_map_tiles.model_dump()} + task_id=self._task_id, output={"map_tiles": image_tiles.model_dump()} ) def _get_model_id(self, model: YOLO) -> str: @@ -230,8 +234,8 @@ def version(self): @property def input_type(self): - return List[MapTile] + return List[ImageTile] @property def output_type(self): - return List[MapTile] + return List[ImageTile] diff --git a/tasks/point_extraction/point_extractor_utils.py b/tasks/point_extraction/point_extractor_utils.py index 725b8107..83558353 100644 --- a/tasks/point_extraction/point_extractor_utils.py +++ b/tasks/point_extraction/point_extractor_utils.py @@ -5,7 +5,7 @@ from PIL import Image from collections import defaultdict -from tasks.point_extraction.entities import MapImage +from tasks.point_extraction.entities import PointLabels from tasks.text_extraction.entities import TextExtraction from shapely.geometry import Polygon from shapely.strtree import STRtree @@ -314,13 +314,13 @@ def mask_ocr_blocks( def convert_preds_to_bitmasks( - map_image: MapImage, + map_image: PointLabels, legend_pt_labels: List[str], w_h: Tuple[int, int], binary_pixel_val=1, ) -> Dict[str, Image.Image]: """ - Convert the MapImage point predictions to CMA contest style bitmasks + Convert the PointLabels point predictions to CMA contest style bitmasks Output is dict: point label -> bitmask image """ if not map_image.labels: diff --git a/tasks/point_extraction/point_orientation_extractor.py b/tasks/point_extraction/point_orientation_extractor.py index c70f5bee..6b10a7c6 100644 --- a/tasks/point_extraction/point_orientation_extractor.py +++ b/tasks/point_extraction/point_orientation_extractor.py @@ -1,4 +1,4 @@ -from tasks.point_extraction.entities import MapImage +from tasks.point_extraction.entities import PointLabels from tasks.common.task import Task, TaskInput, TaskResult from tasks.point_extraction.label_map import POINT_CLASS from tasks.point_extraction.task_config import PointOrientationConfig @@ -178,15 +178,15 @@ def _dip_magnitude_extraction( def run(self, input: TaskInput) -> TaskResult: """ - Run batch predictions over a MapImage object. + Run batch predictions over a PointLabels object. - This modifies the MapImage object predictions in-place. + This modifies the PointLabels object predictions in-place. """ # get result from point extractor task (with point symbol predictions) - map_image = MapImage.model_validate(input.data["map_image"]) + map_image = PointLabels.model_validate(input.data["map_image"]) if map_image.labels is None: - raise RuntimeError("MapImage must have labels to run batch_predict") + raise RuntimeError("PointLabels must have labels to run batch_predict") if len(map_image.labels) == 0: logger.warning( "No point symbol extractions found. Skipping Point orientation extraction." @@ -371,8 +371,8 @@ def _trig_to_compass_angle(self, angle_deg: int, rotate_max: int) -> int: @property def input_type(self): - return MapImage + return PointLabels @property def output_type(self): - return MapImage + return PointLabels diff --git a/tasks/point_extraction/pytorch/utils.py b/tasks/point_extraction/pytorch/utils.py index 2088305a..5e604b2b 100644 --- a/tasks/point_extraction/pytorch/utils.py +++ b/tasks/point_extraction/pytorch/utils.py @@ -1,4 +1,4 @@ -from tasks.point_extraction.entities import MapTile +from tasks.point_extraction.entities import ImageTile import numpy as np import torch from torch.utils.data import Dataset @@ -14,7 +14,7 @@ class PointInferenceDataset(Dataset): def __init__( self, - tiles: List[MapTile], + tiles: List[ImageTile], ) -> None: self.tiles = tiles super().__init__() diff --git a/tasks/point_extraction/template_match_point_extractor.py b/tasks/point_extraction/template_match_point_extractor.py index 0a68eee3..ec51dd38 100644 --- a/tasks/point_extraction/template_match_point_extractor.py +++ b/tasks/point_extraction/template_match_point_extractor.py @@ -16,14 +16,14 @@ TEXT_EXTRACTION_OUTPUT_KEY, ) from tasks.point_extraction.entities import ( - MapImage, - MapPointLabel, + PointLabels, + PointLabel, LegendPointItem, LegendPointItems, LEGEND_ITEMS_OUTPUT_KEY, ) -MODEL_NAME = "uncharted_template_pointextractor" +MODEL_NAME = "uncharted_oneshot_point_extractor" MODEL_VER = "0.0.1" # class labels for map and points legend areas @@ -86,11 +86,11 @@ def run(self, task_input: TaskInput) -> TaskResult: # get existing point predictions from YOLO point extractor if "map_image" in task_input.data: - map_image_results = MapImage.model_validate(task_input.data["map_image"]) + map_image_results = PointLabels.model_validate(task_input.data["map_image"]) if map_image_results.labels is None: map_image_results.labels = [] else: - map_image_results = MapImage( + map_image_results = PointLabels( path="", raster_id=task_input.raster_id, labels=[] ) @@ -401,10 +401,10 @@ def _process_output( map_roi: List[int], legend_bbox: List, bbox_size: int = 90, - ) -> List[MapPointLabel]: + ) -> List[PointLabel]: """ Convert template based detection results - to a list of MapPointLabel objects + to a list of PointLabel objects """ pt_labels = [] bbox_half = bbox_size / 2 @@ -420,9 +420,9 @@ def _process_output( # prepare final result label # note: using hash(label) as class numeric ID pt_labels.append( - MapPointLabel( - classifier_name=MODEL_NAME, - classifier_version=MODEL_VER, + PointLabel( + model_name=MODEL_NAME, + model_version=MODEL_VER, class_id=hash(label), class_name=label, x1=x1, @@ -431,7 +431,7 @@ def _process_output( y2=y2, score=xcorr / 255.0, legend_name=label, - legend_bbox=legend_bbox, + # legend_bbox=legend_bbox, ) ) @@ -439,7 +439,7 @@ def _process_output( def _which_points_need_processing( self, - map_point_labels: List[MapPointLabel], + map_point_labels: List[PointLabel], legend_pt_items: List[LegendPointItem], min_predictions=0, ) -> List[LegendPointItem]: diff --git a/tasks/point_extraction/tiling.py b/tasks/point_extraction/tiling.py index e14bb6cf..df6075df 100644 --- a/tasks/point_extraction/tiling.py +++ b/tasks/point_extraction/tiling.py @@ -7,7 +7,12 @@ from common.task import Task, TaskInput, TaskResult -from tasks.point_extraction.entities import MapTile, MapTiles, MapImage, MapPointLabel +from tasks.point_extraction.entities import ( + ImageTile, + ImageTiles, + PointLabels, + PointLabel, +) from tasks.segmentation.entities import MapSegmentation, SEGMENTATION_OUTPUT_KEY from tasks.segmentation.segmenter_utils import get_segment_bounds, segments_to_mask @@ -49,6 +54,7 @@ def run( x_min = 0 y_min = 0 y_max, x_max, _ = image_array.shape + roi_label = "" # use image segmentation to restrict point extraction to map area only if SEGMENTATION_OUTPUT_KEY in task_input.data: @@ -70,11 +76,13 @@ def run( # restrict tiling to use *only* the bounding rectangle of map area p_map = p_map[0] # use 1st (highest ranked) map segment (x_min, y_min, x_max, y_max) = [int(b) for b in p_map.bounds] + roi_label = SEGMENT_MAP_CLASS + roi_bounds = (x_min, y_min, x_max, y_max) step_x = int(self.tile_size[0] - self.overlap[0]) step_y = int(self.tile_size[1] - self.overlap[1]) - tiles: List[MapTile] = [] + tiles: List[ImageTile] = [] for y in range(y_min, y_max, step_y): for x in range(x_min, x_max, step_x): @@ -95,28 +103,32 @@ def run( padded_tile[:height, :width] = tile_array tile_array = padded_tile - maptile = MapTile( + maptile = ImageTile( x_offset=x, y_offset=y, width=self.tile_size[0], height=self.tile_size[1], - map_bounds=(x_min, y_min, x_max, y_max), image=Image.fromarray(tile_array), - map_path="", + image_path="", ) tiles.append(maptile) - map_tiles = MapTiles(raster_id=task_input.raster_id, tiles=tiles) + map_tiles = ImageTiles( + raster_id=task_input.raster_id, + tiles=tiles, + roi_bounds=roi_bounds, + roi_label=roi_label, + ) return TaskResult( task_id=self._task_id, output={"map_tiles": map_tiles.model_dump()} ) @property def input_type(self): - return MapImage + return PointLabels @property def output_type(self): - return List[MapTile] + return List[ImageTile] class Untiler(Task): @@ -133,21 +145,20 @@ def __init__(self, task_id="", overlap: tuple = TILE_OVERLAP_DEFAULT): def run(self, input: TaskInput) -> TaskResult: """ Reconstructs the original image from the tiles and maps back the bounding boxes and labels. - tile_predictions: List of MapPointLabel objects. Generated by the model. TILES MUST BE FROM ONLY ONE MAP. - returns: List of MapPointLabel objects. These can be mapped directly onto the original map. + tile_predictions: List of PointLabel objects. Generated by the model. TILES MUST BE FROM ONLY ONE MAP. + returns: List of PointLabel objects. These can be mapped directly onto the original map. """ - map_tiles = MapTiles.model_validate(input.get_data("map_tiles")) - tiles = map_tiles.tiles + image_tiles = ImageTiles.model_validate(input.get_data("map_tiles")) assert all( - i.predictions is not None for i in tiles + i.predictions is not None for i in image_tiles.tiles ), "Tiles must have predictions attached to them." all_predictions = [] overlap_predictions = {} num_dedup = 0 - map_path = tiles[0].map_path - for tile in tiles: + map_path = image_tiles.tiles[0].image_path + for tile in image_tiles.tiles: x_offset = tile.x_offset # xmin of tile, absolute value in original map y_offset = tile.y_offset # ymin of tile, absolute value in original map @@ -170,7 +181,7 @@ def run(self, input: TaskInput) -> TaskResult: if self.overlap[0] > 0 or self.overlap[1] > 0: pred_redundant, pred_in_overlap = self._is_prediction_redundant( pred, - tile.map_bounds, + image_tiles.roi_bounds, (tile.x_offset, tile.y_offset), (tile.width, tile.height), ) @@ -178,9 +189,9 @@ def run(self, input: TaskInput) -> TaskResult: if pred_redundant: continue - global_prediction = MapPointLabel( - classifier_name=pred.classifier_name, - classifier_version=pred.classifier_version, + global_prediction = PointLabel( + model_name=pred.model_name, + model_version=pred.model_version, class_id=pred.class_id, class_name=label_name, # Add offset of tile to project onto original map... @@ -192,7 +203,7 @@ def run(self, input: TaskInput) -> TaskResult: direction=pred.direction, dip=pred.dip, legend_name=pred.legend_name, - legend_bbox=pred.legend_bbox, # bbox coords assumed to be in global pixel coords + # legend_bbox=pred.legend_bbox, # bbox coords assumed to be in global pixel coords ) if pred_in_overlap: @@ -217,15 +228,15 @@ def run(self, input: TaskInput) -> TaskResult: logger.info( f"Total point predictions after re-constructing map tiles: {len(all_predictions)}, with {num_dedup} discarded as duplicates" ) - map_image = MapImage( + map_image = PointLabels( path=map_path, raster_id=input.raster_id, labels=all_predictions ) return TaskResult(task_id=self._task_id, output={"map_image": map_image}) def _is_prediction_redundant( self, - pred: MapPointLabel, - map_bbox, + pred: PointLabel, + roi_bbox, tile_offset: tuple, tile_wh: tuple, shape_thres=2, @@ -239,7 +250,7 @@ def _is_prediction_redundant( x2 = pred.x2 y1 = pred.y1 y2 = pred.y2 - (map_xmin, map_ymin, map_xmax, map_ymax) = map_bbox + (roi_xmin, roi_ymin, roi_xmax, roi_ymax) = roi_bbox tile_w, tile_h = tile_wh x_offset, y_offset = tile_offset @@ -263,10 +274,10 @@ def _is_prediction_redundant( # pred bbox is at a tile edge and NOT square, # check if bbox edges correspond to global image bounds if ( - x1 + x_offset > map_xmin - and x2 + x_offset < map_xmax - and y1 + y_offset > map_ymin - and y2 + y_offset < map_ymax + x1 + x_offset > roi_xmin + and x2 + x_offset < roi_xmax + and y1 + y_offset > roi_ymin + and y2 + y_offset < roi_ymax ): # non-square point bbox not at map edges, assume this is a noisy prediction (due to tile overlap) and skip pred_redundant = True @@ -279,8 +290,8 @@ def _is_prediction_redundant( @property def input_type(self): - return List[MapTile] + return List[ImageTile] @property def output_type(self): - return MapImage + return PointLabels From 310780c8ed016c3bf5485dddd6aaae7f52d7489e Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Mon, 15 Jul 2024 14:13:22 -0400 Subject: [PATCH 04/66] append pipeline to system name to avoid overwrite on CDR upload --- cdr/result_subscriber.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cdr/result_subscriber.py b/cdr/result_subscriber.py index 599397cf..c4d9ea69 100644 --- a/cdr/result_subscriber.py +++ b/cdr/result_subscriber.py @@ -361,7 +361,7 @@ def _push_georeferencing(self, result: RequestResult): cog_id=result.request.image_id, georeference_results=[], gcps=[], - system=self._system_name, + system=f"{self._system_name}-georeference", system_version=self._system_version, ) @@ -425,7 +425,9 @@ def _push_segmentation(self, result: RequestResult): cdr_result: Optional[FeatureResults] = None try: lara_result = LARASegmentation.model_validate(segmentation_raw_result) - mapper = get_mapper(lara_result, self._system_name, self._system_version) + mapper = get_mapper( + lara_result, f"{self._system_name}-area", self._system_version + ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except: logger.error( @@ -443,7 +445,9 @@ def _push_points(self, result: RequestResult): cdr_result: Optional[FeatureResults] = None try: lara_result = LARAPoints.model_validate(points_raw_result) - mapper = get_mapper(lara_result, self._system_name, self._system_version) + mapper = get_mapper( + lara_result, f"{self._system_name}-points", self._system_version + ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except: logger.error("bad points result received so unable to send results to cdr") @@ -462,7 +466,9 @@ def _push_metadata(self, result: RequestResult): cdr_result: Optional[CogMetaData] = None try: lara_result = LARAMetadata.model_validate(metadata_result_raw) - mapper = get_mapper(lara_result, self._system_name, self._system_version) + mapper = get_mapper( + lara_result, f"{self._system_name}-metadata", self._system_version + ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except Exception as e: logger.exception( From 3dfee295935bee87781c8a66cde6de1a8e3911ec Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Thu, 18 Jul 2024 12:50:17 -0400 Subject: [PATCH 05/66] Added geocoding approach for maps with many points to average out the points rather than rely on randomness as much. --- pipelines/geo_referencing/factory.py | 36 +- .../geo_referencing/coordinates_extractor.py | 4 +- tasks/geo_referencing/geocode.py | 359 +++++++++++++++--- tasks/geo_referencing/util.py | 28 +- tasks/geo_referencing/utm_extractor.py | 1 + 5 files changed, 362 insertions(+), 66 deletions(-) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index 6f78fd3e..a8cfbc41 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -21,7 +21,7 @@ from tasks.geo_referencing.filter import NaiveFilter, OutlierFilter, UTMStatePlaneFilter from tasks.geo_referencing.geo_fencing import GeoFencer from tasks.geo_referencing.georeference import GeoReference -from tasks.geo_referencing.geocode import Geocoder as rfGeocoder +from tasks.geo_referencing.geocode import PointGeocoder, BoxGeocoder from tasks.geo_referencing.ground_control import CreateGroundControlPoints from tasks.geo_referencing.inference import InferenceCoordinateExtractor from tasks.geo_referencing.roi_extractor import ( @@ -48,8 +48,8 @@ def run_step(input: TaskInput) -> bool: - lats = input.get_data("lats", []) - lons = input.get_data("lons", []) + lats = input.get_data("lats", {}) + lons = input.get_data("lons", {}) lats_distinct = set(map(lambda x: x[1].get_parsed_degree(), lats.items())) lons_distinct = set(map(lambda x: x[1].get_parsed_degree(), lons.items())) @@ -81,6 +81,7 @@ def create_geo_referencing_pipelines( segmentation_cache = os.path.join(working_dir, "segmentation") text_cache = os.path.join(working_dir, "text") metadata_cache = os.path.join(working_dir, f"metadata-gamma-{ocr_gamma_correction}") + geocoder_thresh = 10 p = [] @@ -148,7 +149,11 @@ def create_geo_referencing_pipelines( run_points=True, ) ) - tasks.append(rfGeocoder("geocoded-georeferencing", ["point", "population"])) + tasks.append( + PointGeocoder( + "geocoded-georeferencing", ["point", "population"], geocoder_thresh + ) + ) tasks.append(UTMCoordinatesExtractor("fifth")) tasks.append(CreateGroundControlPoints("sixth")) tasks.append(GeoReference("seventh", 1)) @@ -281,7 +286,14 @@ def create_geo_referencing_pipelines( ) tasks.append(OutlierFilter("utm-outliers")) tasks.append(UTMStatePlaneFilter("utm-state-plane")) - tasks.append(rfGeocoder("geocoded-georeferencing", ["point", "population"])) + tasks.append( + PointGeocoder( + "geocoded-georeferencing", ["point", "population"], geocoder_thresh + ) + ) + tasks.append( + BoxGeocoder("geocoded-box", ["point", "population"], geocoder_thresh) + ) tasks.append(InferenceCoordinateExtractor("coordinate-inference")) tasks.append(ScaleExtractor("scaler", "")) tasks.append(CreateGroundControlPoints("seventh")) @@ -390,7 +402,11 @@ def create_geo_referencing_pipelines( ) tasks.append(OutlierFilter("utm-outliers")) tasks.append(UTMStatePlaneFilter("utm-state-plane")) - tasks.append(rfGeocoder("geocoded-georeferencing", ["point", "population"])) + tasks.append( + PointGeocoder( + "geocoded-georeferencing", ["point", "population"], geocoder_thresh + ) + ) tasks.append(CreateGroundControlPoints("seventh")) tasks.append(GeoReference("eighth", 1)) """p.append( @@ -495,7 +511,11 @@ def create_geo_referencing_pipelines( ) tasks.append(OutlierFilter("utm-outliers")) tasks.append(UTMStatePlaneFilter("utm-state-plane")) - tasks.append(rfGeocoder("geocoded-georeferencing", ["point", "population"])) + tasks.append( + PointGeocoder( + "geocoded-georeferencing", ["point", "population"], geocoder_thresh + ) + ) tasks.append(CreateGroundControlPoints("seventh")) tasks.append(GeoReference("eighth", 1)) """p.append( @@ -650,7 +670,7 @@ def create_geo_referencing_pipeline( ) tasks.append(OutlierFilter("utm-outliers")) tasks.append(UTMStatePlaneFilter("utm-state-plane")) - tasks.append(rfGeocoder("geocoded-georeferencing", ["point", "population"])) + tasks.append(PointGeocoder("geocoded-georeferencing", ["point", "population"], 10)) tasks.append(InferenceCoordinateExtractor("coordinate-inference")) tasks.append(ScaleExtractor("scaler", "")) tasks.append(CreateGroundControlPoints("seventh")) diff --git a/tasks/geo_referencing/coordinates_extractor.py b/tasks/geo_referencing/coordinates_extractor.py index 45a2f02d..c9be6c11 100644 --- a/tasks/geo_referencing/coordinates_extractor.py +++ b/tasks/geo_referencing/coordinates_extractor.py @@ -152,8 +152,8 @@ def _create_coordinate_result( return result def _should_run(self, input: CoordinateInput) -> bool: - lats = input.input.get_data("lats", []) - lons = input.input.get_data("lons", []) + lats = input.input.get_data("lats", {}) + lons = input.input.get_data("lons", {}) num_keypoints = min(len(lons), len(lats)) logger.info(f"checking run condition: {num_keypoints} key points") return num_keypoints < 2 diff --git a/tasks/geo_referencing/geocode.py b/tasks/geo_referencing/geocode.py index db7e5411..41945645 100644 --- a/tasks/geo_referencing/geocode.py +++ b/tasks/geo_referencing/geocode.py @@ -26,7 +26,7 @@ GEOCODED_PLACES_OUTPUT_KEY, ) -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Callable COORDINATE_CONFIDENCE_GEOCODE = 0.8 @@ -38,24 +38,13 @@ def __init__(self, task_id: str, place_types: List[str]): super().__init__(task_id) self._place_types = place_types - def _extract_coordinates( - self, input: CoordinateInput - ) -> Tuple[ - Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] - ]: - geocoded: DocGeocodedPlaces = input.input.parse_data( - GEOCODED_PLACES_OUTPUT_KEY, DocGeocodedPlaces.model_validate - ) - geofence_raw: DocGeoFence = input.input.parse_data( - GEOFENCE_OUTPUT_KEY, DocGeoFence.model_validate - ) - places = [p for p in geocoded.places if p.place_type in self._place_types] + def _should_run(self, input: CoordinateInput) -> bool: + return super()._should_run(input) - # filter places to only consider those within the geofence - # TODO: may need to deep copy the object to not overwrite coordinates - logger.info( - f"extracting coordinates via geocoding with {len(places)} locations" - ) + def _filter_coordinates( + self, geofence_raw: DocGeoFence, geocoded: DocGeocodedPlaces + ) -> List[GeocodedPlace]: + places = [p for p in geocoded.places if p.place_type in self._place_types] places_filtered = [] if geofence_raw.geofence.defaulted: places_filtered = places @@ -77,36 +66,7 @@ def _extract_coordinates( logger.info( f"removing {pc.place_name} from location set since no coordinates fall within the geofence" ) - logger.info( - f"extracting coordinates via geocoding with {len(places_filtered)} locations remaining after filtering" - ) - - # get the coordinates for the points that fall within range - coordinates = self._get_coordinates(places_filtered) - - # create the required coordinate structures - lon_pts: Dict[Tuple[float, float], Coordinate] = input.input.get_data("lons") - lat_pts: Dict[Tuple[float, float], Coordinate] = input.input.get_data("lats") - for c in coordinates: - d = lon_pts - if c.is_lat(): - d = lat_pts - d[c.to_deg_result()[0]] = c - self._add_param( - input.input, - str(uuid.uuid4()), - f"coordinate-{c.get_type()}-geocoded", - { - "text": c.get_text(), - "parsed": c.get_parsed_degree(), - "type": "latitude" if c.is_lat() else "longitude", - "pixel_alignment": c.get_pixel_alignment(), - "confidence": c.get_confidence(), - }, - "geocoded coordinate", - ) - - return lon_pts, lat_pts + return places_filtered def _get_coordinates(self, places: List[GeocodedPlace]) -> List[Coordinate]: # cluster points using the geographic coordinates @@ -149,7 +109,7 @@ def _get_coordinates(self, places: List[GeocodedPlace]) -> List[Coordinate]: Coordinate( "point derived lat", c[1][0].place_name, - abs(c[0][1]), + c[0][1], SOURCE_GEOCODE, True, pixel_alignment=(c[1][1], c[1][2]), @@ -161,7 +121,7 @@ def _get_coordinates(self, places: List[GeocodedPlace]) -> List[Coordinate]: Coordinate( "point derived lon", c[1][0].place_name, - abs(c[0][0]), + c[0][0], SOURCE_GEOCODE, False, pixel_alignment=(c[1][1], c[1][2]), @@ -202,3 +162,302 @@ def _get_point_geo( return (coordinate[0].geo_x + coordinate[2].geo_x) / 2, ( coordinate[0].geo_y + coordinate[2].geo_y ) / 2 + + +class PointGeocoder(Geocoder): + def __init__(self, task_id: str, place_types: List[str], run_limit: int): + super().__init__(task_id, place_types) + self._run_limit = run_limit + + def _should_run(self, input: CoordinateInput) -> bool: + parent_should = super()._should_run(input) + + # dont run if the base condition is not met (sufficient coordinates already parsed) + if not parent_should: + return False + + # check how many geocoded points there are + # only run if a sufficiently large amount of them exist + # only run if a sufficiently large area of the map is covered + geocoded: DocGeocodedPlaces = input.input.parse_data( + GEOCODED_PLACES_OUTPUT_KEY, DocGeocodedPlaces.model_validate + ) + geofence_raw: DocGeoFence = input.input.parse_data( + GEOFENCE_OUTPUT_KEY, DocGeoFence.model_validate + ) + + # filter places to only consider those within the geofence + places_filtered = self._filter_coordinates(geofence_raw, geocoded) + + # TODO: CHECK FOR SPREAD ACROSS MAP AREA + logger.info( + f"point geocoder to run if {len(places_filtered)} < {self._run_limit}" + ) + return len(places_filtered) < self._run_limit + + def _extract_coordinates( + self, input: CoordinateInput + ) -> Tuple[ + Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] + ]: + geocoded: DocGeocodedPlaces = input.input.parse_data( + GEOCODED_PLACES_OUTPUT_KEY, DocGeocodedPlaces.model_validate + ) + geofence_raw: DocGeoFence = input.input.parse_data( + GEOFENCE_OUTPUT_KEY, DocGeoFence.model_validate + ) + + # filter places to only consider those within the geofence + # TODO: may need to deep copy the object to not overwrite coordinates + places_filtered = self._filter_coordinates(geofence_raw, geocoded) + logger.info( + f"extracting coordinates via point geocoding with {len(places_filtered)} locations remaining after filtering" + ) + + # get the coordinates for the points that fall within range + coordinates = self._get_coordinates(places_filtered) + + # create the required coordinate structures + lon_pts: Dict[Tuple[float, float], Coordinate] = input.input.get_data("lons") + lat_pts: Dict[Tuple[float, float], Coordinate] = input.input.get_data("lats") + for c in coordinates: + d = lon_pts + if c.is_lat(): + d = lat_pts + d[c.to_deg_result()[0]] = c + self._add_param( + input.input, + str(uuid.uuid4()), + f"coordinate-{c.get_type()}-geocoded", + { + "text": c.get_text(), + "parsed": c.get_parsed_degree(), + "type": "latitude" if c.is_lat() else "longitude", + "pixel_alignment": c.get_pixel_alignment(), + "confidence": c.get_confidence(), + }, + "geocoded coordinate", + ) + + return lon_pts, lat_pts + + +class BoxGeocoder(Geocoder): + def __init__(self, task_id: str, place_types: List[str], run_limit: int = 10): + super().__init__(task_id, place_types) + self._run_limit = run_limit + + def _should_run(self, input: CoordinateInput) -> bool: + parent_should = super()._should_run(input) + + # dont run if the base condition is not met (sufficient coordinates already parsed) + if not parent_should: + return False + + # check how many geocoded points there are + # only run if a sufficiently large amount of them exist + # only run if a sufficiently large area of the map is covered + geocoded: DocGeocodedPlaces = input.input.parse_data( + GEOCODED_PLACES_OUTPUT_KEY, DocGeocodedPlaces.model_validate + ) + geofence_raw: DocGeoFence = input.input.parse_data( + GEOFENCE_OUTPUT_KEY, DocGeoFence.model_validate + ) + + # filter places to only consider those within the geofence + places_filtered = self._filter_coordinates(geofence_raw, geocoded) + + # TODO: CHECK FOR SPREAD ACROSS MAP AREA + logger.info( + f"box geocoder to run if {len(places_filtered)} >= {self._run_limit}" + ) + return len(places_filtered) >= self._run_limit + + def _extract_coordinates( + self, input: CoordinateInput + ) -> Tuple[ + Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] + ]: + geocoded: DocGeocodedPlaces = input.input.parse_data( + GEOCODED_PLACES_OUTPUT_KEY, DocGeocodedPlaces.model_validate + ) + geofence_raw: DocGeoFence = input.input.parse_data( + GEOFENCE_OUTPUT_KEY, DocGeoFence.model_validate + ) + lon_pts = input.input.get_data("lons") + lat_pts = input.input.get_data("lats") + + # filter places to only consider those within the geofence + places_filtered = self._filter_coordinates(geofence_raw, geocoded) + logger.info( + f"extracting coordinates via box geocoding with {len(places_filtered)} locations remaining after filtering" + ) + + # cluster to figure out which geocodings to consider + coordinates = self._get_coordinates(places_filtered) + logger.info("got all geocoded coordinate for box geocoding") + + # keep the middle 80% of each direction roughly + # a point dropped for one direction cannot be used for the other direction + # TODO: SHOULD PROBABLY BE MORE BOX AND WHISKER STYLE OUTLIER FILTERING + remove_count = max(int(len(coordinates) / 2 * 0.1), 1) + logger.info(f"removing {remove_count} coordinates from each end and direction") + coordinates_lons = self._remove_extreme( + remove_count, + list(filter(lambda x: not x.is_lat(), coordinates)), + lambda x: x.get_parsed_degree(), + ) + coordinates_lats = self._remove_extreme( + remove_count, + list(filter(lambda x: x.is_lat(), coordinates)), + lambda x: x.get_parsed_degree(), + ) + logger.info( + f"after coordinate removal {len(coordinates_lons)} lons and {len(coordinates_lats)} lats remain" + ) + + # keep those that are in both lists only + coordinates_count = {} + for c in coordinates_lats: + coordinates_count[c.get_pixel_alignment()] = 1 + for c in coordinates_lons: + pixels = c.get_pixel_alignment() + if pixels not in coordinates_count: + coordinates_count[pixels] = 0 + coordinates_count[pixels] = coordinates_count[pixels] + 1 + + coordinates_lons: List[Coordinate] = [] + coordinates_lats: List[Coordinate] = [] + for c in coordinates: + pixels = c.get_pixel_alignment() + if pixels in coordinates_count and coordinates_count[pixels] == 2: + if c.is_lat(): + coordinates_lats.append(c) + else: + coordinates_lons.append(c) + logger.info( + f"after harmonizing removals {len(coordinates_lons)} lons and {len(coordinates_lats)} lats remain" + ) + + # determine the pixel range and rough latitude / longitude range (assume x -> lon, y -> lat) + min_lon, max_lon = self._get_min_max(coordinates_lons) + min_lat, max_lat = self._get_min_max(coordinates_lats) + logger.info("obtained the coordinates covering the target range") + coordinates_all = coordinates_lons + coordinates_lats + vals_x = [c.get_pixel_alignment()[0] for c in coordinates_all] + vals_y = [c.get_pixel_alignment()[1] for c in coordinates_all] + min_x, max_x = min(vals_x), max(vals_x) + min_y, max_y = min(vals_y), max(vals_y) + logger.info( + f"creating coordinates between pixels x ({min_x}, {max_x}) and y ({min_y}, {max_y}) using lons {min_lon.get_parsed_degree()}, {max_lon.get_parsed_degree()} and lats {min_lat.get_parsed_degree()}, {max_lat.get_parsed_degree()}" + ) + + # create new points at the extremes of both ranges (ex: min x -> min lon, max x -> max lon) + coords = self._create_coordinates( + (min_x, max_x), (min_y, max_y), (min_lon, max_lon), (min_lat, max_lat) + ) + for c in coords: + self._add_param( + input.input, + str(uuid.uuid4()), + f"coordinate-{c.get_type()}-geocoded", + { + "text": c.get_text(), + "parsed": c.get_parsed_degree(), + "type": "latitude" if c.is_lat() else "longitude", + "pixel_alignment": c.get_pixel_alignment(), + "confidence": c.get_confidence(), + }, + "geocoded coordinate", + ) + if c.is_lat(): + lat_pts[c.to_deg_result()[0]] = c + else: + lon_pts[c.to_deg_result()[0]] = c + + return lon_pts, lat_pts + + def _remove_extreme( + self, n: int, coordinates: List[Coordinate], mapper: Callable + ) -> List[Coordinate]: + # map and sort the coordinates + coordinate_sorted = sorted( + [(mapper(c), c) for c in coordinates], key=lambda x: x[0] + ) + + # remove the top and bottom N values + end_index = len(coordinate_sorted) - n + return [cf[1] for cf in coordinate_sorted[n:end_index]] + + def _get_min_max( + self, coordinates: List[Coordinate] + ) -> Tuple[Coordinate, Coordinate]: + # find the min & max coordinate + degrees = [c.get_parsed_degree() for c in coordinates] + deg_min, deg_max = min(degrees), max(degrees) + coord_min, coord_max = None, None + for c in coordinates: + if c.get_parsed_degree() == deg_min: + coord_min = c + if c.get_parsed_degree() == deg_max: + coord_max = c + assert coord_min is not None + assert coord_max is not None + return coord_min, coord_max + + def _create_coordinates( + self, + minmax_x: Tuple[float, float], + minmax_y: Tuple[float, float], + minmax_lon: Tuple[Coordinate, Coordinate], + minmax_lat: Tuple[Coordinate, Coordinate], + ) -> List[Coordinate]: + # for lon, min and max x always map together + pixels_x = minmax_x + + # for lat, the mapping is reversed (min y -> max lat, max y -> min lat) + pixels_y = (minmax_y[1], minmax_y[0]) + + # create the four new coordinates mapping pixels to degrees + return [ + Coordinate( + "box derived lon", + minmax_lon[0].get_text(), + minmax_lon[0].get_parsed_degree(), + SOURCE_GEOCODE, + False, + pixel_alignment=(pixels_x[0], pixels_y[1]), + confidence=COORDINATE_CONFIDENCE_GEOCODE, + derivation="geocoded", + ), + Coordinate( + "box derived lon", + minmax_lon[1].get_text(), + minmax_lon[1].get_parsed_degree(), + SOURCE_GEOCODE, + False, + pixel_alignment=(pixels_x[1], pixels_y[0]), + confidence=COORDINATE_CONFIDENCE_GEOCODE, + derivation="geocoded", + ), + Coordinate( + "box derived lat", + minmax_lat[0].get_text(), + minmax_lat[0].get_parsed_degree(), + SOURCE_GEOCODE, + True, + pixel_alignment=(pixels_x[0], pixels_y[0]), + confidence=COORDINATE_CONFIDENCE_GEOCODE, + derivation="geocoded", + ), + Coordinate( + "box derived lat", + minmax_lat[1].get_text(), + minmax_lat[1].get_parsed_degree(), + SOURCE_GEOCODE, + True, + pixel_alignment=(pixels_x[1], pixels_y[1]), + confidence=COORDINATE_CONFIDENCE_GEOCODE, + derivation="geocoded", + ), + ] diff --git a/tasks/geo_referencing/util.py b/tasks/geo_referencing/util.py index 4570c4d7..b3157765 100644 --- a/tasks/geo_referencing/util.py +++ b/tasks/geo_referencing/util.py @@ -92,14 +92,10 @@ def get_min_max_count( is_negative_hemisphere: bool, sources: List[str] = [], ) -> Tuple[float, float, int]: - if len(coordinates) == 0: + coords = get_points(coordinates, sources).items() + if len(coords) == 0: return 0, 0, 0 - coords = filter( - lambda x: x[1].get_source() in sources if len(sources) > 0 else True, - coordinates.items(), - ) - # adjust values to be in the right hemisphere multiplier = 1 if is_negative_hemisphere: @@ -108,3 +104,23 @@ def get_min_max_count( values = list(map(lambda x: multiplier * abs(x[1].get_parsed_degree()), coords)) return min(values), max(values), len(values) + + +def get_points( + coordinates: Dict[Tuple[float, float], Coordinate], sources: List[str] = [] +) -> Dict[Tuple[float, float], Coordinate]: + + if len(coordinates) == 0: + return {} + + coords = list( + filter( + lambda x: x[1].get_source() in sources if len(sources) > 0 else True, + coordinates.items(), + ) + ) + + filtered = {} + for c in coords: + filtered[c[0]] = c[1] + return filtered diff --git a/tasks/geo_referencing/utm_extractor.py b/tasks/geo_referencing/utm_extractor.py index 596d6a98..d7e7d63e 100644 --- a/tasks/geo_referencing/utm_extractor.py +++ b/tasks/geo_referencing/utm_extractor.py @@ -64,6 +64,7 @@ def _extract_coordinates( ) -> Tuple[ Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] ]: + logger.info("extracting utm coordinates") geofence_raw: DocGeoFence = input.input.parse_data( GEOFENCE_OUTPUT_KEY, DocGeoFence.model_validate ) From 34b76505f32ca2351bf4be9d3de05ca45f3549e5 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Thu, 18 Jul 2024 16:13:06 -0400 Subject: [PATCH 06/66] add georef tag in correct place --- cdr/result_subscriber.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdr/result_subscriber.py b/cdr/result_subscriber.py index c4d9ea69..a947c508 100644 --- a/cdr/result_subscriber.py +++ b/cdr/result_subscriber.py @@ -329,7 +329,9 @@ def _push_georeferencing(self, result: RequestResult): files_ = [] try: lara_result = LARAGeoreferenceResult.model_validate(georef_result_raw) - mapper = get_mapper(lara_result, self._system_name, self._system_version) + mapper = get_mapper( + lara_result, f"{self._system_name}-georeference", self._system_version + ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore assert cdr_result is not None assert cdr_result.georeference_results is not None From 3048e976aa018424b853a6721df5ef349a589489 Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Mon, 22 Jul 2024 09:06:10 -0400 Subject: [PATCH 07/66] Moved ROI to separate filter. --- pipelines/geo_referencing/factory.py | 8 +- .../geo_referencing/coordinates_extractor.py | 160 ---------------- tasks/geo_referencing/filter.py | 181 +++++++++++++++++- 3 files changed, 185 insertions(+), 164 deletions(-) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index a8cfbc41..ac804258 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -18,7 +18,12 @@ ) from tasks.geo_referencing.state_plane_extractor import StatePlaneExtractor from tasks.geo_referencing.utm_extractor import UTMCoordinatesExtractor -from tasks.geo_referencing.filter import NaiveFilter, OutlierFilter, UTMStatePlaneFilter +from tasks.geo_referencing.filter import ( + NaiveFilter, + OutlierFilter, + ROIFilter, + UTMStatePlaneFilter, +) from tasks.geo_referencing.geo_fencing import GeoFencer from tasks.geo_referencing.georeference import GeoReference from tasks.geo_referencing.geocode import PointGeocoder, BoxGeocoder @@ -235,6 +240,7 @@ def create_geo_referencing_pipelines( ) tasks.append(GeoFencer("geofence")) tasks.append(GeoCoordinatesExtractor("third")) + tasks.append(ROIFilter("roiness")) tasks.append(OutlierFilter("fourth")) tasks.append(NaiveFilter("fun")) if extract_metadata: diff --git a/tasks/geo_referencing/coordinates_extractor.py b/tasks/geo_referencing/coordinates_extractor.py index c9be6c11..2e357826 100644 --- a/tasks/geo_referencing/coordinates_extractor.py +++ b/tasks/geo_referencing/coordinates_extractor.py @@ -158,12 +158,6 @@ def _should_run(self, input: CoordinateInput) -> bool: logger.info(f"checking run condition: {num_keypoints} key points") return num_keypoints < 2 - def _in_polygon( - self, point: Tuple[float, float], polygon: List[Tuple[float, float]] - ) -> bool: - path = mpltPath.Path(polygon) # type: ignore - return path.contains_point(point) - class GeoCoordinatesExtractor(CoordinatesExtractor): def __init__(self, task_id: str): @@ -182,19 +176,6 @@ def _extract_coordinates( lon_pts, lat_pts = self._extract_lonlat( input, ocr_blocks, lon_minmax, lat_minmax, defaulted ) - num_keypoints = min(len(lon_pts), len(lat_pts)) - if num_keypoints > 0: - logger.info(f"filtering via roi") - # ----- do Region-of-Interest analysis (automatic cropping) - roi_xy = input.input.get_data("roi") - self._add_param(input.input, str(uuid.uuid4()), "roi", {"bounds": roi_xy}) - roi_inner_xy = input.input.get_data("roi_inner") - self._add_param( - input.input, str(uuid.uuid4()), "roi_inner", {"bounds": roi_inner_xy} - ) - lon_pts, lat_pts = self._validate_lonlat_extractions( - input, lon_pts, lat_pts, input.input.image.size, roi_xy, roi_inner_xy - ) return lon_pts, lat_pts @@ -792,147 +773,6 @@ def _check_consecutive(self, deg: float, minutes: float, seconds: float) -> bool is_consecutive = abs(minutes - seconds) == 1 return is_consecutive - def _validate_lonlat_extractions( - self, - input: CoordinateInput, - lon_results: Dict[Tuple[float, float], Coordinate], - lat_results: Dict[Tuple[float, float], Coordinate], - im_size: Tuple[float, float], - roi_xy: List[Tuple[float, float]] = [], - roi_inner_xy: List[Tuple[float, float]] = [], - ) -> Tuple[ - Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] - ]: - logger.info("validating lonlat") - - num_lat_pts = len(lat_results) - num_lon_pts = len(lon_results) - - # remove points in secondary maps - # lon_results, _ = self._filter_secondary_map(lon_results) - # lat_results, _ = self._filter_secondary_map(lat_results) - # num_lat_pts = len(lat_results) - # num_lon_pts = len(lon_results) - - if roi_xy and (num_lat_pts > 4 or num_lon_pts > 4): - for (deg, y), coord in list(lat_results.items()): - if not self._in_polygon(coord.get_pixel_alignment(), roi_xy) or ( - len(roi_inner_xy) > 0 - and self._in_polygon(coord.get_pixel_alignment(), roi_inner_xy) - ): - logger.info( - f"Excluding out-of-bounds latitude point: {deg} ({coord.get_pixel_alignment()})" - ) - del lat_results[(deg, y)] - self._add_param( - input.input, - str(uuid.uuid4()), - "coordinate-excluded", - { - "bounds": ocr_to_coordinates(coord.get_bounds()), - "text": coord.get_text(), - "type": "latitude" if coord.is_lat() else "longitude", - "pixel_alignment": coord.get_pixel_alignment(), - "confidence": coord.get_confidence(), - }, - "excluded due to being outside roi", - ) - for (deg, x), coord in list(lon_results.items()): - if not self._in_polygon(coord.get_pixel_alignment(), roi_xy) or ( - len(roi_inner_xy) > 0 - and self._in_polygon(coord.get_pixel_alignment(), roi_inner_xy) - ): - logger.info( - f"Excluding out-of-bounds longitude point: {deg} ({coord.get_pixel_alignment()})" - ) - del lon_results[(deg, x)] - self._add_param( - input.input, - str(uuid.uuid4()), - "coordinate-excluded", - { - "bounds": ocr_to_coordinates(coord.get_bounds()), - "text": coord.get_text(), - "type": "latitude" if coord.is_lat() else "longitude", - "pixel_alignment": coord.get_pixel_alignment(), - "confidence": coord.get_confidence(), - }, - "excluded due to being outside roi", - ) - - num_lat_pts = len(lat_results) - num_lon_pts = len(lon_results) - logger.info(f"point count after exclusion lat,lon: {num_lat_pts},{num_lon_pts}") - - # if num_lon_pts > 4: - # lon_results = self._remove_outlier_pts(input, lon_results, im_size[0], im_size[1]) - # if num_lat_pts > 4: - # lat_results = self._remove_outlier_pts(input, lat_results, im_size[1], im_size[0]) - - # check number of unique lat and lon values - num_lat_pts = len(set([x[0] for x in lat_results])) - num_lon_pts = len(set([x[0] for x in lon_results])) - logger.info(f"distinct after outlier lat,lon: {num_lat_pts},{num_lon_pts}") - - if num_lon_pts >= 2 and num_lat_pts == 1: - # estimate additional lat pt (based on lon pxl resolution) - lst = [ - (k[0], k[1], v.get_pixel_alignment()[1]) for k, v in lon_results.items() - ] - max_pt = max(lst, key=lambda p: p[1]) - min_pt = min(lst, key=lambda p: p[1]) - pxl_range = max_pt[1] - min_pt[1] - deg_range = max_pt[0] - min_pt[0] - if deg_range != 0 and pxl_range != 0: - deg_per_pxl = abs( - deg_range / pxl_range - ) # TODO could use geodesic dist here? - lat_pt = list(lat_results.items())[0] - # new_y = im_size[1]-1 - new_y = 0 if lat_pt[0][1] > im_size[1] / 2 else im_size[1] - 1 - new_lat = -deg_per_pxl * (new_y - lat_pt[0][1]) + lat_pt[0][0] - coord = Coordinate( - "lat keypoint", - "", - new_lat, - SOURCE_LAT_LON, - True, - pixel_alignment=(lat_pt[1].to_deg_result()[1], new_y), - confidence=0.6, - ) - lat_results[(new_lat, new_y)] = coord - - elif num_lat_pts >= 2 and num_lon_pts == 1: - # estimate additional lon pt (based on lat pxl resolution) - lst = [ - (k[0], k[1], v.get_pixel_alignment()[0]) for k, v in lat_results.items() - ] - max_pt = max(lst, key=lambda p: p[1]) - min_pt = min(lst, key=lambda p: p[1]) - pxl_range = max_pt[1] - min_pt[1] - deg_range = max_pt[0] - min_pt[0] - if deg_range != 0 and pxl_range != 0: - deg_per_pxl = abs( - deg_range / pxl_range - ) # TODO could use geodesic dist here? - lon_pt = list(lon_results.items())[0] - # new_x = im_size[0]-1 - new_x = 0 if lon_pt[0][1] > im_size[0] / 2 else im_size[0] - 1 - new_lon = -deg_per_pxl * (new_x - lon_pt[0][1]) + lon_pt[0][0] - coord = Coordinate( - "lon keypoint", - "", - new_lon, - SOURCE_LAT_LON, - False, - pixel_alignment=(new_x, lon_pt[1].to_deg_result()[1]), - confidence=0.6, - ) - lon_results[(new_lon, new_x)] = coord - logger.info("done validating coordinates") - - return (lon_results, lat_results) - def _remove_outlier_pts( self, input: CoordinateInput, diff --git a/tasks/geo_referencing/filter.py b/tasks/geo_referencing/filter.py index e826b56e..b9244f78 100644 --- a/tasks/geo_referencing/filter.py +++ b/tasks/geo_referencing/filter.py @@ -4,13 +4,19 @@ import numpy as np from sklearn.cluster import DBSCAN - -from tasks.geo_referencing.entities import Coordinate, SOURCE_STATE_PLANE, SOURCE_UTM +import matplotlib.path as mpltPath + +from tasks.geo_referencing.entities import ( + Coordinate, + SOURCE_STATE_PLANE, + SOURCE_UTM, + SOURCE_LAT_LON, +) from tasks.common.task import Task, TaskInput, TaskResult from tasks.geo_referencing.geo_projection import PolyRegression from tasks.geo_referencing.util import ocr_to_coordinates -from typing import Dict, Tuple +from typing import Dict, Tuple, List logger = logging.getLogger("coordinates_filter") @@ -353,3 +359,172 @@ def _filter_coarse( "excluded due to naive outlier detection", ) return filtered_coords + + +class ROIFilter(FilterCoordinates): + def __init__(self, task_id: str): + super().__init__(task_id) + + def _filter( + self, + input: TaskInput, + lon_coords: Dict[Tuple[float, float], Coordinate], + lat_coords: Dict[Tuple[float, float], Coordinate], + ) -> Tuple[ + Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] + ]: + logger.info(f"roi filter running against lon and lat coordinates") + roi_xy = input.get_data("roi") + self._add_param(input, str(uuid.uuid4()), "roi", {"bounds": roi_xy}) + roi_inner_xy = input.get_data("roi_inner") + self._add_param(input, str(uuid.uuid4()), "roi_inner", {"bounds": roi_inner_xy}) + + num_keypoints = min(len(lon_coords), len(lat_coords)) + if num_keypoints == 0: + logger.info( + f"roi filter not filtering since {num_keypoints} coord exists along one axis" + ) + return lon_coords, lat_coords + # ----- do Region-of-Interest analysis (automatic cropping) + lon_pts, lat_pts = self._validate_lonlat_extractions( + input, lon_coords, lat_coords, input.image.size, roi_xy, roi_inner_xy + ) + + # apply to the parsed coordinates + return lon_pts, lat_pts + + def _validate_lonlat_extractions( + self, + input: TaskInput, + lon_results: Dict[Tuple[float, float], Coordinate], + lat_results: Dict[Tuple[float, float], Coordinate], + im_size: Tuple[float, float], + roi_xy: List[Tuple[float, float]] = [], + roi_inner_xy: List[Tuple[float, float]] = [], + ) -> Tuple[ + Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] + ]: + logger.info("validating lonlat") + + num_lat_pts = len(lat_results) + num_lon_pts = len(lon_results) + + if roi_xy and (num_lat_pts > 4 or num_lon_pts > 4): + for (deg, y), coord in list(lat_results.items()): + if not self._in_polygon(coord.get_pixel_alignment(), roi_xy) or ( + len(roi_inner_xy) > 0 + and self._in_polygon(coord.get_pixel_alignment(), roi_inner_xy) + ): + logger.info( + f"Excluding out-of-bounds latitude point: {deg} ({coord.get_pixel_alignment()})" + ) + del lat_results[(deg, y)] + self._add_param( + input, + str(uuid.uuid4()), + "coordinate-excluded", + { + "bounds": ocr_to_coordinates(coord.get_bounds()), + "text": coord.get_text(), + "type": "latitude" if coord.is_lat() else "longitude", + "pixel_alignment": coord.get_pixel_alignment(), + "confidence": coord.get_confidence(), + }, + "excluded due to being outside roi", + ) + for (deg, x), coord in list(lon_results.items()): + if not self._in_polygon(coord.get_pixel_alignment(), roi_xy) or ( + len(roi_inner_xy) > 0 + and self._in_polygon(coord.get_pixel_alignment(), roi_inner_xy) + ): + logger.info( + f"Excluding out-of-bounds longitude point: {deg} ({coord.get_pixel_alignment()})" + ) + del lon_results[(deg, x)] + self._add_param( + input, + str(uuid.uuid4()), + "coordinate-excluded", + { + "bounds": ocr_to_coordinates(coord.get_bounds()), + "text": coord.get_text(), + "type": "latitude" if coord.is_lat() else "longitude", + "pixel_alignment": coord.get_pixel_alignment(), + "confidence": coord.get_confidence(), + }, + "excluded due to being outside roi", + ) + + num_lat_pts = len(lat_results) + num_lon_pts = len(lon_results) + logger.info(f"point count after exclusion lat,lon: {num_lat_pts},{num_lon_pts}") + + # check number of unique lat and lon values + num_lat_pts = len(set([x[0] for x in lat_results])) + num_lon_pts = len(set([x[0] for x in lon_results])) + logger.info(f"distinct after outlier lat,lon: {num_lat_pts},{num_lon_pts}") + + if num_lon_pts >= 2 and num_lat_pts == 1: + # estimate additional lat pt (based on lon pxl resolution) + lst = [ + (k[0], k[1], v.get_pixel_alignment()[1]) for k, v in lon_results.items() + ] + max_pt = max(lst, key=lambda p: p[1]) + min_pt = min(lst, key=lambda p: p[1]) + pxl_range = max_pt[1] - min_pt[1] + deg_range = max_pt[0] - min_pt[0] + if deg_range != 0 and pxl_range != 0: + deg_per_pxl = abs( + deg_range / pxl_range + ) # TODO could use geodesic dist here? + lat_pt = list(lat_results.items())[0] + # new_y = im_size[1]-1 + new_y = 0 if lat_pt[0][1] > im_size[1] / 2 else im_size[1] - 1 + new_lat = -deg_per_pxl * (new_y - lat_pt[0][1]) + lat_pt[0][0] + coord = Coordinate( + "lat keypoint", + "", + new_lat, + SOURCE_LAT_LON, + True, + pixel_alignment=(lat_pt[1].to_deg_result()[1], new_y), + confidence=0.6, + ) + lat_results[(new_lat, new_y)] = coord + + elif num_lat_pts >= 2 and num_lon_pts == 1: + # estimate additional lon pt (based on lat pxl resolution) + lst = [ + (k[0], k[1], v.get_pixel_alignment()[0]) for k, v in lat_results.items() + ] + max_pt = max(lst, key=lambda p: p[1]) + min_pt = min(lst, key=lambda p: p[1]) + pxl_range = max_pt[1] - min_pt[1] + deg_range = max_pt[0] - min_pt[0] + if deg_range != 0 and pxl_range != 0: + deg_per_pxl = abs( + deg_range / pxl_range + ) # TODO could use geodesic dist here? + lon_pt = list(lon_results.items())[0] + # new_x = im_size[0]-1 + new_x = 0 if lon_pt[0][1] > im_size[0] / 2 else im_size[0] - 1 + new_lon = -deg_per_pxl * (new_x - lon_pt[0][1]) + lon_pt[0][0] + coord = Coordinate( + "lon keypoint", + "", + new_lon, + SOURCE_LAT_LON, + False, + pixel_alignment=(new_x, lon_pt[1].to_deg_result()[1]), + confidence=0.6, + ) + lon_results[(new_lon, new_x)] = coord + logger.info("done validating coordinates") + + return (lon_results, lat_results) + + def _in_polygon( + self, point: Tuple[float, float], polygon: List[Tuple[float, float]] + ) -> bool: + path = mpltPath.Path(polygon) # type: ignore + return path.contains_point(point) From c9bcb243648b23f146e311204516cde9c0861d46 Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Thu, 25 Jul 2024 09:44:44 -0400 Subject: [PATCH 08/66] initial rework of roi filtering. --- tasks/geo_referencing/filter.py | 138 +++++++++++++++++++------------- 1 file changed, 84 insertions(+), 54 deletions(-) diff --git a/tasks/geo_referencing/filter.py b/tasks/geo_referencing/filter.py index b9244f78..0548633c 100644 --- a/tasks/geo_referencing/filter.py +++ b/tasks/geo_referencing/filter.py @@ -3,6 +3,7 @@ import numpy as np +from copy import deepcopy from sklearn.cluster import DBSCAN import matplotlib.path as mpltPath @@ -380,81 +381,43 @@ def _filter( self._add_param(input, str(uuid.uuid4()), "roi_inner", {"bounds": roi_inner_xy}) num_keypoints = min(len(lon_coords), len(lat_coords)) - if num_keypoints == 0: + if num_keypoints < 2: logger.info( f"roi filter not filtering since {num_keypoints} coord exists along one axis" ) return lon_coords, lat_coords + lon_inputs = deepcopy(lon_coords) + lat_inputs = deepcopy(lat_coords) # ----- do Region-of-Interest analysis (automatic cropping) + lon_pts, lat_pts = self._filter_roi( + input, lon_inputs, lat_inputs, True, roi_inner_xy + ) + lon_pts, lat_pts = self._filter_roi(input, lon_pts, lat_pts, False, roi_xy) lon_pts, lat_pts = self._validate_lonlat_extractions( - input, lon_coords, lat_coords, input.image.size, roi_xy, roi_inner_xy + lon_pts, lat_pts, input.image.size ) + # check if too many points were removed + lats_distinct = set(map(lambda x: x[1].get_parsed_degree(), lat_pts.items())) + lons_distinct = set(map(lambda x: x[1].get_parsed_degree(), lon_pts.items())) + num_keypoints = min(len(lons_distinct), len(lats_distinct)) + if num_keypoints < 2: + logger.info(f"not filtering using roi due to too many points being removed") + return lon_coords, lat_coords + # apply to the parsed coordinates return lon_pts, lat_pts def _validate_lonlat_extractions( self, - input: TaskInput, lon_results: Dict[Tuple[float, float], Coordinate], lat_results: Dict[Tuple[float, float], Coordinate], im_size: Tuple[float, float], - roi_xy: List[Tuple[float, float]] = [], - roi_inner_xy: List[Tuple[float, float]] = [], ) -> Tuple[ Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] ]: logger.info("validating lonlat") - num_lat_pts = len(lat_results) - num_lon_pts = len(lon_results) - - if roi_xy and (num_lat_pts > 4 or num_lon_pts > 4): - for (deg, y), coord in list(lat_results.items()): - if not self._in_polygon(coord.get_pixel_alignment(), roi_xy) or ( - len(roi_inner_xy) > 0 - and self._in_polygon(coord.get_pixel_alignment(), roi_inner_xy) - ): - logger.info( - f"Excluding out-of-bounds latitude point: {deg} ({coord.get_pixel_alignment()})" - ) - del lat_results[(deg, y)] - self._add_param( - input, - str(uuid.uuid4()), - "coordinate-excluded", - { - "bounds": ocr_to_coordinates(coord.get_bounds()), - "text": coord.get_text(), - "type": "latitude" if coord.is_lat() else "longitude", - "pixel_alignment": coord.get_pixel_alignment(), - "confidence": coord.get_confidence(), - }, - "excluded due to being outside roi", - ) - for (deg, x), coord in list(lon_results.items()): - if not self._in_polygon(coord.get_pixel_alignment(), roi_xy) or ( - len(roi_inner_xy) > 0 - and self._in_polygon(coord.get_pixel_alignment(), roi_inner_xy) - ): - logger.info( - f"Excluding out-of-bounds longitude point: {deg} ({coord.get_pixel_alignment()})" - ) - del lon_results[(deg, x)] - self._add_param( - input, - str(uuid.uuid4()), - "coordinate-excluded", - { - "bounds": ocr_to_coordinates(coord.get_bounds()), - "text": coord.get_text(), - "type": "latitude" if coord.is_lat() else "longitude", - "pixel_alignment": coord.get_pixel_alignment(), - "confidence": coord.get_confidence(), - }, - "excluded due to being outside roi", - ) - num_lat_pts = len(lat_results) num_lon_pts = len(lon_results) logger.info(f"point count after exclusion lat,lon: {num_lat_pts},{num_lon_pts}") @@ -528,3 +491,70 @@ def _in_polygon( ) -> bool: path = mpltPath.Path(polygon) # type: ignore return path.contains_point(point) + + def _filter_roi( + self, + input: TaskInput, + lon_results: Dict[Tuple[float, float], Coordinate], + lat_results: Dict[Tuple[float, float], Coordinate], + in_filter: bool, + roi_xy: List[Tuple[float, float]], + ) -> Tuple[ + Dict[Tuple[float, float], Coordinate], Dict[Tuple[float, float], Coordinate] + ]: + logger.info("filtering using roi") + + num_lat_pts = len(lat_results) + num_lon_pts = len(lon_results) + + if roi_xy and (num_lat_pts > 4 or num_lon_pts > 4): + for (deg, y), coord in list(lat_results.items()): + if ( + in_filter and self._in_polygon(coord.get_pixel_alignment(), roi_xy) + ) or ( + not in_filter + and not self._in_polygon(coord.get_pixel_alignment(), roi_xy) + ): + logger.info( + f"removing out-of-bounds latitude point: {deg} ({coord.get_pixel_alignment()})" + ) + del lat_results[(deg, y)] + self._add_param( + input, + str(uuid.uuid4()), + "coordinate-excluded", + { + "bounds": ocr_to_coordinates(coord.get_bounds()), + "text": coord.get_text(), + "type": "latitude" if coord.is_lat() else "longitude", + "pixel_alignment": coord.get_pixel_alignment(), + "confidence": coord.get_confidence(), + }, + "excluded due to being outside roi", + ) + for (deg, x), coord in list(lon_results.items()): + if ( + in_filter and self._in_polygon(coord.get_pixel_alignment(), roi_xy) + ) or ( + not in_filter + and not self._in_polygon(coord.get_pixel_alignment(), roi_xy) + ): + logger.info( + f"removing out-of-bounds longitude point: {deg} ({coord.get_pixel_alignment()})" + ) + del lon_results[(deg, x)] + self._add_param( + input, + str(uuid.uuid4()), + "coordinate-excluded", + { + "bounds": ocr_to_coordinates(coord.get_bounds()), + "text": coord.get_text(), + "type": "latitude" if coord.is_lat() else "longitude", + "pixel_alignment": coord.get_pixel_alignment(), + "confidence": coord.get_confidence(), + }, + "excluded due to being outside roi", + ) + + return (lon_results, lat_results) From 6392a21daca92014e81f24f5d04b6889102905ed Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Fri, 26 Jul 2024 10:28:07 -0400 Subject: [PATCH 09/66] more points' pipeline re-factoring and adding support for legend item annotations (WIP) --- .../point_extraction_pipeline.py | 14 +- pipelines/point_extraction/run_pipeline.py | 4 +- tasks/point_extraction/entities.py | 23 +- tasks/point_extraction/legend_analyzer.py | 27 ++- tasks/point_extraction/legend_item_utils.py | 34 ++- tasks/point_extraction/point_extractor.py | 86 ++++---- .../point_orientation_extractor.py | 18 +- tasks/point_extraction/tiling.py | 201 +++++++++++++----- 8 files changed, 273 insertions(+), 134 deletions(-) diff --git a/pipelines/point_extraction/point_extraction_pipeline.py b/pipelines/point_extraction/point_extraction_pipeline.py index 509fa66a..5865b057 100644 --- a/pipelines/point_extraction/point_extraction_pipeline.py +++ b/pipelines/point_extraction/point_extraction_pipeline.py @@ -3,7 +3,10 @@ from typing import List from schema.mappers.cdr import PointsMapper -from tasks.point_extraction.legend_analyzer import PointLegendAnalyzer +from tasks.point_extraction.legend_analyzer import ( + LegendPreprocessor, + LegendPostprocessor, +) from tasks.point_extraction.point_extractor import YOLOPointDetector from tasks.point_extraction.point_orientation_extractor import PointOrientationExtractor from tasks.point_extraction.point_extractor_utils import convert_preds_to_bitmasks @@ -84,7 +87,7 @@ def __init__( ) tasks.extend( [ - PointLegendAnalyzer("legend_analyzer", ""), + LegendPreprocessor("legend_preprocessor", ""), Tiler("tiling"), YOLOPointDetector( "point_detection", @@ -94,6 +97,7 @@ def __init__( ), Untiler("untiling"), PointOrientationExtractor("point_orientation_extraction"), + LegendPostprocessor("legend_postprocessor", ""), TemplateMatchPointExtractor( "template_match_point_extraction", str(Path(work_dir).joinpath("template_match_points")), @@ -127,7 +131,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: Returns: PointLabel: The map point label extraction object. """ - map_image = MapImage.model_validate(pipeline_result.data["map_image"]) + map_image = PointLabels.model_validate(pipeline_result.data["map_image"]) return BaseModelOutput( pipeline_result.pipeline_id, pipeline_result.pipeline_name, @@ -159,7 +163,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: Returns: Output: The output of the pipeline. """ - map_image = MapImage.model_validate(pipeline_result.data["map_image"]) + map_image = PointLabels.model_validate(pipeline_result.data["map_image"]) mapper = PointsMapper(MODEL_NAME, MODEL_VERSION) cdr_points = mapper.map_to_cdr(map_image) @@ -192,7 +196,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: Returns: Output: The output of the pipeline. """ - map_image = MapImage.model_validate(pipeline_result.data["map_image"]) + map_image = PointLabels.model_validate(pipeline_result.data["map_image"]) legend_labels = [] if LEGEND_ITEMS_OUTPUT_KEY in pipeline_result.data: legend_pt_items = LegendPointItems.model_validate( diff --git a/pipelines/point_extraction/run_pipeline.py b/pipelines/point_extraction/run_pipeline.py index debb30d2..22ef0617 100644 --- a/pipelines/point_extraction/run_pipeline.py +++ b/pipelines/point_extraction/run_pipeline.py @@ -107,9 +107,9 @@ def main(): if p.bitmasks: bitmasks_out_dir = os.path.join(p.output, "bitmasks") os.makedirs(bitmasks_out_dir, exist_ok=True) - if not p.legend_hints_dir: + if not p.legend_hints_dir or not p.legend_annotations_dir: logger.warning( - 'Points pipeline is configured to create CMA contest bitmasks without using legend hints! Setting "legend_hints_dir" param is recommended.' + 'Points pipeline is configured to create CMA contest bitmasks without using legend annotations! Setting "legend_hints_dir" or "legend_annotations_dir" param is recommended.' ) results = pipeline.run(image_input) diff --git a/tasks/point_extraction/entities.py b/tasks/point_extraction/entities.py index 0b57dc31..2db2a066 100644 --- a/tasks/point_extraction/entities.py +++ b/tasks/point_extraction/entities.py @@ -28,11 +28,6 @@ class PointLabel(BaseModel): score: float direction: Optional[float] = None # [deg] orientation of point symbol dip: Optional[float] = None # [deg] dip angle associated with symbol - legend_name: str = Field( - default="", - description="Label for the legend item associated with this extraction", - ) - # legend_bbox: List[Union[float, int]] # TODO -- is this needed here? class PointLabels(BaseModel): @@ -127,9 +122,7 @@ def format_for_caching(self) -> ImageTiles: tiles=tiles_cache, ) - def join_with_cached_predictions( - self, cached_preds: ImageTiles, point_legend_mapping: Dict[str, LegendPointItem] - ) -> bool: + def join_with_cached_predictions(self, cached_preds: ImageTiles) -> bool: """ Append cached point predictions to ImageTiles """ @@ -146,16 +139,6 @@ def join_with_cached_predictions( t_cached: ImageTile = cached_dict[key] t.predictions = t_cached.predictions - if t.predictions is not None: - for pred in t.predictions: - class_name = pred.class_name - # map YOLO class name to legend item name, if available - if pred.class_name in point_legend_mapping: - pred.legend_name = point_legend_mapping[class_name].name - pred.legend_bbox = point_legend_mapping[ - class_name - ].legend_bbox - return True except Exception as e: print(f"Exception in join_with_cached_predictions: {str(e)}") @@ -192,9 +175,7 @@ class LegendPointItem(BaseModel): label. Format is expected to be [x,y] coordinate pairs where the top left is the origin (0,0).""", ) - system: str = Field( - default="", description="System that published this item" - ) # dgdg -- is this needed? (or get rid of provenance below?) + system: str = Field(default="", description="System that published this item") validated: bool = Field(default=False, description="Validated by human") confidence: Optional[float] = Field( default=None, diff --git a/tasks/point_extraction/legend_analyzer.py b/tasks/point_extraction/legend_analyzer.py index 64d25d60..2a35f8ad 100644 --- a/tasks/point_extraction/legend_analyzer.py +++ b/tasks/point_extraction/legend_analyzer.py @@ -13,9 +13,9 @@ logger = logging.getLogger(__name__) -class PointLegendAnalyzer(Task): +class LegendPreprocessor(Task): """ - Analysis of Point Symbol Legend Items + Pre-processing of Point Symbol Legend Items """ def __init__( @@ -69,3 +69,26 @@ def run(self, task_input: TaskInput) -> TaskResult: ) return self._create_result(task_input) + + +class LegendPostprocessor(Task): + """ + Post-processing of Point Symbol Legend Items + """ + + def __init__( + self, + task_id: str, + cache_path: str, + ): + + super().__init__(task_id, cache_path) + + def run(self, task_input: TaskInput) -> TaskResult: + """ + run point symbol legend post-processing + """ + ## WIP! + + return self._create_result(task_input) + diff --git a/tasks/point_extraction/legend_item_utils.py b/tasks/point_extraction/legend_item_utils.py index c7d3a6c2..3c7ac439 100644 --- a/tasks/point_extraction/legend_item_utils.py +++ b/tasks/point_extraction/legend_item_utils.py @@ -5,7 +5,7 @@ from shapely import Polygon, distance from tasks.point_extraction.entities import LegendPointItem, LegendPointItems -from tasks.point_extraction.label_map import LABEL_MAPPING +from tasks.point_extraction.label_map import LABEL_MAPPING, YOLO_TO_CDR_LABEL from schema.cdr_schemas.cdr_responses.legend_items import LegendItemResponse from tasks.segmentation.entities import MapSegmentation @@ -159,6 +159,17 @@ def find_legend_keyword_match(legend_item_name: str, raster_id: str) -> str: ) return symbol_class + # if no matches, then double-check exact matches with CDR ontology terms + cdr_to_yolo = {v: k for k, v in YOLO_TO_CDR_LABEL.items()} + leg_label_norm = legend_item_name.strip().lower() + if leg_label_norm in cdr_to_yolo: + # match found + symbol_class = cdr_to_yolo[leg_label_norm] + logger.info( + f"Legend label: {legend_item_name} matches point class: {symbol_class}" + ) + return symbol_class + logger.info(f"No point class match found for legend label: {legend_item_name}") return "" @@ -256,3 +267,24 @@ def filter_labelme_annotations( # legend item swatch bbox is close to square filtered_leg_items.append(leg) leg_point_items.items = filtered_leg_items + + +def legend_items_use_ontology(leg_point_items: LegendPointItems) -> bool: + """ + Check if all legend items use the feature ontology + (ie the class names are set) + """ + class_labels_ok = True + if len(leg_point_items.items) > 0: + for leg_item in leg_point_items.items: + if not leg_item.class_name: + logger.info( + "Point ontology labels are missing for some of the legend items. Proceeding with tiling and further analysis of legend area..." + ) + class_labels_ok = False + break + if class_labels_ok: + logger.info( + f"*** Point ontology labels are available for ALL legend items. Skipping further legend item analysis." + ) + return class_labels_ok diff --git a/tasks/point_extraction/point_extractor.py b/tasks/point_extraction/point_extractor.py index 33b136ce..b76d83de 100644 --- a/tasks/point_extraction/point_extractor.py +++ b/tasks/point_extraction/point_extractor.py @@ -2,13 +2,8 @@ ImageTile, ImageTiles, PointLabel, - LegendPointItem, - LegendPointItems, - LEGEND_ITEMS_OUTPUT_KEY, ) -from tasks.point_extraction.legend_item_utils import find_legend_label_matches - from tasks.common.s3_data_cache import S3DataCache from tasks.common.task import Task, TaskInput, TaskResult import hashlib @@ -110,9 +105,7 @@ def _prep_model_data(self, model_data_path: str, data_cache_path: str) -> Path: return local_model_data_path - def process_output( - self, predictions: Results, point_legend_mapping: Dict[str, LegendPointItem] - ) -> List[PointLabel]: + def process_output(self, predictions: Results) -> List[PointLabel]: """ Convert point detection inference results from YOLO model format to a list of PointLabel objects @@ -127,28 +120,19 @@ def process_output( continue for box in pred.boxes.data.detach().cpu().tolist(): x1, y1, x2, y2, score, class_id = box - - # map YOLO class name to legend item name, if available class_name = self.model.names[int(class_id)] - legend_name = class_name - # legend_bbox = [] - if class_name in point_legend_mapping: - legend_name = point_legend_mapping[class_name].name - # legend_bbox = point_legend_mapping[class_name].legend_bbox pt_labels.append( PointLabel( model_name=MODEL_NAME, model_version=self._model_id, class_id=int(class_id), - class_name=self.model.names[int(class_id)], + class_name=class_name, x1=int(x1), y1=int(y1), x2=int(x2), y2=int(y2), score=score, - legend_name=legend_name, - # legend_bbox=legend_bbox, ) ) return pt_labels @@ -157,37 +141,61 @@ def run(self, task_input: TaskInput) -> TaskResult: """ run YOLO model inference for point symbol detection """ - image_tiles = ImageTiles.model_validate(task_input.data["map_tiles"]) - - point_legend_mapping: Dict[str, LegendPointItem] = {} - if LEGEND_ITEMS_OUTPUT_KEY in task_input.data: - legend_pt_items = LegendPointItems.model_validate( - task_input.data[LEGEND_ITEMS_OUTPUT_KEY] - ) - # find mappings between legend item labels and YOLO model class names - point_legend_mapping = find_legend_label_matches( - legend_pt_items, task_input.raster_id - ) + map_tiles = ImageTiles.model_validate(task_input.data["map_tiles"]) if self.device == "auto": self.device = "cuda" if torch.cuda.is_available() else "cpu" if self.device not in ["cuda", "cpu"]: raise ValueError(f"Invalid device: {self.device}") - doc_key = f"{task_input.raster_id}_points-{self._model_id}" + # --- run point extraction model on map area tiles + logger.info(f"Running model inference on {len(map_tiles.tiles)} map tiles") + self._process_tiles(map_tiles, task_input.raster_id, "map") + + if "legend_tiles" in task_input.data: + # --- also run point extraction model on legend area tiles, if available + legend_tiles = ImageTiles.model_validate(task_input.data["legend_tiles"]) + logger.info( + f"Also running model inference on {len(legend_tiles.tiles)} legend tiles" + ) + self._process_tiles(legend_tiles, task_input.raster_id, "legend") + + return TaskResult( + task_id=self._task_id, + output={ + "map_tiles": map_tiles.model_dump(), + "legend_tiles": legend_tiles.model_dump(), + }, + ) + + return TaskResult( + task_id=self._task_id, output={"map_tiles": map_tiles.model_dump()} + ) + + def _process_tiles( + self, image_tiles: ImageTiles, raster_id: str, tile_type: str = "map" + ): + """ + do batch inference on image tiles + prediction results are appended in-place to the ImageTiles object + """ + + # get key for points' data cache + doc_key = ( + f"{raster_id}_points-{self._model_id}" + if tile_type == "map" + else f"{raster_id}_points_{tile_type}-{self._model_id}" + ) # check cache and re-use existing file if present json_data = self.fetch_cached_result(doc_key) if json_data and image_tiles.join_with_cached_predictions( - ImageTiles(**json_data), point_legend_mapping + ImageTiles(**json_data) ): # cached point predictions loaded successfully logger.info( - f"Using cached point extractions for raster: {task_input.raster_id}" - ) - return TaskResult( - task_id=self._task_id, - output={"map_tiles": image_tiles}, + f"Using cached point extractions for raster {raster_id} and tile type {tile_type}" ) + return tiles_out: List[ImageTile] = [] # run batch model inference... @@ -205,7 +213,7 @@ def run(self, task_input: TaskInput) -> TaskResult: iou=IOU_THRES, ) for tile, preds in zip(batch, batch_preds): - tile.predictions = self.process_output(preds, point_legend_mapping) + tile.predictions = self.process_output(preds) tiles_out.append(tile) # save tile results with point extraction predictions image_tiles.tiles = tiles_out @@ -215,10 +223,6 @@ def run(self, task_input: TaskInput) -> TaskResult: image_tiles.format_for_caching().model_dump(), doc_key ) - return TaskResult( - task_id=self._task_id, output={"map_tiles": image_tiles.model_dump()} - ) - def _get_model_id(self, model: YOLO) -> str: """ Create a unique string ID for this model, diff --git a/tasks/point_extraction/point_orientation_extractor.py b/tasks/point_extraction/point_orientation_extractor.py index 6b10a7c6..e41affdb 100644 --- a/tasks/point_extraction/point_orientation_extractor.py +++ b/tasks/point_extraction/point_orientation_extractor.py @@ -184,15 +184,16 @@ def run(self, input: TaskInput) -> TaskResult: """ # get result from point extractor task (with point symbol predictions) - map_image = PointLabels.model_validate(input.data["map_image"]) - if map_image.labels is None: + map_point_labels = PointLabels.model_validate(input.data["map_point_labels"]) + if map_point_labels.labels is None: raise RuntimeError("PointLabels must have labels to run batch_predict") - if len(map_image.labels) == 0: + if len(map_point_labels.labels) == 0: logger.warning( "No point symbol extractions found. Skipping Point orientation extraction." ) TaskResult( - task_id=self._task_id, output={"map_image": map_image.model_dump()} + task_id=self._task_id, + output={"map_point_labels": map_point_labels.model_dump()}, ) # get OCR output @@ -212,7 +213,7 @@ def run(self, input: TaskInput) -> TaskResult: # group point extractions by class label match_candidates = defaultdict(list) # class name -> list of tuples - for i, p in enumerate(map_image.labels): + for i, p in enumerate(map_point_labels.labels): # tuple of (original extraction id, pt extraction object) match_candidates[p.class_name].append((i, p)) @@ -241,7 +242,7 @@ def run(self, input: TaskInput) -> TaskResult: ) # save dip angle results for this point class for idx, (dip_angle, _) in dip_magnitudes.items(): - map_image.labels[idx].dip = dip_angle + map_point_labels.labels[idx].dip = dip_angle # --- 2. estimate symbol orientation (using template matching) # --- pre-process the main image and template image, before template matching @@ -347,13 +348,14 @@ def run(self, input: TaskInput) -> TaskResult: for idx, (_, best_angle) in xcorr_results.items(): # convert final result from 'trig' angle convention # to compass angle convention (CW with 0 deg at top) - map_image.labels[idx].direction = self._trig_to_compass_angle( + map_point_labels.labels[idx].direction = self._trig_to_compass_angle( best_angle, task_config.rotate_max ) logger.info(f"Finished point orientation analysis for class {c}") return TaskResult( - task_id=self._task_id, output={"map_image": map_image.model_dump()} + task_id=self._task_id, + output={"map_point_labels": map_point_labels.model_dump()}, ) def _trig_to_compass_angle(self, angle_deg: int, rotate_max: int) -> int: diff --git a/tasks/point_extraction/tiling.py b/tasks/point_extraction/tiling.py index df6075df..56e53041 100644 --- a/tasks/point_extraction/tiling.py +++ b/tasks/point_extraction/tiling.py @@ -1,7 +1,7 @@ from PIL import Image import numpy as np from tqdm import tqdm -from typing import List +from typing import List, Tuple import logging import cv2 @@ -15,8 +15,15 @@ ) from tasks.segmentation.entities import MapSegmentation, SEGMENTATION_OUTPUT_KEY from tasks.segmentation.segmenter_utils import get_segment_bounds, segments_to_mask +from tasks.point_extraction.entities import ( + LegendPointItems, + LEGEND_ITEMS_OUTPUT_KEY, +) +from tasks.point_extraction.legend_item_utils import legend_items_use_ontology -SEGMENT_MAP_CLASS = "map" # class label for map area segmentation +# segmentation class labels for map and points legend areas +SEGMENT_MAP_CLASS = "map" +SEGMENT_POINT_LEGEND_CLASS = "legend_points_lines" TILE_OVERLAP_DEFAULT = ( # default tliing overlap = point bbox + 10% int(1.1 * 90), int(1.1 * 90), @@ -54,16 +61,18 @@ def run( x_min = 0 y_min = 0 y_max, x_max, _ = image_array.shape - roi_label = "" - # use image segmentation to restrict point extraction to map area only + # ---- use image segmentation to restrict point extraction to map area only + poly_legend = [] if SEGMENTATION_OUTPUT_KEY in task_input.data: segmentation = MapSegmentation.model_validate( task_input.data[SEGMENTATION_OUTPUT_KEY] ) # get a binary mask of the regions-of-interest and apply to the input image before tiling binary_mask = segments_to_mask( - segmentation, (x_max, y_max), roi_classes=[SEGMENT_MAP_CLASS] + segmentation, + (x_max, y_max), + roi_classes=[SEGMENT_MAP_CLASS, SEGMENT_POINT_LEGEND_CLASS], ) if binary_mask.size != 0: # apply binary mask to input image prior to tiling @@ -71,56 +80,115 @@ def run( image_array, image_array, mask=binary_mask ) - p_map = get_segment_bounds(segmentation, SEGMENT_MAP_CLASS) - if len(p_map) > 0: + poly_map = get_segment_bounds(segmentation, SEGMENT_MAP_CLASS) + poly_legend = get_segment_bounds(segmentation, SEGMENT_POINT_LEGEND_CLASS) + if len(poly_map) > 0: # restrict tiling to use *only* the bounding rectangle of map area - p_map = p_map[0] # use 1st (highest ranked) map segment - (x_min, y_min, x_max, y_max) = [int(b) for b in p_map.bounds] - roi_label = SEGMENT_MAP_CLASS + poly_map = poly_map[0] # use 1st (highest ranked) map segment + (x_min, y_min, x_max, y_max) = [int(b) for b in poly_map.bounds] roi_bounds = (x_min, y_min, x_max, y_max) - step_x = int(self.tile_size[0] - self.overlap[0]) - step_y = int(self.tile_size[1] - self.overlap[1]) - - tiles: List[ImageTile] = [] + # ---- create tiles for map area + logger.info("Creating map area tiles") + map_tiles = self._create_tiles( + task_input.raster_id, image_array, [roi_bounds], SEGMENT_MAP_CLASS + ) - for y in range(y_min, y_max, step_y): - for x in range(x_min, x_max, step_x): - width = min(self.tile_size[0], x_max - x) - height = min(self.tile_size[1], y_max - y) + # --- load legend item annotations, if available + if LEGEND_ITEMS_OUTPUT_KEY in task_input.data: + legend_pt_items = LegendPointItems.model_validate( + task_input.data[LEGEND_ITEMS_OUTPUT_KEY] + ) + if poly_legend and not legend_items_use_ontology(legend_pt_items): + # legend annotations don't all use the expected ontology + roi_bounds = [] + for p_leg in poly_legend: + roi_bounds.append([int(b) for b in p_leg.bounds]) + logger.info("Also creating legend area tiles") + legend_tiles = self._create_tiles( + task_input.raster_id, + image_array, + roi_bounds, + SEGMENT_POINT_LEGEND_CLASS, + ) - tile_array = image_array[y : y + height, x : x + width] + # prepare task result with both map and legend tiles + return TaskResult( + task_id=self._task_id, + output={ + "map_tiles": map_tiles.model_dump(), + "legend_tiles": legend_tiles.model_dump(), + }, + ) - if ( - tile_array.shape[0] < self.tile_size[1] - or tile_array.shape[1] < self.tile_size[0] - ): - padded_tile = np.zeros( - (self.tile_size[1], self.tile_size[0], 3), - dtype=tile_array.dtype, - ) + # prepare task result with only map tiles + return TaskResult( + task_id=self._task_id, output={"map_tiles": map_tiles.model_dump()} + ) - padded_tile[:height, :width] = tile_array - tile_array = padded_tile + def _create_tiles( + self, + raster_id: str, + image_array: np.ndarray, + roi_bounds: List[Tuple], + roi_label: str, + ) -> ImageTiles: + """ + create tiles for an image regions-of-interest + """ + step_x = int(self.tile_size[0] - self.overlap[0]) + step_y = int(self.tile_size[1] - self.overlap[1]) + tiles: List[ImageTile] = [] - maptile = ImageTile( - x_offset=x, - y_offset=y, - width=self.tile_size[0], - height=self.tile_size[1], - image=Image.fromarray(tile_array), - image_path="", - ) - tiles.append(maptile) - map_tiles = ImageTiles( - raster_id=task_input.raster_id, + for bounds in roi_bounds: + (x_min, y_min, x_max, y_max) = bounds + + for y in range(y_min, y_max, step_y): + for x in range(x_min, x_max, step_x): + width = min(self.tile_size[0], x_max - x) + height = min(self.tile_size[1], y_max - y) + + tile_array = image_array[y : y + height, x : x + width] # type: ignore + + if ( + tile_array.shape[0] < self.tile_size[1] + or tile_array.shape[1] < self.tile_size[0] + ): + padded_tile = np.zeros( + (self.tile_size[1], self.tile_size[0], 3), + dtype=tile_array.dtype, + ) + + padded_tile[:height, :width] = tile_array + tile_array = padded_tile + + maptile = ImageTile( + x_offset=x, + y_offset=y, + width=self.tile_size[0], + height=self.tile_size[1], + image=Image.fromarray(tile_array), + image_path="", + ) + tiles.append(maptile) + # get global bounds, if multiple segments for this roi + bounds_global = [] + if len(roi_bounds) > 0: + bounds_global = list(roi_bounds[0]) + for bounds in roi_bounds: + bounds_global[0] = min(bounds_global[0], bounds[0]) + bounds_global[1] = min(bounds_global[1], bounds[1]) + bounds_global[2] = max(bounds_global[2], bounds[2]) + bounds_global[3] = max(bounds_global[3], bounds[3]) + + image_tiles = ImageTiles( + raster_id=raster_id, tiles=tiles, - roi_bounds=roi_bounds, + roi_bounds=tuple(bounds_global), roi_label=roi_label, ) - return TaskResult( - task_id=self._task_id, output={"map_tiles": map_tiles.model_dump()} - ) + + return image_tiles @property def input_type(self): @@ -142,14 +210,43 @@ def __init__(self, task_id="", overlap: tuple = TILE_OVERLAP_DEFAULT): Note that new images aren't actually constructed here, we are just mapping predictions from tiles onto the original map. """ - def run(self, input: TaskInput) -> TaskResult: + def run(self, task_input: TaskInput) -> TaskResult: """ Reconstructs the original image from the tiles and maps back the bounding boxes and labels. tile_predictions: List of PointLabel objects. Generated by the model. TILES MUST BE FROM ONLY ONE MAP. returns: List of PointLabel objects. These can be mapped directly onto the original map. """ - image_tiles = ImageTiles.model_validate(input.get_data("map_tiles")) + # run untiling on map tiles + logger.info("Untiling map tiles...") + map_tiles = ImageTiles.model_validate(task_input.data["map_tiles"]) + map_point_labels = self._merge_tiles(map_tiles, task_input.raster_id) + + if "legend_tiles" in task_input.data: + # --- also run untiling on legend area tiles, if available + logger.info("Also untiling legend area tiles...") + legend_tiles = ImageTiles.model_validate(task_input.data["legend_tiles"]) + legend_point_labels = self._merge_tiles(legend_tiles, task_input.raster_id) + + # store untiling results for both map and legend areas + return TaskResult( + task_id=self._task_id, + output={ + "map_point_labels": map_point_labels.model_dump(), + "legend_point_labels": legend_point_labels.model_dump(), + }, + ) + + # store untiling results for map area + return TaskResult( + task_id=self._task_id, + output={"map_point_labels": map_point_labels.model_dump()}, + ) + + def _merge_tiles(self, image_tiles: ImageTiles, raster_id: str) -> PointLabels: + """ + Merge tile extractions by converting predictions results to global image pixel coordinates + """ assert all( i.predictions is not None for i in image_tiles.tiles @@ -165,7 +262,7 @@ def run(self, input: TaskInput) -> TaskResult: for pred in tqdm( tile.predictions, - desc="Reconstructing original map with predictions on tiles", + desc="Reconstructing original image with predictions on tiles", ): x1 = pred.x1 @@ -202,8 +299,6 @@ def run(self, input: TaskInput) -> TaskResult: score=score, direction=pred.direction, dip=pred.dip, - legend_name=pred.legend_name, - # legend_bbox=pred.legend_bbox, # bbox coords assumed to be in global pixel coords ) if pred_in_overlap: @@ -226,12 +321,10 @@ def run(self, input: TaskInput) -> TaskResult: all_predictions.extend(list(overlap_predictions.values())) logger.info( - f"Total point predictions after re-constructing map tiles: {len(all_predictions)}, with {num_dedup} discarded as duplicates" + f"Total point predictions after re-constructing image tiles: {len(all_predictions)}, with {num_dedup} discarded as duplicates" ) - map_image = PointLabels( - path=map_path, raster_id=input.raster_id, labels=all_predictions - ) - return TaskResult(task_id=self._task_id, output={"map_image": map_image}) + + return PointLabels(path=map_path, raster_id=raster_id, labels=all_predictions) def _is_prediction_redundant( self, From 69912fe85eadafceaac44aa3d5c090736de0580a Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Fri, 26 Jul 2024 13:19:34 -0400 Subject: [PATCH 10/66] Added re-adding of roi filtered coordinates based on distance to polygon when insufficient coordinates kept. --- tasks/geo_referencing/filter.py | 68 +++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/tasks/geo_referencing/filter.py b/tasks/geo_referencing/filter.py index 0548633c..f8fc8a9e 100644 --- a/tasks/geo_referencing/filter.py +++ b/tasks/geo_referencing/filter.py @@ -6,6 +6,7 @@ from copy import deepcopy from sklearn.cluster import DBSCAN import matplotlib.path as mpltPath +from shapely import distance, Polygon from tasks.geo_referencing.entities import ( Coordinate, @@ -380,7 +381,8 @@ def _filter( roi_inner_xy = input.get_data("roi_inner") self._add_param(input, str(uuid.uuid4()), "roi_inner", {"bounds": roi_inner_xy}) - num_keypoints = min(len(lon_coords), len(lat_coords)) + lon_counts, lat_counts = self._get_distinct_degrees(lon_coords, lat_coords) + num_keypoints = min(lon_counts, lat_counts) if num_keypoints < 2: logger.info( f"roi filter not filtering since {num_keypoints} coord exists along one axis" @@ -393,21 +395,79 @@ def _filter( input, lon_inputs, lat_inputs, True, roi_inner_xy ) lon_pts, lat_pts = self._filter_roi(input, lon_pts, lat_pts, False, roi_xy) + lon_counts, lat_counts = self._get_distinct_degrees(lon_pts, lat_pts) + + # TODO: SHOULD PRIORITIZE CERTAIN COORDS + # EX: DETECT CORNER COORDS, OR COORDS THAT LINE UP WITH COORDS WITHIN THE ROI + + # adjust based on distance to roi if insufficient points + if lon_counts < 2: + logger.info( + f"only {lon_counts} lon coords after roi filtering so re-adding coordinates" + ) + lons_kept = self._adjust_filter(lon_inputs, roi_xy) + for lk in lons_kept: + lon_pts[lk.to_deg_result()[0]] = lk + if lat_counts < 2: + logger.info( + f"only {lon_counts} lat coords after roi filtering so re-adding coordinates" + ) + lats_kept = self._adjust_filter(lat_inputs, roi_xy) + for lk in lats_kept: + lat_pts[lk.to_deg_result()[0]] = lk + lon_pts, lat_pts = self._validate_lonlat_extractions( lon_pts, lat_pts, input.image.size ) # check if too many points were removed - lats_distinct = set(map(lambda x: x[1].get_parsed_degree(), lat_pts.items())) - lons_distinct = set(map(lambda x: x[1].get_parsed_degree(), lon_pts.items())) - num_keypoints = min(len(lons_distinct), len(lats_distinct)) + lon_counts, lat_counts = self._get_distinct_degrees(lon_pts, lat_pts) + num_keypoints = min(lon_counts, lat_counts) if num_keypoints < 2: logger.info(f"not filtering using roi due to too many points being removed") return lon_coords, lat_coords # apply to the parsed coordinates + logger.info(f"done filtering coordinates using roi") return lon_pts, lat_pts + def _adjust_filter( + self, + coords: Dict[Tuple[float, float], Coordinate], + roi_xy: List[Tuple[float, float]], + ) -> List[Coordinate]: + # get distance to roi for all coordinates + coordinates = [x[1] for x in coords.items()] + roi_poly = Polygon(roi_xy) + dist_coordinates = [(distance(c, roi_poly), c) for c in coordinates] + + # rank all coordinates by distance to roi + coords_sorted = sorted(dist_coordinates, key=lambda x: x[0]) + + # include sufficient coordinates to still be able to georeference + degrees = set() + coords_kept = [] + for c in coords_sorted: + if c[0] == 0: + # those within the polygon should already be included + degrees.add(c[1].get_parsed_degree()) + coords_kept.append(c[1]) + continue + degree = c[1].get_parsed_degree() + if len(degrees) < 2 and degree not in degrees: + degrees.add(degree) + coords_kept.append(c[1]) + return coords_kept + + def _get_distinct_degrees( + self, + lon_coords: Dict[Tuple[float, float], Coordinate], + lat_coords: Dict[Tuple[float, float], Coordinate], + ) -> Tuple[int, int]: + lats_distinct = set(map(lambda x: x[1].get_parsed_degree(), lat_coords.items())) + lons_distinct = set(map(lambda x: x[1].get_parsed_degree(), lon_coords.items())) + return len(lons_distinct), len(lats_distinct) + def _validate_lonlat_extractions( self, lon_results: Dict[Tuple[float, float], Coordinate], From 0a0fd944f3a9b32790a20d5d4251377ad482ac22 Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Fri, 26 Jul 2024 15:04:21 -0400 Subject: [PATCH 11/66] Fixed roi readding of filtered coordinates. --- tasks/geo_referencing/filter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tasks/geo_referencing/filter.py b/tasks/geo_referencing/filter.py index f8fc8a9e..10009ec7 100644 --- a/tasks/geo_referencing/filter.py +++ b/tasks/geo_referencing/filter.py @@ -6,7 +6,7 @@ from copy import deepcopy from sklearn.cluster import DBSCAN import matplotlib.path as mpltPath -from shapely import distance, Polygon +from shapely import distance, Polygon, Point from tasks.geo_referencing.entities import ( Coordinate, @@ -410,7 +410,7 @@ def _filter( lon_pts[lk.to_deg_result()[0]] = lk if lat_counts < 2: logger.info( - f"only {lon_counts} lat coords after roi filtering so re-adding coordinates" + f"only {lat_counts} lat coords after roi filtering so re-adding coordinates" ) lats_kept = self._adjust_filter(lat_inputs, roi_xy) for lk in lats_kept: @@ -439,7 +439,9 @@ def _adjust_filter( # get distance to roi for all coordinates coordinates = [x[1] for x in coords.items()] roi_poly = Polygon(roi_xy) - dist_coordinates = [(distance(c, roi_poly), c) for c in coordinates] + dist_coordinates = [ + (distance(Point(c.get_pixel_alignment()), roi_poly), c) for c in coordinates + ] # rank all coordinates by distance to roi coords_sorted = sorted(dist_coordinates, key=lambda x: x[0]) From b1a6da2be4b93af7716d47c22000d242db465c46 Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Tue, 30 Jul 2024 12:49:43 -0400 Subject: [PATCH 12/66] Switched roi filter reduction to use a simple scoring mechanism based on point confidence and distance to roi. --- tasks/geo_referencing/filter.py | 45 +++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/tasks/geo_referencing/filter.py b/tasks/geo_referencing/filter.py index 10009ec7..774aca81 100644 --- a/tasks/geo_referencing/filter.py +++ b/tasks/geo_referencing/filter.py @@ -381,8 +381,10 @@ def _filter( roi_inner_xy = input.get_data("roi_inner") self._add_param(input, str(uuid.uuid4()), "roi_inner", {"bounds": roi_inner_xy}) - lon_counts, lat_counts = self._get_distinct_degrees(lon_coords, lat_coords) - num_keypoints = min(lon_counts, lat_counts) + lon_counts_initial, lat_counts_initial = self._get_distinct_degrees( + lon_coords, lat_coords + ) + num_keypoints = min(lon_counts_initial, lat_counts_initial) if num_keypoints < 2: logger.info( f"roi filter not filtering since {num_keypoints} coord exists along one axis" @@ -401,21 +403,20 @@ def _filter( # EX: DETECT CORNER COORDS, OR COORDS THAT LINE UP WITH COORDS WITHIN THE ROI # adjust based on distance to roi if insufficient points - if lon_counts < 2: + if lon_counts < 2 and lon_counts < lon_counts_initial: logger.info( f"only {lon_counts} lon coords after roi filtering so re-adding coordinates" ) - lons_kept = self._adjust_filter(lon_inputs, roi_xy) + lons_kept = self._adjust_filter(lon_coords, roi_xy) for lk in lons_kept: lon_pts[lk.to_deg_result()[0]] = lk - if lat_counts < 2: + if lat_counts < 2 and lat_counts < lat_counts_initial: logger.info( f"only {lat_counts} lat coords after roi filtering so re-adding coordinates" ) - lats_kept = self._adjust_filter(lat_inputs, roi_xy) + lats_kept = self._adjust_filter(lat_coords, roi_xy) for lk in lats_kept: lat_pts[lk.to_deg_result()[0]] = lk - lon_pts, lat_pts = self._validate_lonlat_extractions( lon_pts, lat_pts, input.image.size ) @@ -439,22 +440,15 @@ def _adjust_filter( # get distance to roi for all coordinates coordinates = [x[1] for x in coords.items()] roi_poly = Polygon(roi_xy) - dist_coordinates = [ - (distance(Point(c.get_pixel_alignment()), roi_poly), c) for c in coordinates - ] + score_coordinates = [(self._get_roi_score(c, roi_poly), c) for c in coordinates] # rank all coordinates by distance to roi - coords_sorted = sorted(dist_coordinates, key=lambda x: x[0]) + coords_sorted = sorted(score_coordinates, key=lambda x: x[0], reverse=True) # include sufficient coordinates to still be able to georeference degrees = set() coords_kept = [] for c in coords_sorted: - if c[0] == 0: - # those within the polygon should already be included - degrees.add(c[1].get_parsed_degree()) - coords_kept.append(c[1]) - continue degree = c[1].get_parsed_degree() if len(degrees) < 2 and degree not in degrees: degrees.add(degree) @@ -620,3 +614,22 @@ def _filter_roi( ) return (lon_results, lat_results) + + def _get_roi_score(self, coordinate: Coordinate, roi_poly: Polygon) -> float: + # combine distance and coordinate confidence for an initial scoring mechanism + coord_dist = distance(Point(coordinate.get_pixel_alignment()), roi_poly) + coord_conf = coordinate.get_confidence() + + # penalize confidence based on ratio of distance and roi size + roi_bounds = roi_poly.bounds + min_dimension = min( + (roi_bounds[2] - roi_bounds[0]), (roi_bounds[3] - roi_bounds[1]) + ) + comparable_size = min_dimension / 10 + + conf_adjustment = max(1 - (coord_dist / comparable_size), 0) + score = coord_conf * conf_adjustment + logger.info( + f"adjusting confidence of point {coordinate.get_pixel_alignment()} by {conf_adjustment} to {score}" + ) + return score From 742cf9babc8e21354455c6c9018936c07a2c945e Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Tue, 30 Jul 2024 13:57:15 -0400 Subject: [PATCH 13/66] updated code for writing CDR and bitmask results for points' pipeline to support legend annotations --- .../point_extraction_pipeline.py | 28 ++- schema/mappers/cdr.py | 107 ++++++++---- tasks/point_extraction/entities.py | 3 + tasks/point_extraction/label_map.py | 4 +- tasks/point_extraction/legend_analyzer.py | 159 +++++++++++++++++- tasks/point_extraction/legend_item_utils.py | 1 + .../point_extraction/point_extractor_utils.py | 59 ++++--- .../template_match_point_extractor.py | 66 ++++---- 8 files changed, 328 insertions(+), 99 deletions(-) diff --git a/pipelines/point_extraction/point_extraction_pipeline.py b/pipelines/point_extraction/point_extraction_pipeline.py index 5865b057..319e7bb4 100644 --- a/pipelines/point_extraction/point_extraction_pipeline.py +++ b/pipelines/point_extraction/point_extraction_pipeline.py @@ -131,11 +131,13 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: Returns: PointLabel: The map point label extraction object. """ - map_image = PointLabels.model_validate(pipeline_result.data["map_image"]) + map_point_labels = PointLabels.model_validate( + pipeline_result.data["map_point_labels"] + ) return BaseModelOutput( pipeline_result.pipeline_id, pipeline_result.pipeline_name, - map_image, + map_point_labels, ) @@ -163,10 +165,18 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: Returns: Output: The output of the pipeline. """ - map_image = PointLabels.model_validate(pipeline_result.data["map_image"]) + map_point_labels = PointLabels.model_validate( + pipeline_result.data["map_point_labels"] + ) + legend_pt_items = LegendPointItems(items=[]) + if LEGEND_ITEMS_OUTPUT_KEY in pipeline_result.data: + legend_pt_items = LegendPointItems.model_validate( + pipeline_result.data[LEGEND_ITEMS_OUTPUT_KEY] + ) + mapper = PointsMapper(MODEL_NAME, MODEL_VERSION) - cdr_points = mapper.map_to_cdr(map_image) + cdr_points = mapper.map_to_cdr(map_point_labels, legend_pt_items) return BaseModelOutput( pipeline_result.pipeline_id, pipeline_result.pipeline_name, cdr_points ) @@ -196,18 +206,20 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: Returns: Output: The output of the pipeline. """ - map_image = PointLabels.model_validate(pipeline_result.data["map_image"]) - legend_labels = [] + map_point_labels = PointLabels.model_validate( + pipeline_result.data["map_point_labels"] + ) if LEGEND_ITEMS_OUTPUT_KEY in pipeline_result.data: legend_pt_items = LegendPointItems.model_validate( pipeline_result.data[LEGEND_ITEMS_OUTPUT_KEY] ) - legend_labels = [pt_type.name for pt_type in legend_pt_items.items] if pipeline_result.image is None: raise ValueError("Pipeline result image is None") (w, h) = pipeline_result.image.size - bitmasks_dict = convert_preds_to_bitmasks(map_image, legend_labels, (w, h)) + bitmasks_dict = convert_preds_to_bitmasks( + map_point_labels, legend_pt_items, (w, h) + ) return ImageDictOutput( pipeline_result.pipeline_id, pipeline_result.pipeline_name, bitmasks_dict diff --git a/schema/mappers/cdr.py b/schema/mappers/cdr.py index d728e927..92affd27 100644 --- a/schema/mappers/cdr.py +++ b/schema/mappers/cdr.py @@ -9,13 +9,15 @@ ProjectionResult, ) from schema.cdr_schemas.area_extraction import Area_Extraction, AreaType -from schema.cdr_schemas.map import Map from schema.cdr_schemas.metadata import ( MapColorSchemeTypes, MapMetaData, CogMetaData, MapShapeTypes, ) + +from schema.cdr_schemas.common import ModelProvenance + from schema.cdr_schemas.feature_results import FeatureResults from schema.cdr_schemas.features.point_features import ( PointFeatureCollection, @@ -32,7 +34,8 @@ MetadataExtraction as LARAMetadata, ) from tasks.point_extraction.entities import PointLabels as LARAPoints -from tasks.point_extraction.label_map import YOLO_TO_CDR_LABEL +from tasks.point_extraction.entities import LegendPointItems as LARALegendItems +from tasks.point_extraction.point_extractor_utils import get_cdr_point_name from tasks.segmentation.entities import MapSegmentation as LARASegmentation from pydantic import BaseModel @@ -233,41 +236,66 @@ def map_from_cdr(self, model: FeatureResults) -> LARASegmentation: class PointsMapper(CDRMapper): - def map_to_cdr(self, model: LARAPoints) -> FeatureResults: - point_features: List[PointLegendAndFeaturesResult] = [] + def map_to_cdr( + self, model: LARAPoints, legend_items: LARALegendItems + ) -> FeatureResults: + """ + Convert LARA point extractions to CDR output format + """ + legend_features: Dict[str, PointLegendAndFeaturesResult] = {} + for leg_item in legend_items.items: + # fill in point name, abbreviation and description fields + name = get_cdr_point_name(leg_item.class_name, leg_item.name) + abbr = leg_item.abbreviation if leg_item.abbreviation else name + desc = leg_item.description + if not desc: + desc = leg_item.name if leg_item.name else name + desc = desc.replace("_", " ").strip().lower() + + legend_provenance = None + if leg_item.system: + legend_provenance = ModelProvenance( + model=leg_item.system, model_version=leg_item.system_version + ) + + legend_feature_result = PointLegendAndFeaturesResult( + id=name, + legend_provenance=legend_provenance, + crs="CRITICALMAAS:pixel", + name=name, + abbreviation=abbr, + description=desc, + legend_bbox=leg_item.legend_bbox, + legend_contour=leg_item.legend_contour, + validated=leg_item.validated, + point_features=None, # points are filled in below + ) + legend_features[name] = legend_feature_result - # create seperate lists for each point class since they are groupded by class + # create seperate lists for each point class since they are grouped by class # in the results - point_features_by_class: Dict[str, List[PointFeature]] = {} + point_features: Dict[str, List[PointFeature]] = {} point_id = 0 if model.labels: for map_pt_label in model.labels: - # point label - pt_label = ( - map_pt_label.legend_name - if map_pt_label.legend_name - else map_pt_label.class_name - ) - if pt_label in YOLO_TO_CDR_LABEL: - # map from YOLO point class to CDR point label - pt_label = YOLO_TO_CDR_LABEL[pt_label] + name = get_cdr_point_name(map_pt_label.class_name, "") - if pt_label not in point_features_by_class: - # init result object for this point type... - # TODO -- fill in legend item info if available in future - # ( we should use legend item annotations here, if possible, to fill in legend fields and then append point extractions afterwards) - point_features_by_class[pt_label] = [] - point_features_result = PointLegendAndFeaturesResult( - id="id", + if name not in legend_features: + # init new legend result object... + legend_feature_result = PointLegendAndFeaturesResult( + id=name, crs="CRITICALMAAS:pixel", - name=pt_label, - abbreviation=pt_label, - description=pt_label.replace("_", " ").strip().lower(), - # legend_bbox=map_pt_label.legend_bbox, + name=name, + abbreviation=name, + description=name.replace("_", " ").strip().lower(), point_features=None, # points are filled in below ) - point_features.append(point_features_result) + legend_features[name] = legend_feature_result + + if name not in point_features: + # init result object for this point type... + point_features[name] = [] # create the point geometry point = Point( @@ -298,27 +326,36 @@ def map_to_cdr(self, model: LARAPoints) -> FeatureResults: # add the point geometry and properties to the point feature point_feature = PointFeature( - id=f"{pt_label}.{point_id}", + id=f"{name}.{point_id}", geometry=point, properties=properties, ) point_id += 1 # add to the list of point features for the class - point_features_by_class[pt_label].append(point_feature) + point_features[name].append(point_feature) # append our final list of feature results and create the output - for pt_feat in point_features: - if pt_feat.name not in point_features_by_class: - logger.warning(f"Point type {pt_feat.name} not found in results!") + point_legend_features = list(legend_features.values()) + for pt_leg_feat in point_legend_features: + if ( + pt_leg_feat.name not in point_features + or len(point_features[pt_leg_feat.name]) == 0 + ): + logger.warning(f"Point type {pt_leg_feat.name} has no extractions!") else: - pt_feat.point_features = PointFeatureCollection( - features=point_features_by_class[pt_feat.name] + pt_leg_feat.point_features = PointFeatureCollection( + features=point_features[pt_leg_feat.name] ) + # filter point types with no extractions + point_legend_features = list( + filter(lambda x: x.point_features is not None, point_legend_features) + ) + return FeatureResults( cog_id=model.raster_id, - point_feature_results=point_features, + point_feature_results=point_legend_features, system=self._system_name, system_version=self._system_version, ) diff --git a/tasks/point_extraction/entities.py b/tasks/point_extraction/entities.py index 2db2a066..43957364 100644 --- a/tasks/point_extraction/entities.py +++ b/tasks/point_extraction/entities.py @@ -176,6 +176,9 @@ class LegendPointItem(BaseModel): where the top left is the origin (0,0).""", ) system: str = Field(default="", description="System that published this item") + system_version: str = Field( + default="", description="System version that published this item" + ) validated: bool = Field(default=False, description="Validated by human") confidence: Optional[float] = Field( default=None, diff --git a/tasks/point_extraction/label_map.py b/tasks/point_extraction/label_map.py index 30a147d0..9e783a1b 100644 --- a/tasks/point_extraction/label_map.py +++ b/tasks/point_extraction/label_map.py @@ -108,9 +108,9 @@ def __str__(self): # --- MINE or QUARRY or OPEN PIT (5_pt -- crossed pick-axes) "mine_quarry": ["mine_quarry", "geo_mosaic_5_pt"], # --- SINK HOLE - "sink_hole": ["sink_hole", "sinkhole_pt"], + "sink_hole": ["sink_hole", "sinkhole", "sinkhole_pt"], # --- LINEATION "lineation": ["lineation", "lineation_pt"], # --- DRILL HOLE - "drill_hole": ["drill_hole", "drillhole_pt"], + "drill_hole": ["drill_hole", "drillhole", "drillhole_pt"], } diff --git a/tasks/point_extraction/legend_analyzer.py b/tasks/point_extraction/legend_analyzer.py index 2a35f8ad..2f5fe878 100644 --- a/tasks/point_extraction/legend_analyzer.py +++ b/tasks/point_extraction/legend_analyzer.py @@ -1,8 +1,12 @@ import logging +from collections import defaultdict +from shapely import Polygon, distance from tasks.common.task import Task, TaskInput, TaskResult from tasks.point_extraction.entities import ( LegendPointItems, + LegendPointItem, LEGEND_ITEMS_OUTPUT_KEY, + PointLabels, ) from tasks.point_extraction.legend_item_utils import ( filter_labelme_annotations, @@ -65,7 +69,8 @@ def run(self, task_input: TaskInput) -> TaskResult: ) legend_pt_items.items = [] return TaskResult( - task_id=self._task_id, output={LEGEND_ITEMS_OUTPUT_KEY: legend_pt_items} + task_id=self._task_id, + output={LEGEND_ITEMS_OUTPUT_KEY: legend_pt_items.model_dump()}, ) return self._create_result(task_input) @@ -86,9 +91,155 @@ def __init__( def run(self, task_input: TaskInput) -> TaskResult: """ - run point symbol legend post-processing + Run point symbol legend post-processing + + The goal is to convert point symbol predictions (from a map's legend area) + to legend item annotation objects (or join with existing) """ - ## WIP! - return self._create_result(task_input) + if not "legend_point_labels" in task_input.data: + logger.info( + "Point predictions not available for the legend area. Skipping legend post-processing." + ) + return self._create_result(task_input) + + # --- load legend area ML point predictions + legend_pt_preds = PointLabels.model_validate( + task_input.data["legend_point_labels"] + ) + if not legend_pt_preds.labels: + return self._create_result(task_input) + + # --- load legend item annotations, if available + legend_pt_items = LegendPointItems(items=[]) + if LEGEND_ITEMS_OUTPUT_KEY in task_input.data: + legend_pt_items = LegendPointItems.model_validate( + task_input.data[LEGEND_ITEMS_OUTPUT_KEY] + ) + join_with_existing = len(legend_pt_items.items) > 0 + + # group legend pt predictions by class name + pred_groups = defaultdict(list) + for label in legend_pt_preds.labels: + pred_groups[label.class_name].append(label) + + # loop over groups + for class_name, preds in pred_groups.items(): + if len(preds) > 1: + # more than 1 legend swatch extracted for this class name + # so choose the highest conf one, and discard the others as noisy + logger.info( + f"{len(preds)} predictions found for point type {class_name}. Choosing the best one." + ) + # sort by model confidence score + preds.sort(key=lambda s: s.score, reverse=True) + + bbox = [preds[0].x1, preds[0].y1, preds[0].x2, preds[0].y2] + xy_pts = [ + [preds[0].x1, preds[0].y1], + [preds[0].x2, preds[0].y1], + [preds[0].x2, preds[0].y2], + [preds[0].x1, preds[0].y2], + ] + confidence = preds[0].score + + if join_with_existing: + # check which legend ann swatch most overlaps with others, + # if multiple overlap, choose the closest one + # ... if this one is already in ontology choose that one + p_pred = Polygon(xy_pts) + leg_matches = list( + filter( + lambda leg: (leg.class_name == class_name), + legend_pt_items.items, + ) + ) + if leg_matches: + # legend swatch matches found + p_leg = Polygon(leg_matches[0].legend_contour) + smallest_dim = min( + [ + p_leg.bounds[2] - p_leg.bounds[0], + p_leg.bounds[3] - p_leg.bounds[1], + p_pred.bounds[2] - p_pred.bounds[0], + p_pred.bounds[3] - p_pred.bounds[1], + ] + ) + dist_norm = distance(p_leg.centroid, p_pred.centroid) / max( + smallest_dim, 1.0 + ) + logger.info( + f"Joining legend swatch prediction with existing legend item annotation for class {class_name}; normalized distance = {dist_norm:.3f}" + ) + else: + # get legend item annotations without any class label, + # and match the one with highest overlap + leg_unmatched = list( + filter( + lambda leg: (not leg.class_name), + legend_pt_items.items, + ) + ) + i_min = -1 + dist_min = 9999999.0 + for i, leg in enumerate(leg_unmatched): + p_leg = Polygon(leg.legend_contour) + smallest_dim = min( + [ + p_leg.bounds[2] - p_leg.bounds[0], + p_leg.bounds[3] - p_leg.bounds[1], + p_pred.bounds[2] - p_pred.bounds[0], + p_pred.bounds[3] - p_pred.bounds[1], + ] + ) + dist_norm = distance(p_leg.centroid, p_pred.centroid) / max( + smallest_dim, 1.0 + ) + if dist_norm < dist_min: + dist_min = dist_norm + i_min = i + if dist_min < 1.0: + # match found + logger.info( + f"Joining legend swatch prediction with existing legend item annotation for class {class_name}; normalized distance = {dist_min:.3f}" + ) + leg_unmatched[i_min].class_name = class_name + else: + # add a new legend point item based on ML legend analysis + # TODO could skip, if yolo confidence is low? + logger.info( + f"Adding new legend item for point class {class_name}" + ) + legend_pt_items.items.append( + LegendPointItem( + name=class_name, + class_name=class_name, + legend_bbox=bbox, + legend_contour=xy_pts, + confidence=confidence, + validated=False, + ) + ) + else: + # add a new legend point item based on ML legend analysis + # TODO could skip, if yolo confidence is low? + logger.info(f"Adding new legend item for point class {class_name}") + legend_pt_items.items.append( + LegendPointItem( + name=class_name, + class_name=class_name, + legend_bbox=bbox, + legend_contour=xy_pts, + confidence=confidence, + validated=False, + ) + ) + + logger.info( + f"Number of Point Legend Items after post-processing: {len(legend_pt_items.items)}" + ) + return TaskResult( + task_id=self._task_id, + output={LEGEND_ITEMS_OUTPUT_KEY: legend_pt_items.model_dump()}, + ) diff --git a/tasks/point_extraction/legend_item_utils.py b/tasks/point_extraction/legend_item_utils.py index 3c7ac439..f0f527ce 100644 --- a/tasks/point_extraction/legend_item_utils.py +++ b/tasks/point_extraction/legend_item_utils.py @@ -215,6 +215,7 @@ def legend_ann_to_legend_items( legend_bbox=leg_ann.px_bbox, legend_contour=xy_pts, system=leg_ann.system, + system_version=leg_ann.system_version, confidence=leg_ann.confidence, validated=leg_ann.validated, ) diff --git a/tasks/point_extraction/point_extractor_utils.py b/tasks/point_extraction/point_extractor_utils.py index 83558353..ca073013 100644 --- a/tasks/point_extraction/point_extractor_utils.py +++ b/tasks/point_extraction/point_extractor_utils.py @@ -5,7 +5,8 @@ from PIL import Image from collections import defaultdict -from tasks.point_extraction.entities import PointLabels +from tasks.point_extraction.entities import PointLabels, LegendPointItems +from tasks.point_extraction.label_map import YOLO_TO_CDR_LABEL from tasks.text_extraction.entities import TextExtraction from shapely.geometry import Polygon from shapely.strtree import STRtree @@ -313,9 +314,22 @@ def mask_ocr_blocks( return im +def get_cdr_point_name(class_name: str, legend_name: str) -> str: + """ + get normalized name for this point symbol type, based on internal ML class name, + legend item name, and CDR point ontology + """ + # use CDR point ontology, if possible... + pt_name = class_name if class_name else legend_name + if pt_name in YOLO_TO_CDR_LABEL: + # map from YOLO point class to CDR point label + pt_name = YOLO_TO_CDR_LABEL[pt_name] + return pt_name + + def convert_preds_to_bitmasks( - map_image: PointLabels, - legend_pt_labels: List[str], + map_pt_labels: PointLabels, + legend_pt_items: LegendPointItems, w_h: Tuple[int, int], binary_pixel_val=1, ) -> Dict[str, Image.Image]: @@ -323,40 +337,43 @@ def convert_preds_to_bitmasks( Convert the PointLabels point predictions to CMA contest style bitmasks Output is dict: point label -> bitmask image """ - if not map_image.labels: + if not map_pt_labels.labels: logger.warning( - f"No point predictions for raster id {map_image.raster_id}. Skipping creation of bitmasks." + f"No point predictions for raster id {map_pt_labels.raster_id}. Skipping creation of bitmasks." ) return {} - # group predictions by legend label or class name + # note, for contest bitmask output names use legend annotations/hints labels in place of + # CDR point ontology, where applicable + pt_class_to_legend_name = {} point_preds_by_class = defaultdict(list) - # initialize with any available legend labels, so we will create an empty bitmask - # even if no extractions were found for a given point type - for pt_label in legend_pt_labels: - point_preds_by_class[pt_label] = [] - for map_pt_label in map_image.labels: - # point label - pt_label = ( - map_pt_label.legend_name - if map_pt_label.legend_name - else map_pt_label.class_name - ) + # create class name to legend item mapping + for leg_item in legend_pt_items.items: + pt_name = get_cdr_point_name(leg_item.class_name, leg_item.name) + pt_class_to_legend_name[pt_name] = leg_item.name + # also create empty point_preds entry, so we will create an empty bitmask + # even if no extractions were found for a given point type + point_preds_by_class[pt_name] = [] + + for map_pt_label in map_pt_labels.labels: + pt_name = get_cdr_point_name(map_pt_label.class_name, "") # bbox center xc = int((map_pt_label.x1 + map_pt_label.x2) / 2) yc = int((map_pt_label.y1 + map_pt_label.y2) / 2) - - point_preds_by_class[pt_label].append((xc, yc)) + point_preds_by_class[pt_name].append((xc, yc)) logger.info( - f"Creating {len(point_preds_by_class)} bitmasks for raster id {map_image.raster_id}" + f"Creating {len(point_preds_by_class)} bitmasks for raster id {map_pt_labels.raster_id}" ) bitmasks = {} - for pt_label, pts_xy in point_preds_by_class.items(): + for class_name, pts_xy in point_preds_by_class.items(): im_binary = np.zeros((w_h[1], w_h[0]), dtype=np.uint8) for x, y in pts_xy: im_binary[y, x] = binary_pixel_val + # generate final bitmask feature label and store result + pt_label = pt_class_to_legend_name.get(class_name, class_name) + pt_label = pt_label.strip().replace(" ", "_") bitmasks[pt_label] = Image.fromarray(im_binary.astype(np.uint8)) return bitmasks diff --git a/tasks/point_extraction/template_match_point_extractor.py b/tasks/point_extraction/template_match_point_extractor.py index ec51dd38..cfe5c9e9 100644 --- a/tasks/point_extraction/template_match_point_extractor.py +++ b/tasks/point_extraction/template_match_point_extractor.py @@ -81,22 +81,24 @@ def run(self, task_input: TaskInput) -> TaskResult: "No Legend item info available. Skipping Template-Match Point Extractor" ) result = self._create_result(task_input) - result.add_output("map_image", task_input.data["map_image"]) + result.add_output("map_point_labels", task_input.data["map_point_labels"]) return result # get existing point predictions from YOLO point extractor - if "map_image" in task_input.data: - map_image_results = PointLabels.model_validate(task_input.data["map_image"]) - if map_image_results.labels is None: - map_image_results.labels = [] + if "map_point_labels" in task_input.data: + map_point_labels = PointLabels.model_validate( + task_input.data["map_point_labels"] + ) + if map_point_labels.labels is None: + map_point_labels.labels = [] else: - map_image_results = PointLabels( + map_point_labels = PointLabels( path="", raster_id=task_input.raster_id, labels=[] ) # --- check which legend points still need to be processed, if any? pt_features = self._which_points_need_processing( - map_image_results.labels, legend_pt_items.items, min_predictions=MIN_MATCHES # type: ignore + map_point_labels.labels, legend_pt_items.items, min_predictions=MIN_MATCHES # type: ignore ) if not pt_features: @@ -105,7 +107,7 @@ def run(self, task_input: TaskInput) -> TaskResult: "No legend items need further processing. Skipping Template-Match Point Extractor" ) result = self._create_result(task_input) - result.add_output("map_image", task_input.data["map_image"]) + result.add_output("map_point_labels", task_input.data["map_point_labels"]) return result # convert image from PIL to opencv (numpy) format -- assumed color channel order is RGB @@ -163,7 +165,10 @@ def run(self, task_input: TaskInput) -> TaskResult: # -------------- # loop through all available point legend items for i, pt_feature in enumerate(pt_features): - logger.info(f"Processing {pt_feature.name}") + feature_name = ( + pt_feature.class_name if pt_feature.class_name else pt_feature.name + ) + logger.info(f"Processing {feature_name}") matches_dedup = [] # --- pre-process the main image and template image, before template matching @@ -314,17 +319,16 @@ def run(self, task_input: TaskInput) -> TaskResult: logger.info( "Final number of unique points extracted for label {}: {}".format( - pt_feature.name, len(matches_dedup) + feature_name, len(matches_dedup) ) ) if len(matches_dedup) > 0: - preds = self._process_output( - matches_dedup, pt_feature.name, map_roi, pt_feature.legend_bbox - ) - map_image_results.labels.extend(preds) # type: ignore + preds = self._process_output(matches_dedup, feature_name, map_roi) + map_point_labels.labels.extend(preds) # type: ignore return TaskResult( - task_id=self._task_id, output={"map_image": map_image_results} + task_id=self._task_id, + output={"map_point_labels": map_point_labels.model_dump()}, ) def _get_template_images( @@ -333,7 +337,7 @@ def _get_template_images( im_templates = [] for feat in feats: - (xmin, ymin, xmax, ymax) = feat.legend_bbox + (xmin, ymin, xmax, ymax) = [int(a) for a in feat.legend_bbox] im_templ = (im[ymin:ymax, xmin:xmax]).copy() im_templates.append(im_templ) @@ -399,7 +403,6 @@ def _process_output( matches: List, label: str, map_roi: List[int], - legend_bbox: List, bbox_size: int = 90, ) -> List[PointLabel]: """ @@ -430,8 +433,6 @@ def _process_output( x2=x2, y2=y2, score=xcorr / 255.0, - legend_name=label, - # legend_bbox=legend_bbox, ) ) @@ -445,33 +446,40 @@ def _which_points_need_processing( ) -> List[LegendPointItem]: """ Check which legend items still need processing - (Since some point classes may've already been handled by the YOLO point extractor) + (Since some point classes may've already been handled by the ML point extractor) """ if min_predictions < 0: - # process all legend items regardless of upstream YOLO predictions + # process all legend items regardless of upstream ML predictions return legend_pt_items legend_items_unprocessed = [] preds_per_class = defaultdict(int) for pred in map_point_labels: - if pred.legend_name: - preds_per_class[pred.legend_name] += 1 - elif pred.class_name: + if pred.class_name: preds_per_class[pred.class_name] += 1 for legend_item in legend_pt_items: if not self._is_template_valid(legend_item): logger.warning( - f"No valid legend template is available for legend item {legend_item.name}" + f"No valid legend template is available for legend item {legend_item.name} - skipping" + ) + continue + + leg_item_name = ( + legend_item.class_name if legend_item.class_name else legend_item.name + ) + if not leg_item_name: + logger.warning( + f"No valid legend name is available for legend item - skipping" ) continue - if legend_item.name not in preds_per_class: - # no YOLO predictions for this point type; needs processing + if leg_item_name not in preds_per_class: + # no ML predictions for this point type; needs processing legend_items_unprocessed.append(legend_item) - elif preds_per_class[legend_item.name] < min_predictions: - # only a few YOLO predictions for this point type; still needs processing + elif preds_per_class[leg_item_name] < min_predictions: + # only a few ML predictions for this point type; still needs processing legend_items_unprocessed.append(legend_item) return legend_items_unprocessed From af0d37fb13ecc90ddbd89893313f7f9a3cbf6982 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Tue, 30 Jul 2024 14:23:25 -0400 Subject: [PATCH 14/66] cleaned up constant definitions for segmentation classes --- pipelines/segmentation/run_pipeline.py | 1 + tasks/point_extraction/legend_item_utils.py | 8 ++------ .../point_extraction/template_match_point_extractor.py | 10 +++++----- tasks/point_extraction/tiling.py | 10 ++++++---- tasks/segmentation/entities.py | 4 ++++ 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pipelines/segmentation/run_pipeline.py b/pipelines/segmentation/run_pipeline.py index 11875bb3..0e736539 100644 --- a/pipelines/segmentation/run_pipeline.py +++ b/pipelines/segmentation/run_pipeline.py @@ -42,6 +42,7 @@ def main(): # run the extraction pipeline for doc_id, image in input: + logger.info(f"Processing doc_id: {doc_id}") image_input = PipelineInput(image=image, raster_id=doc_id) results = pipeline.run(image_input) diff --git a/tasks/point_extraction/legend_item_utils.py b/tasks/point_extraction/legend_item_utils.py index f0f527ce..53962a20 100644 --- a/tasks/point_extraction/legend_item_utils.py +++ b/tasks/point_extraction/legend_item_utils.py @@ -7,15 +7,11 @@ from tasks.point_extraction.entities import LegendPointItem, LegendPointItems from tasks.point_extraction.label_map import LABEL_MAPPING, YOLO_TO_CDR_LABEL from schema.cdr_schemas.cdr_responses.legend_items import LegendItemResponse -from tasks.segmentation.entities import MapSegmentation +from tasks.segmentation.entities import MapSegmentation, SEGMENT_POINT_LEGEND_CLASS logger = logging.getLogger(__name__) -SEGMENT_PT_LEGEND_CLASS = ( - "legend_points_lines" # class label for points legend area segmentation -) - # Legend item annotations "system" or provenance labels class LEGEND_ANNOTATION_PROVENANCE(str, Enum): @@ -238,7 +234,7 @@ def filter_labelme_annotations( segs_point_legend = list( filter( - lambda s: (s.class_label == SEGMENT_PT_LEGEND_CLASS), + lambda s: (s.class_label == SEGMENT_POINT_LEGEND_CLASS), segmentation.segments, ) ) diff --git a/tasks/point_extraction/template_match_point_extractor.py b/tasks/point_extraction/template_match_point_extractor.py index cfe5c9e9..eb4e639c 100644 --- a/tasks/point_extraction/template_match_point_extractor.py +++ b/tasks/point_extraction/template_match_point_extractor.py @@ -6,7 +6,11 @@ from scipy import ndimage from collections import defaultdict -from tasks.segmentation.entities import MapSegmentation, SEGMENTATION_OUTPUT_KEY +from tasks.segmentation.entities import ( + MapSegmentation, + SEGMENTATION_OUTPUT_KEY, + SEGMENT_MAP_CLASS, +) from tasks.segmentation.segmenter_utils import get_segment_bounds, segments_to_mask from tasks.point_extraction import point_extractor_utils as pe_utils from tasks.common.task import Task, TaskInput, TaskResult @@ -26,10 +30,6 @@ MODEL_NAME = "uncharted_oneshot_point_extractor" MODEL_VER = "0.0.1" -# class labels for map and points legend areas -SEGMENT_MAP_CLASS = "map" -SEGMENT_PT_LEGEND_CLASS = "legend_points_lines" - OCR_MIN_LEN = 3 CONTOUR_SIZE_FACTOR = 2.0 CONTOUR_THICKNESS = 2 diff --git a/tasks/point_extraction/tiling.py b/tasks/point_extraction/tiling.py index 56e53041..b6e5d2d8 100644 --- a/tasks/point_extraction/tiling.py +++ b/tasks/point_extraction/tiling.py @@ -13,7 +13,12 @@ PointLabels, PointLabel, ) -from tasks.segmentation.entities import MapSegmentation, SEGMENTATION_OUTPUT_KEY +from tasks.segmentation.entities import ( + MapSegmentation, + SEGMENTATION_OUTPUT_KEY, + SEGMENT_MAP_CLASS, + SEGMENT_POINT_LEGEND_CLASS, +) from tasks.segmentation.segmenter_utils import get_segment_bounds, segments_to_mask from tasks.point_extraction.entities import ( LegendPointItems, @@ -21,9 +26,6 @@ ) from tasks.point_extraction.legend_item_utils import legend_items_use_ontology -# segmentation class labels for map and points legend areas -SEGMENT_MAP_CLASS = "map" -SEGMENT_POINT_LEGEND_CLASS = "legend_points_lines" TILE_OVERLAP_DEFAULT = ( # default tliing overlap = point bbox + 10% int(1.1 * 90), int(1.1 * 90), diff --git a/tasks/segmentation/entities.py b/tasks/segmentation/entities.py index 2c758263..4049a090 100644 --- a/tasks/segmentation/entities.py +++ b/tasks/segmentation/entities.py @@ -3,6 +3,10 @@ SEGMENTATION_OUTPUT_KEY = "segmentation" +# class labels for map and points legend areas +SEGMENT_MAP_CLASS = "map" +SEGMENT_POINT_LEGEND_CLASS = "legend_points_lines" + class SegmentationResult(BaseModel): """ From f9d34e50da2b193c195e4dc1e4664035b5a8ec16 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Tue, 30 Jul 2024 14:50:40 -0400 Subject: [PATCH 15/66] cleaned up constant definitions for point extraction task output keys --- .../point_extraction_pipeline.py | 7 +++--- tasks/point_extraction/entities.py | 7 ++++++ tasks/point_extraction/legend_analyzer.py | 5 ++-- tasks/point_extraction/point_extractor.py | 16 ++++++++----- .../point_orientation_extractor.py | 10 ++++---- .../template_match_point_extractor.py | 15 ++++++++---- tasks/point_extraction/tiling.py | 24 ++++++++++++------- 7 files changed, 55 insertions(+), 29 deletions(-) diff --git a/pipelines/point_extraction/point_extraction_pipeline.py b/pipelines/point_extraction/point_extraction_pipeline.py index 319e7bb4..6e4d1bc3 100644 --- a/pipelines/point_extraction/point_extraction_pipeline.py +++ b/pipelines/point_extraction/point_extraction_pipeline.py @@ -18,6 +18,7 @@ PointLabels, LegendPointItems, LEGEND_ITEMS_OUTPUT_KEY, + MAP_PT_LABELS_OUTPUT_KEY, ) from tasks.common.pipeline import ( BaseModelOutput, @@ -132,7 +133,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: PointLabel: The map point label extraction object. """ map_point_labels = PointLabels.model_validate( - pipeline_result.data["map_point_labels"] + pipeline_result.data[MAP_PT_LABELS_OUTPUT_KEY] ) return BaseModelOutput( pipeline_result.pipeline_id, @@ -166,7 +167,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: Output: The output of the pipeline. """ map_point_labels = PointLabels.model_validate( - pipeline_result.data["map_point_labels"] + pipeline_result.data[MAP_PT_LABELS_OUTPUT_KEY] ) legend_pt_items = LegendPointItems(items=[]) if LEGEND_ITEMS_OUTPUT_KEY in pipeline_result.data: @@ -207,7 +208,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: Output: The output of the pipeline. """ map_point_labels = PointLabels.model_validate( - pipeline_result.data["map_point_labels"] + pipeline_result.data[MAP_PT_LABELS_OUTPUT_KEY] ) if LEGEND_ITEMS_OUTPUT_KEY in pipeline_result.data: legend_pt_items = LegendPointItems.model_validate( diff --git a/tasks/point_extraction/entities.py b/tasks/point_extraction/entities.py index 43957364..566b0204 100644 --- a/tasks/point_extraction/entities.py +++ b/tasks/point_extraction/entities.py @@ -8,7 +8,14 @@ logger = logging.getLogger(__name__) ## Data Objects +# task keys for LegendItems LEGEND_ITEMS_OUTPUT_KEY = "legend_point_items" +# task keys for ImageTiles results +MAP_TILES_OUTPUT_KEY = "map_tiles" +LEGEND_TILES_OUTPUT_KEY = "legend_tiles" +# task keys for PointLabels results +MAP_PT_LABELS_OUTPUT_KEY = "map_point_labels" +LEGEND_PT_LABELS_OUTPUT_KEY = "legend_point_labels" class PointLabel(BaseModel): diff --git a/tasks/point_extraction/legend_analyzer.py b/tasks/point_extraction/legend_analyzer.py index 2f5fe878..d8bd0d96 100644 --- a/tasks/point_extraction/legend_analyzer.py +++ b/tasks/point_extraction/legend_analyzer.py @@ -6,6 +6,7 @@ LegendPointItems, LegendPointItem, LEGEND_ITEMS_OUTPUT_KEY, + LEGEND_PT_LABELS_OUTPUT_KEY, PointLabels, ) from tasks.point_extraction.legend_item_utils import ( @@ -97,7 +98,7 @@ def run(self, task_input: TaskInput) -> TaskResult: to legend item annotation objects (or join with existing) """ - if not "legend_point_labels" in task_input.data: + if not LEGEND_PT_LABELS_OUTPUT_KEY in task_input.data: logger.info( "Point predictions not available for the legend area. Skipping legend post-processing." ) @@ -105,7 +106,7 @@ def run(self, task_input: TaskInput) -> TaskResult: # --- load legend area ML point predictions legend_pt_preds = PointLabels.model_validate( - task_input.data["legend_point_labels"] + task_input.data[LEGEND_PT_LABELS_OUTPUT_KEY] ) if not legend_pt_preds.labels: return self._create_result(task_input) diff --git a/tasks/point_extraction/point_extractor.py b/tasks/point_extraction/point_extractor.py index b76d83de..dacdedc1 100644 --- a/tasks/point_extraction/point_extractor.py +++ b/tasks/point_extraction/point_extractor.py @@ -2,6 +2,8 @@ ImageTile, ImageTiles, PointLabel, + MAP_TILES_OUTPUT_KEY, + LEGEND_TILES_OUTPUT_KEY, ) from tasks.common.s3_data_cache import S3DataCache @@ -141,7 +143,7 @@ def run(self, task_input: TaskInput) -> TaskResult: """ run YOLO model inference for point symbol detection """ - map_tiles = ImageTiles.model_validate(task_input.data["map_tiles"]) + map_tiles = ImageTiles.model_validate(task_input.data[MAP_TILES_OUTPUT_KEY]) if self.device == "auto": self.device = "cuda" if torch.cuda.is_available() else "cpu" @@ -152,9 +154,11 @@ def run(self, task_input: TaskInput) -> TaskResult: logger.info(f"Running model inference on {len(map_tiles.tiles)} map tiles") self._process_tiles(map_tiles, task_input.raster_id, "map") - if "legend_tiles" in task_input.data: + if LEGEND_TILES_OUTPUT_KEY in task_input.data: # --- also run point extraction model on legend area tiles, if available - legend_tiles = ImageTiles.model_validate(task_input.data["legend_tiles"]) + legend_tiles = ImageTiles.model_validate( + task_input.data[LEGEND_TILES_OUTPUT_KEY] + ) logger.info( f"Also running model inference on {len(legend_tiles.tiles)} legend tiles" ) @@ -163,13 +167,13 @@ def run(self, task_input: TaskInput) -> TaskResult: return TaskResult( task_id=self._task_id, output={ - "map_tiles": map_tiles.model_dump(), - "legend_tiles": legend_tiles.model_dump(), + MAP_TILES_OUTPUT_KEY: map_tiles.model_dump(), + LEGEND_TILES_OUTPUT_KEY: legend_tiles.model_dump(), }, ) return TaskResult( - task_id=self._task_id, output={"map_tiles": map_tiles.model_dump()} + task_id=self._task_id, output={MAP_TILES_OUTPUT_KEY: map_tiles.model_dump()} ) def _process_tiles( diff --git a/tasks/point_extraction/point_orientation_extractor.py b/tasks/point_extraction/point_orientation_extractor.py index e41affdb..78ef8ce6 100644 --- a/tasks/point_extraction/point_orientation_extractor.py +++ b/tasks/point_extraction/point_orientation_extractor.py @@ -1,4 +1,4 @@ -from tasks.point_extraction.entities import PointLabels +from tasks.point_extraction.entities import PointLabels, MAP_PT_LABELS_OUTPUT_KEY from tasks.common.task import Task, TaskInput, TaskResult from tasks.point_extraction.label_map import POINT_CLASS from tasks.point_extraction.task_config import PointOrientationConfig @@ -184,7 +184,9 @@ def run(self, input: TaskInput) -> TaskResult: """ # get result from point extractor task (with point symbol predictions) - map_point_labels = PointLabels.model_validate(input.data["map_point_labels"]) + map_point_labels = PointLabels.model_validate( + input.data[MAP_PT_LABELS_OUTPUT_KEY] + ) if map_point_labels.labels is None: raise RuntimeError("PointLabels must have labels to run batch_predict") if len(map_point_labels.labels) == 0: @@ -193,7 +195,7 @@ def run(self, input: TaskInput) -> TaskResult: ) TaskResult( task_id=self._task_id, - output={"map_point_labels": map_point_labels.model_dump()}, + output={MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump()}, ) # get OCR output @@ -355,7 +357,7 @@ def run(self, input: TaskInput) -> TaskResult: return TaskResult( task_id=self._task_id, - output={"map_point_labels": map_point_labels.model_dump()}, + output={MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump()}, ) def _trig_to_compass_angle(self, angle_deg: int, rotate_max: int) -> int: diff --git a/tasks/point_extraction/template_match_point_extractor.py b/tasks/point_extraction/template_match_point_extractor.py index eb4e639c..e5fdb2b5 100644 --- a/tasks/point_extraction/template_match_point_extractor.py +++ b/tasks/point_extraction/template_match_point_extractor.py @@ -25,6 +25,7 @@ LegendPointItem, LegendPointItems, LEGEND_ITEMS_OUTPUT_KEY, + MAP_PT_LABELS_OUTPUT_KEY, ) MODEL_NAME = "uncharted_oneshot_point_extractor" @@ -81,13 +82,15 @@ def run(self, task_input: TaskInput) -> TaskResult: "No Legend item info available. Skipping Template-Match Point Extractor" ) result = self._create_result(task_input) - result.add_output("map_point_labels", task_input.data["map_point_labels"]) + result.add_output( + MAP_PT_LABELS_OUTPUT_KEY, task_input.data[MAP_PT_LABELS_OUTPUT_KEY] + ) return result # get existing point predictions from YOLO point extractor - if "map_point_labels" in task_input.data: + if MAP_PT_LABELS_OUTPUT_KEY in task_input.data: map_point_labels = PointLabels.model_validate( - task_input.data["map_point_labels"] + task_input.data[MAP_PT_LABELS_OUTPUT_KEY] ) if map_point_labels.labels is None: map_point_labels.labels = [] @@ -107,7 +110,9 @@ def run(self, task_input: TaskInput) -> TaskResult: "No legend items need further processing. Skipping Template-Match Point Extractor" ) result = self._create_result(task_input) - result.add_output("map_point_labels", task_input.data["map_point_labels"]) + result.add_output( + MAP_PT_LABELS_OUTPUT_KEY, task_input.data[MAP_PT_LABELS_OUTPUT_KEY] + ) return result # convert image from PIL to opencv (numpy) format -- assumed color channel order is RGB @@ -328,7 +333,7 @@ def run(self, task_input: TaskInput) -> TaskResult: return TaskResult( task_id=self._task_id, - output={"map_point_labels": map_point_labels.model_dump()}, + output={MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump()}, ) def _get_template_images( diff --git a/tasks/point_extraction/tiling.py b/tasks/point_extraction/tiling.py index b6e5d2d8..123aa10c 100644 --- a/tasks/point_extraction/tiling.py +++ b/tasks/point_extraction/tiling.py @@ -23,6 +23,10 @@ from tasks.point_extraction.entities import ( LegendPointItems, LEGEND_ITEMS_OUTPUT_KEY, + MAP_TILES_OUTPUT_KEY, + LEGEND_TILES_OUTPUT_KEY, + MAP_PT_LABELS_OUTPUT_KEY, + LEGEND_PT_LABELS_OUTPUT_KEY, ) from tasks.point_extraction.legend_item_utils import legend_items_use_ontology @@ -118,14 +122,14 @@ def run( return TaskResult( task_id=self._task_id, output={ - "map_tiles": map_tiles.model_dump(), - "legend_tiles": legend_tiles.model_dump(), + MAP_TILES_OUTPUT_KEY: map_tiles.model_dump(), + LEGEND_TILES_OUTPUT_KEY: legend_tiles.model_dump(), }, ) # prepare task result with only map tiles return TaskResult( - task_id=self._task_id, output={"map_tiles": map_tiles.model_dump()} + task_id=self._task_id, output={MAP_TILES_OUTPUT_KEY: map_tiles.model_dump()} ) def _create_tiles( @@ -221,28 +225,30 @@ def run(self, task_input: TaskInput) -> TaskResult: # run untiling on map tiles logger.info("Untiling map tiles...") - map_tiles = ImageTiles.model_validate(task_input.data["map_tiles"]) + map_tiles = ImageTiles.model_validate(task_input.data[MAP_TILES_OUTPUT_KEY]) map_point_labels = self._merge_tiles(map_tiles, task_input.raster_id) - if "legend_tiles" in task_input.data: + if LEGEND_TILES_OUTPUT_KEY in task_input.data: # --- also run untiling on legend area tiles, if available logger.info("Also untiling legend area tiles...") - legend_tiles = ImageTiles.model_validate(task_input.data["legend_tiles"]) + legend_tiles = ImageTiles.model_validate( + task_input.data[LEGEND_TILES_OUTPUT_KEY] + ) legend_point_labels = self._merge_tiles(legend_tiles, task_input.raster_id) # store untiling results for both map and legend areas return TaskResult( task_id=self._task_id, output={ - "map_point_labels": map_point_labels.model_dump(), - "legend_point_labels": legend_point_labels.model_dump(), + MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump(), + LEGEND_PT_LABELS_OUTPUT_KEY: legend_point_labels.model_dump(), }, ) # store untiling results for map area return TaskResult( task_id=self._task_id, - output={"map_point_labels": map_point_labels.model_dump()}, + output={MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump()}, ) def _merge_tiles(self, image_tiles: ImageTiles, raster_id: str) -> PointLabels: From 35a0c04fe57463fa459d83841fdee76d1b99fe1a Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Tue, 30 Jul 2024 17:27:18 -0400 Subject: [PATCH 16/66] minor fixes for using 'polymer' legend annotations as opposed to 'labeme' legend annotations --- .../point_extraction_pipeline.py | 1 + pipelines/point_extraction/run_pipeline.py | 2 +- tasks/point_extraction/legend_analyzer.py | 20 +++++--- tasks/point_extraction/legend_item_utils.py | 47 ++++++++++++++----- .../template_match_point_extractor.py | 2 +- 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/pipelines/point_extraction/point_extraction_pipeline.py b/pipelines/point_extraction/point_extraction_pipeline.py index 6e4d1bc3..882e791f 100644 --- a/pipelines/point_extraction/point_extraction_pipeline.py +++ b/pipelines/point_extraction/point_extraction_pipeline.py @@ -210,6 +210,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: map_point_labels = PointLabels.model_validate( pipeline_result.data[MAP_PT_LABELS_OUTPUT_KEY] ) + legend_pt_items = LegendPointItems(items=[]) if LEGEND_ITEMS_OUTPUT_KEY in pipeline_result.data: legend_pt_items = LegendPointItems.model_validate( pipeline_result.data[LEGEND_ITEMS_OUTPUT_KEY] diff --git a/pipelines/point_extraction/run_pipeline.py b/pipelines/point_extraction/run_pipeline.py index 22ef0617..6c4022b3 100644 --- a/pipelines/point_extraction/run_pipeline.py +++ b/pipelines/point_extraction/run_pipeline.py @@ -107,7 +107,7 @@ def main(): if p.bitmasks: bitmasks_out_dir = os.path.join(p.output, "bitmasks") os.makedirs(bitmasks_out_dir, exist_ok=True) - if not p.legend_hints_dir or not p.legend_annotations_dir: + if not p.legend_hints_dir and not p.legend_annotations_dir: logger.warning( 'Points pipeline is configured to create CMA contest bitmasks without using legend annotations! Setting "legend_hints_dir" or "legend_annotations_dir" param is recommended.' ) diff --git a/tasks/point_extraction/legend_analyzer.py b/tasks/point_extraction/legend_analyzer.py index d8bd0d96..e6c4d610 100644 --- a/tasks/point_extraction/legend_analyzer.py +++ b/tasks/point_extraction/legend_analyzer.py @@ -44,22 +44,28 @@ def run(self, task_input: TaskInput) -> TaskResult: ) elif LEGEND_ITEMS_OUTPUT_KEY in task_input.request: # legend items for point symbols already exist as a request param - # (ie, loaded from a JSON hints file) + # (ie, loaded from a JSON hints or annotations file) # convert to a TaskResult... legend_pt_items = LegendPointItems.model_validate( task_input.request[LEGEND_ITEMS_OUTPUT_KEY] ) - if ( - legend_pt_items - and legend_pt_items.provenance == LEGEND_ANNOTATION_PROVENANCE.LABELME - ): + if legend_pt_items: + if not legend_pt_items.provenance == LEGEND_ANNOTATION_PROVENANCE.LABELME: + # not "labelme" legend items, just output task result + return TaskResult( + task_id=self._task_id, + output={LEGEND_ITEMS_OUTPUT_KEY: legend_pt_items.model_dump()}, + ) + + # "labelme" legend items... + # use segmentation results to filter noisy "labelme" legend annotations + # (needed because all labelme annotations are set to type "polygon" regardless of feature type: polygons, lines or points) if SEGMENTATION_OUTPUT_KEY in task_input.data: segmentation = MapSegmentation.model_validate( task_input.data[SEGMENTATION_OUTPUT_KEY] ) - # use segmentation results to filter noisy "labelme" legend annotations - # (needed because all labelme annotations are set to type "polygon" regardless of feature type: polygons, lines or points) + filter_labelme_annotations(legend_pt_items, segmentation) logger.info( f"Number of legend point annotations after filtering: {len(legend_pt_items.items)}" diff --git a/tasks/point_extraction/legend_item_utils.py b/tasks/point_extraction/legend_item_utils.py index 53962a20..bd6651ba 100644 --- a/tasks/point_extraction/legend_item_utils.py +++ b/tasks/point_extraction/legend_item_utils.py @@ -1,6 +1,6 @@ import logging from enum import Enum -from typing import List +from typing import List, Tuple from collections import defaultdict from shapely import Polygon, distance @@ -66,18 +66,21 @@ def parse_legend_annotations( system_label = system legend_point_items.extend(legend_ann_to_legend_items(leg_anns, raster_id)) if legend_point_items: + logger.info(f"Parsed {len(legend_point_items)} legend point items") return LegendPointItems(items=legend_point_items, provenance=system_label) else: - # try to parse label annotations 2nd (since labelme anns have noisy data for point/line features) + # try to parse labelme annotations 2nd (since labelme anns have noisy data for point/line features) for system, leg_anns in legend_item_resps.items(): if not system == LEGEND_ANNOTATION_PROVENANCE.LABELME: continue legend_point_items.extend(legend_ann_to_legend_items(leg_anns, raster_id)) if legend_point_items: + logger.info(f"Parsed {len(legend_point_items)} labelme legend point items") return LegendPointItems( items=legend_point_items, provenance=LEGEND_ANNOTATION_PROVENANCE.LABELME, ) + logger.info(f"Parsed 0 legend point items") return LegendPointItems(items=[], provenance="") @@ -170,6 +173,23 @@ def find_legend_keyword_match(legend_item_name: str, raster_id: str) -> str: return "" +def get_swatch_contour(bbox: List, xy_pts: List[List]) -> Tuple: + if bbox and xy_pts: + return (bbox, xy_pts) + if xy_pts: + # calc bbox from contour + p = Polygon(xy_pts) + bbox = list(p.bounds) + if bbox: + xy_pts = [ + [bbox[0], bbox[1]], + [bbox[2], bbox[1]], + [bbox[2], bbox[3]], + [bbox[0], bbox[3]], + ] + return (bbox, xy_pts) + + def legend_ann_to_legend_items( legend_anns: List[LegendItemResponse], raster_id: str ) -> List[LegendPointItems]: @@ -190,17 +210,18 @@ def legend_ann_to_legend_items( # skip the 2nd labelme annotation in each pair # (this 2nd entry is just the bbox for the legend item text; TODO -- could extract and include this text too?) continue + if ( + not leg_ann.system == LEGEND_ANNOTATION_PROVENANCE.LABELME + and not leg_ann.category == "point" + ): + # this legend annotation does not represent a point feature; skip + # (Note: LABELME annotations are handled separately, because it labels ALL feature types as "polygons") + continue + class_name = find_legend_keyword_match(label, raster_id) - xy_pts = ( - leg_ann.px_geojson.coordinates[0] - if leg_ann.px_geojson - else [ - [leg_ann.px_bbox[0], leg_ann.px_bbox[1]], - [leg_ann.px_bbox[2], leg_ann.px_bbox[1]], - [leg_ann.px_bbox[2], leg_ann.px_bbox[3]], - [leg_ann.px_bbox[0], leg_ann.px_bbox[3]], - ] - ) + + xy_pts = leg_ann.px_geojson.coordinates[0] if leg_ann.px_geojson else [] + (bbox, xy_pts) = get_swatch_contour(leg_ann.px_bbox, xy_pts) legend_point_items.append( LegendPointItem( @@ -208,7 +229,7 @@ def legend_ann_to_legend_items( class_name=class_name, abbreviation=leg_ann.abbreviation, description=leg_ann.description, - legend_bbox=leg_ann.px_bbox, + legend_bbox=bbox, legend_contour=xy_pts, system=leg_ann.system, system_version=leg_ann.system_version, diff --git a/tasks/point_extraction/template_match_point_extractor.py b/tasks/point_extraction/template_match_point_extractor.py index e5fdb2b5..5bd23171 100644 --- a/tasks/point_extraction/template_match_point_extractor.py +++ b/tasks/point_extraction/template_match_point_extractor.py @@ -78,7 +78,7 @@ def run(self, task_input: TaskInput) -> TaskResult: legend_pt_items = LegendPointItems(items=[]) if not legend_pt_items or not legend_pt_items.items: - logger.warning( + logger.info( "No Legend item info available. Skipping Template-Match Point Extractor" ) result = self._create_result(task_input) From e0f8474cef0803da1212f5aa63f527f9784c4c0f Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 31 Jul 2024 09:44:36 -0400 Subject: [PATCH 17/66] logging statement --- tasks/segmentation/segmenter_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/segmentation/segmenter_utils.py b/tasks/segmentation/segmenter_utils.py index 3ac00667..4604f274 100644 --- a/tasks/segmentation/segmenter_utils.py +++ b/tasks/segmentation/segmenter_utils.py @@ -49,7 +49,7 @@ def get_segment_bounds( filter(lambda s: (s.class_label == segment_class), segmentation.segments) ) if not segments: - logger.warning(f"No {segment_class} segment found") + logger.info(f"No {segment_class} segment found") return [] if max_results > 0: segments = segments[:max_results] From 8b90b3a5802702e6d0a9f89553962154ec1f54ec Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 31 Jul 2024 12:29:19 -0400 Subject: [PATCH 18/66] minor fix for handling overlapping region segments (prior to tiling) --- tasks/point_extraction/tiling.py | 10 ++++++++-- tasks/segmentation/segmenter_utils.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tasks/point_extraction/tiling.py b/tasks/point_extraction/tiling.py index 123aa10c..a64b2d81 100644 --- a/tasks/point_extraction/tiling.py +++ b/tasks/point_extraction/tiling.py @@ -19,7 +19,11 @@ SEGMENT_MAP_CLASS, SEGMENT_POINT_LEGEND_CLASS, ) -from tasks.segmentation.segmenter_utils import get_segment_bounds, segments_to_mask +from tasks.segmentation.segmenter_utils import ( + get_segment_bounds, + segments_to_mask, + merge_overlapping_polygons, +) from tasks.point_extraction.entities import ( LegendPointItems, LEGEND_ITEMS_OUTPUT_KEY, @@ -87,7 +91,9 @@ def run( ) poly_map = get_segment_bounds(segmentation, SEGMENT_MAP_CLASS) - poly_legend = get_segment_bounds(segmentation, SEGMENT_POINT_LEGEND_CLASS) + poly_legend = get_segment_bounds( + segmentation, SEGMENT_POINT_LEGEND_CLASS, merge_overlapping=True + ) if len(poly_map) > 0: # restrict tiling to use *only* the bounding rectangle of map area poly_map = poly_map[0] # use 1st (highest ranked) map segment diff --git a/tasks/segmentation/segmenter_utils.py b/tasks/segmentation/segmenter_utils.py index 4604f274..3ad50f40 100644 --- a/tasks/segmentation/segmenter_utils.py +++ b/tasks/segmentation/segmenter_utils.py @@ -33,7 +33,10 @@ def rank_segments(segmentation: MapSegmentation, class_labels: List): def get_segment_bounds( - segmentation: MapSegmentation, segment_class: str, max_results: int = 0 + segmentation: MapSegmentation, + segment_class: str, + max_results: int = 0, + merge_overlapping: bool = False, ) -> List[Polygon]: """ Parse segmentation result and return the polygon bounds for the desired segmentation class, if present @@ -53,7 +56,10 @@ def get_segment_bounds( return [] if max_results > 0: segments = segments[:max_results] - return [Polygon(s.poly_bounds) for s in segments] + polys = [Polygon(s.poly_bounds) for s in segments] + if merge_overlapping: + polys = merge_overlapping_polygons(polys) + return polys def merge_overlapping_polygons(polys: List[Polygon]) -> List[Polygon]: @@ -61,7 +67,7 @@ def merge_overlapping_polygons(polys: List[Polygon]) -> List[Polygon]: Merge overlapping shapely polygons into single polygon objects If polygons are not overlapping, the original polygon list is returned """ - if not polys: + if not polys or len(polys) < 2: return polys merged_polys = unary_union(polys) From a6a40d38069e86b8fbacb7dcefa0075b0681432c Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 31 Jul 2024 12:42:01 -0400 Subject: [PATCH 19/66] reverted "model_*" internal pydantic fields back to "classifier_*" to fix pydantic lib warnings --- tasks/point_extraction/entities.py | 4 ++-- tasks/point_extraction/point_extractor.py | 4 ++-- tasks/point_extraction/template_match_point_extractor.py | 4 ++-- tasks/point_extraction/tiling.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tasks/point_extraction/entities.py b/tasks/point_extraction/entities.py index 566b0204..69342291 100644 --- a/tasks/point_extraction/entities.py +++ b/tasks/point_extraction/entities.py @@ -24,8 +24,8 @@ class PointLabel(BaseModel): Can be used for either ML Object Detection or One-shot template matching results """ - model_name: str - model_version: str + classifier_name: str + classifier_version: str class_id: int class_name: str x1: int diff --git a/tasks/point_extraction/point_extractor.py b/tasks/point_extraction/point_extractor.py index dacdedc1..ee738021 100644 --- a/tasks/point_extraction/point_extractor.py +++ b/tasks/point_extraction/point_extractor.py @@ -126,8 +126,8 @@ def process_output(self, predictions: Results) -> List[PointLabel]: pt_labels.append( PointLabel( - model_name=MODEL_NAME, - model_version=self._model_id, + classifier_name=MODEL_NAME, + classifier_version=self._model_id, class_id=int(class_id), class_name=class_name, x1=int(x1), diff --git a/tasks/point_extraction/template_match_point_extractor.py b/tasks/point_extraction/template_match_point_extractor.py index 5bd23171..dbfd6f23 100644 --- a/tasks/point_extraction/template_match_point_extractor.py +++ b/tasks/point_extraction/template_match_point_extractor.py @@ -429,8 +429,8 @@ def _process_output( # note: using hash(label) as class numeric ID pt_labels.append( PointLabel( - model_name=MODEL_NAME, - model_version=MODEL_VER, + classifier_name=MODEL_NAME, + classifier_version=MODEL_VER, class_id=hash(label), class_name=label, x1=x1, diff --git a/tasks/point_extraction/tiling.py b/tasks/point_extraction/tiling.py index a64b2d81..75064f9f 100644 --- a/tasks/point_extraction/tiling.py +++ b/tasks/point_extraction/tiling.py @@ -301,8 +301,8 @@ def _merge_tiles(self, image_tiles: ImageTiles, raster_id: str) -> PointLabels: continue global_prediction = PointLabel( - model_name=pred.model_name, - model_version=pred.model_version, + classifier_name=pred.classifier_name, + classifier_version=pred.classifier_version, class_id=pred.class_id, class_name=label_name, # Add offset of tile to project onto original map... From 34eb68c62a8beb51d27b3afef62a48aeb98e3f1c Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 31 Jul 2024 16:20:57 -0400 Subject: [PATCH 20/66] added caching for point orientations task --- .../point_extraction_pipeline.py | 20 ++++-- tasks/point_extraction/entities.py | 8 ++- .../point_orientation_extractor.py | 67 +++++++++++++++---- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/pipelines/point_extraction/point_extraction_pipeline.py b/pipelines/point_extraction/point_extraction_pipeline.py index 882e791f..960d3002 100644 --- a/pipelines/point_extraction/point_extraction_pipeline.py +++ b/pipelines/point_extraction/point_extraction_pipeline.py @@ -64,6 +64,13 @@ def __init__( # extract text from image, segmentation to only keep the map area, # tile, extract points, untile, predict direction logger.info("Initializing Point Extraction Pipeline") + yolo_point_extractor = YOLOPointDetector( + "point_detection", + model_path, + str(Path(work_dir).joinpath("points")), + batch_size=20, + ) + tasks = [] tasks.append( TileTextExtractor( @@ -90,14 +97,13 @@ def __init__( [ LegendPreprocessor("legend_preprocessor", ""), Tiler("tiling"), - YOLOPointDetector( - "point_detection", - model_path, - str(Path(work_dir).joinpath("points")), - batch_size=20, - ), + yolo_point_extractor, Untiler("untiling"), - PointOrientationExtractor("point_orientation_extraction"), + PointOrientationExtractor( + "point_orientation_extraction", + yolo_point_extractor._model_id, + str(Path(work_dir).joinpath("point_orientations")), + ), LegendPostprocessor("legend_postprocessor", ""), TemplateMatchPointExtractor( "template_match_point_extraction", diff --git a/tasks/point_extraction/entities.py b/tasks/point_extraction/entities.py index 69342291..cf1e3bae 100644 --- a/tasks/point_extraction/entities.py +++ b/tasks/point_extraction/entities.py @@ -134,6 +134,12 @@ def join_with_cached_predictions(self, cached_preds: ImageTiles) -> bool: Append cached point predictions to ImageTiles """ try: + if len(self.tiles) != len(cached_preds.tiles): + logger.warning( + f"Number of tiles {len(self.tiles)} doesn't match cached tiles {len(cached_preds.tiles)}; disregarding cached results" + ) + return False + # re-format cached predictions with key as (x_offset, y_offset) cached_dict: Dict[Any, ImageTile] = {} for p in cached_preds.tiles: @@ -148,7 +154,7 @@ def join_with_cached_predictions(self, cached_preds: ImageTiles) -> bool: return True except Exception as e: - print(f"Exception in join_with_cached_predictions: {str(e)}") + logger.warning(f"Exception in join_with_cached_predictions: {str(e)}") return False diff --git a/tasks/point_extraction/point_orientation_extractor.py b/tasks/point_extraction/point_orientation_extractor.py index 78ef8ce6..bbfdd036 100644 --- a/tasks/point_extraction/point_orientation_extractor.py +++ b/tasks/point_extraction/point_orientation_extractor.py @@ -9,7 +9,7 @@ TEXT_EXTRACTION_OUTPUT_KEY, ) -from typing import Dict, List +from typing import Dict, List, Optional import cv2 import logging import math @@ -26,10 +26,10 @@ logger = logging.getLogger(__name__) RE_NONNUMERIC = re.compile(r"[^0-9]") # matches non-numeric chars +CODE_VER = "0.0.1" class PointOrientationExtractor(Task): - _VERSION = 1 # ---- supported point classes and corresponding template image paths POINT_TEMPLATES = { @@ -101,10 +101,11 @@ class PointOrientationExtractor(Task): ), } - def __init__(self, task_id: str): + def __init__(self, task_id: str, points_model_id: str, cache_path: str): + self.points_model_id = points_model_id self.templates = self._load_templates() - super().__init__(task_id) + super().__init__(task_id, cache_path) def _load_templates(self) -> Dict: """ @@ -176,7 +177,7 @@ def _dip_magnitude_extraction( ) return dip_magnitudes - def run(self, input: TaskInput) -> TaskResult: + def run(self, task_input: TaskInput) -> TaskResult: """ Run batch predictions over a PointLabels object. @@ -185,7 +186,7 @@ def run(self, input: TaskInput) -> TaskResult: # get result from point extractor task (with point symbol predictions) map_point_labels = PointLabels.model_validate( - input.data[MAP_PT_LABELS_OUTPUT_KEY] + task_input.data[MAP_PT_LABELS_OUTPUT_KEY] ) if map_point_labels.labels is None: raise RuntimeError("PointLabels must have labels to run batch_predict") @@ -193,16 +194,34 @@ def run(self, input: TaskInput) -> TaskResult: logger.warning( "No point symbol extractions found. Skipping Point orientation extraction." ) - TaskResult( + return TaskResult( task_id=self._task_id, output={MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump()}, ) + # --- check cache and re-use existing result if present + doc_key = ( + f"{task_input.raster_id}_orientations-{self.points_model_id}-{CODE_VER}" + ) + cached_point_labels = self._get_cached_data( + doc_key, len(map_point_labels.labels) + ) + if cached_point_labels: + logger.info( + f"Using cached orientation results for raster: {task_input.raster_id}" + ) + return TaskResult( + task_id=self._task_id, + output={MAP_PT_LABELS_OUTPUT_KEY: cached_point_labels.model_dump()}, + ) + # get OCR output img_text = ( - DocTextExtraction.model_validate(input.data[TEXT_EXTRACTION_OUTPUT_KEY]) - if TEXT_EXTRACTION_OUTPUT_KEY in input.data - else DocTextExtraction(doc_id=input.raster_id, extractions=[]) + DocTextExtraction.model_validate( + task_input.data[TEXT_EXTRACTION_OUTPUT_KEY] + ) + if TEXT_EXTRACTION_OUTPUT_KEY in task_input.data + else DocTextExtraction(doc_id=task_input.raster_id, extractions=[]) ) if len(img_text.extractions) == 0: @@ -249,7 +268,7 @@ def run(self, input: TaskInput) -> TaskResult: # --- 2. estimate symbol orientation (using template matching) # --- pre-process the main image and template image, before template matching im, im_templ = point_extractor_utils.image_pre_processing( - np.array(input.image), + np.array(task_input.image), self.templates[c], np.array([]), ) @@ -355,9 +374,13 @@ def run(self, input: TaskInput) -> TaskResult: ) logger.info(f"Finished point orientation analysis for class {c}") + json_data = map_point_labels.model_dump() + # write to cache + self.write_result_to_cache(json_data, doc_key) + return TaskResult( task_id=self._task_id, - output={MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump()}, + output={MAP_PT_LABELS_OUTPUT_KEY: json_data}, ) def _trig_to_compass_angle(self, angle_deg: int, rotate_max: int) -> int: @@ -373,6 +396,26 @@ def _trig_to_compass_angle(self, angle_deg: int, rotate_max: int) -> int: angle_compass -= rotate_max return angle_compass + def _get_cached_data( + self, doc_key: str, num_predictions: int + ) -> Optional[PointLabels]: + + try: + json_data = self.fetch_cached_result(doc_key) + if json_data: + map_point_labels = PointLabels(**json_data) + if ( + map_point_labels.labels + and len(map_point_labels.labels) == num_predictions + ): + # cached data is ok + return map_point_labels + + except Exception as e: + # legend_pt_items = LegendPointItems(items=[], provenance="") + logger.warning(f"Exception checking cached data: {repr(e)}") + return None + @property def input_type(self): return PointLabels From b2a7fce1cb93667ad1fc7c5529b1ad4fa5be3e9b Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Thu, 1 Aug 2024 11:35:37 -0400 Subject: [PATCH 21/66] added caching for the template-match point extractor task --- .../point_orientation_extractor.py | 3 +- .../template_match_point_extractor.py | 51 ++++++++++++++++++- util/viz/viz_point_extractions.py | 1 + 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/tasks/point_extraction/point_orientation_extractor.py b/tasks/point_extraction/point_orientation_extractor.py index bbfdd036..d900c22b 100644 --- a/tasks/point_extraction/point_orientation_extractor.py +++ b/tasks/point_extraction/point_orientation_extractor.py @@ -412,8 +412,7 @@ def _get_cached_data( return map_point_labels except Exception as e: - # legend_pt_items = LegendPointItems(items=[], provenance="") - logger.warning(f"Exception checking cached data: {repr(e)}") + logger.warning(f"Exception fetching cached data: {repr(e)}") return None @property diff --git a/tasks/point_extraction/template_match_point_extractor.py b/tasks/point_extraction/template_match_point_extractor.py index dbfd6f23..b9d8a3f9 100644 --- a/tasks/point_extraction/template_match_point_extractor.py +++ b/tasks/point_extraction/template_match_point_extractor.py @@ -2,7 +2,7 @@ import logging import math import numpy as np -from typing import List, Tuple +from typing import List, Tuple, Optional from scipy import ndimage from collections import defaultdict @@ -115,6 +115,23 @@ def run(self, task_input: TaskInput) -> TaskResult: ) return result + # --- check cache and re-use existing result if present + doc_key = f"{task_input.raster_id}_templatematch_points-{MODEL_VER}" + templatematch_point_labels = self._get_cached_data(doc_key, pt_features) + if templatematch_point_labels: + logger.info( + f"Using cached template-match point extraction results for raster: {task_input.raster_id}" + ) + map_point_labels.labels.extend(templatematch_point_labels.labels) # type: ignore + return TaskResult( + task_id=self._task_id, + output={MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump()}, + ) + + templatematch_point_labels = PointLabels( + path="", raster_id=task_input.raster_id, labels=[] + ) + # convert image from PIL to opencv (numpy) format -- assumed color channel order is RGB im_in = np.array(task_input.image) @@ -329,8 +346,13 @@ def run(self, task_input: TaskInput) -> TaskResult: ) if len(matches_dedup) > 0: preds = self._process_output(matches_dedup, feature_name, map_roi) - map_point_labels.labels.extend(preds) # type: ignore + templatematch_point_labels.labels.extend(preds) # type: ignore + # write to cache (note: only cache the template matched point extractions; not all of them) + self.write_result_to_cache(templatematch_point_labels.model_dump(), doc_key) + + # append template-match extractions to main point extractions results + map_point_labels.labels.extend(templatematch_point_labels.labels) # type: ignore return TaskResult( task_id=self._task_id, output={MAP_PT_LABELS_OUTPUT_KEY: map_point_labels.model_dump()}, @@ -488,3 +510,28 @@ def _which_points_need_processing( legend_items_unprocessed.append(legend_item) return legend_items_unprocessed + + def _get_cached_data( + self, doc_key: str, legend_items: List[LegendPointItem] + ) -> Optional[PointLabels]: + + try: + json_data = self.fetch_cached_result(doc_key) + if json_data: + templatematch_point_labels = PointLabels(**json_data) + # check that the expected set of point types are in cache + pt_types_legend = set() + for leg in legend_items: + name = leg.class_name if leg.class_name else leg.name + pt_types_legend.add(name) + pt_types_cache = set() + if templatematch_point_labels.labels: + for pt_label in templatematch_point_labels.labels: + pt_types_cache.add(pt_label.class_name) + if len(pt_types_legend - pt_types_cache) == 0: + # pt type sets are the same; cached data is ok + return templatematch_point_labels + + except Exception as e: + logger.warning(f"Exception fetching cached data: {repr(e)}") + return None diff --git a/util/viz/viz_point_extractions.py b/util/viz/viz_point_extractions.py index b18789b1..1e275ca7 100644 --- a/util/viz/viz_point_extractions.py +++ b/util/viz/viz_point_extractions.py @@ -43,6 +43,7 @@ "mine_quarry": "limegreen", "sink_hole": "goldenrod", "lineation": "violet", + "drill_hole": "cyan", } From 9be615e017d514e87edf63afa3f7a363684bfcdd Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Thu, 1 Aug 2024 12:27:22 -0400 Subject: [PATCH 22/66] using try/except around loading points' cache data -- if loading cached point extraction fails, then the cache is disregarded for that map, and point extraction proceeds as usual --- tasks/point_extraction/point_extractor.py | 22 +++++++++++++++---- .../point_orientation_extractor.py | 4 +++- .../template_match_point_extractor.py | 4 +++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tasks/point_extraction/point_extractor.py b/tasks/point_extraction/point_extractor.py index ee738021..1cb21ab2 100644 --- a/tasks/point_extraction/point_extractor.py +++ b/tasks/point_extraction/point_extractor.py @@ -13,7 +13,7 @@ import os from urllib.parse import urlparse from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Optional import torch from tqdm import tqdm @@ -191,9 +191,9 @@ def _process_tiles( else f"{raster_id}_points_{tile_type}-{self._model_id}" ) # check cache and re-use existing file if present - json_data = self.fetch_cached_result(doc_key) - if json_data and image_tiles.join_with_cached_predictions( - ImageTiles(**json_data) + cached_image_tiles = self._get_cached_data(doc_key) + if cached_image_tiles and image_tiles.join_with_cached_predictions( + cached_image_tiles ): # cached point predictions loaded successfully logger.info( @@ -236,6 +236,20 @@ def _get_model_id(self, model: YOLO) -> str: hash_result = hashlib.md5(bytes(state_dict_str, encoding="utf-8")) return hash_result.hexdigest() + def _get_cached_data(self, doc_key: str) -> Optional[ImageTiles]: + try: + json_data = self.fetch_cached_result(doc_key) + if json_data: + cached_image_tiles = ImageTiles(**json_data) + # cached data is ok + return cached_image_tiles + + except Exception as e: + logger.warning( + f"Exception fetching cached data: {repr(e)}; disregarding cached point extractions for this raster" + ) + return None + @property def version(self): return self._model_id diff --git a/tasks/point_extraction/point_orientation_extractor.py b/tasks/point_extraction/point_orientation_extractor.py index d900c22b..71a557eb 100644 --- a/tasks/point_extraction/point_orientation_extractor.py +++ b/tasks/point_extraction/point_orientation_extractor.py @@ -412,7 +412,9 @@ def _get_cached_data( return map_point_labels except Exception as e: - logger.warning(f"Exception fetching cached data: {repr(e)}") + logger.warning( + f"Exception fetching cached data: {repr(e)}; disregarding cached point orientations for this raster" + ) return None @property diff --git a/tasks/point_extraction/template_match_point_extractor.py b/tasks/point_extraction/template_match_point_extractor.py index b9d8a3f9..d722a8ea 100644 --- a/tasks/point_extraction/template_match_point_extractor.py +++ b/tasks/point_extraction/template_match_point_extractor.py @@ -533,5 +533,7 @@ def _get_cached_data( return templatematch_point_labels except Exception as e: - logger.warning(f"Exception fetching cached data: {repr(e)}") + logger.warning( + f"Exception fetching cached data: {repr(e)}; disregarding cached template-match extractions for this raster" + ) return None From f7b457a7f4100b89d730e1fafb501ecb771a9286 Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Thu, 1 Aug 2024 15:49:15 -0400 Subject: [PATCH 23/66] ROI filtering uses scoring approach with limit if too many points get removed. --- tasks/geo_referencing/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/geo_referencing/filter.py b/tasks/geo_referencing/filter.py index 774aca81..1342c63a 100644 --- a/tasks/geo_referencing/filter.py +++ b/tasks/geo_referencing/filter.py @@ -450,7 +450,7 @@ def _adjust_filter( coords_kept = [] for c in coords_sorted: degree = c[1].get_parsed_degree() - if len(degrees) < 2 and degree not in degrees: + if c[0] >= 0.6 and len(degrees) < 2 and degree not in degrees: degrees.add(degree) coords_kept.append(c[1]) return coords_kept From dc44063f81a4613de7735f36007d43c84ddf009e Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Fri, 2 Aug 2024 08:26:19 -0400 Subject: [PATCH 24/66] ROI filtering excludes coordinates within the inner roi polygon. --- tasks/geo_referencing/filter.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tasks/geo_referencing/filter.py b/tasks/geo_referencing/filter.py index 1342c63a..1b836ed0 100644 --- a/tasks/geo_referencing/filter.py +++ b/tasks/geo_referencing/filter.py @@ -407,14 +407,14 @@ def _filter( logger.info( f"only {lon_counts} lon coords after roi filtering so re-adding coordinates" ) - lons_kept = self._adjust_filter(lon_coords, roi_xy) + lons_kept = self._adjust_filter(lon_coords, roi_xy, roi_inner_xy) for lk in lons_kept: lon_pts[lk.to_deg_result()[0]] = lk if lat_counts < 2 and lat_counts < lat_counts_initial: logger.info( f"only {lat_counts} lat coords after roi filtering so re-adding coordinates" ) - lats_kept = self._adjust_filter(lat_coords, roi_xy) + lats_kept = self._adjust_filter(lat_coords, roi_xy, roi_inner_xy) for lk in lats_kept: lat_pts[lk.to_deg_result()[0]] = lk lon_pts, lat_pts = self._validate_lonlat_extractions( @@ -436,11 +436,14 @@ def _adjust_filter( self, coords: Dict[Tuple[float, float], Coordinate], roi_xy: List[Tuple[float, float]], + roi_inner_xy: List[Tuple[float, float]], ) -> List[Coordinate]: # get distance to roi for all coordinates coordinates = [x[1] for x in coords.items()] roi_poly = Polygon(roi_xy) - score_coordinates = [(self._get_roi_score(c, roi_poly), c) for c in coordinates] + score_coordinates = [ + (self._get_roi_score(c, roi_poly, roi_inner_xy), c) for c in coordinates + ] # rank all coordinates by distance to roi coords_sorted = sorted(score_coordinates, key=lambda x: x[0], reverse=True) @@ -615,7 +618,16 @@ def _filter_roi( return (lon_results, lat_results) - def _get_roi_score(self, coordinate: Coordinate, roi_poly: Polygon) -> float: + def _get_roi_score( + self, + coordinate: Coordinate, + roi_poly: Polygon, + roi_inner_xy: List[Tuple[float, float]], + ) -> float: + # if inside inner polygon, then the score is 0 + if self._in_polygon(coordinate.get_pixel_alignment(), roi_inner_xy): + return 0 + # combine distance and coordinate confidence for an initial scoring mechanism coord_dist = distance(Point(coordinate.get_pixel_alignment()), roi_poly) coord_conf = coordinate.get_confidence() From e6333eefaf3c9c3c29908308c6f66445b5891ffc Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Fri, 2 Aug 2024 09:51:25 -0400 Subject: [PATCH 25/66] Added roi filter to server pipeline. --- pipelines/geo_referencing/factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index ac804258..57e74fa6 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -626,6 +626,7 @@ def create_geo_referencing_pipeline( ) tasks.append(GeoFencer("geofence")) tasks.append(GeoCoordinatesExtractor("third")) + tasks.append(ROIFilter("roiness")) tasks.append(OutlierFilter("fourth")) tasks.append(NaiveFilter("fun")) tasks.append( From 41a90a0eb5b8ad9a41002f94e9526e0caef8d63c Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Mon, 29 Jul 2024 23:46:21 -0400 Subject: [PATCH 26/66] registers each pipeline as a CDR system --- cdr/result_subscriber.py | 29 +++++++++++---- cdr/server.py | 79 ++++++++++++++++++++++------------------ 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/cdr/result_subscriber.py b/cdr/result_subscriber.py index 02827f8b..f08707a9 100644 --- a/cdr/result_subscriber.py +++ b/cdr/result_subscriber.py @@ -16,6 +16,7 @@ from pika.exceptions import AMQPChannelError, AMQPConnectionError import pika.spec as spec from pydantic import BaseModel +from regex import P from cdr.json_log import JSONLog from cdr.request_publisher import LaraRequestPublisher from schema.cdr_schemas.feature_results import FeatureResults @@ -79,6 +80,14 @@ class LaraResultSubscriber: GEOREFERENCE_PIPELINE, ] + # map of pipeline name to system name + PIPELINE_SYSTEM_NAMES = { + SEGMENTATION_PIPELINE: "uncharted-area", + METADATA_PIPELINE: "uncharted-metadata", + POINTS_PIPELINE: "uncharted-points", + GEOREFERENCE_PIPELINE: "uncharted-georeference", + } + def __init__( self, request_publisher: Optional[LaraRequestPublisher], @@ -87,7 +96,6 @@ def __init__( cdr_token: str, output: str, workdir: str, - system_name: str, system_version: str, json_log: JSONLog, host="localhost", @@ -101,7 +109,6 @@ def __init__( self._cdr_token = cdr_token self._workdir = workdir self._output = output - self._system_name = system_name self._system_version = system_version self._json_log = json_log self._host = host @@ -330,7 +337,9 @@ def _push_georeferencing(self, result: RequestResult): try: lara_result = LARAGeoreferenceResult.model_validate(georef_result_raw) mapper = get_mapper( - lara_result, f"{self._system_name}-georeference", self._system_version + lara_result, + self.PIPELINE_SYSTEM_NAMES[self.GEOREFERENCE_PIPELINE], + self._system_version, ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore assert cdr_result is not None @@ -363,7 +372,7 @@ def _push_georeferencing(self, result: RequestResult): cog_id=result.request.image_id, georeference_results=[], gcps=[], - system=f"{self._system_name}-georeference", + system=self.PIPELINE_SYSTEM_NAMES[self.GEOREFERENCE_PIPELINE], system_version=self._system_version, ) @@ -428,7 +437,9 @@ def _push_segmentation(self, result: RequestResult): try: lara_result = LARASegmentation.model_validate(segmentation_raw_result) mapper = get_mapper( - lara_result, f"{self._system_name}-area", self._system_version + lara_result, + self.PIPELINE_SYSTEM_NAMES[self.SEGMENTATION_PIPELINE], + self._system_version, ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except: @@ -448,7 +459,9 @@ def _push_points(self, result: RequestResult): try: lara_result = LARAPoints.model_validate(points_raw_result) mapper = get_mapper( - lara_result, f"{self._system_name}-points", self._system_version + lara_result, + self.PIPELINE_SYSTEM_NAMES[self.POINTS_PIPELINE], + self._system_version, ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except: @@ -469,7 +482,9 @@ def _push_metadata(self, result: RequestResult): try: lara_result = LARAMetadata.model_validate(metadata_result_raw) mapper = get_mapper( - lara_result, f"{self._system_name}-metadata", self._system_version + lara_result, + self.PIPELINE_SYSTEM_NAMES[self.METADATA_PIPELINE], + self._system_version, ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except Exception as e: diff --git a/cdr/server.py b/cdr/server.py index 14e247f1..d325f809 100644 --- a/cdr/server.py +++ b/cdr/server.py @@ -24,7 +24,7 @@ from schema.mappers.cdr import get_mapper from schema.cdr_schemas.events import MapEventPayload -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from util.logging import config_logger @@ -34,7 +34,6 @@ CDR_API_TOKEN = os.environ["CDR_API_TOKEN"] CDR_HOST = "https://api.cdr.land" -CDR_SYSTEM_NAME = "uncharted" CDR_SYSTEM_VERSION = "0.0.4" CDR_CALLBACK_SECRET = "maps rock" APP_PORT = 5001 @@ -54,15 +53,14 @@ class Settings: workdir: str imagedir: str output: str - system_name: str system_version: str callback_secret: str callback_url: str - registration_id: str + registration_id: Dict[str, str] = {} rabbitmq_host: str json_log: JSONLog serial: bool - sequence: List[str] + sequence: List[str] = [] def prefetch_image(working_dir: Path, image_id: str, image_url: str) -> None: @@ -212,31 +210,41 @@ def process_image(image_id: str, request_publisher: LaraRequestPublisher): def register_cdr_system(): - logger.info(f"registering system {settings.system_name} with cdr") - headers = {"Authorization": f"Bearer {settings.cdr_api_token}"} - - registration = { - "name": settings.system_name, - "version": settings.system_version, - "callback_url": settings.callback_url, - "webhook_secret": settings.callback_secret, - # Leave blank if callback url has no auth requirement - # "auth_header": "", - # "auth_token": "", - # Registers for ALL events - "events": [], - } - - client = httpx.Client(follow_redirects=True) - r = client.post( - f"{settings.cdr_host}/user/me/register", json=registration, headers=headers - ) + for i, pipeline in enumerate(settings.sequence): + system_name = LaraResultSubscriber.PIPELINE_SYSTEM_NAMES[pipeline] + logger.info(f"registering system {system_name} with cdr") + headers = {"Authorization": f"Bearer {settings.cdr_api_token}"} + + # register for all events on the first pipeline others can ignore + events: Optional[List[str]] = [] if i == 0 else ["ping"] + + registration = { + "name": system_name, + "version": settings.system_version, + "callback_url": settings.callback_url, + "webhook_secret": settings.callback_secret, + # Leave blank if callback url has no auth requirement + # "auth_header": "", + # "auth_token": "", + "events": events, + } + + client = httpx.Client(follow_redirects=True) + + r = client.post( + f"{settings.cdr_host}/user/me/register", json=registration, headers=headers + ) + # check if the request was successful + if r.status_code != 200: + logger.error(f"failed to register system {system_name} with cdr") + logger.error(f"response: {r.text}") + exit(1) - # Log our registration_id such we can delete it when we close the program. - response_raw = r.json() - settings.registration_id = response_raw["id"] - logger.info(f"system {settings.system_name} registered with cdr") + # Log our registration_id such we can delete it when we close the program. + response_raw = r.json() + settings.registration_id[pipeline] = response_raw["id"] + logger.info(f"system {system_name} registered with cdr") def get_cdr_registrations() -> List[Dict[str, Any]]: @@ -266,8 +274,9 @@ def cdr_unregister(registration_id: str): def cdr_clean_up(): logger.info(f"unregistering system {settings.registration_id} with cdr") # delete our registered system at CDR on program end - cdr_unregister(settings.registration_id) - logger.info(f"system {settings.registration_id} no longer registered with cdr") + for pipeline in settings.sequence: + cdr_unregister(settings.registration_id[pipeline]) + logger.info(f"system {settings.registration_id} no longer registered with cdr") def cdr_startup(host: str): @@ -275,8 +284,11 @@ def cdr_startup(host: str): registrations = get_cdr_registrations() if len(registrations) > 0: for r in registrations: - if r["name"] == settings.system_name: - cdr_unregister(r["id"]) + for pipeline in settings.sequence: + if r["name"] == LaraResultSubscriber.PIPELINE_SYSTEM_NAMES[pipeline]: + logger.info(f"unregistering system {r['name']} with cdr") + cdr_unregister(r["id"]) + break # make it accessible from the outside settings.callback_url = f"{host}/process_event" @@ -302,7 +314,6 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--mode", choices=("process", "host"), required=True) - parser.add_argument("--system", type=str, default=CDR_SYSTEM_NAME) parser.add_argument("--workdir", type=str, required=True) parser.add_argument("--imagedir", type=str, required=True) parser.add_argument("--cog_id", type=str, required=False) @@ -322,7 +333,6 @@ def main(): settings.workdir = p.workdir settings.imagedir = p.imagedir settings.output = p.output - settings.system_name = p.system settings.system_version = CDR_SYSTEM_VERSION settings.callback_secret = CDR_CALLBACK_SECRET settings.serial = True @@ -362,7 +372,6 @@ def main(): settings.cdr_api_token, settings.output, settings.workdir, - settings.system_name, settings.system_version, settings.json_log, host=p.host, From 7d8987c7538f65db0cf262cec20041f54c279f7e Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Fri, 2 Aug 2024 13:17:46 -0400 Subject: [PATCH 27/66] adds missing variables to example jinja input --- deploy/vars_example.json | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/deploy/vars_example.json b/deploy/vars_example.json index 5f025e6d..ab75c6bc 100644 --- a/deploy/vars_example.json +++ b/deploy/vars_example.json @@ -4,8 +4,12 @@ "georef_workdir": "", "point_extract_workdir": "", "segmentation_workdir": "", - "image_workdir": "", + "metadata_workdir": "", + "image_dir": "", "ngrok_authtoken": "", "openai_api_key": "", - "google_application_credentials": "" -} \ No newline at end of file + "google_application_credentials": "", + "segmentation_model_weights": "", + "point_model_weights": "", + "tag": "" +} From 7b563d7cbce37a72375dfda9724fe491adeab447 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Fri, 2 Aug 2024 17:02:23 -0400 Subject: [PATCH 28/66] adding support to query CDR and fetch HITL legend annotations for a given COG id --- .../point_extraction_pipeline.py | 3 +- pipelines/point_extraction/run_pipeline.py | 2 + pipelines/point_extraction/run_server.py | 7 ++- tasks/point_extraction/legend_analyzer.py | 63 ++++++++++++++++++- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/pipelines/point_extraction/point_extraction_pipeline.py b/pipelines/point_extraction/point_extraction_pipeline.py index 960d3002..3528cfed 100644 --- a/pipelines/point_extraction/point_extraction_pipeline.py +++ b/pipelines/point_extraction/point_extraction_pipeline.py @@ -57,6 +57,7 @@ def __init__( model_path_segmenter: str, work_dir: str, verbose=False, + fetch_legend_items=False, include_cdr_output=True, include_bitmasks_output=False, gpu=True, @@ -95,7 +96,7 @@ def __init__( ) tasks.extend( [ - LegendPreprocessor("legend_preprocessor", ""), + LegendPreprocessor("legend_preprocessor", "", fetch_legend_items), Tiler("tiling"), yolo_point_extractor, Untiler("untiling"), diff --git a/pipelines/point_extraction/run_pipeline.py b/pipelines/point_extraction/run_pipeline.py index 6c4022b3..cf3184b6 100644 --- a/pipelines/point_extraction/run_pipeline.py +++ b/pipelines/point_extraction/run_pipeline.py @@ -34,6 +34,7 @@ def main(): parser.add_argument("--no_gpu", action="store_true") p = parser.parse_args() + fetch_legend_items = bool(p.legend_annotations_dir) # setup an input stream input = ImageFileInputIterator(p.input) @@ -46,6 +47,7 @@ def main(): p.model_point_extractor, p.model_segmenter, p.workdir, + fetch_legend_items=fetch_legend_items, include_cdr_output=p.cdr_schema, include_bitmasks_output=p.bitmasks, gpu=not p.no_gpu, diff --git a/pipelines/point_extraction/run_server.py b/pipelines/point_extraction/run_server.py index 7ce345f2..c5535f50 100644 --- a/pipelines/point_extraction/run_server.py +++ b/pipelines/point_extraction/run_server.py @@ -89,6 +89,7 @@ def health(): parser.add_argument("--model_segmenter", type=str, default=None) parser.add_argument("--debug", action="store_true") parser.add_argument("--cdr_schema", action="store_true") + parser.add_argument("--fetch_legend_items", action="store_true") parser.add_argument("--rest", action="store_true") parser.add_argument("--rabbit_host", type=str, default="localhost") parser.add_argument("--request_queue", type=str, default=POINTS_REQUEST_QUEUE) @@ -98,7 +99,11 @@ def health(): # init point extraction pipeline point_extraction_pipeline = PointExtractionPipeline( - p.model_point_extractor, p.model_segmenter, p.workdir, gpu=not p.no_gpu + p.model_point_extractor, + p.model_segmenter, + p.workdir, + fetch_legend_items=p.fetch_legend_items, + gpu=not p.no_gpu, ) result_key = ( diff --git a/tasks/point_extraction/legend_analyzer.py b/tasks/point_extraction/legend_analyzer.py index e6c4d610..3f432a95 100644 --- a/tasks/point_extraction/legend_analyzer.py +++ b/tasks/point_extraction/legend_analyzer.py @@ -1,6 +1,8 @@ -import logging +import json, logging, os from collections import defaultdict +import httpx from shapely import Polygon, distance +from typing import Optional from tasks.common.task import Task, TaskInput, TaskResult from tasks.point_extraction.entities import ( LegendPointItems, @@ -11,12 +13,17 @@ ) from tasks.point_extraction.legend_item_utils import ( filter_labelme_annotations, + parse_legend_annotations, LEGEND_ANNOTATION_PROVENANCE, ) from tasks.segmentation.entities import MapSegmentation, SEGMENTATION_OUTPUT_KEY logger = logging.getLogger(__name__) +CDR_API_TOKEN = os.environ.get("CDR_API_TOKEN", "") +CDR_HOST = "https://api.cdr.land" +CDR_LEGEND_SYSTEM_VERSION_DEFAULT = "polymer__0.0.1" + class LegendPreprocessor(Task): """ @@ -27,8 +34,10 @@ def __init__( self, task_id: str, cache_path: str, + fetch_legend_items: bool = False, ): + self.fetch_legend_items = fetch_legend_items super().__init__(task_id, cache_path) def run(self, task_input: TaskInput) -> TaskResult: @@ -50,6 +59,20 @@ def run(self, task_input: TaskInput) -> TaskResult: task_input.request[LEGEND_ITEMS_OUTPUT_KEY] ) + if self.fetch_legend_items: + # try to fetch legend annotations pre COG id from the CDR (via REST) + logger.info( + f"Trying to fetch legend annotations for raster {task_input.raster_id} from the CDR..." + ) + cdr_legend_items = self.fetch_cdr_legend_items(task_input.raster_id) + if cdr_legend_items and cdr_legend_items.items: + # legend items sucessfully fetched from the CDR + # overwriting any pre-loaded legend items, if present + legend_pt_items = cdr_legend_items + logger.info( + f"Sucessfully fetched {len(legend_pt_items.items)} legend annotations from the CDR" + ) + if legend_pt_items: if not legend_pt_items.provenance == LEGEND_ANNOTATION_PROVENANCE.LABELME: # not "labelme" legend items, just output task result @@ -82,6 +105,44 @@ def run(self, task_input: TaskInput) -> TaskResult: return self._create_result(task_input) + def fetch_cdr_legend_items( + self, raster_id: str, system_version: str = CDR_LEGEND_SYSTEM_VERSION_DEFAULT + ) -> Optional[LegendPointItems]: + """ + fetch legend annotations from the CDR for a given COG id + """ + + if not CDR_API_TOKEN: + logger.warning("Unable to fetch legend items; CDR_API_TOKEN not set") + return None + + try: + headers = { + "accept": "application/json", + "Authorization": f"Bearer {CDR_API_TOKEN}", + } + client = httpx.Client(follow_redirects=True) + url = f"{CDR_HOST}/v1/features/{raster_id}/legend_items" + if system_version: + url += f"?system_version={system_version}" + + r = client.get(url, headers=headers) + if r.status_code != 200: + logger.warning( + f"Unable to fetch legend items for raster {raster_id}; cdr response code {r.status_code}" + ) + return None + legend_anns = json.loads(r.content) + legend_pt_items = parse_legend_annotations(legend_anns, raster_id) + if legend_pt_items.items: + return legend_pt_items + + except Exception as e: + logger.warning( + f"Exception fetching legend items from the CDR for raster {raster_id}: {repr(e)}" + ) + return None + class LegendPostprocessor(Task): """ From 4135a76df8f52b17b4a2330bb9e32634c1f4d447 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Tue, 6 Aug 2024 14:37:39 -0400 Subject: [PATCH 29/66] minor changes to points' results viz script to handle CDR output JSON format --- util/viz/viz_point_extractions.py | 98 ++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 14 deletions(-) diff --git a/util/viz/viz_point_extractions.py b/util/viz/viz_point_extractions.py index 1e275ca7..957596c5 100644 --- a/util/viz/viz_point_extractions.py +++ b/util/viz/viz_point_extractions.py @@ -7,11 +7,16 @@ import os from pathlib import Path +from schema.cdr_schemas.cdr_responses.features import PointExtractionResponse +from tasks.point_extraction.label_map import YOLO_TO_CDR_LABEL + # ------------- # Plot Point Extraction location and orientation bboxes on an image RESULTS_DIR = "results_viz/" +BBOX_LEN_DEFAULT = 90 +CDR_JSON_SUFFIX = "" Image.MAX_IMAGE_PIXELS = 500000000 @@ -46,6 +51,43 @@ "drill_hole": "cyan", } +cdr_to_yolo = {v: k for k, v in YOLO_TO_CDR_LABEL.items()} + + +def parse_cdr_features(data: list): + + pt_labels = [] + for d in data: + pt_label = {} + rec = PointExtractionResponse(**d) + bbox = rec.px_bbox + if bbox[0] == bbox[2]: + # 0th size bbox (x axis) + pt_label["x1"] = int(bbox[0] - BBOX_LEN_DEFAULT / 2) + pt_label["x2"] = int(bbox[2] + BBOX_LEN_DEFAULT / 2) + else: + pt_label["x1"] = int(rec.px_bbox[0]) + pt_label["x2"] = int(rec.px_bbox[2]) + + if bbox[1] == bbox[3]: + # 0th size bbox (y axis) + pt_label["y1"] = int(bbox[1] - BBOX_LEN_DEFAULT / 2) + pt_label["y2"] = int(bbox[3] + BBOX_LEN_DEFAULT / 2) + else: + pt_label["y1"] = int(rec.px_bbox[1]) + pt_label["y2"] = int(rec.px_bbox[3]) + pt_label["score"] = rec.confidence if rec.confidence else 0.5 + pt_label["direction"] = rec.dip_direction + pt_label["dip"] = rec.dip + + label = (rec.legend_item.get("label", "") if rec.legend_item else "").strip() + if label in cdr_to_yolo: + label = cdr_to_yolo[label] + pt_label["class_name"] = label + + pt_labels.append(pt_label) + return pt_labels + # finds the straight-line distance between two points def distance(ax, ay, bx, by): @@ -60,8 +102,13 @@ def rotated_about(ax, ay, bx, by, angle): return (round(bx + radius * math.cos(angle)), round(by + radius * math.sin(angle))) -def run(input_path: Path, json_pred_path: Path): - print(f"*** Running viz point extractions on image path : {input_path}") +def run(input_dir: str, json_pred_dir: str, cdr_json_pred_dir: str): + print(f"*** Running viz point extractions on image path : {input_dir}") + + input_path = Path(input_dir) + cdr_json_pred_path = Path(cdr_json_pred_dir) + json_pred_path = Path(json_pred_dir) + parse_cdr_preds = True if cdr_json_pred_dir else False os.makedirs(RESULTS_DIR, exist_ok=True) @@ -85,16 +132,32 @@ def run(input_path: Path, json_pred_path: Path): print(f"Skipping {img_filen}") continue - json_path = json_pred_path / f"{img_filen}_point_extraction.json" + if parse_cdr_preds: + json_path = cdr_json_pred_path / f"{img_filen}{CDR_JSON_SUFFIX}.json" + else: + json_path = json_pred_path / f"{img_filen}_point_extraction.json" print(f"---- {img_filen}") - data = json.load(open(json_path)) - img = Image.open(img_path) - - labels = data.get("labels", []) - if not labels: - print("Oooops!! No predictions! Skipping") - continue + try: + data = json.load(open(json_path)) + img = Image.open(img_path) + + # Create a drawing object to overlay bounding boxes + draw = ImageDraw.Draw(img, mode="RGB") + except Exception as e: + # try converting mode to RGBA + img = img.convert("RGB") + draw = ImageDraw.Draw(img, mode="RGB") + + if parse_cdr_preds: + print("Parsing CDR records") + labels = parse_cdr_features(data) + + else: + labels = data.get("labels", []) + if not labels: + print("Oooops!! No predictions! Skipping") + continue print(f"Visualizing {len(labels)} point predictions") # Create a drawing object to overlay bounding boxes @@ -105,7 +168,7 @@ def run(input_path: Path, json_pred_path: Path): counts_per_class = defaultdict(int) font = ImageFont.load_default(FONT_SIZE) - for label in data["labels"]: + for label in labels: box_coords = (label["x1"], label["y1"], label["x2"], label["y2"]) class_name = label["class_name"] score = label["score"] @@ -182,12 +245,19 @@ def run(input_path: Path, json_pred_path: Path): if __name__ == "__main__": args = ArgumentParser() - args.add_argument("--input_path", type=Path, help="path to input tiffs") + args.add_argument("--input_path", type=str, help="path to input tiffs") args.add_argument( "--json_pred_path", - type=Path, + type=str, help="path to uncharted point extraction json results", + default="", + ) + args.add_argument( + "--cdr_json_pred_path", + type=str, + help="path to uncharted point extraction json results in CDR format", + default="", ) p = args.parse_args() - run(p.input_path, p.json_pred_path) + run(p.input_path, p.json_pred_path, p.cdr_json_pred_path) From 7d629a766c7879ccbc82b61ff3c87310f531d5e9 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Tue, 6 Aug 2024 14:40:45 -0400 Subject: [PATCH 30/66] minor improvements for fetching HITL legend annotations for points pipeline -- handling duplicate legend labels (if present), and checking if annotations have validated=True --- pipelines/point_extraction/run_pipeline.py | 8 ++-- schema/mappers/cdr.py | 1 - tasks/point_extraction/legend_analyzer.py | 53 +++++++++++---------- tasks/point_extraction/legend_item_utils.py | 34 +++++++++---- 4 files changed, 57 insertions(+), 39 deletions(-) diff --git a/pipelines/point_extraction/run_pipeline.py b/pipelines/point_extraction/run_pipeline.py index cf3184b6..e9f7838d 100644 --- a/pipelines/point_extraction/run_pipeline.py +++ b/pipelines/point_extraction/run_pipeline.py @@ -29,12 +29,12 @@ def main(): parser.add_argument("--model_segmenter", type=str, default=None) parser.add_argument("--cdr_schema", action="store_true") # False by default parser.add_argument("--bitmasks", action="store_true") # False by default - parser.add_argument("--legend_annotations_dir", type=str, default="") + parser.add_argument("--fetch_legend_items", action="store_true") + parser.add_argument("--legend_items_dir", type=str, default="") parser.add_argument("--legend_hints_dir", type=str, default="") parser.add_argument("--no_gpu", action="store_true") p = parser.parse_args() - fetch_legend_items = bool(p.legend_annotations_dir) # setup an input stream input = ImageFileInputIterator(p.input) @@ -47,7 +47,7 @@ def main(): p.model_point_extractor, p.model_segmenter, p.workdir, - fetch_legend_items=fetch_legend_items, + fetch_legend_items=p.fetch_legend_items, include_cdr_output=p.cdr_schema, include_bitmasks_output=p.bitmasks, gpu=not p.no_gpu, @@ -69,7 +69,7 @@ def main(): logger.info(f"Processing {doc_id}") image_input = PipelineInput(image=image, raster_id=doc_id) - if p.legend_annotations_dir: + if p.legend_items_dir: # load JSON legend annotations file, if present, parse and add to PipelineInput # expected format is LegendItemResponse CDR pydantic objects try: diff --git a/schema/mappers/cdr.py b/schema/mappers/cdr.py index 92affd27..bef30db1 100644 --- a/schema/mappers/cdr.py +++ b/schema/mappers/cdr.py @@ -267,7 +267,6 @@ def map_to_cdr( description=desc, legend_bbox=leg_item.legend_bbox, legend_contour=leg_item.legend_contour, - validated=leg_item.validated, point_features=None, # points are filled in below ) legend_features[name] = legend_feature_result diff --git a/tasks/point_extraction/legend_analyzer.py b/tasks/point_extraction/legend_analyzer.py index 3f432a95..164c8d94 100644 --- a/tasks/point_extraction/legend_analyzer.py +++ b/tasks/point_extraction/legend_analyzer.py @@ -13,6 +13,7 @@ ) from tasks.point_extraction.legend_item_utils import ( filter_labelme_annotations, + handle_duplicate_labels, parse_legend_annotations, LEGEND_ANNOTATION_PROVENANCE, ) @@ -60,7 +61,7 @@ def run(self, task_input: TaskInput) -> TaskResult: ) if self.fetch_legend_items: - # try to fetch legend annotations pre COG id from the CDR (via REST) + # try to fetch legend annotations for COG id from the CDR (via REST) logger.info( f"Trying to fetch legend annotations for raster {task_input.raster_id} from the CDR..." ) @@ -74,30 +75,27 @@ def run(self, task_input: TaskInput) -> TaskResult: ) if legend_pt_items: - if not legend_pt_items.provenance == LEGEND_ANNOTATION_PROVENANCE.LABELME: - # not "labelme" legend items, just output task result - return TaskResult( - task_id=self._task_id, - output={LEGEND_ITEMS_OUTPUT_KEY: legend_pt_items.model_dump()}, - ) + if legend_pt_items.provenance == LEGEND_ANNOTATION_PROVENANCE.LABELME: + # "labelme" legend items... + # use segmentation results to filter noisy "labelme" legend annotations + # (needed because all labelme annotations are set to type "polygon" regardless of feature type: polygons, lines or points) + if SEGMENTATION_OUTPUT_KEY in task_input.data: + segmentation = MapSegmentation.model_validate( + task_input.data[SEGMENTATION_OUTPUT_KEY] + ) - # "labelme" legend items... - # use segmentation results to filter noisy "labelme" legend annotations - # (needed because all labelme annotations are set to type "polygon" regardless of feature type: polygons, lines or points) - if SEGMENTATION_OUTPUT_KEY in task_input.data: - segmentation = MapSegmentation.model_validate( - task_input.data[SEGMENTATION_OUTPUT_KEY] - ) + filter_labelme_annotations(legend_pt_items, segmentation) + logger.info( + f"Number of legend point annotations after filtering: {len(legend_pt_items.items)}" + ) + else: + logger.warning( + "No segmentation results available. Disregarding labelme legend annotations as noisy." + ) + legend_pt_items.items = [] + + handle_duplicate_labels(legend_pt_items) - filter_labelme_annotations(legend_pt_items, segmentation) - logger.info( - f"Number of legend point annotations after filtering: {len(legend_pt_items.items)}" - ) - else: - logger.warning( - "No segmentation results available. Disregarding labelme legend annotations as noisy." - ) - legend_pt_items.items = [] return TaskResult( task_id=self._task_id, output={LEGEND_ITEMS_OUTPUT_KEY: legend_pt_items.model_dump()}, @@ -106,7 +104,10 @@ def run(self, task_input: TaskInput) -> TaskResult: return self._create_result(task_input) def fetch_cdr_legend_items( - self, raster_id: str, system_version: str = CDR_LEGEND_SYSTEM_VERSION_DEFAULT + self, + raster_id: str, + system_version: str = CDR_LEGEND_SYSTEM_VERSION_DEFAULT, + check_validated: bool = True, ) -> Optional[LegendPointItems]: """ fetch legend annotations from the CDR for a given COG id @@ -133,7 +134,9 @@ def fetch_cdr_legend_items( ) return None legend_anns = json.loads(r.content) - legend_pt_items = parse_legend_annotations(legend_anns, raster_id) + legend_pt_items = parse_legend_annotations( + legend_anns, raster_id, check_validated=check_validated + ) if legend_pt_items.items: return legend_pt_items diff --git a/tasks/point_extraction/legend_item_utils.py b/tasks/point_extraction/legend_item_utils.py index bd6651ba..6c77b791 100644 --- a/tasks/point_extraction/legend_item_utils.py +++ b/tasks/point_extraction/legend_item_utils.py @@ -26,11 +26,8 @@ def __str__(self): def parse_legend_annotations( legend_anns: list, raster_id: str, - system_filter=[ - LEGEND_ANNOTATION_PROVENANCE.POLYMER, - LEGEND_ANNOTATION_PROVENANCE.LABELME, - ], - check_validated=False, + system_filter=[LEGEND_ANNOTATION_PROVENANCE.POLYMER], + check_validated=True, ) -> LegendPointItems: """ parse legend annotations JSON data (CDR LegendItemResponse json format) @@ -43,11 +40,9 @@ def parse_legend_annotations( for leg_ann in legend_anns: try: leg_resp = LegendItemResponse(**leg_ann) - if leg_resp.system in system_filter or ( - check_validated and leg_resp.validated - ): + validated_ok = leg_resp.validated if check_validated else True + if leg_resp.system in system_filter and validated_ok: # only keep legend item responses from desired systems - # or with validated=True legend_item_resps[leg_resp.system].append(leg_resp) count_leg_items += 1 except Exception as e: @@ -306,3 +301,24 @@ def legend_items_use_ontology(leg_point_items: LegendPointItems) -> bool: f"*** Point ontology labels are available for ALL legend items. Skipping further legend item analysis." ) return class_labels_ok + + +def handle_duplicate_labels(leg_point_items: LegendPointItems): + """ + De-duplicate legend item labels, if present + """ + + leg_labels = set() + for leg_item in leg_point_items.items: + if leg_item.name in leg_labels: + suffix = 1 + while str(f"{leg_item.name}_{suffix}") in leg_labels and suffix < 99: + suffix += 1 + dedup_name = f"{leg_item.name}_{suffix}" + logger.info( + f"Multiple legend items named {leg_item.name}; using {dedup_name} for duplicate item" + ) + leg_item.name = dedup_name + leg_labels.add(dedup_name) + else: + leg_labels.add(leg_item.name) From f1cc7821813a4c072f0088fc9e721b353494adbf Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Tue, 6 Aug 2024 16:30:27 -0400 Subject: [PATCH 31/66] added cdr legend id to legend-items parsed from CDR (helps wtih data provenance) --- pipelines/point_extraction/run_pipeline.py | 6 +++--- schema/mappers/cdr.py | 1 + tasks/point_extraction/entities.py | 4 ++++ tasks/point_extraction/legend_item_utils.py | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pipelines/point_extraction/run_pipeline.py b/pipelines/point_extraction/run_pipeline.py index e9f7838d..bd47894c 100644 --- a/pipelines/point_extraction/run_pipeline.py +++ b/pipelines/point_extraction/run_pipeline.py @@ -75,7 +75,7 @@ def main(): try: # check for legend annotations for this image with open( - os.path.join(p.legend_annotations_dir, doc_id + ".json"), "r" + os.path.join(p.legend_items_dir, doc_id + ".json"), "r" ) as fp: legend_anns = json.load(fp) legend_pt_items = parse_legend_annotations(legend_anns, doc_id) @@ -109,9 +109,9 @@ def main(): if p.bitmasks: bitmasks_out_dir = os.path.join(p.output, "bitmasks") os.makedirs(bitmasks_out_dir, exist_ok=True) - if not p.legend_hints_dir and not p.legend_annotations_dir: + if not p.legend_hints_dir and not p.legend_items_dir: logger.warning( - 'Points pipeline is configured to create CMA contest bitmasks without using legend annotations! Setting "legend_hints_dir" or "legend_annotations_dir" param is recommended.' + 'Points pipeline is configured to create CMA contest bitmasks without using legend annotations! Setting "legend_hints_dir" or "legend_items_dir" param is recommended.' ) results = pipeline.run(image_input) diff --git a/schema/mappers/cdr.py b/schema/mappers/cdr.py index bef30db1..bc2c43ed 100644 --- a/schema/mappers/cdr.py +++ b/schema/mappers/cdr.py @@ -267,6 +267,7 @@ def map_to_cdr( description=desc, legend_bbox=leg_item.legend_bbox, legend_contour=leg_item.legend_contour, + reference_id=leg_item.cdr_legend_id, point_features=None, # points are filled in below ) legend_features[name] = legend_feature_result diff --git a/tasks/point_extraction/entities.py b/tasks/point_extraction/entities.py index cf1e3bae..b9b6b399 100644 --- a/tasks/point_extraction/entities.py +++ b/tasks/point_extraction/entities.py @@ -192,6 +192,10 @@ class LegendPointItem(BaseModel): system_version: str = Field( default="", description="System version that published this item" ) + cdr_legend_id: str = Field( + default="", + description="CDR legend id; can be used to reference CDR LegendItemResponse objects", + ) validated: bool = Field(default=False, description="Validated by human") confidence: Optional[float] = Field( default=None, diff --git a/tasks/point_extraction/legend_item_utils.py b/tasks/point_extraction/legend_item_utils.py index 6c77b791..11714f20 100644 --- a/tasks/point_extraction/legend_item_utils.py +++ b/tasks/point_extraction/legend_item_utils.py @@ -229,6 +229,7 @@ def legend_ann_to_legend_items( system=leg_ann.system, system_version=leg_ann.system_version, confidence=leg_ann.confidence, + cdr_legend_id=leg_ann.legend_id, validated=leg_ann.validated, ) ) From 87e7323f1d48699757cf414a97a39750ad811c74 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Thu, 8 Aug 2024 16:17:15 -0400 Subject: [PATCH 32/66] added script to vizualize GCPs offline --- util/viz/viz_georef_gcps.py | 127 ++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 util/viz/viz_georef_gcps.py diff --git a/util/viz/viz_georef_gcps.py b/util/viz/viz_georef_gcps.py new file mode 100644 index 00000000..1a3ff193 --- /dev/null +++ b/util/viz/viz_georef_gcps.py @@ -0,0 +1,127 @@ +from argparse import ArgumentParser +from PIL import Image, ImageDraw, ImageFont +import json +from collections import defaultdict +import os +from pathlib import Path + +# ------------- +# Plot pxl locations and geo-coord labels for georeferencing GCPs + + +RESULTS_DIR = "results_viz/" +BBOX_LEN_DEFAULT = 150 +CDR_JSON_SUFFIX = "" + +Image.MAX_IMAGE_PIXELS = 500000000 + +FONT_SIZE = 24 +COLOUR = "red" +FILL_COLOUR = (128, 0, 128, 128) +TEXT_COLOUR = "darkmagenta" +LINE_WIDTH = 6 + + +def run(input_dir: str, json_gcps_dir: str): + print(f"*** Running Viz Georef GCPs on image path : {input_dir}") + + input_path = Path(input_dir) + json_gcps_path = Path(json_gcps_dir) + + os.makedirs(RESULTS_DIR, exist_ok=True) + + input_files = [] + if input_path.is_dir(): + # collect the ids of the files in the directory + input_files = [file for file in input_path.glob("*.tif")] + else: + input_files = [input_path] + + for img_path in input_files: + + img_filen = Path(img_path).stem + # --- TEMP code needed to run with contest dir-based data + if ( + img_filen.endswith("_pt") + or img_filen.endswith("_poly") + or img_filen.endswith("_line") + or img_filen.endswith("_point") + ): + print(f"Skipping {img_filen}") + continue + + json_path = json_gcps_path / f"{img_filen}.json" + print(f"---- {img_filen}") + + try: + data = json.load(open(json_path)) + img = Image.open(img_path) + + # Create a drawing object to overlay bounding boxes + draw = ImageDraw.Draw(img, mode="RGB") + except Exception as e: + # try converting mode to RGBA + img = img.convert("RGB") + draw = ImageDraw.Draw(img, mode="RGB") + + gcps = data[0]["map"]["projection_info"]["gcps"] + if not gcps: + print("Oooops!! No GCPs! Skipping") + continue + print(f"Visualizing {len(gcps)} GCPs") + + # Create a drawing object to overlay bounding boxes + draw = ImageDraw.Draw(img, mode="RGBA") + + # Draw bounding boxes from the labels onto the image + # create a color map for each unique class_name + counts_per_class = defaultdict(int) + font = ImageFont.load_default(FONT_SIZE) + + for gcp in gcps: + lonlat = gcp["map_geom"] + xy = gcp["px_geom"] + x1 = int(xy[0] - BBOX_LEN_DEFAULT / 2) + y1 = int(xy[1] - BBOX_LEN_DEFAULT / 2) + box_coords = ( + x1, + y1, + x1 + BBOX_LEN_DEFAULT, + y1 + BBOX_LEN_DEFAULT, + ) + + draw.rectangle( + box_coords, outline=COLOUR, fill=FILL_COLOUR, width=LINE_WIDTH + ) + + coord_text = f"lon:{lonlat[0]:.3f}, lat:{lonlat[1]:.3f}" + draw.text( + (x1, y1 - FONT_SIZE * 1.5), + coord_text, + fill=TEXT_COLOUR, + font=font, + font_size=FONT_SIZE, + ) + + print(f"Number of GCPs: {len(gcps)}") + + img.save( + os.path.join(RESULTS_DIR, img_filen + "_gcps_viz.jpg") + ) # save output to a new jpg file + + print("Done!") + + +if __name__ == "__main__": + + args = ArgumentParser() + args.add_argument("--input_path", type=str, help="path to input tiffs") + args.add_argument( + "--json_gcps_dir", + type=str, + help="path to uncharted georef GCP json results", + default="", + ) + p = args.parse_args() + + run(p.input_path, p.json_gcps_dir) From c218f16a2ad93ae79bc4fd6d6d4dac6cf6f35347 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Thu, 8 Aug 2024 16:39:52 -0400 Subject: [PATCH 33/66] added code to create GCPs based on extracted geo-coord pixel locations --- pipelines/geo_referencing/factory.py | 2 +- pipelines/geo_referencing/run_pipeline.py | 2 +- tasks/geo_referencing/ground_control.py | 171 +++++++++++++++++++++- 3 files changed, 165 insertions(+), 10 deletions(-) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index 57e74fa6..38975be3 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -302,7 +302,7 @@ def create_geo_referencing_pipelines( ) tasks.append(InferenceCoordinateExtractor("coordinate-inference")) tasks.append(ScaleExtractor("scaler", "")) - tasks.append(CreateGroundControlPoints("seventh")) + tasks.append(CreateGroundControlPoints("seventh", create_random_pts=False)) tasks.append(GeoReference("eighth", 1)) p.append( Pipeline( diff --git a/pipelines/geo_referencing/run_pipeline.py b/pipelines/geo_referencing/run_pipeline.py index 4f56bd47..97f3a902 100644 --- a/pipelines/geo_referencing/run_pipeline.py +++ b/pipelines/geo_referencing/run_pipeline.py @@ -41,7 +41,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--input", type=str, required=True) parser.add_argument("--output", type=str, required=True) - parser.add_argument("--workdir", type=str, default=None) + parser.add_argument("--workdir", type=str, default="tmp/lara/workdir") parser.add_argument("--clue_dir", type=str, default="") parser.add_argument("--query_dir", type=str, default="") parser.add_argument("--points_dir", type=str, default="") diff --git a/tasks/geo_referencing/ground_control.py b/tasks/geo_referencing/ground_control.py index 0ea57664..6f7609c0 100644 --- a/tasks/geo_referencing/ground_control.py +++ b/tasks/geo_referencing/ground_control.py @@ -4,14 +4,26 @@ from tasks.common.task import Task, TaskInput, TaskResult from tasks.geo_referencing.georeference import QueryPoint +from tasks.segmentation.entities import ( + MapSegmentation, + SEGMENTATION_OUTPUT_KEY, + SEGMENT_MAP_CLASS, +) +from tasks.segmentation.segmenter_utils import get_segment_bounds -from typing import List, Tuple +from typing import List, Tuple, Dict +from shapely import Point, Polygon, MultiPolygon, concave_hull +from shapely.ops import nearest_points + +GEOCOORD_DIST_THRES = 30 +NUM_PTS = 10 logger = logging.getLogger("ground_control_points") class CreateGroundControlPoints(Task): - def __init__(self, task_id: str): + def __init__(self, task_id: str, create_random_pts: bool = True): + self.create_random_pts = create_random_pts super().__init__(task_id) def run(self, input: TaskInput) -> TaskResult: @@ -26,9 +38,41 @@ def run(self, input: TaskInput) -> TaskResult: logger.info("ground control points already exist") return self._create_result(input) - # no query points exist, so create them - query_pts = self._create_query_points(input) - logger.info(f"created {len(query_pts)} ground control points") + # no query points exist, so create them... + # get map segmentation ROI as a shapely polygon (without any dilation buffering) + poly_map = [] + if SEGMENTATION_OUTPUT_KEY in input.data: + segmentation = MapSegmentation.model_validate( + input.data[SEGMENTATION_OUTPUT_KEY] + ) + poly_map = get_segment_bounds(segmentation, SEGMENT_MAP_CLASS) + + if self.create_random_pts or not poly_map: + # create random GCPs... + query_pts = self._create_random_query_points(input, num_pts=NUM_PTS) + logger.info(f"created {len(query_pts)} random ground control points") + else: + # create GCPs based on geo-coord pixel locations... + # use 1st (highest ranked) map segment + poly_map = poly_map[0] + # get extracted lat and lon coords + lon_pts = input.get_data("lons", {}) + lat_pts = input.get_data("lats", {}) + query_pts = self._create_geo_coord_query_points( + input.raster_id, poly_map, lon_pts, lat_pts + ) + logger.info( + f"created {len(query_pts)} geo-coord based ground control points" + ) + if len(query_pts) < NUM_PTS: + logger.info( + f"Also creating up to {NUM_PTS-len(query_pts)} inner ground control points" + ) + query_pts.extend( + self._create_inner_query_points( + input.raster_id, poly_map, num_pts=NUM_PTS - len(query_pts) + ) + ) # add them to the output result = self._create_result(input) @@ -36,8 +80,12 @@ def run(self, input: TaskInput) -> TaskResult: return result - def _create_query_points(self, input: TaskInput) -> List[QueryPoint]: - # create 10 ground control points roughly around the middle of the ROI (or failing that the middle of the image) + def _create_random_query_points( + self, input: TaskInput, num_pts=10 + ) -> List[QueryPoint]: + """ + create N random ground control points roughly around the middle of the ROI (or failing that the middle of the image) + """ min_x = min_y = max_x = max_y = 0 roi = input.get_data("roi") if roi and len(roi) > 0: @@ -49,7 +97,7 @@ def _create_query_points(self, input: TaskInput) -> List[QueryPoint]: else: max_x, max_y = input.image.size - coords = self._create_random_coordinates(min_x, min_y, max_x, max_y) + coords = self._create_random_coordinates(min_x, min_y, max_x, max_y, n=num_pts) return [ QueryPoint( input.raster_id, (c[0], c[1]), None, properties={"label": "random"} @@ -57,6 +105,99 @@ def _create_query_points(self, input: TaskInput) -> List[QueryPoint]: for c in coords ] + def _create_geo_coord_query_points( + self, raster_id: str, poly_map: Polygon, lon_pts: Dict, lat_pts: Dict + ) -> List[QueryPoint]: + """ + create ground control points at approximately the pixel locations where geo-coordinates have been extracted + """ + + # do polygon of roi + gcp_pts = [] + # GCPs based on extracted longitude geo-coords + for coord in lon_pts.values(): + pt = Point(coord.get_pixel_alignment()) + if pt.intersects(poly_map): + if not self._do_pts_overlap(pt, gcp_pts): + gcp_pts.append(pt) + else: + # pt doesn't intersect map ROI, adjust y pixel location + dist1 = poly_map.distance(pt) + pt_on_map = nearest_points(poly_map, pt)[0] + pt2 = Point(pt.x, pt_on_map.y) + dist2 = poly_map.distance(pt2) + # use adjusted pt if it's closer to map roi + if dist2 < dist1: + pt = pt2 + if not self._do_pts_overlap(pt, gcp_pts): + gcp_pts.append(pt) + # GCPs based on extracted latitude geo-coords + for coord in lat_pts.values(): + pt = Point(coord.get_pixel_alignment()) + if pt.intersects(poly_map): + if not self._do_pts_overlap(pt, gcp_pts): + gcp_pts.append(pt) + else: + # pt doesn't intersect map ROI, adjust x pixel location + dist1 = poly_map.distance(pt) + pt_on_map = nearest_points(poly_map, pt)[0] + pt2 = Point(pt_on_map.x, pt.y) + dist2 = poly_map.distance(pt2) + # use adjusted pt if it's closer to map roi + if dist2 < dist1: + pt = pt2 + if not self._do_pts_overlap(pt, gcp_pts): + gcp_pts.append(pt) + + return [ + QueryPoint(raster_id, (pt.x, pt.y), None, properties={"label": "geo_coord"}) + for pt in gcp_pts + ] + + def _create_inner_query_points( + self, raster_id: str, poly_map: Polygon, buffer=0.33, num_pts=10 + ) -> List[QueryPoint]: + """ + create ground control points inside a map polygon with a given buffer + pts will be approx equally spaced + """ + + # calc buffer in pixels, using approx polygon radius + (minx, miny, maxx, maxy) = poly_map.bounds + p_radius = 0.5 * min(maxx - minx, maxy - miny) + buffer_pxl = p_radius * buffer + + # apply negative buffer to map polygon so GCPs will be inside map roi + poly_buffer = poly_map.buffer(-buffer_pxl) + if isinstance(poly_buffer, Polygon): + poly_pts = list(poly_buffer.exterior.coords) + poly_pts.pop() # remove last vertex (same as first) + else: + # unexpected shapely response, create 4 vertices around polygon centroid + c = poly_map.centroid + d = p_radius * (1 - buffer) + poly_pts = [ + (c.x - d, c.y - d), + (c.x + d, c.y - d), + (c.x + d, c.y + d), + (c.x - d, c.y + d), + ] + + # create approx equally spaced GCPs along inner polygon + gcp_pts = [] + step = max(int(len(poly_pts) / num_pts), 1) + for i in range(0, len(poly_pts), step): + pt = poly_pts[i] + pt = Point(pt[0], pt[1]) + if not self._do_pts_overlap(pt, gcp_pts): + gcp_pts.append(pt) + if len(gcp_pts) >= num_pts: + break + return [ + QueryPoint(raster_id, (pt.x, pt.y), None, properties={"label": "map_inner"}) + for pt in gcp_pts + ] + def _create_random_coordinates( self, min_x: float, @@ -82,3 +223,17 @@ def _create_random_coordinates( ) for _ in range(n) ] + + def _do_pts_overlap( + self, + input_pt: Point, + ref_pts: List[Point], + dist_thres: int = GEOCOORD_DIST_THRES, + ) -> bool: + """ + Check if an input point overlaps with any of list of reference points, within a distance threshold + """ + for pt in ref_pts: + if input_pt.dwithin(pt, distance=dist_thres): + return True + return False From 92c08fc524baad6fce886bcf0829d9ed3e3b24c5 Mon Sep 17 00:00:00 2001 From: Philippe Horne Date: Fri, 9 Aug 2024 13:16:34 -0400 Subject: [PATCH 34/66] Updated some filtering to make make use of parsing multiple coordinates with the same degree. Updated geocoding to use iqr to identify outliers when many points are geocoded. --- pipelines/geo_referencing/factory.py | 5 + .../geo_referencing/coordinates_extractor.py | 2 +- tasks/geo_referencing/filter.py | 145 +++++++++++++++++- tasks/geo_referencing/geocode.py | 89 ++++++----- 4 files changed, 201 insertions(+), 40 deletions(-) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index 57e74fa6..73264105 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -23,6 +23,8 @@ OutlierFilter, ROIFilter, UTMStatePlaneFilter, + DistinctDegreeOutlierFilter, + HighQualityCoordinateFilter, ) from tasks.geo_referencing.geo_fencing import GeoFencer from tasks.geo_referencing.georeference import GeoReference @@ -241,6 +243,8 @@ def create_geo_referencing_pipelines( tasks.append(GeoFencer("geofence")) tasks.append(GeoCoordinatesExtractor("third")) tasks.append(ROIFilter("roiness")) + tasks.append(DistinctDegreeOutlierFilter("uniqueness")) + tasks.append(HighQualityCoordinateFilter("goodness")) tasks.append(OutlierFilter("fourth")) tasks.append(NaiveFilter("fun")) if extract_metadata: @@ -627,6 +631,7 @@ def create_geo_referencing_pipeline( tasks.append(GeoFencer("geofence")) tasks.append(GeoCoordinatesExtractor("third")) tasks.append(ROIFilter("roiness")) + tasks.append(DistinctDegreeOutlierFilter("uniqueness")) tasks.append(OutlierFilter("fourth")) tasks.append(NaiveFilter("fun")) tasks.append( diff --git a/tasks/geo_referencing/coordinates_extractor.py b/tasks/geo_referencing/coordinates_extractor.py index 2e357826..81d3a6a1 100644 --- a/tasks/geo_referencing/coordinates_extractor.py +++ b/tasks/geo_referencing/coordinates_extractor.py @@ -39,7 +39,7 @@ # GeoCoordinates # pre-compiled regex patterns RE_DMS = re.compile( - r"^|\b[-+]?([0-9]{1,2}|1[0-7][0-9]|180)( |[o*°⁰˙˚•·º:]|( ?,?[o*°⁰˙˚•·º:]))( ?[0-6]?[0-9])['`′/]?( ?[0-6][0-9])?[\"″'`′/]?($|\b)" + r"^|\b[-+]?([0-9]{1,2}|1[0-7][0-9]|180)( |[o*°⁰˙˚•·º:]|( ?,?[o*°⁰˙˚•·º:]))( ?[0-6]?[0-9])['`′/:]?( ?[0-6][0-9])?[\"″'`′/:]?($|\b)" ) # match degrees with minutes (and optionally seconds) RE_DEG = re.compile( diff --git a/tasks/geo_referencing/filter.py b/tasks/geo_referencing/filter.py index 1b836ed0..71380469 100644 --- a/tasks/geo_referencing/filter.py +++ b/tasks/geo_referencing/filter.py @@ -214,6 +214,149 @@ def _reduce(self, coords: list[Coordinate], index: int) -> float: return sum([abs(degrees[i] - predictions[i]) for i in range(len(predictions))]) +class DistinctDegreeOutlierFilter(FilterAxisCoordinates): + def __init__(self, task_id: str): + super().__init__(task_id) + + def _filter( + self, input: TaskInput, coords: Dict[Tuple[float, float], Coordinate] + ) -> Dict[Tuple[float, float], Coordinate]: + # a more strict outlier filter focusing on coordinates with the same parsed degree + distincts = {} + to_process = False + for _, c in coords.items(): + degree = c.get_parsed_degree() + if degree not in distincts: + distincts[degree] = [] + distincts[degree].append(c) + if len(distincts) >= 3: + to_process = True + + if not to_process: + logger.info( + "skipping distinct degree filtering since there are not enough duplicates" + ) + return coords + + # cycle through all distinct groups + remaining_coords = {} + for k, g in distincts.items(): + to_keep = [] + # if less than 3, simply add to output since filter will not be possible + if len(g) < 3: + to_keep = g + else: + logger.info(f"attempting to filter by distinct degree value for {k}") + # identical degree values should have one of x or y be fairly similar + # if one is misaligned then it is probably from some other context + x, y = input.image.size + size_relevant = (y if g[0].is_lat() else x) / 20 + for c in g: + c_d = c.get_constant_dimension() + for c_i in g: + if c_i.get_pixel_alignment() != c.get_pixel_alignment(): + c_i_d = c_i.get_constant_dimension() + if abs(c_i_d - c_d) < size_relevant and c_i not in to_keep: + to_keep.append(c_i) + if len(to_keep) == 0: + to_keep = g + logger.info(f"kept all coordinates parsed as {k}") + else: + logger.info(f"kept only a subset of coordinates parsed as {k}") + for c in to_keep: + key = c.to_deg_result()[0] + remaining_coords[key] = c + + return remaining_coords + + +class HighQualityCoordinateFilter(FilterAxisCoordinates): + def __init__(self, task_id: str): + super().__init__(task_id) + + def _filter( + self, input: TaskInput, coords: Dict[Tuple[float, float], Coordinate] + ) -> Dict[Tuple[float, float], Coordinate]: + # if two points have the same degree parsed and are roughly aligned + # then throw out everything else that is within the same general area with a different degree + distincts = {} + coordinates = [] + to_process = False + for _, c in coords.items(): + degree = c.get_parsed_degree() + if degree not in distincts: + distincts[degree] = [] + distincts[degree].append(c) + if len(distincts) >= 2: + to_process = True + coordinates.append(c) + + if not to_process: + logger.info( + "skipping high quality coordinate filtering since there are not enough duplicates" + ) + return coords + logger.info( + f"filtering parsed coordinates by excluding coordinates that clash with high quality coordinates" + ) + + remove_coords = {} + remaining_coords = {} + for degree, g in distincts.items(): + if len(g) >= 2: + # check to make sure at least one other coordinate in the group falls within the expected range + x, y = input.image.size + size_relevant = (y if g[0].is_lat() else x) / 20 + can_filter, pixel_range = self._can_filter(g, size_relevant) + if can_filter: + _, rejected = self._filter_range( + coordinates, + degree, + (pixel_range - size_relevant, pixel_range + size_relevant), + ) + for c in rejected: + logger.info( + f"removing {c.get_parsed_degree()} since it falls within the pixel range of high confidence points" + ) + remove_coords[c.get_pixel_alignment()] = c + + for c in coordinates: + key = c.to_deg_result()[0] + if c.get_pixel_alignment() not in remove_coords: + remaining_coords[key] = c + return remaining_coords + + def _can_filter( + self, coordinates: List[Coordinate], pixel_range: float + ) -> Tuple[bool, float]: + for c in coordinates: + c_d = c.get_constant_dimension() + for c_i in coordinates: + if c_i.get_pixel_alignment() != c.get_pixel_alignment(): + c_i_d = c_i.get_constant_dimension() + if abs(c_i_d - c_d) < pixel_range: + return True, c_d + return False, -1 + + def _filter_range( + self, + coordinates: List[Coordinate], + degree: float, + pixel_range: Tuple[float, float], + ) -> Tuple[List[Coordinate], List[Coordinate]]: + # split coordinates found within the range to either be kept (degree matches) or removed (degree does not match) + to_keep = [] + rejected = [] + for c in coordinates: + pixel = c.get_constant_dimension() + if pixel_range[0] <= pixel <= pixel_range[1]: + if c.get_parsed_degree() == degree: + to_keep.append(c) + else: + rejected.append(c) + return to_keep, rejected + + class UTMStatePlaneFilter(FilterCoordinates): def __init__(self, task_id: str): super().__init__(task_id) @@ -484,7 +627,7 @@ def _validate_lonlat_extractions( # check number of unique lat and lon values num_lat_pts = len(set([x[0] for x in lat_results])) num_lon_pts = len(set([x[0] for x in lon_results])) - logger.info(f"distinct after outlier lat,lon: {num_lat_pts},{num_lon_pts}") + logger.info(f"distinct after roi lat,lon: {num_lat_pts},{num_lon_pts}") if num_lon_pts >= 2 and num_lat_pts == 1: # estimate additional lat pt (based on lon pxl resolution) diff --git a/tasks/geo_referencing/geocode.py b/tasks/geo_referencing/geocode.py index 41945645..b2ac55b6 100644 --- a/tasks/geo_referencing/geocode.py +++ b/tasks/geo_referencing/geocode.py @@ -3,6 +3,8 @@ import numpy as np +from statistics import median + from copy import deepcopy from sklearn.cluster import DBSCAN @@ -104,31 +106,35 @@ def _get_coordinates(self, places: List[GeocodedPlace]) -> List[Coordinate]: # create the coordinates for the clustered points (one for lon, one for lat) coordinates = [] + pixels = {} for c in max_cluster: - coordinates.append( - Coordinate( - "point derived lat", - c[1][0].place_name, - c[0][1], - SOURCE_GEOCODE, - True, - pixel_alignment=(c[1][1], c[1][2]), - confidence=COORDINATE_CONFIDENCE_GEOCODE, - derivation="geocoded", + pixel_alignment = (c[1][1], c[1][2]) + if pixel_alignment not in pixels: + coordinates.append( + Coordinate( + "point derived lat", + c[1][0].place_name, + c[0][1], + SOURCE_GEOCODE, + True, + pixel_alignment=pixel_alignment, + confidence=COORDINATE_CONFIDENCE_GEOCODE, + derivation="geocoded", + ) ) - ) - coordinates.append( - Coordinate( - "point derived lon", - c[1][0].place_name, - c[0][0], - SOURCE_GEOCODE, - False, - pixel_alignment=(c[1][1], c[1][2]), - confidence=COORDINATE_CONFIDENCE_GEOCODE, - derivation="geocoded", + coordinates.append( + Coordinate( + "point derived lon", + c[1][0].place_name, + c[0][0], + SOURCE_GEOCODE, + False, + pixel_alignment=pixel_alignment, + confidence=COORDINATE_CONFIDENCE_GEOCODE, + derivation="geocoded", + ) ) - ) + pixels[pixel_alignment] = 1 return coordinates def _in_geofence( @@ -295,20 +301,17 @@ def _extract_coordinates( # cluster to figure out which geocodings to consider coordinates = self._get_coordinates(places_filtered) - logger.info("got all geocoded coordinate for box geocoding") + logger.info(f"got {len(coordinates)} geocoded coordinates for box geocoding") # keep the middle 80% of each direction roughly # a point dropped for one direction cannot be used for the other direction # TODO: SHOULD PROBABLY BE MORE BOX AND WHISKER STYLE OUTLIER FILTERING - remove_count = max(int(len(coordinates) / 2 * 0.1), 1) - logger.info(f"removing {remove_count} coordinates from each end and direction") - coordinates_lons = self._remove_extreme( - remove_count, + logger.info(f"removing outlier coordinates via iqr from each end and direction") + coordinates_lons = self._remove_outliers( list(filter(lambda x: not x.is_lat(), coordinates)), lambda x: x.get_parsed_degree(), ) - coordinates_lats = self._remove_extreme( - remove_count, + coordinates_lats = self._remove_outliers( list(filter(lambda x: x.is_lat(), coordinates)), lambda x: x.get_parsed_degree(), ) @@ -320,11 +323,13 @@ def _extract_coordinates( coordinates_count = {} for c in coordinates_lats: coordinates_count[c.get_pixel_alignment()] = 1 + count = 0 for c in coordinates_lons: pixels = c.get_pixel_alignment() if pixels not in coordinates_count: coordinates_count[pixels] = 0 coordinates_count[pixels] = coordinates_count[pixels] + 1 + count = count + 1 coordinates_lons: List[Coordinate] = [] coordinates_lats: List[Coordinate] = [] @@ -377,17 +382,25 @@ def _extract_coordinates( return lon_pts, lat_pts - def _remove_extreme( - self, n: int, coordinates: List[Coordinate], mapper: Callable + def _remove_outliers( + self, coordinates: List[Coordinate], mapper: Callable ) -> List[Coordinate]: # map and sort the coordinates - coordinate_sorted = sorted( - [(mapper(c), c) for c in coordinates], key=lambda x: x[0] - ) - - # remove the top and bottom N values - end_index = len(coordinate_sorted) - n - return [cf[1] for cf in coordinate_sorted[n:end_index]] + values = sorted([(mapper(c), c) for c in coordinates], key=lambda x: x[0]) + + # find the iqr + lh_index = int(len(values) / 2) + if len(values) % 2 == 0: + lh_index = lh_index + 1 + q1 = median([v[0] for v in values[:lh_index]]) + q3 = median([v[0] for v in values[lh_index:]]) + iqr = q3 - q1 + + # use usual iqr * 1.5 as basis for filtering + remaining_values = [ + cf[1] for cf in values if q1 - (iqr * 1.5) <= cf[0] <= q3 + (iqr * 1.5) + ] + return remaining_values def _get_min_max( self, coordinates: List[Coordinate] From 8a8864c7bcaa3d31f440cb907c442d6b58366675 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Mon, 8 Jul 2024 23:03:15 -0400 Subject: [PATCH 35/66] georef projection code --- __init__.py | 0 util/georef/project.py | 112 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 __init__.py create mode 100644 util/georef/project.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/util/georef/project.py b/util/georef/project.py new file mode 100644 index 00000000..150ae78d --- /dev/null +++ b/util/georef/project.py @@ -0,0 +1,112 @@ +from typing import List +from PIL import Image +from pyproj import Transformer +from rasterio.transform import Affine +from rasterio.warp import Resampling, calculate_default_transform, reproject +import rasterio as rio +import rasterio.transform as riot +from schema.cdr_schemas.georeference import GroundControlPoint +import json +import argparse + + +def project_georeference( + source_image_path: str, + target_image_path: str, + target_crs: str, + gcps: List[GroundControlPoint], +): + # open the image + img = Image.open(source_image_path) + _, height = img.size + + # create the transform + geo_transform = _cps_to_transform(gcps, height=height, to_crs=target_crs) + + # use the transform to project the image + _project_image(source_image_path, target_image_path, geo_transform, target_crs) + + +def _project_image( + source_image_path: str, + target_image_path: str, + geo_transform: Affine, + crs: str, +): + with rio.open(source_image_path) as raw: + bounds = riot.array_bounds(raw.height, raw.width, geo_transform) + pro_transform, pro_width, pro_height = calculate_default_transform( + crs, + crs, + raw.width, + raw.height, + *tuple(bounds), + dst_width=raw.width, + dst_height=raw.height + ) + pro_kwargs = raw.profile.copy() + pro_kwargs.update( + { + "driver": "COG", + "crs": {"init": crs}, + "transform": pro_transform, + "width": pro_width, + "height": pro_height, + } + ) + _raw_data = raw.read() + with rio.open(target_image_path, "w", **pro_kwargs) as pro: + for i in range(raw.count): + _ = reproject( + source=_raw_data[i], + destination=rio.band(pro, i + 1), + src_transform=geo_transform, + src_crs=crs, + dst_transform=pro_transform, + dst_crs=crs, + resampling=Resampling.bilinear, + num_threads=8, + warp_mem_limit=256, + ) + + +def _cps_to_transform( + gcps: List[GroundControlPoint], height: int, to_crs: str +) -> Affine: + cps = [ + { + "row": float(gcp.px_geom.rows_from_top), + "col": float(gcp.px_geom.columns_from_left), + "x": float(gcp.map_geom.longitude), # type: ignore + "y": float(gcp.map_geom.latitude), # type: ignore + "crs": gcp.crs, + } + for gcp in gcps + ] + cps_p = [] + for cp in cps: + proj = Transformer.from_crs(cp["crs"], to_crs, always_xy=True) + x_p, y_p = proj.transform(xx=cp["x"], yy=cp["y"]) + cps_p.append( + riot.GroundControlPoint(row=cp["row"], col=cp["col"], x=x_p, y=y_p) + ) + + return riot.from_gcps(cps_p) + + +def _load_gcps(gcps_path: str) -> List[GroundControlPoint]: + with open(gcps_path, "r") as f: + gcps = json.load(f) + gcps_list = gcps["gcps"] + return [GroundControlPoint(**gcp) for gcp in gcps_list] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=str, help="Path to the source image") + parser.add_argument("--output", type=str, help="Path to the target image") + parser.add_argument("--gcps", type=str, help="Path to the GCPS JSON file") + args = parser.parse_args() + + gcps = _load_gcps(args.gcps) + project_georeference(args.input, args.output, "EPSG:4326", gcps) From 49d672eb465541ae13d20eee43fc72ef1592c363 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sun, 14 Jul 2024 14:17:54 -0400 Subject: [PATCH 36/66] adds test utils for building point geopackage output --- tasks/pyproject.toml | 3 +- util/cdr/__init__.py | 0 util/cdr/build_point_geopackage.py | 45 ++++++++++++++++++++++++++++++ util/georef/__init__.py | 0 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 util/cdr/__init__.py create mode 100644 util/cdr/build_point_geopackage.py create mode 100644 util/georef/__init__.py diff --git a/tasks/pyproject.toml b/tasks/pyproject.toml index 0956e16f..170dd9c0 100644 --- a/tasks/pyproject.toml +++ b/tasks/pyproject.toml @@ -31,7 +31,8 @@ dependencies = [ "langchain", "langchain-openai", "stateplane", - "coloredlogs" + "coloredlogs", + "cdrc @ git+https://github.com/DARPA-CRITICALMAAS/cdrc.git@main", ] [project.optional-dependencies] diff --git a/util/cdr/__init__.py b/util/cdr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/util/cdr/build_point_geopackage.py b/util/cdr/build_point_geopackage.py new file mode 100644 index 00000000..be60d833 --- /dev/null +++ b/util/cdr/build_point_geopackage.py @@ -0,0 +1,45 @@ +from pathlib import Path +import cdrc +import os +import argparse + +token = os.getenv("CDR_API_TOKEN") + + +if __name__ == "__main__": + # Create an argument parser + parser = argparse.ArgumentParser() + parser.add_argument("--cog_id", type=str, help="the COG ID", required=True) + parser.add_argument("--output", type=Path, help="the output file path", default=".") + parser.add_argument( + "--system", type=str, help="the system name", default="uncharted" + ) + parser.add_argument( + "--version", type=str, help="the system version", default="0.0.4" + ) + args = parser.parse_args() + + if token is None: + raise ValueError("CDR_API_TOKEN is not set") + + # assert the supplied directory exists and create it if it doesn't + if not args.output.is_dir(): + raise ValueError("Output path must be a directory") + if not os.path.exists(args.output): + os.makedirs(args.output) + + client = cdrc.CDRClient(token=token, output_dir=args.output) + + # Define the name of the layer to be created + layer_name = f"{args.cog_id}_points" + + cog_id = args.cog_id + system = args.system + system_version = args.version + + client.build_cog_geopackages( + cog_id=args.cog_id, + feature_types=["point"], + system_versions=[(args.system, args.version)], + validated=None, + ) diff --git a/util/georef/__init__.py b/util/georef/__init__.py new file mode 100644 index 00000000..e69de29b From 26b9072db88b5b7a1d1b5d3b8a3dcbca770b1540 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Tue, 16 Jul 2024 11:59:19 -0400 Subject: [PATCH 37/66] generates corner points but doesn't yet output them beyond a json file --- pipelines/geo_referencing/factory.py | 35 +++--- .../geo_referencing/corner_point_extractor.py | 102 ++++++++++++++++++ 2 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 tasks/geo_referencing/corner_point_extractor.py diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index cded270d..80d2fce8 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -16,6 +16,7 @@ from tasks.geo_referencing.coordinates_extractor import ( GeoCoordinatesExtractor, ) +from tasks.geo_referencing.corner_point_extractor import CornerPointExtractor from tasks.geo_referencing.state_plane_extractor import StatePlaneExtractor from tasks.geo_referencing.utm_extractor import UTMCoordinatesExtractor from tasks.geo_referencing.filter import ( @@ -245,6 +246,7 @@ def create_geo_referencing_pipelines( tasks.append(ROIFilter("roiness")) tasks.append(DistinctDegreeOutlierFilter("uniqueness")) tasks.append(HighQualityCoordinateFilter("goodness")) + tasks.append(CornerPointExtractor("corner_point_extractor")) tasks.append(OutlierFilter("fourth")) tasks.append(NaiveFilter("fun")) if extract_metadata: @@ -582,7 +584,10 @@ def create_geo_referencing_pipeline( ) tasks.append( TileTextExtractor( - "first", Path(text_cache), 6000, gamma_correction=ocr_gamma_correction + "tile_text_extractor", + Path(text_cache), + 6000, + gamma_correction=ocr_gamma_correction, ) ) tasks.append( @@ -595,13 +600,13 @@ def create_geo_referencing_pipeline( ) tasks.append( ModelROIExtractor( - "model roi", + "model_roi_buffering", buffer_fixed, ) ) tasks.append( TextFilter( - "text_filter", + "resize_text_filter", input_key="metadata_ocr", output_key="filtered_ocr_text", classes=[ @@ -621,7 +626,7 @@ def create_geo_referencing_pipeline( ) tasks.append( Geocoder( - "geo-bounds", + "geobounds", geocoder_bounds, run_bounds=True, run_points=False, @@ -629,14 +634,14 @@ def create_geo_referencing_pipeline( ) ) tasks.append(GeoFencer("geofence")) - tasks.append(GeoCoordinatesExtractor("third")) + tasks.append(GeoCoordinatesExtractor("geocoodinates_extractor")) tasks.append(ROIFilter("roiness")) tasks.append(DistinctDegreeOutlierFilter("uniqueness")) - tasks.append(OutlierFilter("fourth")) - tasks.append(NaiveFilter("fun")) + tasks.append(OutlierFilter("coordinate_outlier_filter")) + tasks.append(NaiveFilter("coordinate_naive_filter")) tasks.append( TextFilter( - "map_area_filter", + "tiled_map_area_filter", FilterMode.INCLUDE, TEXT_EXTRACTION_OUTPUT_KEY, "map_area_filter", @@ -655,7 +660,7 @@ def create_geo_referencing_pipeline( ) tasks.append( Geocoder( - "geo-places", + "places_geocoder", geocoder_points, run_bounds=False, run_points=True, @@ -664,17 +669,17 @@ def create_geo_referencing_pipeline( ) tasks.append( Geocoder( - "geo-centres", + "popluation_centers_geocoder", geocoder_bounds, run_bounds=False, run_points=False, run_centres=True, ) ) - tasks.append(UTMCoordinatesExtractor("fifth")) + tasks.append(UTMCoordinatesExtractor("utm_coordinate_extractor")) tasks.append( StatePlaneExtractor( - "great-plains", + "state_plain_coordinate_extractor", state_plane_lookup_filename, state_plane_zone_filename, state_code_filename, @@ -684,7 +689,7 @@ def create_geo_referencing_pipeline( tasks.append(UTMStatePlaneFilter("utm-state-plane")) tasks.append(PointGeocoder("geocoded-georeferencing", ["point", "population"], 10)) tasks.append(InferenceCoordinateExtractor("coordinate-inference")) - tasks.append(ScaleExtractor("scaler", "")) - tasks.append(CreateGroundControlPoints("seventh")) - tasks.append(GeoReference("eighth", 1)) + tasks.append(ScaleExtractor("scale_extractor", "")) + tasks.append(CreateGroundControlPoints("gcp_creator")) + tasks.append(GeoReference("geo_referencer", 1)) return Pipeline("wally-finder", "wally-finder", outputs, tasks) diff --git a/tasks/geo_referencing/corner_point_extractor.py b/tasks/geo_referencing/corner_point_extractor.py new file mode 100644 index 00000000..bd44867b --- /dev/null +++ b/tasks/geo_referencing/corner_point_extractor.py @@ -0,0 +1,102 @@ +import json +import logging +import pprint + +from shapely import LineString, Point + +from schema.cdr_schemas.georeference import Geom_Point, GroundControlPoint, Pixel_Point +from tasks.common.task import Task, TaskInput, TaskResult +from tasks.geo_referencing.entities import Coordinate + +from typing import Dict, List, Tuple + +logger = logging.getLogger("corner_point_extractor") + + +class CornerPointExtractor(Task): + def __init__(self, task_id: str): + super().__init__(task_id) + + def run(self, input: TaskInput) -> TaskResult: + logger.info( + f"==========> running corner point extractor task index {input.task_index} with id {self._task_id}" + ) + + # check if query points already defined + corner_points = None + + lon_pts: Dict[Tuple[float, float], Coordinate] = input.get_data("lons") + lat_pts: Dict[Tuple[float, float], Coordinate] = input.get_data("lats") + + intersection_points: List[Tuple[Point, Point]] = [] + + for lon_key, lon_coord in lon_pts.items(): + # generate a vertical segment through the center of the lon label + lon_bounds = lon_coord.get_bounds() + lon_label_width = lon_bounds[1].x - lon_bounds[0].x + lon_center_x = lon_bounds[0].x + (lon_bounds[1].x - lon_bounds[0].x) / 2.0 + lon_center_y = lon_bounds[0].y + (lon_bounds[2].y - lon_bounds[0].y) / 2.0 + lon_line = LineString( + [ + ( + lon_center_x, + lon_center_y + lon_label_width, + ), + (lon_center_x, lon_center_y - lon_label_width), + ] + ) + for lat_key, lat_coord in lat_pts.items(): + # generate a vertical segment through the center of the lat label + lat_bounds = lat_coord.get_bounds() + lat_label_width = lat_bounds[1].x - lat_bounds[0].x + lat_center_x = ( + lat_bounds[0].x + (lat_bounds[1].x - lat_bounds[0].x) / 2.0 + ) + lat_center_y = ( + lat_bounds[0].y + (lat_bounds[2].y - lat_bounds[0].y) / 2.0 + ) + lat_line = LineString( + [ + ( + lat_center_x + lat_label_width, + lat_center_y, + ), + (lat_center_x - lat_label_width, lat_center_y), + ] + ) + # find the intersection of the two lines + intersection = lon_line.intersection(lat_line) + if isinstance(intersection, Point): + intersection_points.append( + (Point(-lon_key[0], lat_key[0]), intersection) + ) + + if len(intersection_points) == 4: + logger.info("Found 4 intersection points") + output = {"gcps": []} + # write out as gcps + for i, point in enumerate(intersection_points): + geo_point = point[0] + pixel_point = point[1] + gcp = GroundControlPoint( + gcp_id=str(i), + map_geom=Geom_Point(longitude=geo_point.x, latitude=geo_point.y), + px_geom=Pixel_Point( + columns_from_left=pixel_point.x, rows_from_top=pixel_point.y + ), + model="corner_point_extractor", + model_version="0.0.1", + crs="EPSG:4267", + ) + output["gcps"].append(gcp.model_dump()) + + # convert output to json + with open("corner_points.json", "w") as outfile: + json.dump(output, outfile, indent=4) + + # if we have intersection points, we can use them as corner points + # add them to the output + result = self._create_result(input) + result.output["corner_points"] = "scaramouche" + + return result From c502dfc5db49ac05e29fb9113458a57fcd58e5c5 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sat, 20 Jul 2024 12:53:43 -0400 Subject: [PATCH 38/66] Generate the geodetic datum as an epsg code. --- tasks/metadata_extraction/metadata_extraction.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/metadata_extraction/metadata_extraction.py b/tasks/metadata_extraction/metadata_extraction.py index 1d3f9edb..1f9ea3ba 100644 --- a/tasks/metadata_extraction/metadata_extraction.py +++ b/tasks/metadata_extraction/metadata_extraction.py @@ -72,13 +72,13 @@ class MetdataLLM(BaseModel): description="The scale of the map. Example: '1:24000'", default="NULL" ) datum: str = Field( - description="The datum of the map." - + "Examples: 'North American Datum of 1927', 'NAD83', 'WGS 84'", + description="The geoditic datum of the map expressed as an EPSG code. If this is not present, it can often be inferred from the map's country and year." + + "Examples of geodetic datums: 'North American Datum of 1927', 'NAD83', 'WGS 84'. Examples of output: 'EPSG:4269', 'EPSG:4326'", default="NULL", ) vertical_datum: str = Field( description="The vertical datum of the map." - + "Examples: 'mean sea level', 'vertical datum of 1901', 'national vertical geoditic datum of 1929'", + + "Examples: 'mean sea level', 'vertical datum of 1901', 'national vertical geodetic datum of 1929'", default="NULL", ) projection: str = Field( From 6188d5c677de61ae5e353f2ce985c3db6fc0d333 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sat, 20 Jul 2024 14:25:18 -0400 Subject: [PATCH 39/66] add datum shift capability --- pipelines/geo_referencing/factory.py | 14 +- tasks/common/pipeline.py | 3 +- .../geo_referencing/corner_point_extractor.py | 84 +++++---- tasks/geo_referencing/entities.py | 1 + tasks/geo_referencing/georeference.py | 162 ++++++++++++++++-- 5 files changed, 213 insertions(+), 51 deletions(-) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index 80d2fce8..e3e90168 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -93,7 +93,7 @@ def create_geo_referencing_pipelines( p = [] - tasks = [] + """tasks = [] tasks.append(ResizeTextExtractor("first", Path(text_cache), False, True, 6000)) tasks.append(EntropyROIExtractor("entropy roi")) if extract_metadata: @@ -102,7 +102,7 @@ def create_geo_referencing_pipelines( tasks.append(UTMCoordinatesExtractor("fourth")) tasks.append(CreateGroundControlPoints("sixth")) tasks.append(GeoReference("seventh", 1)) - """p.append( + p.append( Pipeline( "resize", "resize", @@ -117,6 +117,7 @@ def create_geo_referencing_pipelines( ) )""" + """ tasks = [] tasks.append( TileTextExtractor("first", Path(text_cache), 6000, gamma_correction=0.5) @@ -165,7 +166,7 @@ def create_geo_referencing_pipelines( tasks.append(UTMCoordinatesExtractor("fifth")) tasks.append(CreateGroundControlPoints("sixth")) tasks.append(GeoReference("seventh", 1)) - """p.append( + p.append( Pipeline( "tile", "tile", @@ -306,6 +307,7 @@ def create_geo_referencing_pipelines( tasks.append( BoxGeocoder("geocoded-box", ["point", "population"], geocoder_thresh) ) + tasks.append(CornerPointExtractor("corner_point_extractor")) tasks.append(InferenceCoordinateExtractor("coordinate-inference")) tasks.append(ScaleExtractor("scaler", "")) tasks.append(CreateGroundControlPoints("seventh", create_random_pts=False)) @@ -325,6 +327,7 @@ def create_geo_referencing_pipelines( ) ) + """ tasks = [] tasks.append( TileTextExtractor("first", Path(text_cache), 6000, gamma_correction=0.5) @@ -421,7 +424,7 @@ def create_geo_referencing_pipelines( ) tasks.append(CreateGroundControlPoints("seventh")) tasks.append(GeoReference("eighth", 1)) - """p.append( + p.append( Pipeline( "roi poly image", "roi poly", @@ -436,6 +439,7 @@ def create_geo_referencing_pipelines( ) )""" + """ tasks = [] tasks.append(TileTextExtractor("first", Path(text_cache), 6000)) tasks.append( @@ -530,7 +534,7 @@ def create_geo_referencing_pipelines( ) tasks.append(CreateGroundControlPoints("seventh")) tasks.append(GeoReference("eighth", 1)) - """p.append( + p.append( Pipeline( "roi poly roi", "roi poly", diff --git a/tasks/common/pipeline.py b/tasks/common/pipeline.py index b5607acf..3b62276f 100644 --- a/tasks/common/pipeline.py +++ b/tasks/common/pipeline.py @@ -2,6 +2,7 @@ from typing import Optional, List, Dict, Any, Sequence from PIL.Image import Image as PILImage from pydantic import BaseModel +import traceback class PipelineInput: @@ -149,7 +150,7 @@ def run(self, input: PipelineInput) -> Dict[str, Output]: print( f"EXCEPTION executing pipeline at step {t.get_task_id()} ({task_input.task_index}) for raster {input.raster_id}" ) - print(e) + traceback.print_exc() return self._produce_output(pipeline_result) diff --git a/tasks/geo_referencing/corner_point_extractor.py b/tasks/geo_referencing/corner_point_extractor.py index bd44867b..fb340437 100644 --- a/tasks/geo_referencing/corner_point_extractor.py +++ b/tasks/geo_referencing/corner_point_extractor.py @@ -1,30 +1,54 @@ -import json +from curses import meta import logging -import pprint +from unittest.mock import DEFAULT from shapely import LineString, Point -from schema.cdr_schemas.georeference import Geom_Point, GroundControlPoint, Pixel_Point +from pipelines.metadata_extraction.metadata_extraction_pipeline import ( + MetadataExtractionOutput, +) from tasks.common.task import Task, TaskInput, TaskResult -from tasks.geo_referencing.entities import Coordinate +from tasks.geo_referencing.entities import ( + Coordinate, + GroundControlPoint, + CORNER_POINTS_OUTPUT_KEY, +) -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple logger = logging.getLogger("corner_point_extractor") class CornerPointExtractor(Task): + """ + Extracts corner points from longitude and latitude coordinates. + + This task takes in longitude and latitude coordinates and generates corner points + by finding the intersection of vertical and horizontal lines passing through the + center of each coordinate label. + + Args: + task_id (str): The ID of the task. + + Attributes: + task_id (str): The ID of the task. + + """ + def __init__(self, task_id: str): super().__init__(task_id) def run(self, input: TaskInput) -> TaskResult: - logger.info( - f"==========> running corner point extractor task index {input.task_index} with id {self._task_id}" - ) + """ + Runs the corner point extraction task. - # check if query points already defined - corner_points = None + Args: + input (TaskInput): The input data for the task. + Returns: + TaskResult: The result object updated with the corner points if present. + + """ lon_pts: Dict[Tuple[float, float], Coordinate] = input.get_data("lons") lat_pts: Dict[Tuple[float, float], Coordinate] = input.get_data("lats") @@ -46,7 +70,7 @@ def run(self, input: TaskInput) -> TaskResult: ] ) for lat_key, lat_coord in lat_pts.items(): - # generate a vertical segment through the center of the lat label + # generate a horizontal segment through the center of the lat label lat_bounds = lat_coord.get_bounds() lat_label_width = lat_bounds[1].x - lat_bounds[0].x lat_center_x = ( @@ -68,35 +92,33 @@ def run(self, input: TaskInput) -> TaskResult: intersection = lon_line.intersection(lat_line) if isinstance(intersection, Point): intersection_points.append( - (Point(-lon_key[0], lat_key[0]), intersection) + (Point(lon_key[0], lat_key[0]), intersection) ) - if len(intersection_points) == 4: - logger.info("Found 4 intersection points") - output = {"gcps": []} + if len(intersection_points) >= 3: + logger.info(f"Found {len(intersection_points)} corner points") + + output: List[GroundControlPoint] = [] # write out as gcps for i, point in enumerate(intersection_points): geo_point = point[0] pixel_point = point[1] gcp = GroundControlPoint( - gcp_id=str(i), - map_geom=Geom_Point(longitude=geo_point.x, latitude=geo_point.y), - px_geom=Pixel_Point( - columns_from_left=pixel_point.x, rows_from_top=pixel_point.y - ), - model="corner_point_extractor", - model_version="0.0.1", - crs="EPSG:4267", + id=f"corner.{str(i)}", + longitude=geo_point.x, + latitude=geo_point.y, + pixel_x=pixel_point.x, + pixel_y=pixel_point.y, + confidence=1.0, ) - output["gcps"].append(gcp.model_dump()) - - # convert output to json - with open("corner_points.json", "w") as outfile: - json.dump(output, outfile, indent=4) + output.append(gcp) + else: + logger.info( + f"Found {len(intersection_points)} corner points, require at least 3. Corner point referencing not available." + ) + output = [] - # if we have intersection points, we can use them as corner points - # add them to the output result = self._create_result(input) - result.output["corner_points"] = "scaramouche" + result.output[CORNER_POINTS_OUTPUT_KEY] = output return result diff --git a/tasks/geo_referencing/entities.py b/tasks/geo_referencing/entities.py index 39320bec..02f6a09b 100644 --- a/tasks/geo_referencing/entities.py +++ b/tasks/geo_referencing/entities.py @@ -5,6 +5,7 @@ GEOFENCE_OUTPUT_KEY = "geofence_output" +CORNER_POINTS_OUTPUT_KEY = "corner_points" SOURCE_LAT_LON = "lat-lon parser" SOURCE_STATE_PLANE = "state plane parser" SOURCE_UTM = "utm parser" diff --git a/tasks/geo_referencing/georeference.py b/tasks/geo_referencing/georeference.py index 81cd6ae3..d7c1fc29 100644 --- a/tasks/geo_referencing/georeference.py +++ b/tasks/geo_referencing/georeference.py @@ -1,10 +1,19 @@ import logging import math +import pprint +from unittest.mock import DEFAULT from geopy.distance import geodesic +from regex import R +from tasks.geo_referencing.entities import GroundControlPoint from tasks.common.task import Task, TaskInput, TaskResult -from tasks.geo_referencing.entities import Coordinate, DocGeoFence, GEOFENCE_OUTPUT_KEY +from tasks.geo_referencing.entities import ( + Coordinate, + DocGeoFence, + GEOFENCE_OUTPUT_KEY, + CORNER_POINTS_OUTPUT_KEY, +) from tasks.geo_referencing.geo_projection import GeoProjection from tasks.geo_referencing.util import get_input_geofence from tasks.metadata_extraction.entities import ( @@ -15,6 +24,10 @@ from typing import Any, Dict, List, Optional, Tuple +import rasterio.transform as riot +from pyproj import Transformer + + FALLBACK_RANGE_ADJUSTMENT_FACTOR = 0.05 # used to calculate how far from the edge of the fallback range to anchor points logger = logging.getLogger("geo_referencing") @@ -65,18 +78,43 @@ def __init__(self, pixel_coord, geo_coord): class GeoReference(Task): _poly_order = 1 + # default destination datum for georeferencing output + DEFAULT_DEST_DATUM = "EPSG:4269" + def __init__(self, task_id: str, poly_order: int = 1): super().__init__(task_id) self._poly_order = poly_order def run(self, input: TaskInput) -> TaskResult: + """ + Creates a transformation from pixel coordinates to geographic coordinates + based on exracted map information. The transformation is run against supplied + query points in pixel space to generate a final set of geographic coordinates. + + These pixel/geo point tuples can be used downstream as control points. + + Args: + input (TaskInput): The input data for the georeferencing task. + + Returns: + TaskResult: The result of the georeferencing task. + """ logger.info(f"running georeferencing task for image {input.raster_id}") + # check if the task should run label or corner point georeferencing + if input.get_data(CORNER_POINTS_OUTPUT_KEY): + logger.info(f"running corner point georeferencing") + return self._run_corner_georef(input) + logger.info(f"running label georeferencing") + return self._run_label_georef(input) + + def _run_label_georef(self, input: TaskInput) -> TaskResult: lon_minmax = input.get_request_info("lon_minmax", [0, 180]) lat_minmax = input.get_request_info("lat_minmax", [0, 90]) logger.info(f"initial lon_minmax: {lon_minmax}") lon_pts = input.get_data("lons") lat_pts = input.get_data("lats") + scale_value = input.get_data(SCALE_VALUE_OUTPUT_KEY) im_resize_ratio = input.get_data("im_resize_ratio", 1) @@ -212,19 +250,107 @@ def run(self, input: TaskInput) -> TaskResult: # results = self._clip_query_pts(query_pts, lon_minmax, lat_minmax) results = self._update_hemispheres(query_pts, lon_multiplier, lat_multiplier) + source_datum, projection = self._determine_projection(input) + logger.info( + f"extracted datum: {source_datum}\textracted projection: {projection}" + ) + + if source_datum != self.DEFAULT_DEST_DATUM: + for qp in query_pts: + proj = Transformer.from_crs( + source_datum, self.DEFAULT_DEST_DATUM, always_xy=True + ) + x_p, y_p = proj.transform(qp.lonlat[0], qp.lonlat[1]) + qp.lonlat = (x_p, y_p) + rmse, scale_error = self._score_query_points(query_pts, scale_value) - datum, projection = self._determine_projection(input) - logger.info(f"extracted datum: {datum}\textracted projection: {projection}") + logger.info(f"rmse: {rmse} scale error: {scale_error}") result = super()._create_result(input) result.output["query_pts"] = results result.output["rmse"] = rmse result.output["error_scale"] = scale_error - result.output["datum"] = datum + result.output["datum"] = self.DEFAULT_DEST_DATUM result.output["projection"] = projection result.output["keypoints"] = keypoint_stats return result + def _run_corner_georef(self, input: TaskInput) -> TaskResult: + """ + Runs georeferencing on the input query points assuming using a projection + based on extracted corner points. + + Args: + input (TaskInput): The input for the task - contains the corner points and + query points. + + Returns: + TaskResult: The result of the task. + """ + result = super()._create_result(input) + corner_points: List[GroundControlPoint] = input.get_data( + CORNER_POINTS_OUTPUT_KEY + ) + if not input.get_data(CORNER_POINTS_OUTPUT_KEY): + logger.error("corner points not provided") + return result + + gcps: List[riot.GroundControlPoint] = [] + for cp in corner_points: + gcp = riot.GroundControlPoint( + row=cp.pixel_y, col=cp.pixel_x, x=cp.longitude, y=cp.latitude + ) + gcps.append(gcp) + transform = riot.from_gcps(gcps) + + # get the generated query points, or use those that were passed into the pipeline + # from an external source + query_pts: List[QueryPoint] = input.get_data("query_pts") + if not query_pts: + query_pts = input.get_request_info("query_pts") + + # transform the query pixel points into geo locations and write the result into + # the query point + for qp in query_pts: + lonlat: Tuple[float, float] = riot.xy(transform, qp.xy[1], qp.xy[0]) + qp.lonlat = lonlat + + # correct the hemisphere of the points + lon_multiplier, lat_multiplier = self._determine_hemispheres(input, query_pts) + logger.info( + f"derived hemispheres for georeferencing: {lon_multiplier},{lat_multiplier}" + ) + results = self._update_hemispheres(query_pts, lon_multiplier, lat_multiplier) + + source_datum, projection = self._determine_projection(input) + logger.info( + f"extracted datum: {source_datum}\textracted projection: {projection}" + ) + + # transform to NAD83 when external query points are supplied + if source_datum != self.DEFAULT_DEST_DATUM: + proj = Transformer.from_crs( + source_datum, self.DEFAULT_DEST_DATUM, always_xy=True + ) + for pt in results: + x_proj, y_proj = proj.transform(*pt.lonlat) + pt.lonlat = (x_proj, y_proj) + + scale_value = input.get_data(SCALE_VALUE_OUTPUT_KEY) + if not scale_value: + scale_value = 0 + + rmse, scale_error = self._score_query_points(query_pts, scale_value=scale_value) + logger.info(f"rmse: {rmse} scale error: {scale_error}") + + result = super()._create_result(input) + result.output["query_pts"] = results + result.output["rmse"] = rmse + result.output["error_scale"] = scale_error + result.output["datum"] = self.DEFAULT_DEST_DATUM + result.output["projection"] = projection + return result + def _count_keypoints( self, points: Dict[Tuple[float, float], Coordinate] ) -> Dict[str, int]: @@ -438,14 +564,16 @@ def _update_hemispheres( abs(qp.lonlat[0]) * lon_multiplier, abs(qp.lonlat[1]) * lat_multiplier, ) - qp.lonlat_xp = ( - abs(qp.lonlat_xp[0]) * lon_multiplier, - abs(qp.lonlat_xp[1]) * lat_multiplier, - ) - qp.lonlat_yp = ( - abs(qp.lonlat_yp[0]) * lon_multiplier, - abs(qp.lonlat_yp[1]) * lat_multiplier, - ) + if "lonlat_xp" in qp.properties: + qp.lonlat_xp = ( + abs(qp.lonlat_xp[0]) * lon_multiplier, + abs(qp.lonlat_xp[1]) * lat_multiplier, + ) + if "lonlat_yp" in qp.properties: + qp.lonlat_yp = ( + abs(qp.lonlat_yp[0]) * lon_multiplier, + abs(qp.lonlat_yp[1]) * lat_multiplier, + ) return query_pts @@ -561,8 +689,14 @@ def _score_query_points( latlon = (qp.lonlat[1], qp.lonlat[0]) err_dist = geodesic(latlon_gtruth, latlon).km qp.error_km = err_dist - qp.dist_xp_km = geodesic(latlon, (qp.lonlat_xp[1], qp.lonlat_xp[0])).km - qp.dist_yp_km = geodesic(latlon, (qp.lonlat_yp[1], qp.lonlat_yp[0])).km + if "lonlat_xp" in qp.properties: + qp.dist_xp_km = geodesic( + latlon, (qp.lonlat_xp[1], qp.lonlat_xp[0]) + ).km + if "lonlat_yp" in qp.properties: + qp.dist_yp_km = geodesic( + latlon, (qp.lonlat_yp[1], qp.lonlat_yp[0]) + ).km qp.error_lonlat = ( latlon[1] - latlon_gtruth[1], latlon[0] - latlon_gtruth[0], From 67d79078aaebcdf03b7929db48f9b3da5f708e55 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sun, 21 Jul 2024 11:47:24 -0400 Subject: [PATCH 40/66] cleans up dependencies --- cdr/pyproject.toml | 2 +- tasks/pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cdr/pyproject.toml b/cdr/pyproject.toml index ab7e878f..db171495 100644 --- a/cdr/pyproject.toml +++ b/cdr/pyproject.toml @@ -7,7 +7,7 @@ name = "lara-cdr" version = "0.1.0" description = "LARA CDR integration supporting both one-off processing and webhook event-driven processing" readme = "README.md" -dependencies = ["jsons", "flask", "lara-tasks", "mypy-boto3-s3", "rasterio", "ngrok", "pyproj", "coloredlogs"] +dependencies = ["jsons", "flask", "lara-tasks", "mypy-boto3-s3", "ngrok", "pyproj"] [project.optional-dependencies] development = [ diff --git a/tasks/pyproject.toml b/tasks/pyproject.toml index 170dd9c0..5db3c830 100644 --- a/tasks/pyproject.toml +++ b/tasks/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "stateplane", "coloredlogs", "cdrc @ git+https://github.com/DARPA-CRITICALMAAS/cdrc.git@main", + "rasterio" ] [project.optional-dependencies] From fd480fb2f4fab859d53286db7c5c396c93b8a4da Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Mon, 22 Jul 2024 13:59:30 -0400 Subject: [PATCH 41/66] GCP output now driven by corner points when available + datum/proj/crs terminology cleanup --- pipelines/geo_referencing/factory.py | 3 +- pipelines/geo_referencing/output.py | 163 ++++++++---------- .../geo_referencing/corner_point_extractor.py | 7 +- tasks/geo_referencing/entities.py | 2 +- tasks/geo_referencing/georeference.py | 61 ++++--- .../metadata_extraction.py | 9 +- 6 files changed, 117 insertions(+), 128 deletions(-) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index e3e90168..c3da16e8 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -7,7 +7,6 @@ OutputCreator, GCPOutput, GeoReferencingOutput, - IntegrationOutput, UserLeverOutput, SummaryOutput, ) @@ -321,7 +320,6 @@ def create_geo_referencing_pipelines( SummaryOutput("summary"), UserLeverOutput("levers"), GCPOutput("gcps"), - IntegrationOutput("schema"), ], tasks, ) @@ -692,6 +690,7 @@ def create_geo_referencing_pipeline( tasks.append(OutlierFilter("utm-outliers")) tasks.append(UTMStatePlaneFilter("utm-state-plane")) tasks.append(PointGeocoder("geocoded-georeferencing", ["point", "population"], 10)) + tasks.append(CornerPointExtractor("corner_point_extractor")) tasks.append(InferenceCoordinateExtractor("coordinate-inference")) tasks.append(ScaleExtractor("scale_extractor", "")) tasks.append(CreateGroundControlPoints("gcp_creator")) diff --git a/pipelines/geo_referencing/output.py b/pipelines/geo_referencing/output.py index 8c633a75..54c0fc04 100644 --- a/pipelines/geo_referencing/output.py +++ b/pipelines/geo_referencing/output.py @@ -1,7 +1,10 @@ +from math import pi import jsons import os import uuid +import pip + from tasks.common.pipeline import ( BaseModelOutput, ObjectOutput, @@ -11,6 +14,7 @@ PipelineResult, ) from tasks.geo_referencing.entities import ( + CORNER_POINTS_OUTPUT_KEY, GeoreferenceResult, GroundControlPoint as LARAGroundControlPoint, SOURCE_GEOCODE, @@ -22,14 +26,7 @@ from typing import Any, Dict, List - -def get_projection(datum: str) -> str: - # get espg code via basic lookup of the two frequently seen datums - if "83" in datum: - return "EPSG:4269" - elif "27" in datum: - return "EPSG:4267" - return "EPSG:4326" +from tasks.geo_referencing.georeference import QueryPoint class GeoReferencingOutput(OutputCreator): @@ -233,35 +230,47 @@ def __init__(self, id: str): def create_output(self, pipeline_result: PipelineResult) -> Output: assert pipeline_result.image is not None - query_points = pipeline_result.data["query_pts"] - projection_raw = pipeline_result.data["projection"] - datum_raw = pipeline_result.data["datum"] - projection_mapped = get_projection(datum_raw) + crs = pipeline_result.data["crs"] + query_points: List[QueryPoint] = pipeline_result.data["query_pts"] + corner_points: List[LARAGroundControlPoint] = pipeline_result.data[ + CORNER_POINTS_OUTPUT_KEY + ] res = ObjectOutput(pipeline_result.pipeline_id, pipeline_result.pipeline_name) res.data = { "map": pipeline_result.raster_id, - "crs": [projection_mapped], - "datum_raw": datum_raw, - "projection_raw": projection_raw, + "crs": [crs], "image_height": pipeline_result.image.size[1], "image_width": pipeline_result.image.size[0], "gcps": [], "levers": [], } - for qp in query_points: - o = { - "crs": projection_mapped, - "gcp_id": uuid.uuid4(), - "rowb": qp.xy[1], - "coll": qp.xy[0], - "x": qp.lonlat[0], - "y": qp.lonlat[1], - } - if qp.properties and len(qp.properties) > 0: - o["properties"] = qp.properties - res.data["gcps"].append(o) + + if corner_points: + for cp in corner_points: + o = { + "crs": crs, + "gcp_id": cp.id, + "rowb": cp.pixel_y, + "coll": cp.pixel_x, + "x": cp.longitude, + "y": cp.latitude, + } + res.data["gcps"].append(o) + else: + for qp in query_points: + o = { + "crs": crs, + "gcp_id": uuid.uuid4(), + "rowb": qp.xy[1], + "coll": qp.xy[0], + "x": qp.lonlat[0], + "y": qp.lonlat[1], + } + if qp.properties and len(qp.properties) > 0: + o["properties"] = qp.properties + res.data["gcps"].append(o) # extract the levers available via params for p in pipeline_result.params: @@ -269,46 +278,6 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: return res -class IntegrationOutput(OutputCreator): - def __init__(self, id: str): - super().__init__(id) - - def create_output(self, pipeline_result: PipelineResult) -> Output: - # capture query points as output - query_points = pipeline_result.data["query_pts"] - projection_raw = pipeline_result.data["projection"] - datum_raw = pipeline_result.data["datum"] - projection_mapped = get_projection(datum_raw) - - res = ObjectOutput(pipeline_result.pipeline_id, pipeline_result.pipeline_name) - - # need to format the data to meet the schema definition - gcps = [] - count = 0 - for qp in query_points: - count = count + 1 - o = { - "id": count, - "map_geom": (qp.lonlat[0], qp.lonlat[1]), - "px_geom": (qp.xy[0], qp.xy[1]), - "confidence": qp.confidence, - "provenance": "modelled", - } - gcps.append(o) - res.data = { - "map": { - "name": pipeline_result.raster_id, - "projection_info": { - "projection": projection_mapped, - "provenance": "modelled", - "gcps": gcps, - }, - } - } - - return res - - class LARAModelOutput(OutputCreator): def __init__(self, id: str): super().__init__(id) @@ -316,33 +285,45 @@ def __init__(self, id: str): def create_output(self, pipeline_result: PipelineResult) -> Output: # capture query points as output query_points = pipeline_result.data["query_pts"] - projection_raw = pipeline_result.data["projection"] - datum_raw = pipeline_result.data["datum"] - projection_mapped = get_projection(datum_raw) - - gcps = [] - count = 0 + crs = pipeline_result.data["crs"] + corner_points: List[LARAGroundControlPoint] = pipeline_result.data[ + CORNER_POINTS_OUTPUT_KEY + ] confidence = 0 - for qp in query_points: - count = count + 1 - gcp = LARAGroundControlPoint( - id=f"gcp-{count}", - pixel_x=qp.xy[0], - pixel_y=qp.xy[1], - latitude=qp.lonlat[1], - longitude=qp.lonlat[0], - confidence=qp.confidence, - ) - gcps.append(gcp) - confidence = qp.confidence + + gcps: List[LARAGroundControlPoint] = [] result = GeoreferenceResult( map_id=pipeline_result.raster_id, gcps=gcps, - projection=projection_mapped, + crs=crs, provenance="modelled", confidence=confidence, ) + + if corner_points: + for i, cp in enumerate(corner_points): + o = LARAGroundControlPoint( + id=f"gpc.{i}", + pixel_x=cp.pixel_x, + pixel_y=cp.pixel_y, + latitude=cp.latitude, + longitude=cp.longitude, + confidence=cp.confidence, + ) + gcps.append(o) + else: + for i, qp in enumerate(query_points): + o = LARAGroundControlPoint( + id=f"gcp.{i}", + pixel_x=qp.xy[0], + pixel_y=qp.xy[1], + latitude=qp.lonlat[1], + longitude=qp.lonlat[0], + confidence=qp.confidence, + ) + gcps.append(o) + return BaseModelOutput( pipeline_result.pipeline_id, pipeline_result.pipeline_name, result ) @@ -355,9 +336,7 @@ def __init__(self, id: str): def create_output(self, pipeline_result: PipelineResult) -> Output: assert pipeline_result.image is not None query_points = pipeline_result.data["query_pts"] - projection_raw = pipeline_result.data["projection"] - datum_raw = pipeline_result.data["datum"] - projection_mapped = get_projection(datum_raw) + crs = pipeline_result.data["crs"] res = ObjectOutput(pipeline_result.pipeline_id, pipeline_result.pipeline_name) @@ -370,7 +349,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: "confidence": qp.confidence, "model": "uncharted", "model_version": "0.0.1", - "crs": projection_mapped, + "crs": crs, } gcps.append(o) @@ -378,11 +357,11 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: "cog_id": pipeline_result.raster_id, "georeference_results": [ { - "likely_CRSs": [projection_mapped], + "likely_CRSs": [crs], "map_area": None, "projections": [ { - "crs": projection_mapped, + "crs": crs, "gcp_ids": [gcp["gcp_id"] for gcp in gcps], "file_name": f"lara-{pipeline_result.raster_id}.tif", } diff --git a/tasks/geo_referencing/corner_point_extractor.py b/tasks/geo_referencing/corner_point_extractor.py index fb340437..0225543d 100644 --- a/tasks/geo_referencing/corner_point_extractor.py +++ b/tasks/geo_referencing/corner_point_extractor.py @@ -4,9 +4,6 @@ from shapely import LineString, Point -from pipelines.metadata_extraction.metadata_extraction_pipeline import ( - MetadataExtractionOutput, -) from tasks.common.task import Task, TaskInput, TaskResult from tasks.geo_referencing.entities import ( Coordinate, @@ -14,7 +11,7 @@ CORNER_POINTS_OUTPUT_KEY, ) -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Tuple logger = logging.getLogger("corner_point_extractor") @@ -104,7 +101,7 @@ def run(self, input: TaskInput) -> TaskResult: geo_point = point[0] pixel_point = point[1] gcp = GroundControlPoint( - id=f"corner.{str(i)}", + id=f"corner_point.{str(i)}", longitude=geo_point.x, latitude=geo_point.y, pixel_x=pixel_point.x, diff --git a/tasks/geo_referencing/entities.py b/tasks/geo_referencing/entities.py index 02f6a09b..7c2ed989 100644 --- a/tasks/geo_referencing/entities.py +++ b/tasks/geo_referencing/entities.py @@ -35,7 +35,7 @@ class GroundControlPoint(BaseModel): class GeoreferenceResult(BaseModel): map_id: str - projection: str + crs: str gcps: List[GroundControlPoint] provenance: str confidence: float diff --git a/tasks/geo_referencing/georeference.py b/tasks/geo_referencing/georeference.py index d7c1fc29..ae498a3a 100644 --- a/tasks/geo_referencing/georeference.py +++ b/tasks/geo_referencing/georeference.py @@ -79,7 +79,7 @@ class GeoReference(Task): _poly_order = 1 # default destination datum for georeferencing output - DEFAULT_DEST_DATUM = "EPSG:4269" + DEFAULT_DEST_CRS = "EPSG:4269" def __init__(self, task_id: str, poly_order: int = 1): super().__init__(task_id) @@ -250,15 +250,15 @@ def _run_label_georef(self, input: TaskInput) -> TaskResult: # results = self._clip_query_pts(query_pts, lon_minmax, lat_minmax) results = self._update_hemispheres(query_pts, lon_multiplier, lat_multiplier) - source_datum, projection = self._determine_projection(input) + crs = self._determine_crs(input) logger.info( - f"extracted datum: {source_datum}\textracted projection: {projection}" + f"extracted CRS: {crs}" ) - if source_datum != self.DEFAULT_DEST_DATUM: + if crs != self.DEFAULT_DEST_CRS: for qp in query_pts: proj = Transformer.from_crs( - source_datum, self.DEFAULT_DEST_DATUM, always_xy=True + crs, self.DEFAULT_DEST_CRS, always_xy=True ) x_p, y_p = proj.transform(qp.lonlat[0], qp.lonlat[1]) qp.lonlat = (x_p, y_p) @@ -270,8 +270,7 @@ def _run_label_georef(self, input: TaskInput) -> TaskResult: result.output["query_pts"] = results result.output["rmse"] = rmse result.output["error_scale"] = scale_error - result.output["datum"] = self.DEFAULT_DEST_DATUM - result.output["projection"] = projection + result.output["crs"] = self.DEFAULT_DEST_CRS result.output["keypoints"] = keypoint_stats return result @@ -322,15 +321,15 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: ) results = self._update_hemispheres(query_pts, lon_multiplier, lat_multiplier) - source_datum, projection = self._determine_projection(input) + crs, projection = self._determine_crs(input) logger.info( - f"extracted datum: {source_datum}\textracted projection: {projection}" + f"extracted crs: {crs}" ) # transform to NAD83 when external query points are supplied - if source_datum != self.DEFAULT_DEST_DATUM: + if crs != self.DEFAULT_DEST_CRS: proj = Transformer.from_crs( - source_datum, self.DEFAULT_DEST_DATUM, always_xy=True + crs, self.DEFAULT_DEST_CRS, always_xy=True ) for pt in results: x_proj, y_proj = proj.transform(*pt.lonlat) @@ -347,8 +346,7 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: result.output["query_pts"] = results result.output["rmse"] = rmse result.output["error_scale"] = scale_error - result.output["datum"] = self.DEFAULT_DEST_DATUM - result.output["projection"] = projection + result.output["crs"] = self.DEFAULT_DEST_CRS return result def _count_keypoints( @@ -577,8 +575,8 @@ def _update_hemispheres( return query_pts - def _determine_projection(self, input: TaskInput) -> Tuple[str, str]: - logger.info("determining projection for georeferencing") + def _determine_crs(self, input: TaskInput) -> str: + logger.info("determining CRS for georeferencing") # parse extracted metadata metadata = input.parse_data( METADATA_EXTRACTION_OUTPUT_KEY, MetadataExtraction.model_validate @@ -586,18 +584,31 @@ def _determine_projection(self, input: TaskInput) -> Tuple[str, str]: # make sure there is metadata if not metadata: - return "", "" + return self.DEFAULT_DEST_CRS + # we we assume geographic coordinates and combine that with the datum to + # come up with a CRS datum = metadata.datum - if not datum or len(datum) == 0: - year = metadata.year - if year >= "1985": - datum = "NAD83" - if year >= "1930": - datum = "NAD27" - - # return the datum and the projection - return datum, metadata.projection + if datum is not None and datum != "NULL": + if datum.contains("NAD") and metadata.year >= "1985": + return "EPSG:4269" + if datum.contains("NAD") and metadata.year >= "1930": + return "EPSG:4267" + # default to a WGS84 CRS + return "EPSG:4326" + + # no datum info in the metadata so we will use the country and year + if not datum or datum == "NULL" or len(datum) == 0: + if metadata.country is not "NULL" and ( + metadata.country == "US" or metadata.country == "CA" + ): + if metadata.year >= "1985": + return "EPSG:4269" + if metadata.year >= "1930": + return "EPSG:4267" + + # default to a WGS84 CRS when all else fails + return "EPSG:4326" def _build_fallback( self, diff --git a/tasks/metadata_extraction/metadata_extraction.py b/tasks/metadata_extraction/metadata_extraction.py index 1f9ea3ba..e0fb4e91 100644 --- a/tasks/metadata_extraction/metadata_extraction.py +++ b/tasks/metadata_extraction/metadata_extraction.py @@ -72,8 +72,8 @@ class MetdataLLM(BaseModel): description="The scale of the map. Example: '1:24000'", default="NULL" ) datum: str = Field( - description="The geoditic datum of the map expressed as an EPSG code. If this is not present, it can often be inferred from the map's country and year." - + "Examples of geodetic datums: 'North American Datum of 1927', 'NAD83', 'WGS 84'. Examples of output: 'EPSG:4269', 'EPSG:4326'", + description="The geoditic datum of the map. If this is not present, it can often be inferred from the map's country and year." + + "Examples of geodetic datums: 'North American Datum of 1927', 'NAD83', 'WGS 84'", default="NULL", ) vertical_datum: str = Field( @@ -299,6 +299,7 @@ def run(self, input: TaskInput) -> TaskResult: logger.info(f"Running metadata extraction task for '{input.raster_id}'") if self._should_run and not self._should_run(input): + logging.info("Skipping metadata extraction task") return self._create_result(input) task_result = TaskResult(self._task_id) @@ -308,7 +309,9 @@ def run(self, input: TaskInput) -> TaskResult: self._create_empty_extraction, ) if not doc_text: - logger.info("returning empty metadata extraction result") + logger.info( + "OCR output not available - returning empty metadata extraction result" + ) return task_result doc_id = self._generate_doc_key(input, doc_text) From 094b3a327f9390a7bf8ad8ad132c788460a84e3c Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Tue, 23 Jul 2024 01:38:49 -0400 Subject: [PATCH 42/66] various fixes for CDR output --- cdr/result_subscriber.py | 27 +++++++++++++------ pipelines/geo_referencing/factory.py | 12 +++------ pipelines/geo_referencing/run_pipeline.py | 5 ---- schema/mappers/cdr.py | 6 ++--- .../geo_referencing/corner_point_extractor.py | 6 +++-- tasks/geo_referencing/georeference.py | 20 +++++--------- 6 files changed, 36 insertions(+), 40 deletions(-) diff --git a/cdr/result_subscriber.py b/cdr/result_subscriber.py index f08707a9..a053257a 100644 --- a/cdr/result_subscriber.py +++ b/cdr/result_subscriber.py @@ -1,6 +1,7 @@ import json import logging import os +import pprint import threading from time import sleep from typing import List, Optional @@ -362,9 +363,10 @@ def _push_georeferencing(self, result: RequestResult): files_.append( ("files", (output_file_name, open(output_file_name_full, "rb"))) ) - except: - logger.error( - "bad georeferencing result received so creating an empty result to send to cdr" + except Exception as e: + logger.exception( + "bad georeferencing result received so creating an empty result to send to cdr", + e, ) # create an empty result to send to cdr @@ -576,10 +578,19 @@ def _cps_to_transform( ] cps_p = [] for cp in cps: - proj = Transformer.from_crs(cp["crs"], to_crs, always_xy=True) - x_p, y_p = proj.transform(xx=cp["x"], yy=cp["y"]) - cps_p.append( - riot.GroundControlPoint(row=cp["row"], col=cp["col"], x=x_p, y=y_p) - ) + if cp["crs"] != to_crs: + proj = Transformer.from_crs(cp["crs"], to_crs, always_xy=True) + x_p, y_p = proj.transform(xx=cp["x"], yy=cp["y"]) + cps_p.append( + riot.GroundControlPoint(row=cp["row"], col=cp["col"], x=x_p, y=y_p) + ) + else: + cps_p.append( + riot.GroundControlPoint( + row=cp["row"], col=cp["col"], x=cp["x"], y=cp["y"] + ) + ) + print("cps_p:") + pprint.pprint(cps_p) return riot.from_gcps(cps_p) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index c3da16e8..0dcee29c 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -109,8 +109,7 @@ def create_geo_referencing_pipelines( GeoReferencingOutput("geo"), SummaryOutput("summary"), UserLeverOutput("levers"), - GCPOutput("gcps"), - IntegrationOutput("schema"), + GCPOutput("gcps") ], tasks, ) @@ -173,8 +172,7 @@ def create_geo_referencing_pipelines( GeoReferencingOutput("geo"), SummaryOutput("summary"), UserLeverOutput("levers"), - GCPOutput("gcps"), - IntegrationOutput("schema"), + GCPOutput("gcps") ], tasks, ) @@ -430,8 +428,7 @@ def create_geo_referencing_pipelines( GeoReferencingOutput("geo"), SummaryOutput("summary"), UserLeverOutput("levers"), - GCPOutput("gcps"), - IntegrationOutput("schema"), + GCPOutput("gcps") ], tasks, ) @@ -540,8 +537,7 @@ def create_geo_referencing_pipelines( GeoReferencingOutput("geo"), SummaryOutput("summary"), UserLeverOutput("levers"), - GCPOutput("gcps"), - IntegrationOutput("schema"), + GCPOutput("gcps") ], tasks, ) diff --git a/pipelines/geo_referencing/run_pipeline.py b/pipelines/geo_referencing/run_pipeline.py index 97f3a902..571d2023 100644 --- a/pipelines/geo_referencing/run_pipeline.py +++ b/pipelines/geo_referencing/run_pipeline.py @@ -152,11 +152,6 @@ def run_pipelines(parsed, input_data: ImageFileInputIterator): results_summary[pipeline.id].append(output["summary"]) results_levers[pipeline.id].append(output["levers"]) results_gcps[pipeline.id].append(output["gcps"]) - results_integration[pipeline.id].append(output["schema"]) - schema_output_path = os.path.join( - parsed.output, "maps", f"{pipeline.id}", f"{raster_id}.json" - ) - writer_json.output([output["schema"]], {"path": schema_output_path}) # type: ignore logger.info(f"done pipeline {pipeline.id}\n\n") for p in pipelines: diff --git a/schema/mappers/cdr.py b/schema/mappers/cdr.py index bc2c43ed..911568df 100644 --- a/schema/mappers/cdr.py +++ b/schema/mappers/cdr.py @@ -75,7 +75,7 @@ def map_to_cdr(self, model: LARAGeoferenceResult) -> CDRGeoreferenceResults: confidence=gcp.confidence, model=MODEL_NAME, model_version=MODEL_VERSION, - crs=model.projection, + crs=model.crs, ) gcps.append(cdr_gcp) @@ -83,11 +83,11 @@ def map_to_cdr(self, model: LARAGeoferenceResult) -> CDRGeoreferenceResults: cog_id=model.map_id, georeference_results=[ GeoreferenceResult( - likely_CRSs=[model.projection], + likely_CRSs=[model.crs], map_area=None, projections=[ ProjectionResult( - crs=model.projection, + crs=model.crs, gcp_ids=[gcp.gcp_id for gcp in gcps], file_name=f"lara-{model.map_id}.tif", ) diff --git a/tasks/geo_referencing/corner_point_extractor.py b/tasks/geo_referencing/corner_point_extractor.py index 0225543d..d1cc307f 100644 --- a/tasks/geo_referencing/corner_point_extractor.py +++ b/tasks/geo_referencing/corner_point_extractor.py @@ -1,5 +1,6 @@ from curses import meta import logging +import pprint from unittest.mock import DEFAULT from shapely import LineString, Point @@ -49,12 +50,13 @@ def run(self, input: TaskInput) -> TaskResult: lon_pts: Dict[Tuple[float, float], Coordinate] = input.get_data("lons") lat_pts: Dict[Tuple[float, float], Coordinate] = input.get_data("lats") + intersection_points: List[Tuple[Point, Point]] = [] for lon_key, lon_coord in lon_pts.items(): # generate a vertical segment through the center of the lon label lon_bounds = lon_coord.get_bounds() - lon_label_width = lon_bounds[1].x - lon_bounds[0].x + lon_label_width = 2 * (lon_bounds[1].x - lon_bounds[0].x) lon_center_x = lon_bounds[0].x + (lon_bounds[1].x - lon_bounds[0].x) / 2.0 lon_center_y = lon_bounds[0].y + (lon_bounds[2].y - lon_bounds[0].y) / 2.0 lon_line = LineString( @@ -69,7 +71,7 @@ def run(self, input: TaskInput) -> TaskResult: for lat_key, lat_coord in lat_pts.items(): # generate a horizontal segment through the center of the lat label lat_bounds = lat_coord.get_bounds() - lat_label_width = lat_bounds[1].x - lat_bounds[0].x + lat_label_width = 2 * (lat_bounds[1].x - lat_bounds[0].x) lat_center_x = ( lat_bounds[0].x + (lat_bounds[1].x - lat_bounds[0].x) / 2.0 ) diff --git a/tasks/geo_referencing/georeference.py b/tasks/geo_referencing/georeference.py index ae498a3a..9496660e 100644 --- a/tasks/geo_referencing/georeference.py +++ b/tasks/geo_referencing/georeference.py @@ -251,15 +251,11 @@ def _run_label_georef(self, input: TaskInput) -> TaskResult: results = self._update_hemispheres(query_pts, lon_multiplier, lat_multiplier) crs = self._determine_crs(input) - logger.info( - f"extracted CRS: {crs}" - ) + logger.info(f"extracted CRS: {crs}") if crs != self.DEFAULT_DEST_CRS: for qp in query_pts: - proj = Transformer.from_crs( - crs, self.DEFAULT_DEST_CRS, always_xy=True - ) + proj = Transformer.from_crs(crs, self.DEFAULT_DEST_CRS, always_xy=True) x_p, y_p = proj.transform(qp.lonlat[0], qp.lonlat[1]) qp.lonlat = (x_p, y_p) @@ -321,16 +317,12 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: ) results = self._update_hemispheres(query_pts, lon_multiplier, lat_multiplier) - crs, projection = self._determine_crs(input) - logger.info( - f"extracted crs: {crs}" - ) + crs = self._determine_crs(input) + logger.info(f"extracted crs: {crs}") # transform to NAD83 when external query points are supplied if crs != self.DEFAULT_DEST_CRS: - proj = Transformer.from_crs( - crs, self.DEFAULT_DEST_CRS, always_xy=True - ) + proj = Transformer.from_crs(crs, self.DEFAULT_DEST_CRS, always_xy=True) for pt in results: x_proj, y_proj = proj.transform(*pt.lonlat) pt.lonlat = (x_proj, y_proj) @@ -599,7 +591,7 @@ def _determine_crs(self, input: TaskInput) -> str: # no datum info in the metadata so we will use the country and year if not datum or datum == "NULL" or len(datum) == 0: - if metadata.country is not "NULL" and ( + if metadata.country != "NULL" and ( metadata.country == "US" or metadata.country == "CA" ): if metadata.year >= "1985": From e8729062950620a047355f7688ee3c185bba6305 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sat, 10 Aug 2024 14:50:32 -0400 Subject: [PATCH 43/66] assigns correct hemisphere to corner points --- pipelines/geo_referencing/factory.py | 1 - pipelines/geo_referencing/output.py | 1 + tasks/geo_referencing/georeference.py | 23 ++++++++++++++++++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index 0dcee29c..7ae37be8 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -244,7 +244,6 @@ def create_geo_referencing_pipelines( tasks.append(ROIFilter("roiness")) tasks.append(DistinctDegreeOutlierFilter("uniqueness")) tasks.append(HighQualityCoordinateFilter("goodness")) - tasks.append(CornerPointExtractor("corner_point_extractor")) tasks.append(OutlierFilter("fourth")) tasks.append(NaiveFilter("fun")) if extract_metadata: diff --git a/pipelines/geo_referencing/output.py b/pipelines/geo_referencing/output.py index 54c0fc04..2a9c5d02 100644 --- a/pipelines/geo_referencing/output.py +++ b/pipelines/geo_referencing/output.py @@ -324,6 +324,7 @@ def create_output(self, pipeline_result: PipelineResult) -> Output: ) gcps.append(o) + result.gcps = gcps return BaseModelOutput( pipeline_result.pipeline_id, pipeline_result.pipeline_name, result ) diff --git a/tasks/geo_referencing/georeference.py b/tasks/geo_referencing/georeference.py index 9496660e..0468ec94 100644 --- a/tasks/geo_referencing/georeference.py +++ b/tasks/geo_referencing/georeference.py @@ -316,6 +316,9 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: f"derived hemispheres for georeferencing: {lon_multiplier},{lat_multiplier}" ) results = self._update_hemispheres(query_pts, lon_multiplier, lat_multiplier) + corner_points = self._update_hemispheres_corners( + corner_points, lon_multiplier, lat_multiplier + ) crs = self._determine_crs(input) logger.info(f"extracted crs: {crs}") @@ -339,6 +342,7 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: result.output["rmse"] = rmse result.output["error_scale"] = scale_error result.output["crs"] = self.DEFAULT_DEST_CRS + result.output[CORNER_POINTS_OUTPUT_KEY] = corner_points return result def _count_keypoints( @@ -567,10 +571,23 @@ def _update_hemispheres( return query_pts + def _update_hemispheres_corners( + self, + corner_points: List[GroundControlPoint], + lon_multiplier: float, + lat_multiplier: float, + ) -> List[GroundControlPoint]: + logger.info("updating corner point hemispheres for georeferencing") + for cp in corner_points: + cp.longitude = abs(cp.longitude) * lon_multiplier + cp.latitude = abs(cp.latitude) * lat_multiplier + + return corner_points + def _determine_crs(self, input: TaskInput) -> str: logger.info("determining CRS for georeferencing") # parse extracted metadata - metadata = input.parse_data( + metadata: MetadataExtraction = input.parse_data( METADATA_EXTRACTION_OUTPUT_KEY, MetadataExtraction.model_validate ) @@ -582,9 +599,9 @@ def _determine_crs(self, input: TaskInput) -> str: # come up with a CRS datum = metadata.datum if datum is not None and datum != "NULL": - if datum.contains("NAD") and metadata.year >= "1985": + if "NAD" in datum and metadata.year >= "1985": return "EPSG:4269" - if datum.contains("NAD") and metadata.year >= "1930": + if "NAD" in datum and metadata.year >= "1930": return "EPSG:4267" # default to a WGS84 CRS return "EPSG:4326" From fd63c038d136a9cc7b766de2b91ac1e3f002c0c1 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sat, 10 Aug 2024 18:17:52 -0400 Subject: [PATCH 44/66] adds bounds check to corner point detector --- tasks/geo_referencing/corner_point_extractor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tasks/geo_referencing/corner_point_extractor.py b/tasks/geo_referencing/corner_point_extractor.py index d1cc307f..aeaf48bb 100644 --- a/tasks/geo_referencing/corner_point_extractor.py +++ b/tasks/geo_referencing/corner_point_extractor.py @@ -50,12 +50,14 @@ def run(self, input: TaskInput) -> TaskResult: lon_pts: Dict[Tuple[float, float], Coordinate] = input.get_data("lons") lat_pts: Dict[Tuple[float, float], Coordinate] = input.get_data("lats") - intersection_points: List[Tuple[Point, Point]] = [] for lon_key, lon_coord in lon_pts.items(): # generate a vertical segment through the center of the lon label lon_bounds = lon_coord.get_bounds() + if len(lon_bounds) < 4: + continue + lon_label_width = 2 * (lon_bounds[1].x - lon_bounds[0].x) lon_center_x = lon_bounds[0].x + (lon_bounds[1].x - lon_bounds[0].x) / 2.0 lon_center_y = lon_bounds[0].y + (lon_bounds[2].y - lon_bounds[0].y) / 2.0 @@ -71,6 +73,9 @@ def run(self, input: TaskInput) -> TaskResult: for lat_key, lat_coord in lat_pts.items(): # generate a horizontal segment through the center of the lat label lat_bounds = lat_coord.get_bounds() + if len(lat_bounds) < 4: + continue + lat_label_width = 2 * (lat_bounds[1].x - lat_bounds[0].x) lat_center_x = ( lat_bounds[0].x + (lat_bounds[1].x - lat_bounds[0].x) / 2.0 From cd37d832dc1779dd7795ac47add1b54560372358 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sat, 10 Aug 2024 18:18:23 -0400 Subject: [PATCH 45/66] improves parsing of datum until metadata extract is generating properly --- tasks/geo_referencing/georeference.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tasks/geo_referencing/georeference.py b/tasks/geo_referencing/georeference.py index 0468ec94..46f254af 100644 --- a/tasks/geo_referencing/georeference.py +++ b/tasks/geo_referencing/georeference.py @@ -597,12 +597,17 @@ def _determine_crs(self, input: TaskInput) -> str: # we we assume geographic coordinates and combine that with the datum to # come up with a CRS - datum = metadata.datum + datum = metadata.datum.lower() if datum is not None and datum != "NULL": - if "NAD" in datum and metadata.year >= "1985": - return "EPSG:4269" - if "NAD" in datum and metadata.year >= "1930": - return "EPSG:4267" + if "nad" in datum or "north american" in datum: + if "27" or "1927" in datum: + return "EPSG:4267" + if "83" or "1983" in datum: + return "EPSG:4269" + if metadata.year >= "1985": + return "EPSG:4269" + if metadata.year >= "1930": + return "EPSG:4267" # default to a WGS84 CRS return "EPSG:4326" From 0444c98356bd2aaeef3a1850a8de0cb720ad111f Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sat, 10 Aug 2024 21:09:42 -0400 Subject: [PATCH 46/66] only enable geolocation via corner point when we have 4 available --- tasks/geo_referencing/corner_point_extractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/geo_referencing/corner_point_extractor.py b/tasks/geo_referencing/corner_point_extractor.py index aeaf48bb..09a607e9 100644 --- a/tasks/geo_referencing/corner_point_extractor.py +++ b/tasks/geo_referencing/corner_point_extractor.py @@ -99,7 +99,7 @@ def run(self, input: TaskInput) -> TaskResult: (Point(lon_key[0], lat_key[0]), intersection) ) - if len(intersection_points) >= 3: + if len(intersection_points) >= 4: logger.info(f"Found {len(intersection_points)} corner points") output: List[GroundControlPoint] = [] @@ -118,7 +118,7 @@ def run(self, input: TaskInput) -> TaskResult: output.append(gcp) else: logger.info( - f"Found {len(intersection_points)} corner points, require at least 3. Corner point referencing not available." + f"Found {len(intersection_points)} corner points, require 4. Corner point referencing not available." ) output = [] From 4552e505283da089644dbf2880a00d803c53c323 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sun, 11 Aug 2024 00:49:52 -0400 Subject: [PATCH 47/66] allows gpu to be forced off for georef pipeline runs --- pipelines/geo_referencing/factory.py | 4 ++++ pipelines/geo_referencing/run_pipeline.py | 2 ++ pipelines/geo_referencing/run_server.py | 2 ++ 3 files changed, 8 insertions(+) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index 7ae37be8..b58b1a70 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -75,6 +75,7 @@ def create_geo_referencing_pipelines( state_code_filename: str, country_code_filename: str, ocr_gamma_correction: float, + gpu_enabled: bool, ) -> List[Pipeline]: geocoding_cache_bounds = os.path.join(working_dir, "geocoding_cache_bounds.json") geocoding_cache_points = os.path.join(working_dir, "geocoding_cache_points.json") @@ -201,6 +202,7 @@ def create_geo_referencing_pipelines( segmentation_model_path, segmentation_cache, confidence_thres=0.25, + gpu=gpu_enabled, ) ) tasks.append( @@ -554,6 +556,7 @@ def create_geo_referencing_pipeline( state_code_filename: str, country_code_filename: str, ocr_gamma_correction: float, + gpu_enabled: bool, ) -> Pipeline: geocoding_cache_bounds = os.path.join(working_dir, "geocoding_cache_bounds.json") geocoding_cache_points = os.path.join(working_dir, "geocoding_cache_points.json") @@ -593,6 +596,7 @@ def create_geo_referencing_pipeline( segmentation_model_path, segmentation_cache, confidence_thres=0.25, + gpu=gpu_enabled, ) ) tasks.append( diff --git a/pipelines/geo_referencing/run_pipeline.py b/pipelines/geo_referencing/run_pipeline.py index 571d2023..94e910e5 100644 --- a/pipelines/geo_referencing/run_pipeline.py +++ b/pipelines/geo_referencing/run_pipeline.py @@ -72,6 +72,7 @@ def main(): type=float, default=0.5, ) + parser.add_argument("--no_gpu", action="store_true") p = parser.parse_args() # setup an input stream @@ -115,6 +116,7 @@ def run_pipelines(parsed, input_data: ImageFileInputIterator): parsed.state_code_filename, parsed.country_code_filename, parsed.ocr_gamma_correction, + not parsed.no_gpu, ) # get file paths diff --git a/pipelines/geo_referencing/run_server.py b/pipelines/geo_referencing/run_server.py index 5f723a9f..49b76dae 100644 --- a/pipelines/geo_referencing/run_server.py +++ b/pipelines/geo_referencing/run_server.py @@ -158,6 +158,7 @@ def start_server(): type=float, default=0.5, ) + parser.add_argument("--no_gpu", action="store_true") p = parser.parse_args() global georef_pipeline @@ -170,6 +171,7 @@ def start_server(): p.state_code_filename, p.country_code_filename, p.ocr_gamma_correction, + not p.no_gpu, ) #### start flask server or startup up the message queue From ec28cef5b6ef04e71aec0d4a28dde607f0eaf51e Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sun, 11 Aug 2024 12:01:37 -0400 Subject: [PATCH 48/66] uses non-random geo points and enforces upper case in CRS derivation --- pipelines/geo_referencing/factory.py | 2 +- tasks/geo_referencing/georeference.py | 43 ++++++++++++++++++--------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/pipelines/geo_referencing/factory.py b/pipelines/geo_referencing/factory.py index b58b1a70..1b9591e1 100644 --- a/pipelines/geo_referencing/factory.py +++ b/pipelines/geo_referencing/factory.py @@ -692,6 +692,6 @@ def create_geo_referencing_pipeline( tasks.append(CornerPointExtractor("corner_point_extractor")) tasks.append(InferenceCoordinateExtractor("coordinate-inference")) tasks.append(ScaleExtractor("scale_extractor", "")) - tasks.append(CreateGroundControlPoints("gcp_creator")) + tasks.append(CreateGroundControlPoints("gcp_creator", create_random_pts=False)) tasks.append(GeoReference("geo_referencer", 1)) return Pipeline("wally-finder", "wally-finder", outputs, tasks) diff --git a/tasks/geo_referencing/georeference.py b/tasks/geo_referencing/georeference.py index 46f254af..b1c1bf77 100644 --- a/tasks/geo_referencing/georeference.py +++ b/tasks/geo_referencing/georeference.py @@ -282,6 +282,8 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: Returns: TaskResult: The result of the task. """ + + # get the corner points if they exist result = super()._create_result(input) corner_points: List[GroundControlPoint] = input.get_data( CORNER_POINTS_OUTPUT_KEY @@ -290,6 +292,8 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: logger.error("corner points not provided") return result + # convert the corner points into raster io GCP objects and create a transformation + # that we will use to transform the query points gcps: List[riot.GroundControlPoint] = [] for cp in corner_points: gcp = riot.GroundControlPoint( @@ -305,40 +309,45 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: query_pts = input.get_request_info("query_pts") # transform the query pixel points into geo locations and write the result into - # the query point + # the query points for qp in query_pts: lonlat: Tuple[float, float] = riot.xy(transform, qp.xy[1], qp.xy[0]) qp.lonlat = lonlat - # correct the hemisphere of the points + # correct the hemisphere of the resulting query points lon_multiplier, lat_multiplier = self._determine_hemispheres(input, query_pts) logger.info( f"derived hemispheres for georeferencing: {lon_multiplier},{lat_multiplier}" ) - results = self._update_hemispheres(query_pts, lon_multiplier, lat_multiplier) + proj_query_points = self._update_hemispheres( + query_pts, lon_multiplier, lat_multiplier + ) + + # correct the hemisphere of the corner points since it hasnt' been done before now corner_points = self._update_hemispheres_corners( corner_points, lon_multiplier, lat_multiplier ) + # get the source CRS for corner points crs = self._determine_crs(input) logger.info(f"extracted crs: {crs}") - # transform to NAD83 when external query points are supplied + # perform a datum shift on the query points if the source CRS is not the default if crs != self.DEFAULT_DEST_CRS: proj = Transformer.from_crs(crs, self.DEFAULT_DEST_CRS, always_xy=True) - for pt in results: + for pt in proj_query_points: x_proj, y_proj = proj.transform(*pt.lonlat) pt.lonlat = (x_proj, y_proj) + # calculate the RMSE and scale error for the query points scale_value = input.get_data(SCALE_VALUE_OUTPUT_KEY) if not scale_value: scale_value = 0 - rmse, scale_error = self._score_query_points(query_pts, scale_value=scale_value) logger.info(f"rmse: {rmse} scale error: {scale_error}") result = super()._create_result(input) - result.output["query_pts"] = results + result.output["query_pts"] = proj_query_points result.output["rmse"] = rmse result.output["error_scale"] = scale_error result.output["crs"] = self.DEFAULT_DEST_CRS @@ -587,7 +596,7 @@ def _update_hemispheres_corners( def _determine_crs(self, input: TaskInput) -> str: logger.info("determining CRS for georeferencing") # parse extracted metadata - metadata: MetadataExtraction = input.parse_data( + metadata: Optional[MetadataExtraction] = input.parse_data( METADATA_EXTRACTION_OUTPUT_KEY, MetadataExtraction.model_validate ) @@ -595,18 +604,24 @@ def _determine_crs(self, input: TaskInput) -> str: if not metadata: return self.DEFAULT_DEST_CRS + # grab the year from the metadata if present + try: + year = int(metadata.year) + except: + year = -1 + # we we assume geographic coordinates and combine that with the datum to # come up with a CRS - datum = metadata.datum.lower() + datum = metadata.datum.upper() if datum is not None and datum != "NULL": - if "nad" in datum or "north american" in datum: + if "NAD" in datum or "NORTH AMERICAN" in datum: if "27" or "1927" in datum: return "EPSG:4267" if "83" or "1983" in datum: return "EPSG:4269" - if metadata.year >= "1985": + if year >= 1985: return "EPSG:4269" - if metadata.year >= "1930": + if year >= 1930: return "EPSG:4267" # default to a WGS84 CRS return "EPSG:4326" @@ -616,9 +631,9 @@ def _determine_crs(self, input: TaskInput) -> str: if metadata.country != "NULL" and ( metadata.country == "US" or metadata.country == "CA" ): - if metadata.year >= "1985": + if year >= 1985: return "EPSG:4269" - if metadata.year >= "1930": + if year >= 1930: return "EPSG:4267" # default to a WGS84 CRS when all else fails From 7766af6fa927d00b12930556bd3ee572f06f7705 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sun, 11 Aug 2024 17:35:02 -0400 Subject: [PATCH 49/66] applies datum shift only when using external query points --- cdr/result_subscriber.py | 6 ++++- tasks/geo_referencing/georeference.py | 39 ++++++++++++++++++++------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/cdr/result_subscriber.py b/cdr/result_subscriber.py index a053257a..2be76f02 100644 --- a/cdr/result_subscriber.py +++ b/cdr/result_subscriber.py @@ -89,6 +89,10 @@ class LaraResultSubscriber: GEOREFERENCE_PIPELINE: "uncharted-georeference", } + + # output CRS to use for projected maps that are pushed to CDR + DEFAULT_OUTPUT_CRS = "EPSG:3857" + def __init__( self, request_publisher: Optional[LaraRequestPublisher], @@ -357,7 +361,7 @@ def _push_georeferencing(self, result: RequestResult): f"projecting image {result.image_path} to {output_file_name_full} using crs {projection.crs}" ) self._project_georeference( - result.image_path, output_file_name_full, projection.crs, gcps + result.image_path, output_file_name_full, self.DEFAULT_OUTPUT_CRS, gcps ) files_.append( diff --git a/tasks/geo_referencing/georeference.py b/tasks/geo_referencing/georeference.py index b1c1bf77..6b294d9d 100644 --- a/tasks/geo_referencing/georeference.py +++ b/tasks/geo_referencing/georeference.py @@ -78,8 +78,8 @@ def __init__(self, pixel_coord, geo_coord): class GeoReference(Task): _poly_order = 1 - # default destination datum for georeferencing output - DEFAULT_DEST_CRS = "EPSG:4269" + # externally supplied query points will be transformed to this CRS (NAD83 for contest data) + EXTERNAL_QUERY_POINT_CRS = "EPSG:4269" def __init__(self, task_id: str, poly_order: int = 1): super().__init__(task_id) @@ -136,9 +136,11 @@ def _run_label_georef(self, input: TaskInput) -> TaskResult: scale_value = 0 query_pts = None + external_query_pts = False if "query_pts" in input.request: logger.info("reading query points from request") query_pts = input.request["query_pts"] + external_query_pts = True if not query_pts or len(query_pts) < 1: logger.info("reading query points from task input") query_pts = input.get_data("query_pts") @@ -253,9 +255,16 @@ def _run_label_georef(self, input: TaskInput) -> TaskResult: crs = self._determine_crs(input) logger.info(f"extracted CRS: {crs}") - if crs != self.DEFAULT_DEST_CRS: + # perform a datum shift if we're using externally supplied + # query points (ie. eval scenario where we are passed query points from a file) + if crs != self.EXTERNAL_QUERY_POINT_CRS and external_query_pts: + logger.info( + f"performing datum shift from {crs} to {self.EXTERNAL_QUERY_POINT_CRS}" + ) for qp in query_pts: - proj = Transformer.from_crs(crs, self.DEFAULT_DEST_CRS, always_xy=True) + proj = Transformer.from_crs( + crs, self.EXTERNAL_QUERY_POINT_CRS, always_xy=True + ) x_p, y_p = proj.transform(qp.lonlat[0], qp.lonlat[1]) qp.lonlat = (x_p, y_p) @@ -266,7 +275,9 @@ def _run_label_georef(self, input: TaskInput) -> TaskResult: result.output["query_pts"] = results result.output["rmse"] = rmse result.output["error_scale"] = scale_error - result.output["crs"] = self.DEFAULT_DEST_CRS + result.output["crs"] = ( + self.EXTERNAL_QUERY_POINT_CRS if external_query_pts else crs + ) result.output["keypoints"] = keypoint_stats return result @@ -304,8 +315,10 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: # get the generated query points, or use those that were passed into the pipeline # from an external source + external_points = False query_pts: List[QueryPoint] = input.get_data("query_pts") if not query_pts: + external_points = True query_pts = input.get_request_info("query_pts") # transform the query pixel points into geo locations and write the result into @@ -332,9 +345,15 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: crs = self._determine_crs(input) logger.info(f"extracted crs: {crs}") - # perform a datum shift on the query points if the source CRS is not the default - if crs != self.DEFAULT_DEST_CRS: - proj = Transformer.from_crs(crs, self.DEFAULT_DEST_CRS, always_xy=True) + # perform a datum shift on the query points if the we're using externally supplied + # query points (eval scenario where we are passed query points from a file) + if crs != self.EXTERNAL_QUERY_POINT_CRS and external_points: + logger.info( + f"performing datum shift from {crs} to {self.EXTERNAL_QUERY_POINT_CRS} on query points" + ) + proj = Transformer.from_crs( + crs, self.EXTERNAL_QUERY_POINT_CRS, always_xy=True + ) for pt in proj_query_points: x_proj, y_proj = proj.transform(*pt.lonlat) pt.lonlat = (x_proj, y_proj) @@ -350,7 +369,7 @@ def _run_corner_georef(self, input: TaskInput) -> TaskResult: result.output["query_pts"] = proj_query_points result.output["rmse"] = rmse result.output["error_scale"] = scale_error - result.output["crs"] = self.DEFAULT_DEST_CRS + result.output["crs"] = self.EXTERNAL_QUERY_POINT_CRS if external_points else crs result.output[CORNER_POINTS_OUTPUT_KEY] = corner_points return result @@ -602,7 +621,7 @@ def _determine_crs(self, input: TaskInput) -> str: # make sure there is metadata if not metadata: - return self.DEFAULT_DEST_CRS + return self.EXTERNAL_QUERY_POINT_CRS # grab the year from the metadata if present try: From 4a1e3fcbe1c0629bee095c4928f338bb06c95ac0 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sun, 11 Aug 2024 17:36:03 -0400 Subject: [PATCH 50/66] allow system versions to be set per-pipeline --- cdr/result_subscriber.py | 21 ++++++++++++++------- cdr/server.py | 7 ++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cdr/result_subscriber.py b/cdr/result_subscriber.py index 2be76f02..d0be02b1 100644 --- a/cdr/result_subscriber.py +++ b/cdr/result_subscriber.py @@ -89,6 +89,13 @@ class LaraResultSubscriber: GEOREFERENCE_PIPELINE: "uncharted-georeference", } + # map of pipeline name to system version + PIPELINE_SYSTEM_VERSIONS = { + SEGMENTATION_PIPELINE: "0.0.4", + METADATA_PIPELINE: "0.0.4", + POINTS_PIPELINE: "0.0.4", + GEOREFERENCE_PIPELINE: "0.0.5", + } # output CRS to use for projected maps that are pushed to CDR DEFAULT_OUTPUT_CRS = "EPSG:3857" @@ -101,7 +108,6 @@ def __init__( cdr_token: str, output: str, workdir: str, - system_version: str, json_log: JSONLog, host="localhost", pipeline_sequence: List[str] = DEFAULT_PIPELINE_SEQUENCE, @@ -114,7 +120,6 @@ def __init__( self._cdr_token = cdr_token self._workdir = workdir self._output = output - self._system_version = system_version self._json_log = json_log self._host = host self._pipeline_sequence = ( @@ -344,7 +349,7 @@ def _push_georeferencing(self, result: RequestResult): mapper = get_mapper( lara_result, self.PIPELINE_SYSTEM_NAMES[self.GEOREFERENCE_PIPELINE], - self._system_version, + self.PIPELINE_SYSTEM_VERSIONS[self.GEOREFERENCE_PIPELINE], ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore assert cdr_result is not None @@ -379,7 +384,9 @@ def _push_georeferencing(self, result: RequestResult): georeference_results=[], gcps=[], system=self.PIPELINE_SYSTEM_NAMES[self.GEOREFERENCE_PIPELINE], - system_version=self._system_version, + system_version=self.PIPELINE_SYSTEM_VERSIONS[ + self.GEOREFERENCE_PIPELINE + ], ) assert cdr_result is not None @@ -445,7 +452,7 @@ def _push_segmentation(self, result: RequestResult): mapper = get_mapper( lara_result, self.PIPELINE_SYSTEM_NAMES[self.SEGMENTATION_PIPELINE], - self._system_version, + self.PIPELINE_SYSTEM_VERSIONS[self.SEGMENTATION_PIPELINE], ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except: @@ -467,7 +474,7 @@ def _push_points(self, result: RequestResult): mapper = get_mapper( lara_result, self.PIPELINE_SYSTEM_NAMES[self.POINTS_PIPELINE], - self._system_version, + self.PIPELINE_SYSTEM_VERSIONS[self.POINTS_PIPELINE], ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except: @@ -490,7 +497,7 @@ def _push_metadata(self, result: RequestResult): mapper = get_mapper( lara_result, self.PIPELINE_SYSTEM_NAMES[self.METADATA_PIPELINE], - self._system_version, + self.PIPELINE_SYSTEM_VERSIONS[self.METADATA_PIPELINE], ) cdr_result = mapper.map_to_cdr(lara_result) # type: ignore except Exception as e: diff --git a/cdr/server.py b/cdr/server.py index d325f809..8948ec3f 100644 --- a/cdr/server.py +++ b/cdr/server.py @@ -34,7 +34,6 @@ CDR_API_TOKEN = os.environ["CDR_API_TOKEN"] CDR_HOST = "https://api.cdr.land" -CDR_SYSTEM_VERSION = "0.0.4" CDR_CALLBACK_SECRET = "maps rock" APP_PORT = 5001 CDR_EVENT_LOG = "events.log" @@ -53,7 +52,6 @@ class Settings: workdir: str imagedir: str output: str - system_version: str callback_secret: str callback_url: str registration_id: Dict[str, str] = {} @@ -213,6 +211,7 @@ def register_cdr_system(): for i, pipeline in enumerate(settings.sequence): system_name = LaraResultSubscriber.PIPELINE_SYSTEM_NAMES[pipeline] + system_version = LaraResultSubscriber.PIPELINE_SYSTEM_VERSIONS[pipeline] logger.info(f"registering system {system_name} with cdr") headers = {"Authorization": f"Bearer {settings.cdr_api_token}"} @@ -221,7 +220,7 @@ def register_cdr_system(): registration = { "name": system_name, - "version": settings.system_version, + "version": system_version, "callback_url": settings.callback_url, "webhook_secret": settings.callback_secret, # Leave blank if callback url has no auth requirement @@ -333,7 +332,6 @@ def main(): settings.workdir = p.workdir settings.imagedir = p.imagedir settings.output = p.output - settings.system_version = CDR_SYSTEM_VERSION settings.callback_secret = CDR_CALLBACK_SECRET settings.serial = True settings.sequence = p.sequence @@ -372,7 +370,6 @@ def main(): settings.cdr_api_token, settings.output, settings.workdir, - settings.system_version, settings.json_log, host=p.host, pipeline_sequence=settings.sequence, From 152fd575d6f94bc3b262a4ca59bcb9d12107bf08 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Sun, 11 Aug 2024 18:14:12 -0400 Subject: [PATCH 51/66] fixes projection info for transformed map CDR metadata --- cdr/result_subscriber.py | 12 ++++++------ schema/mappers/cdr.py | 6 +++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cdr/result_subscriber.py b/cdr/result_subscriber.py index d0be02b1..9d185ca3 100644 --- a/cdr/result_subscriber.py +++ b/cdr/result_subscriber.py @@ -33,7 +33,7 @@ Request, RequestResult, ) -from schema.mappers.cdr import get_mapper +from schema.mappers.cdr import GeoreferenceMapper, get_mapper from tasks.geo_referencing.entities import GeoreferenceResult as LARAGeoreferenceResult from tasks.metadata_extraction.entities import MetadataExtraction as LARAMetadata from tasks.point_extraction.entities import PointLabels as LARAPoints @@ -97,9 +97,6 @@ class LaraResultSubscriber: GEOREFERENCE_PIPELINE: "0.0.5", } - # output CRS to use for projected maps that are pushed to CDR - DEFAULT_OUTPUT_CRS = "EPSG:3857" - def __init__( self, request_publisher: Optional[LaraRequestPublisher], @@ -363,10 +360,13 @@ def _push_georeferencing(self, result: RequestResult): assert gcps is not None logger.info( - f"projecting image {result.image_path} to {output_file_name_full} using crs {projection.crs}" + f"projecting image {result.image_path} to {output_file_name_full} using crs {GeoreferenceMapper.DEFAULT_OUTPUT_CRS}" ) self._project_georeference( - result.image_path, output_file_name_full, self.DEFAULT_OUTPUT_CRS, gcps + result.image_path, + output_file_name_full, + GeoreferenceMapper.DEFAULT_OUTPUT_CRS, + gcps, ) files_.append( diff --git a/schema/mappers/cdr.py b/schema/mappers/cdr.py index 911568df..5d700322 100644 --- a/schema/mappers/cdr.py +++ b/schema/mappers/cdr.py @@ -1,4 +1,5 @@ import logging +from unittest.mock import DEFAULT from schema.cdr_schemas.georeference import ( GeoreferenceResults as CDRGeoreferenceResults, @@ -63,6 +64,9 @@ def map_from_cdr(self, model: BaseModel) -> BaseModel: class GeoreferenceMapper(CDRMapper): + # output CRS to use for projected maps that are pushed to CDR + DEFAULT_OUTPUT_CRS = "EPSG:3857" + def map_to_cdr(self, model: LARAGeoferenceResult) -> CDRGeoreferenceResults: gcps = [] for gcp in model.gcps: @@ -87,7 +91,7 @@ def map_to_cdr(self, model: LARAGeoferenceResult) -> CDRGeoreferenceResults: map_area=None, projections=[ ProjectionResult( - crs=model.crs, + crs=self.DEFAULT_OUTPUT_CRS, gcp_ids=[gcp.gcp_id for gcp in gcps], file_name=f"lara-{model.map_id}.tif", ) From 175014d43f02f3380cf9b14b6fd9e5dde6fbbf2c Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Tue, 13 Aug 2024 15:09:23 -0400 Subject: [PATCH 52/66] remove unused sympy dependency --- tasks/segmentation/detectron_segmenter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/segmentation/detectron_segmenter.py b/tasks/segmentation/detectron_segmenter.py index fafc1931..3ec07a79 100644 --- a/tasks/segmentation/detectron_segmenter.py +++ b/tasks/segmentation/detectron_segmenter.py @@ -2,13 +2,12 @@ import cv2 from cv2.typing import MatLike import numpy as np -from sympy import LM import torch import hashlib from pathlib import Path import os from urllib.parse import urlparse -from typing import List, Optional, Tuple, Sequence +from typing import List, Tuple, Sequence from tasks.segmentation.ditod import add_vit_config from tasks.segmentation.entities import ( From 9c3fc5010f60011dcedbf02fa40e02c732a5eef1 Mon Sep 17 00:00:00 2001 From: Chris Bethune Date: Tue, 13 Aug 2024 15:09:40 -0400 Subject: [PATCH 53/66] adds text pipeline to makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index be516740..cf9cf50f 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ build_cdr: # Target: build # Description: Builds all components. -build: build_segmentation build_metadata build_points build_georef build_cdr +build: build_segmentation build_metadata build_points build_georef build_text build_cdr # Target: tag_dev # Description: Tags images with the dev tag. From 787494169d43704f7e39f61703be70608a522b9b Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 14 Aug 2024 09:34:07 -0400 Subject: [PATCH 54/66] updated root project readme --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2c72ad3b..2a099473 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,14 @@ ![example workflow](https://github.com/uncharted-lara/lara-models/actions/workflows/build_test.yml/badge.svg) +## LARA - Layered Atlas Reconstruction Analytics +This repository contains Uncharted's TA1 contributions for DARPA's CriticalMAAS program. The main goals are automated feature extraction and georeferencing of geologic maps. + This repository contains five pipelines: -* [Map Segmentation](pipelines/segmentation/README.md) - detects and extracts the main map area, polygon legend, point/line legend and geologic cross section from maps +* [Map Segmentation](pipelines/segmentation/README.md) - detects and extracts the main map area, polygon legend, point/line legend and geologic cross sections from maps * [Metadata Extraction](pipelines/metadata_extraction/README.md) - extracts metadata values such as title, author, year and scale from an input map image -* [Point Extraction](pipelines/point_extraction/README.md) - detects and extracts geologic point symbols from an input map image +* [Point Extraction](pipelines/point_extraction/README.md) - detects and extracts the location and orientation of geologic point symbols from an input map image * [Georeferencing](pipelines/geo_referencing/README.md) - computes an image space to geo space transform given an input map image * [Text Extraction](pipelines/text_extraction/README.md) - extracts text as individual words, lines or paragraphs/blocks from an input image From ded53a695fdc714f7e32499d74e37ce432b90175 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 14 Aug 2024 13:17:29 -0400 Subject: [PATCH 55/66] updated segmentation pipeline readme --- README.md | 2 +- pipelines/geo_referencing/deploy/run.sh | 2 +- pipelines/geo_referencing/deploy/run_gpu.sh | 2 +- pipelines/segmentation/README.md | 34 ++++++++++++++------- pipelines/segmentation/deploy/run.sh | 2 +- pipelines/segmentation/deploy/run_gpu.sh | 2 +- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2a099473..4dab3262 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ This repository contains five pipelines: * [Georeferencing](pipelines/geo_referencing/README.md) - computes an image space to geo space transform given an input map image * [Text Extraction](pipelines/text_extraction/README.md) - extracts text as individual words, lines or paragraphs/blocks from an input image -The `tasks` directory contains the `pip` installable library of tasks and supporting utilities, with each pipeline found in the `pipelines` directory being composed of these tasks. Each pipeline is itself `pip` installable, and is accompanied by a wrapper to support command line execution (`run_pipeline.py`), and a server wrapper to support execution as a REST service (`run_sever.py`). Scripts to build the server wrapper into a Docker container are also included. +The `tasks` directory contains the `pip` installable library of tasks and supporting utilities, with each pipeline found in the `pipelines` directory being composed of these tasks. Each pipeline is itself `pip` installable, and is accompanied by a wrapper to support command line execution (`run_pipeline.py`), and a server wrapper to support execution as a REST service (`run_server.py`). Scripts to build the server wrapper into a Docker container are also included. diff --git a/pipelines/geo_referencing/deploy/run.sh b/pipelines/geo_referencing/deploy/run.sh index 12f3effc..ca41d16c 100755 --- a/pipelines/geo_referencing/deploy/run.sh +++ b/pipelines/geo_referencing/deploy/run.sh @@ -16,5 +16,5 @@ docker run \ uncharted/lara-georef:latest \ --workdir /workdir \ --imagedir /imagedir \ - --model pipelines/segmentation_weights/layoutlmv3_xsection_20231201 \ + --model pipelines/segmentation_weights/layoutlmv3_20240531 \ --rabbit_host rabbitmq \ No newline at end of file diff --git a/pipelines/geo_referencing/deploy/run_gpu.sh b/pipelines/geo_referencing/deploy/run_gpu.sh index 7b3d9c39..ef3bd2b7 100755 --- a/pipelines/geo_referencing/deploy/run_gpu.sh +++ b/pipelines/geo_referencing/deploy/run_gpu.sh @@ -18,6 +18,6 @@ docker run \ uncharted/lara-georef:latest \ --workdir /workdir \ --imagedir /imagedir \ - --model pipelines/segmentation_weights/layoutlmv3_xsection_20231201 \ + --model pipelines/segmentation_weights/layoutlmv3_20240531 \ --rabbit_host rabbitmq diff --git a/pipelines/segmentation/README.md b/pipelines/segmentation/README.md index cdc07c3b..f1bdf478 100644 --- a/pipelines/segmentation/README.md +++ b/pipelines/segmentation/README.md @@ -2,7 +2,7 @@ ## LARA Image Segmentation Pipeline -This pipeline performs segmentation to isolate the map and legend regions on an image +This pipeline performs segmentation to isolate the map, legend and cross-section regions on an image Segmentation is done using a fine-tuned version of the `LayoutLMv3` model: https://github.com/microsoft/unilm/tree/master/layoutlmv3 @@ -21,6 +21,7 @@ The model currently supports 4 segmentation classes: * python 3.10 or higher is required * Installation of Detectron2 requires `torch` already be present in the environment, so it must be installed manually. +* Note: for python virtual environments, `conda` is more reliable for installing torch==2.0.x than `venv` To install from the current directory: ``` @@ -42,12 +43,12 @@ pip install -e .[segmentation] * Pipeline is defined in `segmentation_pipeline.py` and is suitable for integration into other systems * Model weights can be input from S3 or local drive * Input is a image (ie binary image file buffer) -* Ouput is the set of map polygons capturing the map region, legend areas and cross sections materialized as a: - * `SegmentationResults` JSON object - * `FeatureResults` JSON object (the latter being part of the CMA TA1 CDR schema) +* Output is the set of map polygons capturing the map region, legend areas and cross sections materialized as: + * `MapSegmentation` JSON object (LARA's internal data schema) and/or + * `FeatureResults` JSON object (part of the CDR TA1 schema) ### Command Line Execution ### -`run_pipeline.py` provides a command line wrapper around the segmentation pipeline, and allows for a directory map images to be processed serially. +`run_pipeline.py` provides a command line wrapper around the segmentation pipeline, and allows for a directory of map images to be processed serially. To run from the repository root directory: ``` @@ -56,9 +57,12 @@ export AWS_SECRET_ACCESS_KEY= python3 -m pipelines.segmentation.run_pipeline \ --input /image/input/dir \ - --output /model/output/dir \ - --workdir /model/working/dir \ - --model https://s3/compatible/endpoint/layoutlmv3_20230 + --output /results/output/dir \ + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --model /path/to/segmentation/model/weights \ + --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ + --no_gpu (if set, pipeline will force CPU-only processing) + ``` Note that when the `model` parameter can point to a folder in the local file system, or to a resource on an S3-compatible endpoint. The folder/resource must contain the following files: @@ -73,14 +77,22 @@ In the S3 case, the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment * ```POST: /api/process_image``` - Sends an image (as binary file buffer) to the segmenter pipeline for analysis. Results are JSON string. * ```GET /healthcheck``` - Healthcheck endpoint +The server can also be configured to run with a request queue, using RabbitMQ, if the `rest` flag is not set. + To start the server: ``` export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= python3 -m pipelines.segmentation.run_server \ - --workdir /model/workingdir \ - --model https://s3/compatible/endpoint/layoutlmv3_20230 + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --model /path/to/segmentation/model/weights \ + --rest (if set, run the server in REST mode, instead of resquest-queue mode) + --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ + --no_gpu (if set, pipeline will force CPU-only processing) \ + --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --rabbit_host (rabbitmq host; only needed for request-queue mode) + ``` ### Dockerized deployment @@ -92,6 +104,6 @@ cd deploy export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= -./run.sh /model/workingdir https://s3/compatible/endpoint/layoutlmv3_20230 +./run.sh /pipeline/working/dir /pipline/images/working/dir ``` diff --git a/pipelines/segmentation/deploy/run.sh b/pipelines/segmentation/deploy/run.sh index 93fa1a87..10237e68 100755 --- a/pipelines/segmentation/deploy/run.sh +++ b/pipelines/segmentation/deploy/run.sh @@ -15,6 +15,6 @@ docker run \ uncharted/lara-segmentation:test \ --workdir /workdir \ --imagedir /imagedir \ - --model pipelines/segmentation_weights/layoutlmv3_xsection_20231201 \ + --model pipelines/segmentation_weights/layoutlmv3_20240531 \ --cdr_schema \ --rabbit_host rabbitmq diff --git a/pipelines/segmentation/deploy/run_gpu.sh b/pipelines/segmentation/deploy/run_gpu.sh index 5cb88fd9..1702958b 100755 --- a/pipelines/segmentation/deploy/run_gpu.sh +++ b/pipelines/segmentation/deploy/run_gpu.sh @@ -17,6 +17,6 @@ docker run \ uncharted/lara-segmentation:latest \ --workdir /workdir \ --imagedir /imagedir \ - --model pipelines/segmentation_weights/layoutlmv3_xsection_20231201 \ + --model pipelines/segmentation_weights/layoutlmv3_20240531 \ --cdr_schema \ --rabbit_host rabbitmq From 77dffd2f08d2af284c2fd9f08b5d55d008911fc6 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 14 Aug 2024 13:51:12 -0400 Subject: [PATCH 56/66] updated ocr pipeline readme --- pipelines/text_extraction/README.md | 41 +++++++++++++++++------------ 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/pipelines/text_extraction/README.md b/pipelines/text_extraction/README.md index f30f6f61..2cce1a29 100644 --- a/pipelines/text_extraction/README.md +++ b/pipelines/text_extraction/README.md @@ -6,7 +6,7 @@ This pipeline performs OCR-based text extraction on an image This module currently uses Google-Vision OCR API by default: https://cloud.google.com/vision/docs/ocr#vision_text_detection_gcs-python -See more info on pipeline tasks here: [../../tasks/text_extraction/README.md](../../tasks/README.md) +See more info on pipeline tasks here: [../../tasks/README.md](../../tasks/README.md) ### Installation @@ -27,27 +27,29 @@ pip install -e . * Pipeline is defined in `text_extraction_pipeline.py` and is suitable for integration into other systems * Input is a image (ie binary image file buffer) -* Output is the set of extracted text items materialized as a: - * `DocTextExtraction` JSON object - * List of `FeatureResults` objects as defined in the CMA TA1 CDR schema +* Output is the set of extracted text items materialized as: + * `DocTextExtraction` JSON object (LARA's internal data schema) and/or + * `FeatureResults` JSON object (part of the CDR TA1 schema) ### Command Line Execution ### -`run_pipeline.py` provides a command line wrapper around the map extraction pipeline, and allows for a directory map images to be processed serially. +`run_pipeline.py` provides a command line wrapper around the text extraction pipeline, and allows for a directory map images to be processed serially. To run from the repository root directory: ``` -export GOOGLE_APPLICATION_CREDENTIALS=/credentials/google_api_credentials.json +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json python3 -m pipelines.text_extraction.run_pipeline \ --input /image/input/dir \ - --output /model/output/dir \ - --workdir /model/working/dir \ + --output /results/output/dir \ + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ --tile True \ - --pixel_limit 1024 + --pixel_limit 6000 \ + --gamma_corr 1.0 ``` -Where `tile` inidicates whether the image should be tiled or resized when it is larger than `pixel_limit`. - +* `tile` Indicates whether the image should be tiled or resized when it is larger than `pixel_limit`. +* `gamma_corr` Controls optional image gamma correction as pre-processing. Must be <= 1.0; default value is 1.0 (ie gamma correction disabled) ### REST Service ### @@ -57,12 +59,17 @@ Where `tile` inidicates whether the image should be tiled or resized when it is To start the server: ``` -export GOOGLE_APPLICATION_CREDENTIALS=/credentinals/google_api_credentials.json +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json -python3 -m pipelines.metadata_extraction.run_server \ - --workdir /model/workingdir +python3 -m pipelines.text_extraction.run_server \ + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ --tile True \ - --pixel_limit 1024 + --pixel_limit 6000 \ + --gamma_corr 1.0 \ + --rest (if set, run the server in REST mode, instead of resquest-queue mode) \ + --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` ### Dockerized deployment @@ -71,9 +78,9 @@ The `deploy/build.sh` script can be used to build the server above into a Docker ``` cd deploy -export GOOGLE_APPLICATION_CREDENTIALS=/credentials/google_api_credentials.json +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json -./run.sh /model/working/dir +./run.sh /model/working/dir /pipline/images/working/dir ``` From 8ac42e03e3915aa9642a1de1376515c9516d7e52 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 14 Aug 2024 14:33:59 -0400 Subject: [PATCH 57/66] updated metadata pipeline readme --- pipelines/metadata_extraction/README.md | 50 +++++++++++++------ pipelines/metadata_extraction/deploy/run.sh | 2 +- .../metadata_extraction/deploy/run_gpu.sh | 2 +- pipelines/metadata_extraction/run_server.py | 4 +- pipelines/text_extraction/README.md | 2 + pipelines/text_extraction/run_server.py | 7 ++- 6 files changed, 49 insertions(+), 18 deletions(-) diff --git a/pipelines/metadata_extraction/README.md b/pipelines/metadata_extraction/README.md index 1045c2d6..79945322 100644 --- a/pipelines/metadata_extraction/README.md +++ b/pipelines/metadata_extraction/README.md @@ -3,8 +3,7 @@ This pipeline extracts metadata such as title, year and scale from an input raster map image. Extracted text is -combined with a request for the fields of interest into a prompt, which is passed to an [OpenAI GPT-3.5](https://platform.openai.com/docs/models/gpt-3-5) -for analysis and extraction. +combined with a request for the fields of interest into a prompt, which is passed to an [OpenAI GPT-4.0](https://platform.openai.com/docs/models/gpt-4-0) for analysis and extraction. See more info on pipeline tasks here: [../../tasks/README.md](../../tasks/README.md) @@ -30,10 +29,16 @@ The following are the currently extracted fields along with an example of each: ### Installation -python 3.10 or higher required +* python 3.10 or higher is required +* Installation of Detectron2 requires `torch` already be present in the environment, so it must be installed manually. +* Note: for python virtual environments, `conda` is more reliable for installing torch==2.0.x than `venv` To install from the current directory: ``` +# manually install torch - this is necessary due to issues with detectron2 dependencies +# (see https://github.com/facebookresearch/detectron2/issues/4472) +pip install torch==2.0.1 + # install the task library cd ../../tasks pip install -e .[segmentation] @@ -51,39 +56,56 @@ The segmentation dependencies are required due to the use of the map segmentatio * Pipeline is defined in `metdata_extraction_pipeline.py` and is suitable for integration into other systems * Input is a image (ie binary image file buffer) -* Output is the set of metadata fields materialized as a: - * `MetadataExtraction` JSON object - * `CogMetaData` JSON object adhering to the CMA CDR TA1 schema) +* Output is the set of metadata fields materialized as: + * `MetadataExtraction` JSON object (LARA's internal data schema) and/or + * `CogMetaData` JSON object (part of the CDR TA1 schema) ### Command Line Execution ### -`run_pipeline.py` provides a command line wrapper around the map extraction pipeline, and allows for a directory map images to be processed serially. +`run_pipeline.py` provides a command line wrapper around the metadata extraction pipeline, and allows for a directory map images to be processed serially. To run from the repository root directory: ``` -export GOOGLE_APPLICATION_CREDENTIALS=/credentinals/google_api_credentials.json +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json export OPENAI_API_KEY= python3 -m pipelines.metadata_extraction.run_pipeline \ --input /image/input/dir \ --output /model/output/dir \ - --workdir /model/working/dir \ - --model /segmentation/model/weights + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --model /path/to/segmentation/model/weights \ + --llm gpt-4o (which gpt model version to use) \ + --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ + --no_gpu (if set, pipeline will force CPU-only processing) ``` -The `model` argument should point to the latest segmentation model weights, which can be stored on the local file system, or accessed through a URL. +Note that when the segmentation `model` parameter can point to a folder in the local file system, or to a resource on an S3-compatible endpoint. The folder/resource must contain the following files: +* `config.yaml` +* `config.json` +* `model_final.pth` + +In the S3 case, the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables must be set accordingly. The model weights an configuration files will be fetched from the S3 endpoint and cached. ### REST Service ### `run_server.py` provides the pipeline as a REST service with the following endpoints: * ```POST: /api/process_image``` - Sends an image (as binary file buffer) to the metadata extraction pipeline for analysis. Results are JSON string. * ```GET /healthcheck``` - Healthcheck endpoint +The server can also be configured to run with a request queue, using RabbitMQ, if the `rest` flag is not set. + To start the server: ``` -export GOOGLE_APPLICATION_CREDENTIALS=/credentinals/google_api_credentials.json +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json export OPENAI_API_KEY= python3 -m pipelines.metadata_extraction.run_server \ - --workdir /model/workingdir + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --model /path/to/segmentation/model/weights \ + --llm gpt-4o (which gpt model version to use) \ + --rest (if set, run the server in REST mode, instead of resquest-queue mode) + --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ + --no_gpu (if set, pipeline will force CPU-only processing) \ + --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` ### Dockerized deployment @@ -95,7 +117,7 @@ cd deploy export GOOGLE_APPLICATION_CREDENTIALS=/credentials/google_api_credentials.json export OPENAI_API_KEY= -./run.sh /model/working/dir +./run.sh /pipeline/working/dir /pipline/images/working/dir ``` diff --git a/pipelines/metadata_extraction/deploy/run.sh b/pipelines/metadata_extraction/deploy/run.sh index c7fff1f8..bf64b97c 100755 --- a/pipelines/metadata_extraction/deploy/run.sh +++ b/pipelines/metadata_extraction/deploy/run.sh @@ -16,6 +16,6 @@ docker run \ uncharted/lara-metadata-extract:latest \ --workdir /workdir \ --imagedir /imagedir \ - --model pipelines/segmentation_weights/layoutlmv3_xsection_20231201 \ + --model pipelines/segmentation_weights/layoutlmv3_20240531 \ --cdr_schema \ --rabbit_host rabbitmq diff --git a/pipelines/metadata_extraction/deploy/run_gpu.sh b/pipelines/metadata_extraction/deploy/run_gpu.sh index c9b36383..2dd41eaf 100755 --- a/pipelines/metadata_extraction/deploy/run_gpu.sh +++ b/pipelines/metadata_extraction/deploy/run_gpu.sh @@ -18,6 +18,6 @@ docker run \ uncharted/lara-metadata-extract:latest \ --workdir /workdir \ --imagedir /imagedir \ - --model pipelines/segmentation_weights/layoutlmv3_xsection_20231201 \ + --model pipelines/segmentation_weights/layoutlmv3_20240531 \ --cdr_schema \ --rabbit_host rabbitmq diff --git a/pipelines/metadata_extraction/run_server.py b/pipelines/metadata_extraction/run_server.py index 984034a0..f4d89f3f 100644 --- a/pipelines/metadata_extraction/run_server.py +++ b/pipelines/metadata_extraction/run_server.py @@ -15,6 +15,7 @@ METADATA_RESULT_QUEUE, ) from tasks.common.pipeline import PipelineInput, BaseModelOutput, BaseModelListOutput +from tasks.metadata_extraction.metadata_extraction import LLM from tasks.common import image_io from tasks.metadata_extraction.entities import METADATA_EXTRACTION_OUTPUT_KEY from util import logging as logging_util @@ -92,6 +93,7 @@ def health(): action="store_true", help="Output results as TA1 json schema format", ) + parser.add_argument("--llm", type=LLM, choices=list(LLM), default=LLM.GPT_4_O) parser.add_argument("--rest", action="store_true") parser.add_argument("--rabbit_host", type=str, default="localhost") parser.add_argument("--request_queue", type=str, default=METADATA_REQUEST_QUEUE) @@ -101,7 +103,7 @@ def health(): # init segmenter metadata_extraction = MetadataExtractorPipeline( - p.workdir, p.model, cdr_schema=p.cdr_schema, gpu=not p.no_gpu + p.workdir, p.model, cdr_schema=p.cdr_schema, model=p.llm, gpu=not p.no_gpu ) metadata_result_key = ( diff --git a/pipelines/text_extraction/README.md b/pipelines/text_extraction/README.md index 2cce1a29..3a5b8e6d 100644 --- a/pipelines/text_extraction/README.md +++ b/pipelines/text_extraction/README.md @@ -57,6 +57,8 @@ python3 -m pipelines.text_extraction.run_pipeline \ * ```POST: /api/process_image``` - Sends an image (as binary file buffer) to the metadata extraction pipeline for analysis. Results are JSON string. * ```GET /healthcheck``` - Healthcheck endpoint +The server can also be configured to run with a request queue, using RabbitMQ, if the `rest` flag is not set. + To start the server: ``` export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json diff --git a/pipelines/text_extraction/run_server.py b/pipelines/text_extraction/run_server.py index 1213adea..f12a08e9 100644 --- a/pipelines/text_extraction/run_server.py +++ b/pipelines/text_extraction/run_server.py @@ -83,13 +83,18 @@ def health(): parser.add_argument("--imagedir", type=Path, default="tmp/lara/workdir") parser.add_argument("--debug", action="store_true") parser.add_argument("--cdr_schema", action="store_true") + parser.add_argument("--tile", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--pixel_limit", type=int, default=6000) + parser.add_argument("--gamma_corr", type=float, default=1.0) parser.add_argument("--rest", action="store_true") parser.add_argument("--rabbit_host", type=str, default="localhost") parser.add_argument("--request_queue", type=str, default=TEXT_REQUEST_QUEUE) parser.add_argument("--result_queue", type=str, default=TEXT_RESULT_QUEUE) p = parser.parse_args() - pipeline = TextExtractionPipeline(p.workdir, tile=True) + pipeline = TextExtractionPipeline( + p.workdir, p.tile, p.pixel_limit, p.gamma_corr, p.debug + ) result_key = ( "doc_text_extracction_output" From 776487a2d9999e26a02abc87dfc1007e3f7b2bcc Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 14 Aug 2024 15:27:15 -0400 Subject: [PATCH 58/66] updated points pipeline readme --- pipelines/point_extraction/README.md | 104 ++++++++++++++++----------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/pipelines/point_extraction/README.md b/pipelines/point_extraction/README.md index 683c7809..b2fc3ef3 100644 --- a/pipelines/point_extraction/README.md +++ b/pipelines/point_extraction/README.md @@ -2,39 +2,60 @@ ## LARA Point Extraction Pipeline -This pipeline extracts bedding point symbols from a map, along with their orientation and associated incline information. The model leverages [YOLOv8](https://github.com/ultralytics/ultralytics) for the baseline object detection task. +This pipeline extracts bedding point symbols from a map, along with their orientation and associated incline (dip) information. The model leverages [YOLO](https://github.com/ultralytics/ultralytics) for extraction of high priority / common point symbol types. In addition, a CV-based One-Shot algorithm can be used to extract less common point symbols. See more info on pipeline tasks here: [../../tasks/README.md](../../tasks/README.md) ### Extracted Symbols -Initial efforts have focused on identifying and extracting the following symbols: +#### Object Detection Model +The YOLO object detection model has been trained to extract common geologic point symbols as follows: * Inclined Bedding (aka strike/dip) * Vertical Bedding * Horizontal Bedding * Overturned Bedding * Inclined Foliation +* Inclined Foliation (Igneous) * Vertical Foliation * Vertical Joint * Sink Hole +* Lineation +* Drill Hole * Gravel Borrow Pit * Mine Shaft * Prospect * Mine Tunnel * Mine Quarry +#### One-Shot Model +The One-shot CV algorithm can be used to extract any leftover less common point symbols that may be present. This algorithm requires legend swatches to be available as a template (either via HITL annotation or some other manner) + ### Point Symbol Orientation -Some point symbols also contain directional information. +Many point symbols also contain directional information. Point orientation (ie "strike" direction) and the "dip" magnitude are also extracted for applicable symbol types: * Inclined Bedding (strike/dip) +* Vertical Bedding +* Overturned Bedding +* Inclined Foliation +* Inclined Foliation (Igneous) +* Vertical Foliation +* Vertical Joint +* Lineation +* Mine Tunnel ### Installation -python 3.10 or higher is required +* python 3.10 or higher is required +* Installation of Detectron2 requires `torch` already be present in the environment, so it must be installed manually. +* Note: for python virtual environments, `conda` is more reliable for installing torch==2.0.x than `venv` To install from the current directory: ``` +# manually install torch - this is necessary due to issues with detectron2 dependencies +# (see https://github.com/facebookresearch/detectron2/issues/4472) +pip install torch==2.0.1 + # install the task library cd ../../tasks pip install -e .[point,segmentation] @@ -52,43 +73,46 @@ The segmentation dependencies are required due to the use of the map segmentatio * Pipeline is defined in `point_extraction_pipeline.py` and is suitable for integration into other systems * Input is a image (ie binary image file buffer) -* Ouput is the set of extracted points materialized as a: - * `PointLabels` JSON object, which contains a list of `PointLabel` capturing the point information - * List of `FeatureResults` JSON objects as defined in the CMA TA1 CDR schema +* Ouput is the set of extracted points materialized as: + * `PointLabels` JSON object (LARA's internal data schema) and/or + * `FeatureResults` JSON object (part of the CDR TA1 schema) ### Command Line Execution ### -`run_pipeline.py` provides a command line wrapper around the point extraction pipeline, and allows for a directory map images to be processed serially. +`run_pipeline.py` provides a command line wrapper around the point extraction pipeline, and allows for a directory of map images to be processed serially. To run from the repository root directory: ``` export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= -export GOOGLE_APPLICATION_CREDENTIALS=/credentinals/google_api_credentials.json +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json python3 -m pipelines.point_extraction.run_pipeline \ --input /image/input/dir \ - --output /model/output/dir \ - --workdir /model/working/dir \ - --model_point_extractor https://s3/compatible/endpoint/point_extractor_model_weights_dir \ - --model_segmenter https://s3/compatible/endpoint/segmentation_model_weights_dir \ - --cdr_schema=True \ - --bitmasks=True \ - --legend_hints_dir /legend/hints/dir + --output /results/output/dir \ + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --model_point_extractor /path/to/points/yolo/model/weights \ + --model_segmenter /path/to/segmentation/model/weights \ + --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ + --fetch_legend_items (if set, the pipeline will query the CDR for validated legend annotations for a given map input) \ + --no_gpu (if set, pipeline will force CPU-only processing) \ + --bitmasks (if set, pipeline will also output legacy CMAAS contest-style bitmasks) \ + --legend_hints_dir /input/legend/hints/dir (to input legacy CMAAS contest legend hints) ``` Note that the `model_point_extractor` and `model_segmenter` parameters can point to a folder in the local file system, or to a resource on an S3-compatible endpoint. The first file with a `.pt` extension will be loaded as the model weights. In the S3 case, the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables must be set accordingly. The model weights be fetched from the S3 endpoint and cached. -The `model_segmenter` param is optional. If present each image will be segmented and only the map area will be used for point symbol extraction. - OCR text extraction is used to extract the "dip" magnitude labels for strike/dip point symbols. For this functionality, the `GOOGLE_APPLICATION_CREDENTIALS` environment variables must be set. If `cdr_schema` is set, the pipeline will also produce point extractions in the CDR JSON format. -If `bitmasks` is set, the pipeline will produce points extractions in bitmask image format (AI4CMA Contest format) +The `fetch_legend_items` option requires the `CDR_API_TOKEN` environment variable to be set. The pipeline will query the CDR to check for validated legend annotations for a given map input. This is optional. It is not needed for the main YOLO model, but required for the secondary One-Shot algorithm. + +#### Legacy AI4CMA Contest Processing -Legend hints JSON files (ie from the AI4CMA Contest) can be used to improve points extractions, using the `legend_hints_dir` parameter. +* If `bitmasks` is set, the pipeline will produce points extractions in bitmask image format (AI4CMA Contest format) +* Legend hints JSON files (ie from the AI4CMA Contest) can be used to improve points extractions, using the `legend_hints_dir` parameter. ### REST Service ### @@ -96,43 +120,39 @@ Legend hints JSON files (ie from the AI4CMA Contest) can be used to improve poin * ```POST: /api/process_image``` - Sends an image (as binary file buffer) to the segmenter pipeline for analysis. Results are JSON string. * ```GET /healthcheck``` - Healthcheck endpoint +The server can also be configured to run with a request queue, using RabbitMQ, if the `rest` flag is not set. + To start the server: ``` export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= -export GOOGLE_APPLICATION_CREDENTIALS=/credentials/google_api_credentials.json +export GOOGLE_APPLICATION_CREDENTIALS=/path/to//google_api_credentials.json +export CDR_API_TOKEN= python3 -m pipelines.point_extraction.run_server \ - --workdir /model/workingdir \ - --model_point_extractor https://s3/compatible/endpoint/point_extractor_model_weights_dir \ - --model_segmenter https://s3/compatible/endpoint/segmentation_model_weights_dir -``` - -To test the server: - -``` -curl localhost:5000/healthcheck -``` - -To extract points from an image: - -``` -curl http://localhost:5000/api/process_image \ - --header 'Content-Type: image/tiff' \ - --data-binary @/path/to/map/image.tif - --output output/file/path.json + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --model_point_extractor /path/to/points/yolo/model/weights \ + --model_segmenter /path/to/segmentation/model/weights \ + --rest (if set, run the server in REST mode, instead of resquest-queue mode) \ + --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ + --fetch_legend_items (if set, the pipeline will query the CDR for validated legend annotations for a given map input) \ + --no_gpu (if set, pipeline will force CPU-only processing) \ + --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` ### Dockerized deployment A dockerized version REST service described above is available that includes model weights. Similar to the steps above, -OCR text extraction for "dip" magnitude labels need the `GOOGLE_APPLICATION_CREDENTIALS` environment variables must be set, and -a local directory for caching results must be provided. To run: +OCR text extraction for "dip" magnitude labels need the `GOOGLE_APPLICATION_CREDENTIALS` environment variables must be set, and a local directory for caching results must be provided. To run: ``` cd deploy + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json -./run.sh /path/to/workdir +export CDR_API_TOKEN= + +./run.sh /path/to/workdir /pipline/images/working/dir ``` The `deploy/build.sh` script can also be used to build the Docker image from source. From 1f437caca122efbb25633d53b3eb98bdeff3d7da Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Wed, 14 Aug 2024 16:10:18 -0400 Subject: [PATCH 59/66] updated georef pipeline readme --- pipelines/geo_referencing/README.md | 94 ++++++++++++++--------------- 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/pipelines/geo_referencing/README.md b/pipelines/geo_referencing/README.md index f2537dca..7e1de8c5 100644 --- a/pipelines/geo_referencing/README.md +++ b/pipelines/geo_referencing/README.md @@ -4,33 +4,19 @@ This pipeline georeferences an input raster map image. The georeferencing approach revolves around parsing text in the image to extract coordinates, and then using those coordinates to georeference ground control points. It currently attempts to parse Degrees, Minutes, Seconds style coordinates and UTM style coordinates. There is initial support to extract potential geocoding options as basis for extraction of latitude and longitude transformations. -If run through CLI, the georeferencing system is currently structured to execute 5 independent pipelines, each producing 5 outputs. +The georeferencing pipeline uses info from many other LARA tasks to improve its results: +* Text extraction (OCR) is used to extract map geo-coord labels and other metadata +* Metadata Extraction is used to help determine additional info about a map's general region and scale (eg state, county, UTM zone) +* Segmentation is used to determine to the map's bounds within an image. See more info on pipeline tasks here: [../../tasks/README.md](../../tasks/README.md) -### Pipelines -As it stands, there are 5 pipelines that are executed for each map. The pipelines follow the same general process, with slight variations as not all maps are created equal. The differences are in how the OCR process handles larger images and how the Region Of Interest is obtained. -**Resize** -The Resize pipeline will resize images to fit under the maximum allowable size limit for the OCR processing. The Region Of Interest is determined via an entropy based approach. +### CLI Output -**Tile** -The Tile pipeline will tile the image using the minimum amount of tiles necessary to have them all fit under the allowable size limit for the OCR processing. The Region Of Interest is determined via an entropy based approach. +Five different outputs are produced when georeferencing is executed at the command line. -**Fixed ROI** -The Tile pipeline will tile the image using the minimum amount of tiles necessary to have them all fit under the allowable size limit for the OCR processing. The Region Of Interest is determined by the image segmentation model, with the map area being buffered by a fixed amount of pixels (150 for now) - -**Image ROI** -The Tile pipeline will tile the image using the minimum amount of tiles necessary to have them all fit under the allowable size limit for the OCR processing. The Region Of Interest is determined by the image segmentation model, with the map area being buffered by a percentage of the overall image size (3% for now) - -**ROI ROI** -The Tile pipeline will tile the image using the minimum amount of tiles necessary to have them all fit under the allowable size limit for the OCR processing. The Region Of Interest is determined by the image segmentation model, with the map area being buffered by a percentage of the ROI size (5% for now) - -### Output - -5 different outputs are produced when georeferencing is executed at the command line. - -**GCP List** +**GCP List** A list of georeferenced GCPs along with distance from ground truth and other error indications if available. The data is output as a CSV file with the filename being `test-{pipeline name}.csv`. **Summary** @@ -47,81 +33,89 @@ A list of georeferenced gcps output specifically for downstream integration cont ### Installation -python 3.10 or higher is required +* python 3.10 or higher is required +* Installation of Detectron2 requires `torch` already be present in the environment, so it must be installed manually. +* Note: for python virtual environments, `conda` is more reliable for installing torch==2.0.x than `venv` To install from the current directory: ``` +# manually install torch - this is necessary due to issues with detectron2 dependencies +# (see https://github.com/facebookresearch/detectron2/issues/4472) +pip install torch==2.0.1 + # install the task library cd ../../tasks -pip install -e . +pip install -e .[segmentation] -# install the metadata extraction pipeline +# install the georeferencing pipeline cd ../pipelines/geo_referencing pip install -e . ``` ### Overview ### -* Pipelines are defined in `factory.py` and cannot be composed into other pipelines +* The georeferencing pipeline is defined in `factory.py` and is suitable for integration into other systems * Input is a image (ie binary image file buffer) -* Output are files output to disk +* Output is the georeferencing results materialized as: + * `GeoreferenceResult` JSON object (part of the CDR TA1 schema) ### Command Line Execution ### -`run_pipeline.py` provides a command line wrapper around the georeferencing pipelines, allowing for a directory of map images to be processed serially. +`run_pipeline.py` provides a command line wrapper around the georeferencing pipeline, allowing for a directory of map images to be processed serially. To run from the repository root directory: ``` -export GOOGLE_APPLICATION_CREDENTIALS=/credentinals/google_api_credentials.json +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json export OPENAI_API_KEY= +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= python3 -m pipelines.geo_referencing.run_pipeline \ --input /image/input/dir \ - --output /model/output/dir \ - --workdir /model/working/dir \ - --extract_metadata=True + --output /results/output/dir \ + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --model /path/to/segmentation/model/weights \ + --extract_metadata True \ + --no_gpu (if set, pipeline will force CPU-only processing) ``` -### REST Service ### -The REST service runs the Fixed ROI pipeline only to produce a single schema formatted output. +* `extract_metadata` Indicates whether to use metadata extraction to aid with georeferencing. +### REST Service ### `run_server.py` provides the pipeline as a REST service with the following endpoints: * ```POST: /api/process_image``` - Sends an image (as binary file buffer) to the georeferencing pipeline for analysis. Results are JSON string. * ```GET /healthcheck``` - Healthcheck endpoint +The server can also be configured to run with a request queue, using RabbitMQ, if the `rest` flag is not set. + To start the server: ``` export GOOGLE_APPLICATION_CREDENTIALS=/credentinals/google_api_credentials.json export OPENAI_API_KEY= +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= python3 -m pipelines.geo_referencing.run_server \ - --workdir /model/workingdir -``` -To test the server: - -``` -curl localhost:5000/healthcheck + --workdir /pipeline/working/dir (default is tmp/lara/workdir) \ + --model /path/to/segmentation/model/weights \ + --rest (if set, run the server in REST mode, instead of resquest-queue mode) + --no_gpu (if set, pipeline will force CPU-only processing) \ + --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` -To georeference an image: - -``` -curl http://localhost:5000/api/process_image \ - --header 'Content-Type: image/tiff' \ - --data-binary @/path/to/map/image.tif - --output output/file/path.json -``` ### Dockerized deployment -A dockerized version REST service described above is available that includes model weights. OCR text and metadata extraction require -the `GOOGLE_APPLICATION_CREDENTIALS` and `OPENAI_API_KEY` environment variables be set, and a +A dockerized version REST service described above is available that includes model weights. OCR text and metadata extraction require the `GOOGLE_APPLICATION_CREDENTIALS` and `OPENAI_API_KEY` environment variables be set, and a a local directory for caching results must be provided. To run: ``` cd deploy + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json export OPENAI_API_KEY= -./run.sh /path/to/workdir + +./run.sh /path/to/workdir /pipline/images/working/dir ``` The `deploy/build.sh` script can also be used to build the Docker image from source. \ No newline at end of file From f184f041f5615d162b93808f82223320e5fc218e Mon Sep 17 00:00:00 2001 From: Dan Desroches Date: Thu, 15 Aug 2024 09:22:30 -0400 Subject: [PATCH 60/66] use buildx instead --- pipelines/geo_referencing/deploy/build.sh | 2 +- pipelines/metadata_extraction/deploy/build.sh | 2 +- pipelines/segmentation/deploy/build.sh | 2 +- pipelines/text_extraction/deploy/build.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pipelines/geo_referencing/deploy/build.sh b/pipelines/geo_referencing/deploy/build.sh index e5ae5d68..57bc1519 100755 --- a/pipelines/geo_referencing/deploy/build.sh +++ b/pipelines/geo_referencing/deploy/build.sh @@ -23,7 +23,7 @@ else fi # run the build -docker build -t uncharted/lara-georef:latest . +docker buildx build --platform linux/amd64,linux/arm64 -t uncharted/lara-georef:latest . # cleanup the temp files rm -rf pipelines diff --git a/pipelines/metadata_extraction/deploy/build.sh b/pipelines/metadata_extraction/deploy/build.sh index c1d183f4..b8c1cc8b 100755 --- a/pipelines/metadata_extraction/deploy/build.sh +++ b/pipelines/metadata_extraction/deploy/build.sh @@ -23,7 +23,7 @@ else fi # run the build -docker build -t uncharted/lara-metadata-extract:latest . +docker buildx build --platform linux/amd64,linux/arm64 -t uncharted/lara-metadata-extract:latest . --push # cleanup the temp files rm -rf pipelines diff --git a/pipelines/segmentation/deploy/build.sh b/pipelines/segmentation/deploy/build.sh index b0bd5278..8cd93089 100755 --- a/pipelines/segmentation/deploy/build.sh +++ b/pipelines/segmentation/deploy/build.sh @@ -22,7 +22,7 @@ else fi # run the build -docker build -t uncharted/lara-segmentation:latest . +docker buildx build --platform linux/amd64,linux/arm64 -t uncharted/lara-segmentation:latest . --push # cleanup the temp files rm -rf pipelines diff --git a/pipelines/text_extraction/deploy/build.sh b/pipelines/text_extraction/deploy/build.sh index 78a7118f..656f8ebd 100755 --- a/pipelines/text_extraction/deploy/build.sh +++ b/pipelines/text_extraction/deploy/build.sh @@ -10,7 +10,7 @@ cp -r ../../../util . cp -r ../../../tasks . # run the build -docker build -t uncharted/lara-text-extract:latest . +docker buildx build --platform linux/amd64,linux/arm64 -t uncharted/lara-text-extract:latest . --push # cleanup the temp files rm -rf pipelines From 8fd7686e6c05b7cffa822d1a0e8e4fe55a6cbdce Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Thu, 15 Aug 2024 09:26:10 -0400 Subject: [PATCH 61/66] fixed some typos, and updated `tasks` README --- pipelines/geo_referencing/README.md | 8 +++--- pipelines/metadata_extraction/README.md | 4 +-- pipelines/point_extraction/README.md | 10 +++---- pipelines/segmentation/README.md | 4 +-- pipelines/text_extraction/README.md | 4 +-- tasks/README.md | 35 ++++++++++++++++--------- 6 files changed, 37 insertions(+), 28 deletions(-) diff --git a/pipelines/geo_referencing/README.md b/pipelines/geo_referencing/README.md index 7e1de8c5..9b8e6b11 100644 --- a/pipelines/geo_referencing/README.md +++ b/pipelines/geo_referencing/README.md @@ -6,7 +6,7 @@ This pipeline georeferences an input raster map image. The georeferencing approa The georeferencing pipeline uses info from many other LARA tasks to improve its results: * Text extraction (OCR) is used to extract map geo-coord labels and other metadata -* Metadata Extraction is used to help determine additional info about a map's general region and scale (eg state, county, UTM zone) +* Metadata Extraction is used to help determine additional info about a map's general region and scale (e.g. state, county, UTM zone) * Segmentation is used to determine to the map's bounds within an image. See more info on pipeline tasks here: [../../tasks/README.md](../../tasks/README.md) @@ -34,7 +34,7 @@ A list of georeferenced gcps output specifically for downstream integration cont ### Installation * python 3.10 or higher is required -* Installation of Detectron2 requires `torch` already be present in the environment, so it must be installed manually. +* Installation of Detectron2 requires `torch` to already be present in the environment, so it must be installed manually. * Note: for python virtual environments, `conda` is more reliable for installing torch==2.0.x than `venv` To install from the current directory: @@ -99,7 +99,7 @@ python3 -m pipelines.geo_referencing.run_server \ --model /path/to/segmentation/model/weights \ --rest (if set, run the server in REST mode, instead of resquest-queue mode) --no_gpu (if set, pipeline will force CPU-only processing) \ - --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --imagedir /pipeline/images/working/dir (only needed for request-queue mode) \ --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` @@ -115,7 +115,7 @@ cd deploy export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json export OPENAI_API_KEY= -./run.sh /path/to/workdir /pipline/images/working/dir +./run.sh /path/to/workdir /pipeline/images/working/dir ``` The `deploy/build.sh` script can also be used to build the Docker image from source. \ No newline at end of file diff --git a/pipelines/metadata_extraction/README.md b/pipelines/metadata_extraction/README.md index 79945322..a6e0177c 100644 --- a/pipelines/metadata_extraction/README.md +++ b/pipelines/metadata_extraction/README.md @@ -104,7 +104,7 @@ python3 -m pipelines.metadata_extraction.run_server \ --rest (if set, run the server in REST mode, instead of resquest-queue mode) --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ --no_gpu (if set, pipeline will force CPU-only processing) \ - --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --imagedir /pipeline/images/working/dir (only needed for request-queue mode) \ --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` @@ -117,7 +117,7 @@ cd deploy export GOOGLE_APPLICATION_CREDENTIALS=/credentials/google_api_credentials.json export OPENAI_API_KEY= -./run.sh /pipeline/working/dir /pipline/images/working/dir +./run.sh /pipeline/working/dir /pipeline/images/working/dir ``` diff --git a/pipelines/point_extraction/README.md b/pipelines/point_extraction/README.md index b2fc3ef3..c8cbfce4 100644 --- a/pipelines/point_extraction/README.md +++ b/pipelines/point_extraction/README.md @@ -2,7 +2,7 @@ ## LARA Point Extraction Pipeline -This pipeline extracts bedding point symbols from a map, along with their orientation and associated incline (dip) information. The model leverages [YOLO](https://github.com/ultralytics/ultralytics) for extraction of high priority / common point symbol types. In addition, a CV-based One-Shot algorithm can be used to extract less common point symbols. +This pipeline extracts point symbols from a map, along with their orientation and associated incline (dip) information. The model leverages [YOLO](https://github.com/ultralytics/ultralytics) for extracting high priority / common point symbol types. In addition, a CV-based One-Shot algorithm can be used to extract less common point symbols. See more info on pipeline tasks here: [../../tasks/README.md](../../tasks/README.md) @@ -28,7 +28,7 @@ The YOLO object detection model has been trained to extract common geologic poin * Mine Quarry #### One-Shot Model -The One-shot CV algorithm can be used to extract any leftover less common point symbols that may be present. This algorithm requires legend swatches to be available as a template (either via HITL annotation or some other manner) +The One-shot CV algorithm can be used to extract any "leftover" less common point symbols that may be present. This algorithm requires legend swatches to be available as a template (either via human-in-the-loop annotation or some other manner) ### Point Symbol Orientation Many point symbols also contain directional information. @@ -47,7 +47,7 @@ Point orientation (ie "strike" direction) and the "dip" magnitude are also extra ### Installation * python 3.10 or higher is required -* Installation of Detectron2 requires `torch` already be present in the environment, so it must be installed manually. +* Installation of Detectron2 requires `torch` to already be present in the environment, so it must be installed manually. * Note: for python virtual environments, `conda` is more reliable for installing torch==2.0.x than `venv` To install from the current directory: @@ -137,7 +137,7 @@ python3 -m pipelines.point_extraction.run_server \ --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ --fetch_legend_items (if set, the pipeline will query the CDR for validated legend annotations for a given map input) \ --no_gpu (if set, pipeline will force CPU-only processing) \ - --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --imagedir /pipeline/images/working/dir (only needed for request-queue mode) \ --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` @@ -152,7 +152,7 @@ cd deploy export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json export CDR_API_TOKEN= -./run.sh /path/to/workdir /pipline/images/working/dir +./run.sh /path/to/workdir /pipeline/images/working/dir ``` The `deploy/build.sh` script can also be used to build the Docker image from source. diff --git a/pipelines/segmentation/README.md b/pipelines/segmentation/README.md index f1bdf478..f2d1002a 100644 --- a/pipelines/segmentation/README.md +++ b/pipelines/segmentation/README.md @@ -90,7 +90,7 @@ python3 -m pipelines.segmentation.run_server \ --rest (if set, run the server in REST mode, instead of resquest-queue mode) --cdr_schema (if set, pipeline will also output CDR schema JSON objects) \ --no_gpu (if set, pipeline will force CPU-only processing) \ - --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --imagedir /pipeline/images/working/dir (only needed for request-queue mode) \ --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` @@ -104,6 +104,6 @@ cd deploy export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= -./run.sh /pipeline/working/dir /pipline/images/working/dir +./run.sh /pipeline/working/dir /pipeline/images/working/dir ``` diff --git a/pipelines/text_extraction/README.md b/pipelines/text_extraction/README.md index 3a5b8e6d..9b32c852 100644 --- a/pipelines/text_extraction/README.md +++ b/pipelines/text_extraction/README.md @@ -70,7 +70,7 @@ python3 -m pipelines.text_extraction.run_server \ --pixel_limit 6000 \ --gamma_corr 1.0 \ --rest (if set, run the server in REST mode, instead of resquest-queue mode) \ - --imagedir /pipline/images/working/dir (only needed for request-queue mode) \ + --imagedir /pipeline/images/working/dir (only needed for request-queue mode) \ --rabbit_host (rabbitmq host; only needed for request-queue mode) ``` @@ -82,7 +82,7 @@ cd deploy export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json -./run.sh /model/working/dir /pipline/images/working/dir +./run.sh /model/working/dir /pipeline/images/working/dir ``` diff --git a/tasks/README.md b/tasks/README.md index e94fc2df..9f129d78 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -1,5 +1,7 @@ ## LARA Tasks LARA Pipeline Tasks +* In general, a `task` is considered a processing step in a given LARA `pipeline`. +* All `tasks` are executed via their `run` method ### Installation @@ -33,7 +35,7 @@ https://cloud.google.com/vision/docs/ocr#vision_text_detection_gcs-python. * Text extraction is done via the `TextExtractor` child classes * To access the Google Vision API, the `GOOGLE_APPLICATION_CREDENTIALS` environment variable must be set to the google vision credentials json file -* Input is a map raster as an OpenCV image +* Input is a map raster image * Ouput is OCR results as a `DocTextExtraction` object A pipeline using this task, along with a CLI and sever wrapper are available at [../pipelines/text_extraction](../pipelines/text_extraction) @@ -55,12 +57,12 @@ A pipeline using this task, along with a CLI and sever wrapper are available at ### Image Segmentation Task -**Goal:** to perform segmentation to isolate the map and legend regions on an image +**Goal:** to perform segmentation to isolate the map, legend and cross-section regions on an image Segmentation is done using a fine-tuned version of the `LayoutLMv3` model: https://github.com/microsoft/unilm/tree/master/layoutlmv3 -See more info on pipeline deployment here: ../pipelines/segmentation](../pipelines/segmentation) +See more info on pipeline deployment here: [../pipelines/segmentation](../pipelines/segmentation) #### Segmentation categories (classes) @@ -79,12 +81,14 @@ A pipeline using this task, along with a CLI and sever wrapper are available at ### Point Extraction Tasks ### -**Goal:** Extracts point symbols from a map, along with their orientation and associated incline information +**Goal:** Extracts point symbols from a map, along with their orientation and associated incline (dip) information -The model leverages [YOLOv8](https://github.com/ultralytics/ultralytics) for the baseline object detection task +The model leverages [YOLO](https://github.com/ultralytics/ultralytics) for extracting high priority / common point symbol types. In addition, a CV-based One-Shot algorithm can be used to extract less common point symbols. -#### Extracted Point Types #### -Initial efforts have focused on identifying and extracting the following 15 symbols: +### Extracted Symbols + +#### Object Detection Model +The YOLO object detection model has been trained to extract common geologic point symbols as follows: * Inclined Bedding (aka strike/dip) * Vertical Bedding * Horizontal Bedding @@ -102,8 +106,11 @@ Initial efforts have focused on identifying and extracting the following 15 symb * Mine Tunnel * Mine Quarry -#### Point Symbol Orientation #### -Some point symbols also contain directional information. +#### One-Shot Model +The One-shot CV algorithm can be used to extract any "leftover" less common point symbols that may be present. This algorithm requires legend swatches to be available as a template (either via human-in-the-loop annotation or some other manner) + +### Point Symbol Orientation +Many point symbols also contain directional information. Point orientation (ie "strike" direction) and the "dip" magnitude are also extracted for applicable symbol types: * Inclined Bedding (strike/dip) * Vertical Bedding @@ -115,10 +122,13 @@ Point orientation (ie "strike" direction) and the "dip" magnitude are also extra * Lineation * Mine Tunnel + #### Using the Point Extraction Tasks #### -* The main point extraction is available in the `YOLOPointDetector` task -* Ouput is a`PointLabels` JSON object, which contains a list of `PointLabel` capturing the point information. -* Both dectector tasks take `ImageTiles` objects as inputs - `ImageTiles` are produced by the `Tiler` task +* The main YOLO point extraction is available in the `YOLOPointDetector` task +* The One-Shot point extraction is in the `TemplateMatchPointExtractor` task +* Point orientation (strike/dip angle) detection is in the `PointOrientationExtractor` task +* Output is a`PointLabels` JSON object, which contains a list of `PointLabel` capturing the point information. +* Extraction tasks take `ImageTiles` objects as inputs - `ImageTiles` are produced by the `Tiler` task * `ImageTiles` can be re-assembled into a `PointLabels` using the `Untiler` task A pipeline using these task, along with a CLI and sever wrapper are available at [../pipelines/point_extraction](../pipelines/point_extraction) @@ -139,4 +149,3 @@ This module relies on image segmentation to identify the map area, OCR output as A pipeline using this task, along with a CLI and sever wrapper are available at [../pipelines/geo_referencing](../pipelines/geo_referencing) -... \ No newline at end of file From d04323e1b2a88ec402f3bba3d038e6c0a32fb991 Mon Sep 17 00:00:00 2001 From: Dan Desroches Date: Thu, 15 Aug 2024 09:31:14 -0400 Subject: [PATCH 62/66] Update build.sh --- pipelines/point_extraction/deploy/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/point_extraction/deploy/build.sh b/pipelines/point_extraction/deploy/build.sh index 49e92d58..611ba5f5 100755 --- a/pipelines/point_extraction/deploy/build.sh +++ b/pipelines/point_extraction/deploy/build.sh @@ -37,7 +37,7 @@ fi # run the build -docker build -t uncharted/lara-point-extract:latest . +docker buildx build --platform linux/amd64,linux/arm64 -t uncharted/lara-point-extract:latest . --push # cleanup the temp files rm -rf pipelines From c22baf6b310a5a10136047e86ce95cbd59f10365 Mon Sep 17 00:00:00 2001 From: David Giesbrecht Date: Thu, 15 Aug 2024 09:45:09 -0400 Subject: [PATCH 63/66] added notes/links to READMEs about Makefile --- README.md | 3 +-- pipelines/geo_referencing/README.md | 4 +++- pipelines/metadata_extraction/README.md | 1 + pipelines/point_extraction/README.md | 2 +- pipelines/segmentation/README.md | 1 + pipelines/text_extraction/README.md | 1 + 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4dab3262..e98f6e60 100644 --- a/README.md +++ b/README.md @@ -15,5 +15,4 @@ This repository contains five pipelines: The `tasks` directory contains the `pip` installable library of tasks and supporting utilities, with each pipeline found in the `pipelines` directory being composed of these tasks. Each pipeline is itself `pip` installable, and is accompanied by a wrapper to support command line execution (`run_pipeline.py`), and a server wrapper to support execution as a REST service (`run_server.py`). Scripts to build the server wrapper into a Docker container are also included. - - +A [Makefile](./Makefile) is also available to handle building and deploying Docker containers for the various LARA pipelines. diff --git a/pipelines/geo_referencing/README.md b/pipelines/geo_referencing/README.md index 9b8e6b11..ae8d2d0d 100644 --- a/pipelines/geo_referencing/README.md +++ b/pipelines/geo_referencing/README.md @@ -118,4 +118,6 @@ export OPENAI_API_KEY= ./run.sh /path/to/workdir /pipeline/images/working/dir ``` -The `deploy/build.sh` script can also be used to build the Docker image from source. \ No newline at end of file +The `deploy/build.sh` script can also be used to build the Docker image from source. + +Alternatively, a [Makefile](../../Makefile) is available to handle the building and deploying the various LARA pipeline containers. \ No newline at end of file diff --git a/pipelines/metadata_extraction/README.md b/pipelines/metadata_extraction/README.md index a6e0177c..e013677e 100644 --- a/pipelines/metadata_extraction/README.md +++ b/pipelines/metadata_extraction/README.md @@ -120,4 +120,5 @@ export OPENAI_API_KEY= ./run.sh /pipeline/working/dir /pipeline/images/working/dir ``` +Alternatively, a [Makefile](../../Makefile) is available to handle the building and deploying the various LARA pipeline containers. diff --git a/pipelines/point_extraction/README.md b/pipelines/point_extraction/README.md index c8cbfce4..6f3e77d8 100644 --- a/pipelines/point_extraction/README.md +++ b/pipelines/point_extraction/README.md @@ -157,4 +157,4 @@ export CDR_API_TOKEN= The `deploy/build.sh` script can also be used to build the Docker image from source. - +Alternatively, a [Makefile](../../Makefile) is available to handle the building and deploying the various LARA pipeline containers. diff --git a/pipelines/segmentation/README.md b/pipelines/segmentation/README.md index f2d1002a..d033ce4b 100644 --- a/pipelines/segmentation/README.md +++ b/pipelines/segmentation/README.md @@ -107,3 +107,4 @@ export AWS_SECRET_ACCESS_KEY= ./run.sh /pipeline/working/dir /pipeline/images/working/dir ``` +Alternatively, a [Makefile](../../Makefile) is available to handle the building and deploying the various LARA pipeline containers. diff --git a/pipelines/text_extraction/README.md b/pipelines/text_extraction/README.md index 9b32c852..26984850 100644 --- a/pipelines/text_extraction/README.md +++ b/pipelines/text_extraction/README.md @@ -85,5 +85,6 @@ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/google_api_credentials.json ./run.sh /model/working/dir /pipeline/images/working/dir ``` +Alternatively, a [Makefile](../../Makefile) is available to handle the building and deploying the various LARA pipeline containers. From c19dd3aad6ff8dddda573b7097d00bd673f4805a Mon Sep 17 00:00:00 2001 From: rking Date: Thu, 15 Aug 2024 11:58:51 -0400 Subject: [PATCH 64/66] handle cases where input data is missing `SEGMENTATION_OUTPUT_KEY` --- tasks/metadata_extraction/metadata_extraction.py | 3 ++- tasks/metadata_extraction/text_filter.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/metadata_extraction/metadata_extraction.py b/tasks/metadata_extraction/metadata_extraction.py index e0fb4e91..b55b0d64 100644 --- a/tasks/metadata_extraction/metadata_extraction.py +++ b/tasks/metadata_extraction/metadata_extraction.py @@ -367,7 +367,7 @@ def run(self, input: TaskInput) -> TaskResult: metadata.utm_zone = str(self._extract_utm_zone(metadata)) # compute map shape from the segmentation output - segments = input.data[SEGMENTATION_OUTPUT_KEY] + segments = input.data.get(SEGMENTATION_OUTPUT_KEY, None) metadata.map_shape = self._compute_shape(segments) # compute map chroma from the image @@ -779,6 +779,7 @@ def _compute_shape(self, segments) -> MapShape: Returns: MapShape: The shape of the map """ + map_shape = MapShape.UNKNOWN if segments: map_segmentation = MapSegmentation.model_validate(segments) for segment in map_segmentation.segments: diff --git a/tasks/metadata_extraction/text_filter.py b/tasks/metadata_extraction/text_filter.py index 29bca6b7..f57cba5f 100644 --- a/tasks/metadata_extraction/text_filter.py +++ b/tasks/metadata_extraction/text_filter.py @@ -47,7 +47,7 @@ def run(self, input: TaskInput) -> TaskResult: doc_text = DocTextExtraction.model_validate(text_data) # get map segments - segments = input.data[SEGMENTATION_OUTPUT_KEY] + segments = input.data.get(SEGMENTATION_OUTPUT_KEY, {"doc_id": "", "segments": []}) map_segmentation = MapSegmentation.model_validate(segments) output_text: Dict[str, TextExtraction] = {} From 34f4213dff8199935e9e5d300867e7c168a7b71a Mon Sep 17 00:00:00 2001 From: rking Date: Thu, 15 Aug 2024 15:43:21 -0400 Subject: [PATCH 65/66] change doc_id --- tasks/metadata_extraction/text_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/metadata_extraction/text_filter.py b/tasks/metadata_extraction/text_filter.py index f57cba5f..4de764c9 100644 --- a/tasks/metadata_extraction/text_filter.py +++ b/tasks/metadata_extraction/text_filter.py @@ -47,7 +47,7 @@ def run(self, input: TaskInput) -> TaskResult: doc_text = DocTextExtraction.model_validate(text_data) # get map segments - segments = input.data.get(SEGMENTATION_OUTPUT_KEY, {"doc_id": "", "segments": []}) + segments = input.data.get(SEGMENTATION_OUTPUT_KEY, {"doc_id": input.raster_id, "segments": []}) map_segmentation = MapSegmentation.model_validate(segments) output_text: Dict[str, TextExtraction] = {} From dfcea7bfad90bcc7ad1212ea953c0e499764be91 Mon Sep 17 00:00:00 2001 From: rking Date: Thu, 15 Aug 2024 15:48:50 -0400 Subject: [PATCH 66/66] formatting --- tasks/metadata_extraction/text_filter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tasks/metadata_extraction/text_filter.py b/tasks/metadata_extraction/text_filter.py index 4de764c9..ba30d0aa 100644 --- a/tasks/metadata_extraction/text_filter.py +++ b/tasks/metadata_extraction/text_filter.py @@ -47,7 +47,9 @@ def run(self, input: TaskInput) -> TaskResult: doc_text = DocTextExtraction.model_validate(text_data) # get map segments - segments = input.data.get(SEGMENTATION_OUTPUT_KEY, {"doc_id": input.raster_id, "segments": []}) + segments = input.data.get( + SEGMENTATION_OUTPUT_KEY, {"doc_id": input.raster_id, "segments": []} + ) map_segmentation = MapSegmentation.model_validate(segments) output_text: Dict[str, TextExtraction] = {}