Merge pull request #51 from kamangir/modality-refactors-2024-11-28-40590

modality refactors

README.md

@@ -22,4 +22,4 @@ pip install blue-geo
 
 [![pylint](https://github.com/kamangir/blue-geo/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-geo/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-geo/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-geo/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-geo/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-geo/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-geo.svg)](https://pypi.org/project/blue-geo/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-geo)](https://pypistats.org/packages/blue-geo)
 
-built by 🌀 [`blue_options-4.168.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌐 [`blue_geo-4.801.1`](https://github.com/kamangir/blue-geo).
+built by 🌀 [`blue_options-4.171.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌐 [`blue_geo-4.809.1`](https://github.com/kamangir/blue-geo).

blue_geo/.abcli/QGIS/expressions.sh

@@ -5,7 +5,7 @@ function blue_geo_QGIS_expressions() {
 
     if [[ "$task" == pull ]]; then
         rsync \
-            -avv --progress \
+            -avv \
             "$BLUE_GEO_QGIS_PATH_EXPRESSIONS_GIT/" \
             "$BLUE_GEO_QGIS_PATH_EXPRESSIONS/"
         return
@@ -16,7 +16,7 @@ function blue_geo_QGIS_expressions() {
         local do_push=$(abcli_option_int "$options" push 0)
 
         rsync \
-            -avv --progress \
+            -avv \
             --exclude='__pycache__' \
             --exclude='default.py' \
             --exclude='__init__.py' \

blue_geo/.abcli/QGIS/templates.sh

@@ -10,15 +10,15 @@ function blue_geo_QGIS_templates() {
         abcli_download - $object_name
 
         rsync \
-            -avv --progress \
+            -avv \
             $object_path/ \
             "$BLUE_GEO_QGIS_PATH_TEMPLATES/"
         return
     fi
 
     if [ "$task" == "upload" ]; then
         rsync \
-            -avv --progress \
+            -avv \
             "$BLUE_GEO_QGIS_PATH_TEMPLATES/" \
             $object_path/
 

blue_geo/.abcli/tests/datacube_crop.sh

@@ -7,6 +7,7 @@ function test_blue_geo_datacube_crop() {
     blue_geo_datacube_ingest \
         scope=rgbx,$options \
         $datacube_id
+    [[ $? -ne 0 ]] && return 1
 
     local suffix=test_blue_geo_datacube_crop-$(abcli_string_timestamp_short)
 

blue_geo/.abcli/tests/datacube_generate.sh

@@ -2,15 +2,25 @@
 
 function test_blue_geo_datacube_generate() {
     local options=$1
+    local list_of_product=$(abcli_option "$options" product FRE+SRE)
 
     local datacube_id=$BLUE_GEO_TEST_DATACUBE_SKYFOX_VENUS
 
-    blue_geo_datacube_ingest \
-        scope=rgbx,$options \
-        $datacube_id
+    local product
+    for product in $(echo $list_of_product | tr + " "); do
+        abcli_log "product: $product"
 
-    blue_geo_datacube_generate \
-        ,$options \
-        $datacube_id \
-        --modality rgb
+        blue_geo_datacube_ingest \
+            scope=rgbx+_${product}_,$options \
+            $datacube_id
+        [[ $? -ne 0 ]] && return 1
+
+        blue_geo_datacube_generate \
+            ,$options \
+            $datacube_id \
+            --modality rgb@$product
+        [[ $? -ne 0 ]] && return 1
+
+        abcli_hr
+    done
 }

blue_geo/.abcli/watch/algo/modality/map.sh

@@ -22,12 +22,19 @@ function blue_geo_watch_algo_modality_map() {
 
     abcli_log "🌐 @geo watch $algo map $query_object_name @ $offset==$datacube_id -> /$suffix"
 
+    local product=$(python3 -c "print('$modality'.split('@',1)[1] if '@' in '$modality' else '')")
+    [[ ! -z "$product" ]] &&
+        abcli_log "product: $product"
+
     if [[ "$datacube_id" == *"DERIVED"* ]]; then
         abcli_download - \
             $datacube_id
     else
+        local scope="rgbx"
+        [[ ! -z "$product" ]] && scope=$scope+_${product}_
+
         blue_geo_datacube_ingest \
-            dryrun=$do_dryrun,scope=rgbx \
+            dryrun=$do_dryrun,scope=$scope \
             $datacube_id
     fi
 
@@ -52,8 +59,10 @@ function blue_geo_watch_algo_modality_map() {
         --modality $modality
     [[ $? -ne 0 ]] && return 1
 
+    local scope="rgb"
+    [[ ! -z "$product" ]] && scope=$scope+_${product}_
     local filename=$(blue_geo_datacube_list $cropped_datacube_id \
-        --scope rgb \
+        --scope $scope \
         --log 0 \
         --count 1 \
         --exists 1)

blue_geo/.abcli/watch/targets/download.sh

@@ -3,9 +3,13 @@
 function blue_geo_watch_targets_download() {
     local options=$1
 
-    abcli_download overwrite \
+    abcli_download - \
         $BLUE_GEO_WATCH_TARGET_LIST \
         "$@"
 
-    python3 -m blue_geo.watch.targets list
+    abcli_list_log $(python3 -m blue_geo.watch.targets \
+        list \
+        --log 0) \
+        --before "downloaded" \
+        --after "target(s)"
 }

blue_geo/__init__.py

@@ -9,7 +9,7 @@
 
 DESCRIPTION = f"{ICON} AI for a Blue Planet."
 
-VERSION = "4.801.1"
+VERSION = "4.809.1"
 
 REPO_NAME = "blue-geo"
 

blue_geo/catalog/SkyFox/Venus/classes.py

@@ -8,8 +8,6 @@
 from blue_geo import env
 from blue_geo.logger import logger
 
-rgb_suffixes = [f"SRE_B{band_index}.tif" for band_index in [7, 4, 3]]
-
 
 class SkyFoxVenusDatacube(STACDatacube):
     catalog = SkyFoxCatalog()
@@ -20,28 +18,70 @@ class SkyFoxVenusDatacube(STACDatacube):
 
     QGIS_template = env.BLUE_GEO_QGIS_TEMPLATE_DATACUBE_SKYFOX_VENUS
 
+    band_names: Dict[str, str] = {
+        7: "red",
+        4: "green",
+        3: "blue",
+    }
+
+    @classmethod
+    def band_index(cls, band_name: str) -> int:
+        return {
+            band_name: band_index for band_index, band_name in cls.band_names.items()
+        }.get(band_name, -1)
+
+    @staticmethod
+    def band_suffix(
+        band_index: int,
+        product: str = "SRE",  # SRE | FRE
+    ):
+        return f"{product}_B{band_index}.tif"
+
     def generate(
         self,
         modality: str,
         overwrite: bool = False,
     ) -> str:
+        product = ""
+        if modality.startswith("rgb@"):
+            modality, product = modality.split("@", 1)
+        if not product:
+            product = "SRE"
+
         # TODO: clean-up in the next refactor 🤦🏽
         if modality != "rgb":
             logger.error(f"{modality}: modality is not implemented.")
             return ""
 
+        logger.info(
+            "{}.generate({} : {})".format(
+                self.__class__.__name__,
+                modality,
+                product,
+            )
+        )
+
         list_of_colors = ["red", "green", "blue"]
 
-        filenames: Dict[str, str] = {}
-        for suffix, color in zip(rgb_suffixes, list_of_colors):
+        rgb_filename: str = ""
+        list_of_band_files: List[str] = []
+        for color in list_of_colors:
+            suffix = self.band_suffix(
+                band_index=self.band_index(band_name=color),
+                product=product,
+            )
+
             candidates = self.list_of_files(DatacubeScope(suffix))
             if not candidates:
                 logger.error(f"cannot find {suffix}.")
                 return ""
 
-            filenames[color] = self.full_filename(candidates[0])
+            full_filename = self.full_filename(candidates[0])
+            list_of_band_files += [full_filename]
+
+            if not rgb_filename:
+                rgb_filename = full_filename.replace(suffix, f"{product}_RGB.tif")
 
-        rgb_filename = filenames["red"].replace(rgb_suffixes[0], "SRE_RGB.tif")
         if file.exists(rgb_filename) and not overwrite:
             logger.info(f"✅ {rgb_filename}")
             return ""
@@ -52,7 +92,7 @@ def generate(
                 "-separate",
                 f"-o {rgb_filename}",
             ]
-            + [filenames[color] for color in list_of_colors]
+            + list_of_band_files
         )
 
     def ingest_filename(
@@ -90,6 +130,8 @@ def list_of_files(
     ) -> List[str]:
         raw_datacube_id = self.raw_datacube_id()
 
+        product: str = "FRE" if "_FRE_" in scope.contains else "SRE"
+
         output = scope.filter(
             [
                 {
@@ -99,16 +141,33 @@ def list_of_files(
                 if raw_datacube_id in value.href
             ],
             needed_for_rgb=lambda filename: any(
-                filename.endswith(suffix) for suffix in rgb_suffixes
+                filename.endswith(suffix)
+                for suffix in [
+                    self.band_suffix(
+                        band_index,
+                        product,
+                    )
+                    for band_index in [
+                        self.band_index(color)
+                        for color in [
+                            "red",
+                            "green",
+                            "blue",
+                        ]
+                    ]
+                ]
             ),
             verbose=verbose,
         )
 
         if scope.rgb:
-            suffix = rgb_suffixes[0]
+            suffix = self.band_suffix(
+                self.band_index("red"),
+                product,
+            )
             candidates = self.list_of_files(DatacubeScope(suffix))
             if candidates:
-                rgb_filename = candidates[0].replace(suffix, "SRE_RGB.tif")
+                rgb_filename = candidates[0].replace(suffix, f"{product}_RGB.tif")
                 output += [rgb_filename]
 
         return output

blue_geo/catalog/generic/generic/classes.py

@@ -114,12 +114,10 @@ def list_of_files(
         return []
 
     @staticmethod
-    def load_modality(
+    def load_geoimage(
         filename: str,
-        modality: str,
         ignore_error: bool = False,
         log: bool = False,
-        verbose: bool = False,
     ) -> Tuple[bool, np.ndarray, Dict[str, Any]]:
         success, frame, frame_file_metadata = file.load_geoimage(
             filename,

blue_geo/catalog/generic/generic/scope.py

@@ -37,7 +37,7 @@ def __init__(self, what: str):
 
         self.raster = "raster" in list_of_what
 
-        self.suffix = [
+        self.contains = [
             item for item in list_of_what if item not in self.special_options
         ]
 
@@ -91,8 +91,8 @@ def includes(
         ):
             return True
 
-        if self.suffix and any(
-            item_filename.endswith(suffix) for suffix in self.suffix
+        if self.contains and any(
+            item_filename.endswith(item) for item in self.contains
         ):
             return True
 

blue_geo/datacube/modalities.py

@@ -1,8 +1,8 @@
 from typing import List
 
 options: List[str] = [
-    "rgb",
-    "NDVI",
-    "NBR",
-    "NDVI-NBR",
+    "rgb[@<keyword>]",
+    #    "NDVI",
+    #    "NBR",
+    #    "NDVI-NBR",
 ]

blue_geo/help/watch/__init__.py

@@ -41,14 +41,15 @@ def help_(
 
     map_options = "".join(
         [
-            xtra("dryrun", mono),
+            xtra("dryrun,<map-options>", mono),
         ]
     )
 
     reduce_options = "".join(
         [
             xtra("dryrun,~gif,", mono),
             "publish",
+            xtra(",<reduce-options>", mono=mono),
         ]
     )
 
@@ -66,8 +67,9 @@ def help_(
         ],
         "watch target -{}> <object-name>.".format("aws-batch-" if help_batch else ""),
         {
+            "algo: diff | modality": [],
             "<algo-options>:": [
-                "diff: range=<100.0>",
+                "diff: modality=<modality>,range=<100.0>",
                 "modality: modality=<modality>",
             ],
             "modality: {}".format(" | ".join(modality_options)): [],