Merge pull request #2 from kamangir/firms-api-area-2024-07-15

firms/api/area - 1

README.md

@@ -10,12 +10,11 @@ pip install blue-geo
 |-|-|-|
 | 🌐 [QGIS](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/QGIS/README.md) | 🇺🇦 [ukraine-timemap](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/ukraine-timemap/README.md) | 🌈 [vancouver-watching](https://github.com/kamangir/Vancouver-Watching) |
 | [![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/QGIS.jpg)](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/QGIS/README.md) | [![image](https://github.com/kamangir/assets/blob/main/nbs/ukraine-timemap/QGIS.png?raw=true)](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/ukraine-timemap/README.md) | [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true)](https://github.com/kamangir/Vancouver-Watching)  |
-| an AI terraform for [QGIS](https://www.qgis.org/). | ingests the [Bellingcat](https://www.bellingcat.com/) [Civilian Harm in Ukraine TimeMap](https://github.com/bellingcat/ukraine-timemap) dataset, available through [this UI](https://ukraine.bellingcat.com/) and [this API](https://bellingcat-embeds.ams3.cdn.digitaloceanspaces.com/production/ukr/timemap/api.json), and generates a `geojson`, a QGIS project, and [more](https://kamangir-public.s3.ca-central-1.amazonaws.com/ukraine_timemap/ukraine_timemap.png).  | 🌈 Vancouver watching with AI, last build: [🔗](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif). |  |
-
-
-
-
-
+| an AI terraform for [QGIS](https://www.qgis.org/). | `ingest` for the [Bellingcat](https://www.bellingcat.com/) [Civilian Harm in Ukraine TimeMap](https://github.com/bellingcat/ukraine-timemap) dataset, available through [this UI](https://ukraine.bellingcat.com/) and [this API](https://bellingcat-embeds.ams3.cdn.digitaloceanspaces.com/production/ukr/timemap/api.json).  | 🌈 Vancouver watching with AI, last build: [🔗](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif). |  |
+| | | |
+| 🌐 [firms](https://github.com/kamangir/blue-geo/blob/main/blue_geo/firms/README.md) | | |
+| 🔥 | | |
+| `ingest` for [FIRMS](https://firms.modaps.eosdis.nasa.gov): Fire Information for Resource Management System. | | |
 
 ---
 

blue_geo/.abcli/blue_geo.sh

@@ -5,6 +5,8 @@ function blue_geo() {
 
     if [ $task == "help" ]; then
         ukraine_timemap "$@"
+        blue_geo_QGIS "$@"
+        blue_geo_ingest "$@"
         return
     fi
 

blue_geo/.abcli/ingest.sh

@@ -0,0 +1,21 @@
+#! /usr/bin/env bash
+
+function blue_geo_ingest() {
+    local task=$(abcli_unpack_keyword $1 help)
+
+    if [ "$task" == "help" ]; then
+        blue_geo_ingest_firms "$@"
+        return
+    fi
+
+    local function_name=blue_geo_ingest_$task
+    if [[ $(type -t $function_name) == "function" ]]; then
+        $function_name "${@:2}"
+        return
+    fi
+
+    abcli_log_error "-blue_geo: ingest: $task: command not found."
+    return 1
+}
+
+abcli_source_path - caller,suffix=/ingest

blue_geo/.abcli/ingest/firms.sh

@@ -0,0 +1,41 @@
+#! /usr/bin/env bash
+
+function blue_geo_ingest_firms() {
+    local options=$1
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options="dryrun,~upload"
+        local date=$(abcli_string_timestamp_short \
+            --include_time 0 \
+            --unique 0)
+        local area=$(python3 -m blue_geo.firms.api.area \
+            get \
+            --what area \
+            --delim \|)
+        local source=$(python3 -m blue_geo.firms.api.area \
+            get \
+            --what source \
+            --values 1 \
+            --delim \|)
+        local args="[--date $date]$ABCUL[depth 1]$ABCUL[--area $area]$ABCUL[--source $source]$ABCUL[--log 1]"
+        abcli_show_usage "blue_geo ingest firms$ABCUL[$options]$ABCUL[.|<object-name>]$ABCUL$args" \
+            "firms -ingest-> <object-name>."
+        return
+    fi
+
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
+
+    local object_name=$(abcli_clarify_object $2 .)
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m blue_geo.firms.api.area \
+        ingest \
+        --object_name $object_name \
+        "${@:4}"
+
+    [[ "$do_dryrun" == 1 ]] &&
+        abcli_upload - $object_name
+
+    return 0
+}

blue_geo/.abcli/tests/ingest.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+
+function test_blue_geo_ingest() {
+    local object_name=ingest-$(abcli_string_timestamp_short)
+
+    abcli_eval dryrun=$do_dryrun \
+        blue_geo ingest firms - \
+        $object_name \
+        "${@:2}"
+}

blue_geo/__init__.py

@@ -4,7 +4,7 @@
 
 DESCRIPTION = f"{ICON} AI for precise geospatial data analysis and visualization."
 
-VERSION = "4.43.1"
+VERSION = "4.54.1"
 
 REPO_NAME = "blue-geo"
 

blue_geo/firms/README.md

@@ -0,0 +1,23 @@
+# 🌐 firms
+
+[FIRMS](https://firms.modaps.eosdis.nasa.gov): Fire Information for Resource Management System.
+
+map-key: https://firms.modaps.eosdis.nasa.gov/api/map_key/
+
+area api: https://firms.modaps.eosdis.nasa.gov/api/area/
+
+```bash
+pip install blue-geo
+```
+
+```bash
+ > blue_geo ingest firms \
+	[dryrun,~upload] \
+	[.|<object-name>] \
+	[--date 2024-07-16] \
+	[depth 1] \
+	[--area east|north|south|west|world] \
+	[--source LANDSAT_NRT|MODIS_NRT|MODIS_SP|VIIRS_NOAA20_NRT|VIIRS_NOAA21_NRT|VIIRS_SNPP_NRT|VIIRS_SNPP_SP] \
+	[--log 1]
+ . firms -ingest-> <object-name>.
+```

blue_geo/firms/__init__.py

@@ -0,0 +1,3 @@
+from blue_geo import NAME
+
+NAME = f"{NAME}.firms"

blue_geo/firms/api/__init__.py

@@ -0,0 +1,3 @@
+from blue_geo.firms import NAME
+
+NAME = f"{NAME}.api"

blue_geo/firms/api/area/__init__.py

@@ -0,0 +1,5 @@
+from blue_geo.firms.api import NAME
+
+NAME = f"{NAME}.area"
+
+REFERENCE = "https://firms.modaps.eosdis.nasa.gov/api/area/"

blue_geo/firms/api/area/__main__.py

@@ -0,0 +1,83 @@
+import argparse
+from datetime import datetime, timedelta
+from blue_geo import VERSION
+from blue_geo.firms.api.area import NAME
+from blue_geo.firms.api.area.enums import Area, Source
+from blue_geo.firms.api.area.classes import APIRequest
+from blue_geo.logger import logger
+from blueness.argparse.generic import sys_exit
+
+parser = argparse.ArgumentParser(NAME, description=f"{NAME}-{VERSION}")
+parser.add_argument(
+    "task",
+    type=str,
+    help="get|ingest",
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+)
+parser.add_argument(
+    "--source",
+    type=str,
+    default=Source.default().name,
+    help="|".join(Source.values()),
+)
+parser.add_argument(
+    "--area",
+    type=str,
+    default=Area.default().name,
+    help="|".join(Area.values()),
+)
+parser.add_argument(
+    "--date",
+    type=str,
+    default=(datetime.now() - timedelta(days=5)).strftime("%Y-%m-%d"),
+    help="yyyy-mm-dd",
+)
+parser.add_argument(
+    "--depth",
+    type=int,
+    default=1,
+    help="1..10",
+)
+parser.add_argument(
+    "--what",
+    type=str,
+    default="area",
+    help="area|source",
+)
+parser.add_argument(
+    "--values",
+    type=int,
+    default=1,
+    help="0|1",
+)
+parser.add_argument(
+    "--delim",
+    type=str,
+    default="|",
+)
+args = parser.parse_args()
+
+delim = " " if args.delim == "space" else args.delim
+
+success = False
+if args.task == "get":
+    what = Area if args.what == "area" else Source if args.what == "source" else None
+    what = what.default() if what else None
+    print((delim.join(what.values()) if args.values else what.name) if what else None)
+    success = True
+elif args.task == "ingest":
+    api_request = APIRequest(
+        area=Area[args.area],
+        source=Source[args.source],
+        depth=args.depth,
+        date=args.date,
+    )
+
+    success, _ = api_request.ingest(object_name=args.object_name)
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

blue_geo/firms/api/area/classes.py

@@ -0,0 +1,112 @@
+from typing import Tuple
+from datetime import datetime, timedelta
+import pandas as pd
+import geopandas as gpd
+from shapely.geometry import Point
+from . import NAME
+from .enums import Area, Source
+from abcli import file
+from abcli.modules import objects
+from blue_geo import env
+from blue_geo.logger import logger
+
+
+class APIRequest:
+    def __init__(
+        self,
+        source: Source = Source.default(),
+        area: Area = Area.default(),
+        date: str = "",
+        depth: int = 1,
+        log: bool = True,
+    ):
+        self.prefix = "https://firms.modaps.eosdis.nasa.gov"
+        self.map_key = env.FIRMS_MAP_KEY
+
+        self.area: Area = area
+        self.source: Source = source
+
+        self.depth: int = depth
+
+        self.date: str = (
+            date if date else (datetime.now() - timedelta(days=5)).strftime("%Y-%m-%d")
+        )
+
+        if log:
+            logger.info(self.as_str())
+
+    def as_str(self) -> str:
+        return "{}.{}, area:{}, source: {} ({})".format(
+            NAME,
+            self.__class__.__name__,
+            self.area.name.lower(),
+            self.source.name,
+            self.source.description,
+        )
+
+    @property
+    def datacube_id(self) -> str:
+        return "blue-geo-firms-{}-{}".format(
+            self.area.name.lower(),
+            self.source.name,
+        )
+
+    def ingest(self, object_name: str) -> Tuple[
+        bool,
+        gpd.GeoDataFrame,
+    ]:
+        logger.info(
+            "{}.{} -> {}".format(
+                NAME,
+                self.__class__.__name__,
+                object_name,
+            )
+        )
+
+        csv_filename = objects.path_of(
+            f"{object_name}.csv",
+            object_name,
+            create=True,
+        )
+        if not file.download(
+            self.url(),
+            csv_filename,
+        ):
+            return False, gpd.GeoDataFrame()
+
+        data = pd.read_csv(csv_filename)
+        logger.info(f"loaded {len(data):,} point(s).")
+
+        gdf = gpd.GeoDataFrame(
+            data,
+            geometry=[
+                Point(xy)
+                for xy in zip(
+                    data.longitude,
+                    data.latitude,
+                )
+            ],
+            crs="EPSG:4326",  # WGS84
+        )
+
+        return (
+            file.save_geojson(
+                objects.path_of(
+                    f"{object_name}.geojson",
+                    object_name,
+                ),
+                gdf,
+            ),
+            gdf,
+        )
+
+    def url(self, html: bool = False) -> str:
+        return "{}/api/area/{}/{}/{}/{}/{}/{}".format(
+            self.prefix,
+            "html" if html else "csv",
+            self.map_key,
+            self.source.name,
+            self.area.name.lower(),
+            self.depth,  # day_range
+            self.date,
+        )