firms ingest refactor - kamangir/bolt#746

README.md

@@ -12,10 +12,11 @@ pip install blue-geo
 | [![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/QGIS.jpg)](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/QGIS/README.md) | [![image](https://github.com/kamangir/assets/blob/main/nbs/ukraine-timemap/QGIS.png?raw=true)](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/ukraine-timemap/README.md) | [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true)](https://github.com/kamangir/Vancouver-Watching)  |
 | an AI terraform for [QGIS](https://www.qgis.org/). | ingests the [Bellingcat](https://www.bellingcat.com/) [Civilian Harm in Ukraine TimeMap](https://github.com/bellingcat/ukraine-timemap) dataset, available through [this UI](https://ukraine.bellingcat.com/) and [this API](https://bellingcat-embeds.ams3.cdn.digitaloceanspaces.com/production/ukr/timemap/api.json), and generates a `geojson`, a QGIS project, and [more](https://kamangir-public.s3.ca-central-1.amazonaws.com/ukraine_timemap/ukraine_timemap.png).  | 🌈 Vancouver watching with AI, last build: [🔗](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif). |  |
 
-
-
-
-
+| | | |
+|-|-|-|
+| 🌐 [firms](https://github.com/kamangir/blue-geo/blob/main/blue_geo/firms/README.md) | | |
+| 🔥 | | |
+| `ingest` command for [FIRMS](https://firms.modaps.eosdis.nasa.gov): Fire Information for Resource Management System. | | |
 
 ---
 

blue_geo/__init__.py

@@ -4,7 +4,7 @@
 
 DESCRIPTION = f"{ICON} AI for precise geospatial data analysis and visualization."
 
-VERSION = "4.51.1"
+VERSION = "4.52.1"
 
 REPO_NAME = "blue-geo"
 

blue_geo/firms/README.md

@@ -0,0 +1,23 @@
+# 🌐 firms
+
+[FIRMS](https://firms.modaps.eosdis.nasa.gov): Fire Information for Resource Management System.
+
+map-key: https://firms.modaps.eosdis.nasa.gov/api/map_key/
+
+area api: https://firms.modaps.eosdis.nasa.gov/api/area/
+
+```bash
+pip install blue-geo
+```
+
+```bash
+ > blue_geo ingest firms \
+	[dryrun,~upload] \
+	[.|<object-name>] \
+	[--date 2024-07-16] \
+	[depth 1] \
+	[--area east|north|south|west|world] \
+	[--source LANDSAT_NRT|MODIS_NRT|MODIS_SP|VIIRS_NOAA20_NRT|VIIRS_NOAA21_NRT|VIIRS_SNPP_NRT|VIIRS_SNPP_SP] \
+	[--log 1]
+ . firms -ingest-> <object-name>.
+```

blue_geo/firms/api/area/__main__.py

@@ -76,7 +76,7 @@
         date=args.date,
     )
 
-    success = api_request.ingest(object_name=args.object_name)
+    success, _ = api_request.ingest(object_name=args.object_name)
 else:
     success = None
 

blue_geo/firms/api/area/classes.py

@@ -1,4 +1,8 @@
+from typing import Tuple
 from datetime import datetime, timedelta
+import pandas as pd
+import geopandas as gpd
+from shapely.geometry import Point
 from . import NAME
 from .enums import Area, Source
 from abcli import file
@@ -47,7 +51,10 @@ def datacube_id(self) -> str:
             self.source.name,
         )
 
-    def ingest(self, object_name: str) -> bool:
+    def ingest(self, object_name: str) -> Tuple[
+        bool,
+        gpd.GeoDataFrame,
+    ]:
         logger.info(
             "{}.{} -> {}".format(
                 NAME,
@@ -56,19 +63,42 @@ def ingest(self, object_name: str) -> bool:
             )
         )
 
+        csv_filename = objects.path_of(
+            f"{object_name}.csv",
+            object_name,
+            create=True,
+        )
         if not file.download(
             self.url(),
-            objects.path_of(
-                f"{object_name}.csv",
-                object_name,
-                create=True,
-            ),
+            csv_filename,
         ):
-            return False
+            return False, gpd.GeoDataFrame()
 
-        logger.info("🪄")
+        data = pd.read_csv(csv_filename)
+        logger.info(f"loaded {len(data):,} point(s).")
 
-        return True
+        gdf = gpd.GeoDataFrame(
+            data,
+            geometry=[
+                Point(xy)
+                for xy in zip(
+                    data.longitude,
+                    data.latitude,
+                )
+            ],
+            crs="EPSG:4326",  # WGS84
+        )
+
+        return (
+            file.save_geojson(
+                objects.path_of(
+                    f"{object_name}.geojson",
+                    object_name,
+                ),
+                gdf,
+            ),
+            gdf,
+        )
 
     def url(self, html: bool = False) -> str:
         return "{}/api/area/{}/{}/{}/{}/{}/{}".format(

blue_geo/notebooks.py

@@ -0,0 +1,19 @@
+import folium
+import geopandas as gpd
+from IPython.display import display
+
+
+def show_on_map(
+    gdf: gpd.GeoDataFrame,
+    zoom_start: int = 12,
+):
+    if gdf.empty:
+        return
+
+    map = folium.Map(
+        location=gdf.unary_union.centroid.coords[0][::-1],
+        zoom_start=zoom_start,
+    )
+    folium.GeoJson(gdf).add_to(map)
+
+    display(map)

blue_geo/tests/test_firms_api_area.py

@@ -2,6 +2,7 @@
 from blue_geo.firms.api.area import enums
 from abcli.modules.objects import unique_object
 from blue_geo.firms.api.area.classes import APIRequest
+import geopandas as gpd
 
 
 @pytest.mark.parametrize(
@@ -27,6 +28,8 @@ def test_blue_geo_firms_api_area(
 
     assert api_request.url(html=True)
 
-    assert api_request.ingest(object_name)
+    success, gdf = api_request.ingest(object_name)
+    assert success
+    assert isinstance(gdf, gpd.GeoDataFrame)
 
     assert api_request.datacube_id