+= datacube 🧊

README.md

@@ -6,14 +6,16 @@ AI for precise geospatial data analysis and visualization.
 pip install blue-geo
 ```
 
+🔹 🧊 [datacube](https://github.com/kamangir/blue-geo/tree/2024-07-18-datacubes-a/blue_geo/.abcli/datacube) 🔹
+
 | | | |
 |-|-|-|
 | 🌐 [QGIS](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/QGIS/README.md) | 🇺🇦 [ukraine-timemap](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/ukraine-timemap/README.md) | 🌈 [vancouver-watching](https://github.com/kamangir/Vancouver-Watching) |
 | [![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/QGIS.jpg)](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/QGIS/README.md) | [![image](https://github.com/kamangir/assets/blob/main/nbs/ukraine-timemap/QGIS.png?raw=true)](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/ukraine-timemap/README.md) | [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true)](https://github.com/kamangir/Vancouver-Watching)  |
 | an AI terraform for [QGIS](https://www.qgis.org/). | `ingest` for the [Bellingcat](https://www.bellingcat.com/) [Civilian Harm in Ukraine TimeMap](https://github.com/bellingcat/ukraine-timemap) dataset, available through [this UI](https://ukraine.bellingcat.com/) and [this API](https://bellingcat-embeds.ams3.cdn.digitaloceanspaces.com/production/ukr/timemap/api.json).  | 🌈 Vancouver watching with AI, last build: [🔗](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif). |  |
 | | | |
-| 🌐 [firms](https://github.com/kamangir/blue-geo/blob/main/blue_geo/firms/README.md) | | |
-| [![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/firms.jpg)](https://github.com/kamangir/blue-geo/blob/main/blue_geo/firms/README.md) | | |
+| 🌐 [firms](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/datacube/firms/README.md) | | |
+| [![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/firms.jpg)](https://github.com/kamangir/blue-geo/blob/main/blue_geo/.abcli/datacube/firms/README.md) | | |
 | `ingest` for [FIRMS](https://firms.modaps.eosdis.nasa.gov): Fire Information for Resource Management System. | | |
 
 ---

blue_geo/.abcli/alias.sh

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+
+alias @datacube=blue_geo_datacube

blue_geo/.abcli/datacube.sh

@@ -0,0 +1,25 @@
+#! /usr/bin/env bash
+
+function blue_geo_datacube() {
+    local task=$(abcli_unpack_keyword $1 help)
+
+    if [ "$task" == "help" ]; then
+        blue_geo_datacube_browse "$@"
+        blue_geo_datacube_get "$@"
+        blue_geo_datacube_ingest "$@"
+        blue_geo_datacube_list "$@"
+        blue_geo_datacube_query "$@"
+        return
+    fi
+
+    local function_name=blue_geo_datacube_$task
+    if [[ $(type -t $function_name) == "function" ]]; then
+        $function_name "${@:2}"
+        return
+    fi
+
+    abcli_log_error "-blue_geo: datacube: $task: command not found."
+    return 1
+}
+
+abcli_source_path - caller,suffix=/datacube

blue_geo/.abcli/datacube/README.md

@@ -0,0 +1,44 @@
+# 🧊
+
+`datacube`s are [objects](https://kamangir-public.s3.ca-central-1.amazonaws.com/giza-v1/giza.pdf) that can be [queried](#query) and [ingested](#ingest).
+
+## query
+
+run an `@datacube query`, then `@datacube query read` the `datacube-id`(s) as object names and use them with commands.
+
+```bash
+ > @datacube query help
+@datacube query \
+	<catalog> \
+	download,ingest,select,upload \
+	-|<object-name> \
+	<query-options> \
+	<args>
+ . <catalog> -query-> <object-name>.
+@datacube query read \
+	[all,download,len] \
+	[.|<object-name>] \
+	[--count <count>] \
+	[--delim <delim>] \
+	[--index <index>] \
+	[--prefix <prefix>] \
+	[--suffix <suffix>] \
+	[--contains <contains>] \
+	[--notcontains <not-contains>]
+ . read query results in <object-name>.
+```
+
+supported `<catalog>`s: [firms_area](./firms_area/).
+
+## ingest
+
+run `@datacube ingest` to ingest a datacube, or `@datacube query <catalog> ingest,...`. 
+
+```bash
+ > @datacube ingest help
+@datacube ingest \
+	[assets=all|<item-1+item-2>,dryrun,suffix=<suffix>,upload] \
+	[.|<object-name>] \
+	<args>
+ . ingest <object-name>.
+ ```

blue_geo/.abcli/datacube/browse.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+function blue_geo_datacube_browse() {
+    local catalog=${1:firms_area}
+
+    if [ $(abcli_option_int "$catalog" help 0) == 1 ]; then
+        abcli_show_usage "@datacube browse$ABCUL<catalog>$ABCUL<args>" \
+            "browse <catalog>."
+        return
+    fi
+
+    if [[ ",$blue_geo_datacube_list_of_catalogs," != *",$catalog,"* ]]; then
+        abcli_log_error "-@datacube: browse: $catalog: catalog not found."
+        return 1
+    fi
+
+    blue_geo_datacube_${catalog}_browse "${@:2}"
+}

blue_geo/.abcli/datacube/catalogs.sh

@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+
+function blue_geo_datacube_list() {
+    local options=$1
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options="catalogs"
+        local args="[--delim space]$ABCUL[--log 0]"
+        abcli_show_usage "@datacube list$ABCUL[$options]$ABCUL$args" \
+            "list catalogs."
+        return
+    fi
+
+    python3 -m blue_geo.datacube \
+        list_of_catalogs \
+        "${@:2}"
+}
+
+function blue_geo_datacube_ls() {
+    blue_geo_datacube_list "$@"
+}
+
+function blue_geo_datacube_load_catalogs() {
+    abcli_log_list $blue_geo_datacube_list_of_catalogs \
+        --delim , \
+        --before "🌐 loading" \
+        --after "datacube catalog(s)"
+
+    local catalog
+    for catalog in $(echo $blue_geo_datacube_list_of_catalogs | tr , " "); do
+        abcli_log "🧊 $catalog"
+        abcli_source_path - caller,civilized,suffix=/$catalog
+    done
+
+    return 0
+}
+
+export blue_geo_datacube_list_of_catalogs=$(blue_geo_datacube_list catalogs --log 0)
+
+blue_geo_datacube_load_catalogs

blue_geo/.abcli/datacube/firms_area/README.md

@@ -0,0 +1,53 @@
+# 🌐 firms-area
+
+the `firms-area` catalog covers [FIRMS](https://firms.modaps.eosdis.nasa.gov): Fire Information for Resource Management System. see [datacube](../README.md) for usage instructions.
+
+## query
+
+```bash
+ > @datacube query firms_area help
+@datacube query firms_area \
+	[download,ingest,select,upload] \
+	[-|<object-name>] \
+	[dryrun] \
+	[--date 2024-07-21] \
+	[--depth 1] \
+	[--area east|north|south|west|world] \
+	[--source LANDSAT_NRT|MODIS_NRT|MODIS_SP|VIIRS_NOAA20_NRT|VIIRS_NOAA21_NRT|VIIRS_SNPP_NRT|VIIRS_SNPP_SP] \
+	[--log 1]
+ . firms_area -query-> <object-name>.
+```
+
+## example use
+
+```bash
+@datacube query firms_area \
+  ingest,select - - \
+  --date 2024-07-18
+
+@open QGIS .
+@publish tar .
+```
+
+```yaml
+datacube:
+  area: WORLD
+  date: '2024-07-18'
+  depth: 1
+  id: datacube-firms_area-world-MODIS_NRT-2024-07-18-1
+  len: 23627
+  source: MODIS_NRT
+```
+
+
+![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/datacube-firms_area-ingest.png)
+
+[datacube-firms_area-world-MODIS_NRT-2024-07-18-1.tar.gz](https://kamangir-public.s3.ca-central-1.amazonaws.com/datacube-firms_area-world-MODIS_NRT-2024-07-18-1.tar.gz)
+
+![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/datacube-firms_area.jpg)
+
+---
+
+map-key: https://firms.modaps.eosdis.nasa.gov/api/map_key/
+
+area api: https://firms.modaps.eosdis.nasa.gov/api/area/

blue_geo/.abcli/datacube/firms_area/browse.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+function blue_geo_datacube_firms_area_browse() {
+    local options=$1
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options="|map_key|api"
+        abcli_show_usage "@datacube browse firms_area$ABCUL$options" \
+            "browse firms_area datacubes."
+        return
+    fi
+
+    local do_map_key=$(abcli_option_int "$options" map_key 0)
+    local do_area_api=$(abcli_option_int "$options" api 0)
+
+    local url="https://firms.modaps.eosdis.nasa.gov/"
+    [[ "$do_map_key" == 1 ]] &&
+        url="https://firms.modaps.eosdis.nasa.gov/api/map_key/"
+    [[ "$do_area_api" == 1 ]] &&
+        url="https://firms.modaps.eosdis.nasa.gov/api/area/"
+
+    abcli_browse $url
+}

blue_geo/.abcli/datacube/firms_area/ingest.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+function blue_geo_datacube_ingest_firms_area() {
+    local options=$1
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options=$blue_geo_datacube_ingest_options
+        "$EOP~copy_template,dryrun,$EOPE"
+        abcli_show_usage "blue_geo ingest firms_area$ABCUL[$options]$ABCUL[.|<object-name>]$ABCUL" \
+            "firms -ingest-> <object-name>."
+        return
+    fi
+
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+
+    local object_name=$(abcli_clarify_object $2 .)
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m blue_geo.datacube.firms.area \
+        ingest \
+        --object_name $object_name \
+        "${@:3}"
+}

blue_geo/.abcli/datacube/firms_area/query.sh

@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+
+function blue_geo_datacube_firms_area_query() {
+    local options=$1
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options="dryrun"
+
+        local date=$(abcli_string_timestamp_short \
+            --include_time 0 \
+            --unique 0)
+        local area=$(python3 -m blue_geo.datacube.firms.area \
+            get \
+            --what area \
+            --delim \|)
+        local source=$(python3 -m blue_geo.datacube.firms.area \
+            get \
+            --what source \
+            --values 1 \
+            --delim \|)
+
+        local args="[--date $date]$ABCUL[--depth 1]$ABCUL[--area $area]$ABCUL[--source $source]$ABCUL[--log 1]"
+
+        abcli_show_usage "@datacube query firms_area$ABCUL[$blue_geo_datacube_query_options]$ABCUL[-|<object-name>]$ABCUL[$options]$ABCUL$args" \
+            "firms_area -query-> <object-name>."
+        return
+    fi
+
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+
+    local object_name=$(abcli_clarify_object $2 -)
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m blue_geo.datacube.firms.area \
+        query \
+        --object_name $object_name \
+        "${@:3}"
+
+    return 0
+}

blue_geo/.abcli/datacube/get.sh

@@ -0,0 +1,39 @@
+#! /usr/bin/env bash
+
+function blue_geo_datacube_get() {
+    local what=$1
+
+    if [[ "$what" == "help" ]]; then
+        options="catalog"
+        abcli_show_usage "@datacube get$ABCUL[$options]$ABCUL.|<object-name>" \
+            "get <object-name> catalog."
+
+        options="template"
+        abcli_show_usage "@datacube get$ABCUL[$options]$ABCUL<catalog>" \
+            "get <catalog> template."
+
+        return
+    fi
+
+    local extra_args=""
+
+    if [[ "$what" == "catalog" ]]; then
+        local object_name=$(abcli_clarify_object $2 .)
+        python3 -m blue_geo.datacube \
+            get \
+            --what "$what" \
+            --object_name $object_name \
+            "${@:3}"
+    elif [[ "$what" == "template" ]]; then
+        python3 -m blue_geo.datacube \
+            get \
+            --what "$what" \
+            --catalog "$2" \
+            "${@:3}"
+    else
+        python3 -m blue_geo.datacube \
+            get \
+            --what "$what" \
+            "${@:2}"
+    fi
+}

blue_geo/.abcli/datacube/ingest.sh

@@ -0,0 +1,45 @@
+#! /usr/bin/env bash
+
+export blue_geo_datacube_ingest_options="assets=all|<item-1+item-2>,dryrun,suffix=<suffix>,upload"
+
+function blue_geo_datacube_ingest() {
+    local options=$1
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options=$blue_geo_datacube_ingest_options
+        abcli_show_usage "@datacube ingest$ABCUL[$options]$ABCUL[.|<object-name>]$ABCUL<args>" \
+            "ingest <object-name>."
+        return
+    fi
+
+    local do_upload=$(abcli_option_int "$options" upload 0)
+
+    local object_name=$(abcli_clarify_object $2 .)
+
+    local catalog=$(blue_geo_datacube get catalog $object_name)
+    if [[ ",$blue_geo_datacube_list_of_catalogs," != *",$catalog,"* ]]; then
+        abcli_log_error "-@datacube: ingest: $catalog: catalog not found."
+        return 1
+    fi
+
+    abcli_log "🧊 catalog: $catalog"
+
+    local template_object_name=$(blue_geo_datacube get template $catalog)
+    local do_copy_template=1
+    [[ "$template_object_name" == "unknown-template" ]] &&
+        do_copy_template=0
+    do_copy_template=$(abcli_option_int "$options" copy_template $do_copy_template)
+
+    [[ "$do_copy_template" == 1 ]] &&
+        abcli_clone \
+            $template_object_name \
+            $object_name \
+            ~meta
+
+    blue_geo_datacube_ingest_${catalog} "$@"
+
+    [[ "$do_upload" == 1 ]] &&
+        abcli_upload - $object_name
+
+    return 0
+}