Merge pull request #5 from kamangir/catalogs-2024-07-24

+= catalogs

blue_geo/.abcli/alias.sh

@@ -1,3 +1,4 @@
 #! /usr/bin/env bash
 
+alias @catalog=blue_geo_catalog
 alias @datacube=blue_geo_datacube

blue_geo/.abcli/blue_geo.sh

@@ -4,9 +4,10 @@ function blue_geo() {
     local task=$(abcli_unpack_keyword $1 help)
 
     if [ $task == "help" ]; then
-        ukraine_timemap "$@"
+        blue_geo_catalog "$@"
+        blue_geo_datacube "$@"
         blue_geo_QGIS "$@"
-        blue_geo_ingest "$@"
+        ukraine_timemap "$@"
         return
     fi
 

blue_geo/.abcli/catalog.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+function blue_geo_catalog() {
+    local task=$(abcli_unpack_keyword $1 help)
+
+    if [ "$task" == "help" ]; then
+        blue_geo_catalog_browse "$@"
+        blue_geo_catalog_get "$@"
+        blue_geo_catalog_list "$@"
+        blue_geo_catalog_query "$@"
+        return
+    fi
+
+    local function_name=blue_geo_catalog_$task
+    if [[ $(type -t $function_name) == "function" ]]; then
+        $function_name "${@:2}"
+        return
+    fi
+
+    abcli_log_error "-@catalog: $task: command not found."
+    return 1
+}
+
+abcli_source_path - caller,suffix=/catalog

blue_geo/.abcli/catalog/README.md

@@ -0,0 +1,36 @@
+# 🌐 catalog
+
+`catalog`s can be listed and browsed and queried for [`datacube`](../datacube/)s,
+
+```bash
+ > @catalog help
+@catalog browse \
+	<catalog> \
+	[<args>]
+ . browse <catalog>.
+@catalog list \
+	[-] \
+	[--delim space] \
+	[--log 0]
+ . list catalogs.
+@catalog query \
+	<catalog> \
+	[download,ingest,select,upload] \
+	[-|<object-name>] \
+	[<query-options>] \
+	[<args>]
+ . <catalog> -query-> <object-name>.
+@catalog query read \
+	[all,download,len] \
+	[.|<object-name>] \
+	[--count <count>] \
+	[--delim <delim>] \
+	[--index <index>] \
+	[--prefix <prefix>] \
+	[--suffix <suffix>] \
+	[--contains <contains>] \
+	[--notcontains <not-contains>]
+ . read query results in <object-name>.
+```
+
+supported `<catalog>`s: [firms](./firms/).

blue_geo/.abcli/catalog/browse.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+
+function blue_geo_catalog_browse() {
+    local options=$1
+
+    local catalog=$(abcli_option "$options" $blue_geo_catalog_list firms)
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        for catalog in $(echo $blue_geo_catalog_list | tr , " "); do
+            blue_geo_catalog_browse_${catalog} "$@"
+        done
+        return
+    fi
+
+    abcli_log "@catalog: browsing $catalog ..."
+    blue_geo_catalog_browse_${catalog} "${@:2}"
+}

blue_geo/.abcli/datacube/firms_area/README.md → blue_geo/.abcli/catalog/firms/README.md

@@ -1,29 +1,31 @@
-# 🌐 firms-area
+# 🌐 firms
 
-the `firms-area` catalog covers [FIRMS](https://firms.modaps.eosdis.nasa.gov): Fire Information for Resource Management System. see [datacube](../README.md) for usage instructions.
+the `firms` catalog covers [FIRMS](https://firms.modaps.eosdis.nasa.gov): Fire Information for Resource Management System. see [datacube](../README.md) for usage instructions.
 
 ## query
 
 ```bash
- > @datacube query firms_area help
-@datacube query firms_area \
+ > @catalog query firms help
+@catalog query firms \
 	[download,ingest,select,upload] \
 	[-|<object-name>] \
-	[dryrun] \
+	[area,dryrun] \
 	[--date 2024-07-21] \
 	[--depth 1] \
 	[--area east|north|south|west|world] \
 	[--source LANDSAT_NRT|MODIS_NRT|MODIS_SP|VIIRS_NOAA20_NRT|VIIRS_NOAA21_NRT|VIIRS_SNPP_NRT|VIIRS_SNPP_SP] \
 	[--log 1]
- . firms_area -query-> <object-name>.
+ . firms -query-> <object-name>.
 ```
 
 ## example use
 
 ```bash
-@datacube query firms_area \
-  ingest,select - - \
-  --date 2024-07-18
+@catalog query firms \
+	ingest,select \
+	- \
+	area,dryrun \
+	--date 2024-07-24
 
 @open QGIS .
 @publish tar .
@@ -32,17 +34,17 @@ the `firms-area` catalog covers [FIRMS](https://firms.modaps.eosdis.nasa.gov): F
 ```yaml
 datacube:
   area: WORLD
-  date: '2024-07-18'
+  date: '2024-07-24'
   depth: 1
-  id: datacube-firms_area-world-MODIS_NRT-2024-07-18-1
-  len: 23627
+  id: datacube-firms-area-world-MODIS_NRT-2024-07-24-1
+  len: 28543
   source: MODIS_NRT
 ```
 
 
 ![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/datacube-firms_area-ingest.png)
 
-[datacube-firms_area-world-MODIS_NRT-2024-07-18-1.tar.gz](https://kamangir-public.s3.ca-central-1.amazonaws.com/datacube-firms_area-world-MODIS_NRT-2024-07-18-1.tar.gz)
+[datacube-firms-area-world-MODIS_NRT-2024-07-24-1.tar.gz](https://kamangir-public.s3.ca-central-1.amazonaws.com/datacube-firms-area-world-MODIS_NRT-2024-07-24-1.tar.gz)
 
 ![image](https://raw.githubusercontent.com/kamangir/assets/main/blue-geo/datacube-firms_area.jpg)
 

blue_geo/.abcli/datacube/firms_area/browse.sh → blue_geo/.abcli/catalog/firms/browse.sh

@@ -1,23 +1,23 @@
 #! /usr/bin/env bash
 
-function blue_geo_datacube_firms_area_browse() {
+function blue_geo_catalog_browse_firms() {
     local options=$1
 
     if [ $(abcli_option_int "$options" help 0) == 1 ]; then
-        options="|map_key|api"
-        abcli_show_usage "@datacube browse firms_area$ABCUL$options" \
-            "browse firms_area datacubes."
+        options="map_key|area"
+        abcli_show_usage "@catalog browse firms$ABCUL[$options]" \
+            "browse firms."
         return
     fi
 
     local do_map_key=$(abcli_option_int "$options" map_key 0)
-    local do_area_api=$(abcli_option_int "$options" api 0)
+    local collection=$(abcli_option_choice "$options" area area)
 
     local url="https://firms.modaps.eosdis.nasa.gov/"
     [[ "$do_map_key" == 1 ]] &&
         url="https://firms.modaps.eosdis.nasa.gov/api/map_key/"
-    [[ "$do_area_api" == 1 ]] &&
-        url="https://firms.modaps.eosdis.nasa.gov/api/area/"
+    [[ ! -z "$collection" ]] &&
+        url="https://firms.modaps.eosdis.nasa.gov/api/$collection/"
 
     abcli_browse $url
 }

blue_geo/.abcli/datacube/firms_area/ingest.sh → blue_geo/.abcli/catalog/firms/ingest.sh

@@ -1,12 +1,14 @@
 #! /usr/bin/env bash
 
-function blue_geo_datacube_ingest_firms_area() {
-    local options=$1
+function blue_geo_catalog_ingest_firms() {
+    local ingest_options=$1
+
+    local options=$2
 
     if [ $(abcli_option_int "$options" help 0) == 1 ]; then
-        options=$blue_geo_datacube_ingest_options
-        "$EOP~copy_template,dryrun,$EOPE"
-        abcli_show_usage "blue_geo ingest firms_area$ABCUL[$options]$ABCUL[.|<object-name>]$ABCUL" \
+        ingest_options=$blue_geo_datacube_ingest_options
+        options="dryrun"
+        abcli_show_usage "@datacube ingest firms$ABCUL[$ingest_options]$ABCUL[.|<object-name>]$ABCUL[$options]$ABCUL[<args>]" \
             "firms -ingest-> <object-name>."
         return
     fi
@@ -16,7 +18,7 @@ function blue_geo_datacube_ingest_firms_area() {
     local object_name=$(abcli_clarify_object $2 .)
 
     abcli_eval dryrun=$do_dryrun \
-        python3 -m blue_geo.datacube.firms.area \
+        python3 -m blue_geo.catalog.firms \
         ingest \
         --object_name $object_name \
         "${@:3}"

blue_geo/.abcli/catalog/firms/query.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+function blue_geo_catalog_query_firms() {
+    local options=$1
+
+    blue_geo_catalog_query_generic \
+        $options,catalog=firms \
+        "${@:2}"
+}
+
+abcli_source_path - caller,suffix=/query

blue_geo/.abcli/catalog/firms/query/area.sh

@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+
+function blue_geo_catalog_query_firms_area() {
+    local options=$1
+
+    local catalog="firms"
+    local collection="area"
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options="$collection,dryrun"
+        local args=$(python3 -m blue_geo.catalog.$catalog.$collection get --what list_of_args)
+        abcli_show_usage "@catalog query $catalog$ABCUL[$blue_geo_catalog_query_options]$ABCUL[-|<object-name>]$ABCUL[$options]$ABCUL$args" \
+            "$catalog/$collection -query-> <object-name>."
+        return
+    fi
+
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+
+    local object_name=$(abcli_clarify_object $2 -)
+
+    abcli_log "🌐 query: $catalog/$collection -> $object_name"
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m blue_geo.catalog.$catalog.$collection \
+        query \
+        --object_name $object_name \
+        "${@:3}"
+
+    return 0
+}

blue_geo/.abcli/catalog/generic/query.sh

@@ -0,0 +1,50 @@
+#! /usr/bin/env bash
+
+function blue_geo_catalog_query_generic() {
+    local options=$1
+
+    local catalog=$(abcli_option "$options" catalog generic)
+
+    local list_of_collectios=$(blue_geo_catalog get list_of_collections \
+        --catalog $catalog \
+        --delim , \
+        --log 0)
+    local default_collection=$(blue_geo_catalog get list_of_collections \
+        --catalog $catalog \
+        --count 1 \
+        --delim , \
+        --log 0)
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        local collection
+        for collection in $(echo $list_of_collectios | tr , " "); do
+            blue_geo_catalog_query_${catalog}_${collection} "$@"
+        done
+        return 0
+    fi
+
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local collection=$(abcli_option_choice "$options" $list_of_collectios $default_collection)
+
+    if [[ ",$list_of_collectios," != *",$collection,"* ]]; then
+        abcli_log_error "-@catalog: query: $catalog: $collection: collection not found."
+        return 1
+    fi
+
+    if [[ "$2" == "help" ]]; then
+        blue_geo_catalog_query_${catalog}_${collection} help
+        return
+    fi
+
+    abcli_log "🌐 query: $catalog/$collection"
+
+    blue_geo_catalog_query_${catalog}_${collection} "$@"
+}
+
+function blue_geo_catalog_query_generic_generic() {
+    : # no query available.
+}
+
+function blue_geo_catalog_browse_generic() {
+    : # no browse available.
+}

blue_geo/.abcli/catalog/get.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+function blue_geo_catalog_get() {
+    local options=$1
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options="list_of_collections"
+        local args="[--catalog <catalog>]$ABCUL[--count 1]$ABCUL[--delim ,]$ABCUL[--log 0]"
+        abcli_show_usage "@catalog get$ABCUL[$options]$ABCUL$args" \
+            "get list of collections in <catalog>."
+        return
+    fi
+
+    local what=$(abcli_option_choice "$options" list_of_collections list_of_collections)
+
+    python3 -m blue_geo.catalog \
+        get \
+        --what "$what" \
+        "${@:2}"
+}

blue_geo/.abcli/catalog/list.sh

@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+
+function blue_geo_catalog_list() {
+    local options=$1
+
+    if [ $(abcli_option_int "$options" help 0) == 1 ]; then
+        options="-"
+        local args="[--delim space]$ABCUL[--log 0]"
+        abcli_show_usage "@catalog list$ABCUL[$options]$ABCUL$args" \
+            "list catalogs."
+        return
+    fi
+
+    python3 -m blue_geo.catalog \
+        list \
+        "${@:2}"
+}
+
+function blue_geo_catalog_ls() {
+    blue_geo_catalog_list "$@"
+}
+
+function blue_geo_catalog_load_all() {
+    abcli_log_list $blue_geo_catalog_list \
+        --delim , \
+        --before "🌐 loading" \
+        --after "catalog(s)"
+
+    local catalog
+    for catalog in $(echo $blue_geo_catalog_list | tr , " "); do
+        abcli_log "🧊 $catalog"
+        abcli_source_path - caller,civilized,suffix=/$catalog
+    done
+
+    return 0
+}
+
+export blue_geo_catalog_list=$(blue_geo_catalog_list - --log 0)
+
+blue_geo_catalog_load_all