Merge pull request #11 from digitalearthafrica/update-cli

Update deafrica-waterbodies command line tools
digitalearthafrica · Nov 27, 2023 · 895a18c · 895a18c
2 parents 2e75fa0 + 6fc9295
commit 895a18c
Show file tree

Hide file tree

Showing 74 changed files with 3,309 additions and 5,233 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -9,11 +9,11 @@ jobs:
       max-parallel: 5
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10.6
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.6
+        python-version: 3.10.6
     - name: Add conda to system path
       run: |
         # $CONDA is an environment variable pointing to the root of the miniconda directory

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -0,0 +1,36 @@
+
+name: Publish deafrica-waterbodies to PyPI
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'deafrica_waterbodies/**'
+
+  workflow_dispatch:
+
+jobs:
+  build-n-publish:
+    name: Build and publish deafrica-waterbodies to PyPI
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./deafrica_waterbodies
+
+    steps:
+      - name: Checkout the digitalearthafrica/deafrica-waterbodiesrepository
+        uses: actions/checkout@v3
+      - name: Set up Python 3.10.6
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10.6" # Version range or exact version of a Python version to use, using SemVer's version range syntax
+          architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified.
+      - name: Install pypa/build
+        run: python3 -m pip install build --user
+      - name: Build a binary wheel and a source tarball
+        run: python3 -m build --sdist --wheel --outdir dist/ .
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          packages-dir: deafrica_waterbodies/dist/
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -1,26 +1,27 @@
 name: Push
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - stable
     paths:
-      - 'dea_waterbodies/**'
+      - 'deafrica_waterbodies/**'
       - '.github/workflows/push.yml'
       - 'Dockerfile'
 
   release:
     types: [created, edited]
 
 env:
-  IMAGE_NAME: geoscienceaustralia/dea-waterbodies
+  IMAGE_NAME: digitalearthafrica/deafrica-waterbodies
 
 jobs:
   push:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
@@ -29,18 +30,18 @@ jobs:
       run: |
         echo "RELEASE=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV
     - name: Build and Push semver tagged Docker Image for Release
-      uses: whoan/docker-build-with-cache-action@v4
+      uses: whoan/docker-build-with-cache-action@v6
       if: github.event_name == 'release'
       with:
         image_name: ${{ env.IMAGE_NAME }}
-        username: ${{ secrets.DOCKER_USER }}
-        password: ${{ secrets.DOCKER_PASS }}
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DEAFRICA_DOCKER_PASSWORD}}
         image_tag: ${{ env.RELEASE }}
     - name: Run Trivy vulnerability scanner for Release
       uses: aquasecurity/trivy-action@master
       if: github.event_name == 'release'
       with:
-        image-ref: 'docker.io/geoscienceaustralia/dea-waterbodies:${{ env.RELEASE }}'
+        image-ref: 'docker.io/digitalearthafrica/deafrica-waterbodies:${{ env.RELEASE }}'
         format: 'table'
         exit-code: '1'
         ignore-unfixed: true
@@ -53,18 +54,18 @@ jobs:
         git fetch --all --tags
         echo "RELEASE=$(git describe --tags)" >> $GITHUB_ENV
     - name: Build and Push unstable Docker Image for push to main
-      uses: whoan/docker-build-with-cache-action@v4
+      uses: whoan/docker-build-with-cache-action@v6
       if: github.event_name != 'release'
       with:
         image_name: ${{ env.IMAGE_NAME }}
-        username: ${{ secrets.DOCKER_USER }}
-        password: ${{ secrets.DOCKER_PASS }}
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DEAFRICA_DOCKER_PASSWORD}}
         image_tag: latest,${{ env.RELEASE }}
     - name: Run Trivy vulnerability scanner for push to main
       uses: aquasecurity/trivy-action@master
       if: github.event_name != 'release'
       with:
-        image-ref: 'docker.io/geoscienceaustralia/dea-waterbodies:${{ env.RELEASE }}'
+        image-ref: 'docker.io/digitalearthafrica/deafrica-waterbodies:${{ env.RELEASE }}'
         format: 'table'
         exit-code: '1'
         ignore-unfixed: true

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -3,37 +3,33 @@ name: Test
 on: [push]
 
 env:
-  ORG: geoscienceaustralia
-  IMAGE: dea-waterbodies
-  METADATA_CATALOG: https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/a4f39b485b33608a016032d9987251881fec4b6f/workspaces/sandbox-metadata.yaml
-  PRODUCT_CATALOG: https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/a4f39b485b33608a016032d9987251881fec4b6f/workspaces/sandbox-products.csv
+  ORG: digitalearthafrica
+  IMAGE: deafrica-waterbodies
+  PRODUCT_CATALOG: https://raw.githubusercontent.com/digitalearthafrica/config/master/prod/products_prod.csv
 
 jobs:
   test:
     runs-on: ubuntu-latest
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
-
-    - name: Pre-pull layers
-      run: docker-compose pull
-
+      uses: actions/[email protected]
 
     - name: Activate Docker cache
-      uses: satackey/[email protected].8
+      uses: satackey/[email protected].11  
       # Ignore the failure of a step and avoid terminating the job.
       continue-on-error: true
 
-    - name: Build dea-waterbodies image
+    - name: Setup deafrica-waterbodies test environment
       run: |
-        docker-compose build
+          make test-env
     
-    - name: Test dea-waterbodies image
+    - name: Run deafrica-waterbodies test
       run: |
-          docker-compose up -d
-          ./setup_test_datacube.sh
-          docker-compose exec -T waterbodies bash -c "pytest ."
-          docker-compose down
-      
+          make run-tests
+          make clean
     
+    - name: Clean deafrica-waterbodies image
+      run: |
+          (echo y) | docker container prune
+          (echo y) | docker image prune
diff --git a/.gitignore b/.gitignore
@@ -11,13 +11,6 @@ watercourses
 *.shp
 *.shx
 
-# Explicitly allow test data
-!tests/data/*.cpg
-!tests/data/*.dbf
-!tests/data/*.prj
-!tests/data/*.shp
-!tests/data/*.shx
-
 # graphs
 *.graphml
 *.gml
@@ -26,3 +19,6 @@ watercourses
 *.csv
 *.geojson
 *.json
+
+# Explicitly allow test data
+!tests/data/*.geojson
diff --git a/Makefile b/Makefile
@@ -49,9 +49,6 @@ run-tests:
 	docker compose exec -T waterbodies bash -c "coverage xml"
 	docker compose exec -T waterbodies bash -c "coverage html"
 
-test:
-	docker-compose exec waterbodies pytest tests
-
 down: ## Bring down the system
 	docker compose down
 
@@ -71,4 +68,5 @@ pip_compile:
 		requirements.in
 
 lint:
-	docker-compose exec waterbodies black --check dea_waterbodies
+	docker-compose exec waterbodies black --check deafrica_waterbodies
+	docker-compose exec waterbodies isort --check deafrica_waterbodies
diff --git a/README.rst b/README.rst
@@ -1,64 +1,63 @@
-.. image:: figures/dea_logo_wide.jpg
+.. image:: figures/deafrica_logo_wide.jpg
   :width: 900
-  :alt: Digital Earth Australia logo
+  :alt: Digital Earth Africa logo
 
-Digital Earth Australia Waterbodies
+Digital Earth Africa Waterbodies
 ###################################
 
 .. image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg
   :target: https://opensource.org/licenses/Apache-2.0
-  :alt: Digital Earth Australia logo
+  :alt: Digital Earth Africa logo
 
-.. image:: https://github.com/GeoscienceAustralia/dea-waterbodies/actions/workflows/lint.yml/badge.svg
-  :target: https://github.com/GeoscienceAustralia/dea-waterbodies/actions/workflows/lint.yml
+.. image:: https://github.com/digitalearthafrica/deafrica-waterbodies/actions/workflows/lint.yml/badge.svg
+  :target: https://github.com/digitalearthafrica/deafrica-waterbodies/actions/workflows/lint.yml
   :alt: Linting status
 
-.. image:: https://github.com/GeoscienceAustralia/dea-waterbodies/actions/workflows/test.yml/badge.svg
-  :target: https://github.com/GeoscienceAustralia/dea-waterbodies/actions/workflows/test.yml
+.. image:: https://github.com/digitalearthafrica/deafrica-waterbodies/actions/workflows/test.yml/badge.svg
+  :target: https://github.com/digitalearthafrica/deafrica-waterbodies/actions/workflows/test.yml
   :alt: Testing status
 
-**License:** The code in this repository is licensed under the `Apache License, Version 2.0 <https://www.apache.org/licenses/LICENSE-2.0>`_. Digital Earth Australia data is licensed under the `Creative Commons by Attribution 4.0 license <https://creativecommons.org/licenses/by/4.0/>`_.
+**License:** The code in this repository is licensed under the `Apache License, Version 2.0 <https://www.apache.org/licenses/LICENSE-2.0>`_. Digital Earth Africa data is licensed under the `Creative Commons by Attribution 4.0 license <https://creativecommons.org/licenses/by/4.0/>`_.
 
-**Contact:** If you need assistance with any of the Jupyter Notebooks or Python code in this repository, please post a question on the `Open Data Cube Slack channel <http://slack.opendatacube.org/>`_. If you would like to report an issue with this repo, or suggest feature requests, you can `open an issue on this repository <https://github.com/GeoscienceAustralia/dea-waterbodies/issues>`_. Non-technical questions about Digital Earth Australia Waterbodies can be sent to [email protected]. 
+**Contact:** If you need assistance with any of the Jupyter Notebooks or Python code in this repository, please post a question on the `Open Data Cube Slack channel <http://slack.opendatacube.org/>`_. If you would like to report an issue with this repo, or suggest feature requests, you can `open an issue on this repository <https://github.com/digitalearthafrica/deafrica-waterbodies/issues>`_. Non-technical questions about Digital Earth Africa Waterbodies can be sent to [email protected]. 
 
-**Citing Digital Earth Australia Waterbodies:**
+**Citing Digital Earth Africa Waterbodies:**
 
     Krause, Claire E.; Newey, Vanessa; Alger, Matthew J.; Lymburner, Leo. 2021. "Mapping and Monitoring the Multi-Decadal Dynamics of Australia’s Open Waterbodies Using Landsat" Remote Sens. 13, no. 8: 1437. https://doi.org/10.3390/rs13081437
 
 ----------
 
-Up to date information about the extent and location of surface water provides all Australians with a common understanding of this valuable and increasingly scarce resource. Water detection algorithms are now being routinely applied to continental and global archives of satellite imagery. However, water resource management decisions typically take place at the waterbody rather than pixel scale. 
+Up to date information about the extent and location of surface water across Africa provides stakeholders with a common understanding of this valuable and increasingly scarce resource. Water detection algorithms are now being routinely applied to continental and global archives of satellite imagery. However, water resource management decisions typically take place at the waterbody rather than pixel scale. 
 
-This repository presents a workflow for generating polygons of persistent waterbodies from Landsat observations, enabling improved monitoring and management of water assets across Australia. We use `Digital Earth Australia’s (DEA) Water Observations from Space (WOfS) water classifier <https://www.ga.gov.au/dea/products/wofs>`_, which provides a water classified output for every available Landsat scene, to determine the spatial locations and extents of waterbodies across Australia. DEA Waterbodies uses Geoscience Australia’s archive of over 30 years of Landsat satellite imagery to identify where almost 300,000 waterbodies are in the Australian landscape. 
+This repository presents a workflow for generating polygons of persistent waterbodies from Landsat observations, enabling improved monitoring and management of water assets across Africa. We use `Digital Earth Africa’s (DE Africa) Water Observations from Space (WOfS) water classifier <https://docs.digitalearthafrica.org/en/latest/data_specs/Landsat_WOfS_specs.html>`_, which provides a water classified output for every available Landsat scene, to determine the spatial locations and extents of waterbodies across Africa. DE Africa Waterbodies uses Digital Earth Africa’s archive of over 30 years of Landsat satellite imagery to identify where over 700,000 waterbodies are in the African landscape. 
 
 .. image:: figures/WorkflowDiagram.JPG
   :width: 900
-  :alt: Digital Earth Australia Waterbodies workflow diagram
+  :alt: Digital Earth Africa Waterbodies workflow diagram
 
-*Digital Earth Australia Waterbodies workflow*
+*Digital Earth Africa Waterbodies workflow*
 
-Each polygon was then used to generate a time series of WOfS, providing a history of the change in the wet surface area of each waterbody every ~16 days since 1987.
+Each polygon was then used to generate a time series of WOfS, providing a history of the change in the wet surface area of each waterbody every ~16 days since 1984.
 
-.. image:: figures/DEAWaterbodiesESRIBasemap.jpeg
+.. image:: figures/DEAfricaWaterbodiesESRIBasemap.png
   :width: 900
   :alt: Digital Earth Australia Waterbodies
 
-*Digital Earth Australia Waterbodies. Waterbody polygons mapped by this product are shown in blue. There are almost 300,000 across Australia.*
+*Digital Earth Africa Waterbodies. Waterbody polygons mapped by this product are shown in blue. There are over 700,000 across Africa.*
 
-DEA Waterbodies supports users to understand and manage water across Australia. DEA Waterbodies provides new insights into local through to national-scale surface water spatio-temporal dynamics by enabling the monitoring of important landscape features such as lakes and dams, improving our ability to use earth observation data to make meaningful decisions. It can be used to gain insights into the severity and spatial distribution of drought, or identify potential water sources for aerial firefighting during bushfires. 
-
-For more information about the DEA Waterbodies product, including instructions for accessing the product, frequently asked questions and data download links, see the `Digital Earth Australia website <https://www.ga.gov.au/dea/products/dea-waterbodies>`_.
+DE Africa Waterbodies supports users to understand and manage water across Africa. DE Africa Waterbodies provides new insights into local through to continental-scale surface water spatio-temporal dynamics by enabling the monitoring of important landscape features such as lakes and dams, improving our ability to use earth observation data to make meaningful decisions. It can be used to gain insights into the severity and spatial distribution of drought, or identify potential water sources.
+For more information about the DE Africa Waterbodies product, including instructions for accessing the product, frequently asked questions and data download links, see the `Digital Earth Africa Data Catalogue <https://docs.digitalearthafrica.org/en/latest/data_specs/Waterbodies_specs.html>`_.
 
 Installation
 ------------
 
-DEA Waterbodies has some requirements which can be installed with pip:
+DE Africa Waterbodies has some requirements which can be installed with pip:
 
 .. code-block:: bash
 
     pip install --extra-index-url="https://packages.dea.ga.gov.au" -r requirements.txt
     
-Once you have installed the requirements for DEA Waterbodies, install the module locally:
+Once you have installed the requirements for DE Africa Waterbodies, install the module locally:
 
 .. code-block:: bash
 
@@ -70,4 +69,4 @@ A command line interface is available for generating wet area time series for a
 
 .. code-block:: bash
 
-    waterbodies-ts --help
+    deafrica-waterbodies --help