From 3115c4810d10a941c28cc8fe235c145a19de5c96 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Fri, 2 Aug 2024 01:22:15 +0000 Subject: [PATCH 01/14] change awscli install back to unzip --- docker/Dockerfile | 9 +++++++++ docker/env.yaml | 1 - 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8e9b01e..76aed48 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,6 +48,15 @@ RUN echo "Enable server extensions" \ && jupyter server extension list \ && echo "...done" +RUN echo "Install AWS CLI v2" \ + && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \ + && unzip /tmp/awscliv2.zip -d /tmp \ + && /tmp/aws/install -i /env/bin/aws-cli -b /env/bin \ + && aws --version \ + && rm -f /tmp/awscliv2.zip \ + && rm -fr /tmp/aws \ + && echo "...done" + COPY assets/sync_repo assets/jupyterhub-singleuser /usr/local/bin/ COPY assets/overrides.json /env/share/jupyter/lab/settings/ diff --git a/docker/env.yaml b/docker/env.yaml index 2570e42..a3776e6 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -21,7 +21,6 @@ dependencies: - rio-cogeo - access - aiobotocore - - awscliv2 - boto3 - s5cmd - affine From 47d8da70637abd6482a075e08ee1c0e5631f54ac Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Fri, 2 Aug 2024 01:44:05 +0000 Subject: [PATCH 02/14] pin jupyterlab_code_formatter for compatibility --- docker/env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/env.yaml b/docker/env.yaml index a3776e6..58babce 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -213,7 +213,7 @@ dependencies: - ecdsa - jupyter - jupyterlab - - jupyterlab_code_formatter + - jupyterlab_code_formatter<3.0 - jupyterlab-spellchecker - jupyterlab-code-snippets - jupyterlab-dash From 7898de4265ce927c5ad998072c1c62e5c715e9be Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Mon, 5 Aug 2024 07:46:21 +0000 Subject: [PATCH 03/14] upgrade jupyterlab>4 --- docker/Dockerfile | 5 +++-- docker/env.yaml | 19 ++++++++----------- docker/requirements.txt | 8 ++++---- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 76aed48..2000afc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,9 +36,9 @@ ENV LC_ALL=C.UTF-8 \ PATH=/env/bin:$PATH USER $nb_user + # NOTE - JupyterLab extensions can now be distributed as prebuilt extensions and available to PyPI RUN echo "Enable jupyter lab extensions" \ - && jupyter labextension install jupyterlab-topbar-extension jupyterlab-theme-toggle \ && jupyter labextension list \ && echo "...done" @@ -46,11 +46,12 @@ RUN echo "Enable server extensions" \ && jupyter server extension enable --py jupyterlab_iframe --sys-prefix \ && jupyter server extension enable --py jupyter_resource_usage --sys-prefix \ && jupyter server extension list \ + && jupyter lab build \ && echo "...done" RUN echo "Install AWS CLI v2" \ && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \ - && unzip /tmp/awscliv2.zip -d /tmp \ + && unzip /tmp/awscliv2.zip -d /tmp > /dev/null \ && /tmp/aws/install -i /env/bin/aws-cli -b /env/bin \ && aws --version \ && rm -f /tmp/awscliv2.zip \ diff --git a/docker/env.yaml b/docker/env.yaml index 58babce..23f9851 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -168,6 +168,7 @@ dependencies: # Scientific Stack - gsl - cgal-cpp + - cgal - boost - openmp - muparser @@ -178,7 +179,7 @@ dependencies: - tensorflow>=2.16 - xgboost - zarr - - bokeh=3.2.2 + - bokeh - descartes - matplotlib - seaborn @@ -211,22 +212,17 @@ dependencies: - black - python-jose - ecdsa - - jupyter - - jupyterlab - - jupyterlab_code_formatter<3.0 + - jupyterlab>=4.0 + - jupyterlab_code_formatter<3.0 # Non-caught error in >= 3.0 - jupyterlab-spellchecker - - jupyterlab-code-snippets - - jupyterlab-dash - jupyterlab-geojson - jupyterlab-git - jupyterlab-github - jupyterlab_iframe - jupyterlab_widgets - - jupyterlab-topbar - - jupyterlab-system-monitor - - jupyterhub - - jupytext<1.16 - - jupyterlab_pygments<0.3.0 + - jupyterhub=4.1.5 # Match hub version + - jupytext + - jupyterlab_pygments - jupyter-resource-usage - jupyter_server - jupyter-server-proxy @@ -241,6 +237,7 @@ dependencies: - isort - nbdime - nbgitpuller + - notebook<7.0 # Compatible with nbconvert - mypy - prompt-toolkit # because of line_profiler/ipython - nbconvert # because of jupyter_contrib_nbextensions diff --git a/docker/requirements.txt b/docker/requirements.txt index 348af0e..412ac5c 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,14 +1,14 @@ cognitojwt nbval -cgal line-profiler jupyterlab-logout jupyter-contrib-core jupyter-contrib-nbextensions jupyter-nbextensions-configurator -# pin for jupyterlab=3.x -jupyterlab-topbar-text==0.6.2 -jupyterlab-logout==0.5.0 +jupyterlab-code-snippets +jupyterlab-topbar-text +jupyterlab-logout +jupyterlab-theme-toggler # ODC/DEA: these are installed in builder stage otps From de861874ece46cfccb24c7be63d85d5998ed81df Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Mon, 5 Aug 2024 08:16:47 +0000 Subject: [PATCH 04/14] note the gotchas --- docker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2000afc..1f3fcbc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,7 +37,8 @@ ENV LC_ALL=C.UTF-8 \ USER $nb_user -# NOTE - JupyterLab extensions can now be distributed as prebuilt extensions and available to PyPI +# NOTE - Install lab extension from pypi NOT here. +# Leave the layer here for potential disable/enable and non-working check RUN echo "Enable jupyter lab extensions" \ && jupyter labextension list \ && echo "...done" @@ -49,6 +50,7 @@ RUN echo "Enable server extensions" \ && jupyter lab build \ && echo "...done" +# Install from unzip instead of conda as conda version is not the tool but an "invoker" RUN echo "Install AWS CLI v2" \ && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \ && unzip /tmp/awscliv2.zip -d /tmp > /dev/null \ From 219c8bbceae56a854ddebd48167f5876032ca14f Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Mon, 26 Aug 2024 14:56:15 +1000 Subject: [PATCH 05/14] Attempt to fix Docker compose --- .github/workflows/integration-test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 48d2ce2..8485b7f 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -51,16 +51,16 @@ jobs: aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide - - name: Start docker-compose + - name: Start docker compose run: | sudo chown -R 1000:100 ./dea-notebooks cd ./dea-sandbox/integration-testing - CURRENT_UID=1000:100 docker-compose up -d + CURRENT_UID=1000:100 docker compose up -d - name: Extract package versions run: | cd ./dea-sandbox/integration-testing - docker-compose exec -T sandbox cat /tmp/requirements.txt > requirements.txt + docker compose exec -T sandbox cat /tmp/requirements.txt > requirements.txt - name: Read package versions id: versions @@ -85,5 +85,5 @@ jobs: - name: Set up Datacube and Test run: | cd ./dea-sandbox/integration-testing - docker-compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh - docker-compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh + docker compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh + docker compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh From 922cb3f6d3786e0781ddb8ed82806059cbd15d56 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Thu, 12 Sep 2024 04:09:53 +0000 Subject: [PATCH 06/14] add spark --- docker/env.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/env.yaml b/docker/env.yaml index 23f9851..4b3c16c 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -94,6 +94,10 @@ dependencies: - nodejs - numexpr - numpy + # required by spark + - openjdk + - apache-sedona + - pyspark - ordered-set - packaging - pandas From 961f98acb7a6be667862d884edfbba527b78d4a1 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Wed, 18 Sep 2024 02:35:33 +0000 Subject: [PATCH 07/14] update numexpr with fix --- docker/Dockerfile | 1 + docker/env.yaml | 1 - docker/requirements.txt | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1f3fcbc..bc0704f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,6 +12,7 @@ COPY requirements.txt /conf/ # required to build hdmedians # or any --no-binary ENV CC=/env/bin/x86_64-conda_cos6-linux-gnu-gcc \ + CXX=/env/bin/x86_64-conda_cos6-linux-gnu-g++ \ LDSHARED="/env/bin/x86_64-conda_cos6-linux-gnu-gcc -pthread -shared -B /env/compiler_compat -L/env/lib -Wl,-rpath=/env/lib -Wl,--no-as-needed" RUN micromamba run -p /env pip install --no-cache-dir \ --no-build-isolation -r /conf/requirements.txt diff --git a/docker/env.yaml b/docker/env.yaml index 4b3c16c..b4ad40d 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -92,7 +92,6 @@ dependencies: - netCDF4 - networkx - nodejs - - numexpr - numpy # required by spark - openjdk diff --git a/docker/requirements.txt b/docker/requirements.txt index 412ac5c..c0e8e3a 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -9,6 +9,7 @@ jupyterlab-code-snippets jupyterlab-topbar-text jupyterlab-logout jupyterlab-theme-toggler +numexpr @ git+https://github.com/pydata/numexpr@a99412e # ODC/DEA: these are installed in builder stage otps From 723495193b1e702c3ab605020e3152e1d87c288d Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Wed, 9 Oct 2024 13:40:54 +1100 Subject: [PATCH 08/14] Pin pyTMD 2.1.6 --- docker/env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/env.yaml b/docker/env.yaml index b4ad40d..3dd16bc 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -208,7 +208,7 @@ dependencies: - spyndex - urbanaccess - contextily - - pyTMD + - pyTMD==2.1.6 - xarray-spatial # jupyter things - autopep8 From 3ad19fe9c62aaae4385884d5105e97b260582065 Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Wed, 9 Oct 2024 13:51:24 +1100 Subject: [PATCH 09/14] Fix syntax --- docker/env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/env.yaml b/docker/env.yaml index 3dd16bc..747bc23 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -208,7 +208,7 @@ dependencies: - spyndex - urbanaccess - contextily - - pyTMD==2.1.6 + - pyTMD=2.1.6 - xarray-spatial # jupyter things - autopep8 From be76fca316430733e64a44ee93dec14f1a7feb89 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Wed, 9 Oct 2024 03:07:12 +0000 Subject: [PATCH 10/14] pin numpy < 2.0 --- docker/env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/env.yaml b/docker/env.yaml index 747bc23..c6eb61f 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -92,7 +92,7 @@ dependencies: - netCDF4 - networkx - nodejs - - numpy + - numpy<2.0 # required by spark - openjdk - apache-sedona From 24f1fcdf0bc0a902a15128f5b0e13bfa560911da Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Wed, 9 Oct 2024 03:26:46 +0000 Subject: [PATCH 11/14] update fc --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index c0e8e3a..e4b0a70 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -37,5 +37,5 @@ thredds-crawler hdstats==0.1.8.post1 hdmedians datacube-stats -fc +fc>=1.3.10 --extra-index-url="https://packages.dea.ga.gov.au" From 6620faeff2e51032abb595c7bd7b6cc547b980e1 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Wed, 9 Oct 2024 04:37:26 +0000 Subject: [PATCH 12/14] adapt to new naming of fc --- docker/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index e4b0a70..87a4e82 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -37,5 +37,6 @@ thredds-crawler hdstats==0.1.8.post1 hdmedians datacube-stats -fc>=1.3.10 +fractional_cover>=1.3.10 --extra-index-url="https://packages.dea.ga.gov.au" +--find-links="https://packages.dea.ga.gov.au/fc" From 9aded869fcd7bf2485ddbd24d22aba8a2d3ec16b Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Thu, 10 Oct 2024 07:55:00 +1100 Subject: [PATCH 13/14] Pin latest DEA Tools --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 87a4e82..64b32a1 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -31,7 +31,7 @@ odc-stac odc-stats[ows] odc-ui odc-geo >= 0.4.8 # min version required for ESRI/GDAL nodata fix -dea-tools +DEA-Tools >= 0.3.5 thredds-crawler hdstats==0.1.8.post1 From a5155e35cf169b777fc1f0db234726576814bc88 Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Thu, 10 Oct 2024 07:56:10 +1100 Subject: [PATCH 14/14] Fix syntax --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 64b32a1..efed0ce 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -31,7 +31,7 @@ odc-stac odc-stats[ows] odc-ui odc-geo >= 0.4.8 # min version required for ESRI/GDAL nodata fix -DEA-Tools >= 0.3.5 +dea-tools >= 0.3.5 thredds-crawler hdstats==0.1.8.post1