diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 48d2ce2..8485b7f 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -51,16 +51,16 @@ jobs: aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide - - name: Start docker-compose + - name: Start docker compose run: | sudo chown -R 1000:100 ./dea-notebooks cd ./dea-sandbox/integration-testing - CURRENT_UID=1000:100 docker-compose up -d + CURRENT_UID=1000:100 docker compose up -d - name: Extract package versions run: | cd ./dea-sandbox/integration-testing - docker-compose exec -T sandbox cat /tmp/requirements.txt > requirements.txt + docker compose exec -T sandbox cat /tmp/requirements.txt > requirements.txt - name: Read package versions id: versions @@ -85,5 +85,5 @@ jobs: - name: Set up Datacube and Test run: | cd ./dea-sandbox/integration-testing - docker-compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh - docker-compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh + docker compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh + docker compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 8e9b01e..bc0704f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,6 +12,7 @@ COPY requirements.txt /conf/ # required to build hdmedians # or any --no-binary ENV CC=/env/bin/x86_64-conda_cos6-linux-gnu-gcc \ + CXX=/env/bin/x86_64-conda_cos6-linux-gnu-g++ \ LDSHARED="/env/bin/x86_64-conda_cos6-linux-gnu-gcc -pthread -shared -B /env/compiler_compat -L/env/lib -Wl,-rpath=/env/lib -Wl,--no-as-needed" RUN micromamba run -p /env pip install --no-cache-dir \ --no-build-isolation -r /conf/requirements.txt @@ -36,9 +37,10 @@ ENV LC_ALL=C.UTF-8 \ PATH=/env/bin:$PATH USER $nb_user -# NOTE - JupyterLab extensions can now be distributed as prebuilt extensions and available to PyPI + +# NOTE - Install lab extension from pypi NOT here. +# Leave the layer here for potential disable/enable and non-working check RUN echo "Enable jupyter lab extensions" \ - && jupyter labextension install jupyterlab-topbar-extension jupyterlab-theme-toggle \ && jupyter labextension list \ && echo "...done" @@ -46,6 +48,17 @@ RUN echo "Enable server extensions" \ && jupyter server extension enable --py jupyterlab_iframe --sys-prefix \ && jupyter server extension enable --py jupyter_resource_usage --sys-prefix \ && jupyter server extension list \ + && jupyter lab build \ + && echo "...done" + +# Install from unzip instead of conda as conda version is not the tool but an "invoker" +RUN echo "Install AWS CLI v2" \ + && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \ + && unzip /tmp/awscliv2.zip -d /tmp > /dev/null \ + && /tmp/aws/install -i /env/bin/aws-cli -b /env/bin \ + && aws --version \ + && rm -f /tmp/awscliv2.zip \ + && rm -fr /tmp/aws \ && echo "...done" COPY assets/sync_repo assets/jupyterhub-singleuser /usr/local/bin/ diff --git a/docker/env.yaml b/docker/env.yaml index 2570e42..c6eb61f 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -21,7 +21,6 @@ dependencies: - rio-cogeo - access - aiobotocore - - awscliv2 - boto3 - s5cmd - affine @@ -93,8 +92,11 @@ dependencies: - netCDF4 - networkx - nodejs - - numexpr - - numpy + - numpy<2.0 + # required by spark + - openjdk + - apache-sedona + - pyspark - ordered-set - packaging - pandas @@ -169,6 +171,7 @@ dependencies: # Scientific Stack - gsl - cgal-cpp + - cgal - boost - openmp - muparser @@ -179,7 +182,7 @@ dependencies: - tensorflow>=2.16 - xgboost - zarr - - bokeh=3.2.2 + - bokeh - descartes - matplotlib - seaborn @@ -205,29 +208,24 @@ dependencies: - spyndex - urbanaccess - contextily - - pyTMD + - pyTMD=2.1.6 - xarray-spatial # jupyter things - autopep8 - black - python-jose - ecdsa - - jupyter - - jupyterlab - - jupyterlab_code_formatter + - jupyterlab>=4.0 + - jupyterlab_code_formatter<3.0 # Non-caught error in >= 3.0 - jupyterlab-spellchecker - - jupyterlab-code-snippets - - jupyterlab-dash - jupyterlab-geojson - jupyterlab-git - jupyterlab-github - jupyterlab_iframe - jupyterlab_widgets - - jupyterlab-topbar - - jupyterlab-system-monitor - - jupyterhub - - jupytext<1.16 - - jupyterlab_pygments<0.3.0 + - jupyterhub=4.1.5 # Match hub version + - jupytext + - jupyterlab_pygments - jupyter-resource-usage - jupyter_server - jupyter-server-proxy @@ -242,6 +240,7 @@ dependencies: - isort - nbdime - nbgitpuller + - notebook<7.0 # Compatible with nbconvert - mypy - prompt-toolkit # because of line_profiler/ipython - nbconvert # because of jupyter_contrib_nbextensions diff --git a/docker/requirements.txt b/docker/requirements.txt index 348af0e..efed0ce 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,14 +1,15 @@ cognitojwt nbval -cgal line-profiler jupyterlab-logout jupyter-contrib-core jupyter-contrib-nbextensions jupyter-nbextensions-configurator -# pin for jupyterlab=3.x -jupyterlab-topbar-text==0.6.2 -jupyterlab-logout==0.5.0 +jupyterlab-code-snippets +jupyterlab-topbar-text +jupyterlab-logout +jupyterlab-theme-toggler +numexpr @ git+https://github.com/pydata/numexpr@a99412e # ODC/DEA: these are installed in builder stage otps @@ -30,11 +31,12 @@ odc-stac odc-stats[ows] odc-ui odc-geo >= 0.4.8 # min version required for ESRI/GDAL nodata fix -dea-tools +dea-tools >= 0.3.5 thredds-crawler hdstats==0.1.8.post1 hdmedians datacube-stats -fc +fractional_cover>=1.3.10 --extra-index-url="https://packages.dea.ga.gov.au" +--find-links="https://packages.dea.ga.gov.au/fc"