Upgrade odc libs (#210)

* upgrade awscli, boto and aiobotocore libs version

* install odc libs stable releases from pypi

* upgrade tensorflow, numpy and rio

* fix dependencies

* upgrade libs

* fix numpy version conflict

* added note

docker/Dockerfile

@@ -119,12 +119,7 @@ RUN echo "Adding odc-dependencies" \
 
 
 USER root
-# Patch env when needed here
-# RUN echo "Patching python env" \
-#   && pip install --no-cache-dir \
-#   --extra-index-url="https://packages.dea.ga.gov.au" \
-#   --no-deps --upgrade odc-algo odc-ui 'datacube[performance,s3]'
-
+## Patch env when needed here
 # override hdstats - install faster version from dea packages
 RUN echo "Patching python env - hdstats" \
   && pip install --force-reinstall \
@@ -133,7 +128,7 @@ RUN echo "Patching python env - hdstats" \
 # upgrade numpy - to resove issue - https://github.com/scikit-image/scikit-image/issues/5270
 RUN echo "Patching python env - numpy" \
   && pip install --no-cache-dir \
-    --no-deps --upgrade numpy==1.21.2
+    --no-deps --upgrade numpy==1.21.4
 
 COPY assets/sync_repo assets/with_bootstrap assets/jupyterhub-singleuser /usr/local/bin/
 COPY assets/overrides.json $py_env_path/share/jupyter/lab/settings/

docker/constraints-jupyter.txt

@@ -27,7 +27,7 @@ ipython==7.26.0
 ipywidgets==7.6.3
 isort==5.9.3
 line-profiler==3.3.0
-nbdime==3.1.0
+nbdime==3.1.1
 nbgitpuller==1.0.2
 # nbresuse==0.4.0
 mypy==0.910

docker/constraints-odc.txt

@@ -1,10 +1,10 @@
 datacube==1.8.6
 datacube-stats==1.0.2
 fc==1.3.5
-odc-algo==0.2.1.dev2954
-odc-cloud==0.2.1.dev2954
-odc-dscache==0.2.1.dev2954
-odc-io==0.2.1.dev2954
-odc-stac==0.2.1.dev2954
-odc-stats==0.2.1.dev2954
-odc-ui==0.2.1.dev2954
+odc-algo==0.2.1
+odc-cloud==0.2.1
+odc-dscache==0.2.0a3
+odc-io==0.2.1
+odc-stac==0.2.2
+odc-stats==1.0.7
+odc-ui==0.2.0a2

docker/constraints.txt

@@ -12,7 +12,7 @@ pydash==5.0.2
 PyYAML==5.4.1
 pyrsistent==0.18.0
 python-dateutil==2.8.0
-s3fs==2021.9.0
+s3fs==2021.11.0
 tqdm==4.62.3
 urllib3==1.26.6
 voluptuous==0.12.2
@@ -40,21 +40,22 @@ dask==2.30.0
 distributed==2.30.0
 dask-image==0.6.0
 dask-ml==1.9.0
-numba==0.54.0
+numba==0.54.1
 numexpr==2.7.3
 numexpr3==3.0.1a6
+numpy==1.20.3
 pandas==1.3.4
 pathos==0.2.8
 Pillow==8.3.2
 scikit-image==0.18.3
 scikit-learn==1.0
 scipy==1.7.1
-tensorflow==2.6.0
+tensorflow==2.7.0
 xgboost==1.4.2
 xarray==0.19.0
 zarr==2.10.1
 ### plotting
-bokeh==2.3.3
+bokeh==2.4.2
 descartes==1.1.0
 matplotlib==3.4.3
 seaborn==0.11.2
@@ -65,22 +66,22 @@ cmocean==2.0
 geohash2==1.1
 geojson==2.5.0
 geopy==2.2.0
-h5py==3.1.0
+h5py==3.6.0
 mapclassify==2.4.3
 OWSLib==0.25.0
 pandana==0.6.1
-protobuf==3.17.3
+protobuf==3.19.1
 pyproj==3.2.1
 pysal==2.5.0
 pysheds==0.2.7
 python-geohash==0.8.5
-rioxarray==0.7.1
-rio-cogeo==3.0.0
+rioxarray==0.8.0
+rio-cogeo==3.0.1
 Rtree==0.9.7
 urbanaccess==0.2.2
 
 # nobinary
-aiohttp==3.7.4.post0
+aiohttp==3.8.1
 cffi==1.14.6
 cftime==1.5.1
 cryptography==3.4.8
@@ -114,7 +115,7 @@ aioitertools==0.8.0
 appdirs==1.4.4
 asciitree==0.3.3
 astunparse==1.6.3
-async-timeout==3.0.1
+async-timeout==4.0.1
 attrs==21.2.0
 aws-requests-auth==0.4.3
 awscli==1.19.106
@@ -141,7 +142,7 @@ Flask==2.0.1
 Flask-Cors==3.0.10
 Flask-JSON==0.3.4
 flatbuffers==1.12
-fsspec==2021.9.0
+fsspec==2021.11.0
 future==0.18.2
 gast==0.4.0
 geographiclib==1.52
@@ -161,7 +162,7 @@ Jinja2==3.0.1
 jmespath==0.10.0
 joblib==1.0.1
 jsonlines==1.2.0
-keras==2.6.0
+keras==2.7.0
 Keras-Preprocessing==1.1.2
 kiwisolver==1.3.2
 libpysal==4.5.1
@@ -178,7 +179,6 @@ multiprocess==0.70.12.2
 munch==2.5.0
 networkx==2.6.3
 numcodecs==0.9.1
-numpy==1.19.5
 oauthlib==3.1.1
 opencv-contrib-python==4.5.3.56
 opt-einsum==3.3.0
@@ -233,15 +233,15 @@ tenacity==8.0.1
 tensorboard==2.6.0
 tensorboard-data-server==0.6.1
 tensorboard-plugin-wit==1.8.0
-tensorflow-estimator==2.6.0
+tensorflow-estimator==2.7.0
 termcolor==1.1.0
 threadpoolctl==3.0.0
 tifffile==2021.8.30
 tobler==0.8.2
 toml==0.10.2
 tomli==1.2.1
 tornado==6.1
-typing-extensions==3.7.4.3
+typing-extensions==4.0.0
 Werkzeug==2.0.1
 wrapt==1.12.1
 zict==2.0.0

docker/constraints0.txt

@@ -1,6 +1,5 @@
 # tensorflow dependencies
 Cython==0.29.24
-numpy==1.19.5
 
 ## The following requirements were added by pip freeze:
 certifi==2021.5.30

docker/requirements-jupyter.txt

@@ -30,7 +30,7 @@ ipython==7.26.0
 ipywidgets==7.6.3
 isort==5.9.3
 line-profiler==3.3.0
-nbdime==3.1.0
+nbdime==3.1.1
 nbgitpuller==1.0.2
 # nbresuse==0.4.0
 mypy==0.910

docker/requirements-odc.txt

@@ -1,14 +1,14 @@
 # PyPi
 datacube[performance,s3]==1.8.6
-
-# currently only available from dea packages
---extra-index-url="https://packages.dea.ga.gov.au"
 odc-algo
 odc-cloud[ASYNC]
 odc-dscache
 odc-io
 odc-stac
-odc-stats
+odc-stats[ows]
 odc-ui
+
+# currently only available from dea packages
+--extra-index-url="https://packages.dea.ga.gov.au"
 datacube-stats==1.0.2
 fc==1.3.5

docker/requirements.txt

@@ -1,7 +1,7 @@
 # misc
 access
 affine
-aiobotocore[boto3,awscli]
+aiobotocore[boto3,awscli]==1.4.1 # due to s3fs
 alabaster
 colorama
 dill
@@ -43,6 +43,7 @@ dask-ml
 numba
 numexpr
 numexpr3
+numpy==1.20.3 # due to numba, patching version to '1.21.4' in later step (check Dockerfile)
 pandas
 pathos
 Pillow
@@ -54,7 +55,7 @@ xgboost # used by Dale's s2cloudmask
 xarray
 zarr
 ### plotting
-bokeh==2.3.3 # due to tensorflow/typing-extensions
+bokeh
 descartes
 matplotlib
 seaborn
@@ -65,11 +66,11 @@ cmocean
 geohash2
 geojson
 geopy
-h5py==3.1.0 # due to tensorflow
+h5py
 mapclassify
 OWSLib
 pandana
-protobuf==3.17.3 # due to tensorflow
+protobuf
 pyproj
 pysal
 pysheds

docker/requirements0.txt

@@ -1,3 +1,2 @@
 # tensorflow dependencies
-cython --no-binary=cython
-numpy==1.19.5
+cython --no-binary=cython