Migrate JupterLab from 2.x to 3.x (#188)

* remove comments

* JupyterLab migration from 2.x to 3.x

* enable JupyterLab extentions

* use stable release for jupyterlab-widgets

* cleanup

* update libs

* enable jupyter jupyterlab_iframe serverextension

* pinning dask related libs

* updated contraints

* update dask version to 2021.7.1

* update odc libs version

* install JupyterLab extensions to display system information - memory and cpu usage

* install jupyter top bar jupyterlab-logout extension

* upgrade bokeh, install pyviz_comms - Jupyterlab pyviz extension

* downgrade nbconvert

* commented out panel and nbresuse

* add jupyter-server and panel libs

* pin dask and distributed to 2.30.0 due to dask-gateway dependency

* add cognitojwt

* install odc packages from pypi

* fix odc-aio installation

* install odc-stac from pypi

* revised odc requirements

* remove odc-stac from the list

* try disabling jupyter-resource-usage

* try disabling jupyter-resource-usage - take2

* disable jupyter_resource_usage - take3

* revert previous chage - keep jupyter_resource_usage needed for jupyterlab-system-monitor

* install jupyterlab-theme-toggle

* revise odc libs as per odc-tools docs - pulling from PyPi

* upgrade numpy to latest 1.21.2

* updated odc-stac and odc-stats versions

* updated constraint file

* install french language pack

* odc-tools packages is behind so pulling it from dea-packages repo

* fix constraints

* install jupyter-resource-usage - require for resource usage info

* update jupyter server extention - getting no module jupyter_resource_usage in singleserver app

Co-authored-by: Ubuntu <[email protected]>

docker/Dockerfile

@@ -21,26 +21,17 @@ RUN echo "Building python environment: stage Jupyter" \
   && cat /conf/requirements-jupyter.txt \
   && env-build-tool extend /conf/requirements-jupyter.txt /conf/constraints-jupyter.txt ${py_env_path}
 
-COPY jupyter-extensions.txt /conf/
-# TAB_NAME -- name to appear in the browser tab
-ARG TAB_NAME="DEA Sandbox"
-RUN echo "Adding jupyter lab extensions" \
-  && while read ext; do \
-      if [ -n "${ext}" ]; then \
-        echo "Installing ${ext}"; \
-        jupyter labextension install --no-build "${ext}"; \
-      fi; \
-     done < /conf/jupyter-extensions.txt \
-  && jupyter lab build --name="${TAB_NAME}" \
-  && jupyter lab clean \
+# NOTE - JupyterLab extensions can now be distributed as prebuilt extensions and available to PyPI
+RUN echo "Enable jupyter lab extensions" \
+  && jupyter labextension install jupyterlab-theme-toggle \
   && jupyter labextension list \
   && echo "...done"
 
 RUN echo "Enable server extensions" \
-  && jupyter serverextension enable --py --sys-prefix nbresuse \
-  && jupyter serverextension enable --py --sys-prefix jupyterlab_code_formatter \
-  && jupyter serverextension enable --py --sys-prefix jupyterlab_iframe \
-  && echo done
+  && jupyter server extension enable --py jupyterlab_iframe --sys-prefix \
+  && jupyter server extension enable --py jupyter_resource_usage  --sys-prefix \
+  && jupyter server extension list \
+  && echo "...done"
 
 COPY requirements-odc-static.txt constraints-odc-static.txt /conf/
 RUN echo "Adding odc-dependencies (static)" \
@@ -142,7 +133,7 @@ RUN echo "Patching python env - hdstats" \
 # upgrade numpy - to resove issue - https://github.com/scikit-image/scikit-image/issues/5270
 RUN echo "Patching python env - numpy" \
   && pip install --no-cache-dir \
-    --no-deps --upgrade numpy==1.20.1
+    --no-deps --upgrade numpy==1.21.2
 
 COPY assets/sync_repo assets/with_bootstrap assets/jupyterhub-singleuser /usr/local/bin/
 COPY assets/overrides.json $py_env_path/share/jupyter/lab/settings/

docker/constraints-jupyter.txt

@@ -1,57 +1,66 @@
 # jupyter things
 autopep8==1.5.7
 black==21.6b0
+cognitojwt==1.4.1
 jupyter==1.0.0
-jupyterlab==2.2.9
-jupyterlab-code-formatter==1.3.6
-jupyterlab_iframe==0.2.2
-jupyterlab-git==0.20.0
+jupyterlab==3.1.7
+jupyterlab-code-formatter==1.4.10
+jupyterlab-code-snippets==2.1.0
+jupyterlab-geojson==3.1.2
+jupyterlab-git==0.32.2
+jupyterlab-iframe==0.4.0
+jupyterlab-widgets==1.0.0
 jupyterhub==1.4.2
-jupytext==1.5.2
+jupytext==1.11.4
 jupyter-contrib-core==0.3.3
 jupyter-contrib-nbextensions==0.5.1
 jupyter-nbextensions-configurator==0.4.1
-jupyter-server-proxy==1.5.0
+jupyter-resource-usage==0.6.0
+jupyter-server==1.10.2
+jupyter-server-proxy==3.1.0
 jupyter-ui-poll==0.1.2
-ipycanvas==0.4.6
-ipyevents==0.7.1
-ipyfilechooser==0.3.4
-ipyleaflet==0.13.0
-ipython==7.7.0 # because of line_profiler
-ipywidgets==7.5.1
-isort==5.9.1
-line_profiler==3.3.0
-nbdime==2.0.0
+ipycanvas==0.9.0
+ipyevents==2.0.1
+ipyfilechooser==0.4.4
+ipyleaflet==0.14.0
+ipython==7.26.0
+ipywidgets==7.6.3
+isort==5.9.3
+line-profiler==3.3.0
+nbdime==3.1.0
 nbgitpuller==0.10.1
-nbresuse==0.3.3
+# nbresuse==0.4.0
 mypy==0.910
-prompt-toolkit==2.0.10 # because of line_profiler/ipython
+prompt-toolkit==3.0.19 # because of line_profiler/ipython
 nbconvert==5.6.1
-nbformat==5.0.8
+nbformat==5.1.3
 nbval==0.9.6
-sidecar==0.4.0
+sidecar==0.5.1
 yapf==0.31.0
-# fix for auto-complete breaking with jedi 0.18
-jedi==0.17.2
-parso==0.7.1
+jedi==0.18.0
+parso==0.8.2
 
 # jupyter + matplotlib
 # matplotlib itself is in requirements.txt as it's pulled in by other libs there
 folium==0.12.1
-ipympl==0.5.6
-graphviz==0.16
-plotly==5.1.0
+ipympl==0.7.0
+graphviz==0.17
+plotly==5.2.1
 pydotplus==2.0.2
 
 # jupyter + bokeh
-cartopy==0.18.0
-dask-labextension==2.0.2
-datashader==0.12.1
-geoviews==1.8.1
-holoviews==1.13.2
-jupyter-bokeh==2.0.2
-panel==0.9.7
+cartopy==0.19.0.post1
+dask-labextension==5.1.0
+datashader==0.13.0
+geoviews==1.9.1
+holoviews==1.14.5
+jupyter-bokeh==3.0.2
+panel==0.12.1
+pyviz_comms==2.0.1
 
 # jupyterhub + dask
 dask-gateway==0.9.0
 dask-gateway-server==0.9.0
+
+# jupyterlab + language packages
+jupyterlab-language-pack-fr-FR==3.1.post0

docker/constraints-odc.txt

@@ -1,16 +1,10 @@
 datacube==1.8.5
 datacube-stats==1.0.2
 fc==1.3.5
-odc_algo==0.2.1.dev2305
-odc_ui==0.2.1.dev2305
-odc_index==0.2.1.dev2305
-odc_aws==0.2.1.dev2305
-odc_geom==0.2.1.dev2305
-odc_io==0.2.1.dev2305
-odc_aio==0.2.1.dev2305
-odc_ppt==0.2.1.dev2305
-odc_dscache==0.2.1.dev2305
-odc_dtools==0.2.1.dev2305
-odc_stats==0.2.1.dev2305
-odc-apps-dc-tools==0.2.1.dev2305
-odc-apps-cloud==0.2.1.dev2305
+odc-algo==0.2.1.dev2856
+odc-cloud==0.2.1.dev2856
+odc-dscache==0.2.1.dev2856
+odc-io==0.2.1.dev2856
+odc-stac==0.2.1.dev2856
+odc-stats==0.2.1.dev2856
+odc-ui==0.2.1.dev2856

docker/constraints.txt

@@ -49,7 +49,7 @@ ruamel.yaml==0.17.10
 ruamel.yaml.clib==0.2.4
 s3fs==2021.6.1
 tqdm==4.56.0
-urllib3==1.26.6  # for botocore, but conflicts with pysal, which can not be updated as it dropped 3.6
+urllib3==1.26.6
 voluptuous==0.12.1
 # pre-install misc datacube dependencies
 bottleneck==1.3.2
@@ -90,10 +90,10 @@ scipy==1.5.4
 statsmodels==0.12.2
 tensorflow==2.6.0
 xarray==0.16.1
-xgboost==1.4.2       # used by Dale's s2cloudmask
+xgboost==1.4.2
 zarr==2.8.3
 ### plotting
-bokeh==2.1.1
+bokeh==2.3.3
 descartes==1.1.0
 matplotlib==3.2.1
 seaborn==0.11.1

docker/requirements-jupyter.txt

@@ -1,57 +1,69 @@
 # jupyter things
 autopep8==1.5.7
 black==21.6b0
+cognitojwt==1.4.1 # Used for user authentication
 jupyter==1.0.0
-jupyterlab==2.2.9
-jupyterlab-code-formatter==1.3.6
-jupyterlab_iframe==0.2.2
-jupyterlab-git==0.20.0
+jupyterlab==3.1.7
+jupyterlab-code-formatter==1.4.10
+jupyterlab-code-snippets==2.1.0
+jupyterlab-geojson==3.1.2
+jupyterlab-git==0.32.2
+jupyterlab-iframe==0.4.0
+jupyterlab-widgets==1.0.0
+jupyterlab-topbar==0.6.1
+jupyterlab-logout==0.5.0
+jupyterlab-system-monitor==0.8.0
 jupyterhub==1.4.2
-jupytext==1.5.2
+jupytext==1.11.4
 jupyter-contrib-core==0.3.3
 jupyter-contrib-nbextensions==0.5.1
 jupyter-nbextensions-configurator==0.4.1
-jupyter-server-proxy==1.5.0
+jupyter-resource-usage==0.6.0
+jupyter-server==1.10.2
+jupyter-server-proxy==3.1.0
 jupyter-ui-poll==0.1.2
-ipycanvas==0.4.6
-ipyevents==0.7.1
-ipyfilechooser==0.3.4
-ipyleaflet==0.13.0
-ipython==7.7.0 # because of line_profiler
-ipywidgets==7.5.1
-isort==5.9.1
-line_profiler==3.3.0
-nbdime==2.0.0
+ipycanvas==0.9.0
+ipyevents==2.0.1
+ipyfilechooser==0.4.4
+ipyleaflet==0.14.0
+ipython==7.26.0
+ipywidgets==7.6.3
+isort==5.9.3
+line-profiler==3.3.0
+nbdime==3.1.0
 nbgitpuller==0.10.1
-nbresuse==0.3.3
+# nbresuse==0.4.0
 mypy==0.910
-prompt-toolkit==2.0.10 # because of line_profiler/ipython
-nbconvert==5.6.1
-nbformat==5.0.8
+prompt-toolkit==3.0.19 # because of line_profiler/ipython
+nbconvert==5.6.1 # because of jupyter_contrib_nbextensions
+nbformat==5.1.3
 nbval==0.9.6
-sidecar==0.4.0
+sidecar==0.5.1
 yapf==0.31.0
-# fix for auto-complete breaking with jedi 0.18
-jedi==0.17.2
-parso==0.7.1
+jedi==0.18.0
+parso==0.8.2
 
 # jupyter + matplotlib
 # matplotlib itself is in requirements.txt as it's pulled in by other libs there
 folium==0.12.1
-ipympl==0.5.6
-graphviz==0.16
-plotly==5.1.0
+ipympl==0.7.0
+graphviz==0.17
+plotly==5.2.1
 pydotplus==2.0.2
 
 # jupyter + bokeh
-cartopy==0.18.0
-dask-labextension==2.0.2
-datashader==0.12.1
-geoviews==1.8.1
-holoviews==1.13.2
-jupyter-bokeh==2.0.2
-panel==0.9.7
+cartopy==0.19.0.post1
+dask-labextension==5.1.0
+datashader==0.13.0
+geoviews==1.9.1
+holoviews==1.14.5
+jupyter-bokeh==3.0.2
+panel==0.12.1
+pyviz_comms==2.0.1
 
 # jupyterhub + dask
 dask-gateway==0.9.0
 dask-gateway-server==0.9.0
+
+# jupyterlab + language packages
+jupyterlab-language-pack-fr-FR==3.1.post0

docker/requirements-odc.txt

@@ -1,19 +1,14 @@
-# ODC/DEA: frequently changing
-# bumps: 27-01-2020 update odc-tools
+# PyPi
+datacube[performance,s3]==1.8.5
+
+# currently only available from dea packages
 --extra-index-url="https://packages.dea.ga.gov.au"
-datacube[performance,s3]>=1.8.5.dev
-datacube-stats>=1.0.2
-fc>=1.3.4
-odc_algo>=0.2.dev
-odc_ui>=0.2.dev
-odc_index>=0.2.dev
-odc_aws>=0.2.dev
-odc_geom>=0.2.dev
-odc_io>=0.2.dev
-odc_aio>=0.2.dev
-odc_ppt>=0.2.dev
-odc_dscache>=0.2.dev
-odc_dtools>=0.2.dev
-odc_stats>=0.2.dev
-odc-apps-dc-tools>=0.2.dev
-odc-apps-cloud>=0.2.dev
+odc-algo==0.2.1.dev2856
+odc-cloud[ASYNC]==0.2.1.dev2856
+odc-dscache==0.2.1.dev2856
+odc-io==0.2.1.dev2856
+odc-stac==0.2.1.dev2856
+odc-stats==0.2.1.dev2856
+odc-ui==0.2.1.dev2856
+datacube-stats==1.0.2
+fc==1.3.5

docker/requirements.txt

@@ -39,10 +39,10 @@ cattrs
 structlog
 
 # Scientific Stack
-dask[complete]
-distributed
-dask-image
-dask-ml
+dask[complete]==2.30.0 # due to dask-gateway
+distributed==2.30.0 # due to dask-gateway
+dask-image==0.6.0
+dask-ml==1.9.0
 msgpack
 numba
 numexpr