From 62ea81bbb82dbbc23dae6580b78df85b6263c835 Mon Sep 17 00:00:00 2001 From: tobfer Date: Mon, 11 Dec 2023 14:31:24 +0000 Subject: [PATCH 1/5] correct zarr installation on workflows --- .github/workflows/markdown-general.yml | 2 +- .github/workflows/unit_test_contents.yml | 1 + notebook_to_md.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/markdown-general.yml b/.github/workflows/markdown-general.yml index aa519520..5bc147e9 100644 --- a/.github/workflows/markdown-general.yml +++ b/.github/workflows/markdown-general.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 with: repository: British-Oceanographic-Data-Centre/COAsT - ref: develop + ref: bug/zarr-notebooks path: external - name: Prepare environment diff --git a/.github/workflows/unit_test_contents.yml b/.github/workflows/unit_test_contents.yml index 68952e54..8f3ef21a 100644 --- a/.github/workflows/unit_test_contents.yml +++ b/.github/workflows/unit_test_contents.yml @@ -22,6 +22,7 @@ jobs: sudo apt-get update sudo apt-get install -y libgeos-dev pip install cartopy==0.21.0 + pip install zarr python unit_testing/generate_unit_test_contents.py - name: Commit changes run: | diff --git a/notebook_to_md.sh b/notebook_to_md.sh index b1952dfa..312c760c 100644 --- a/notebook_to_md.sh +++ b/notebook_to_md.sh @@ -5,7 +5,7 @@ directory=$1 # Convert ipynb files to markdown. echo "starting notebook execute for ${directory}" -jupyter nbconvert --to notebook --execute example_scripts/notebook_tutorials/runnable_notebooks/${directory}/*.ipynb --allow-errors --output-dir example_scripts/notebook_tutorials/runnable_notebooks/executed/${directory}/ +jupyter nbconvert --to notebook --execute example_scripts/notebook_tutorials/runnable_notebooks/${directory}/*.ipynb --log-level=DEBUG --allow-errors --output-dir example_scripts/notebook_tutorials/runnable_notebooks/executed/${directory}/ echo "starting notebook convert" jupyter nbconvert --to markdown example_scripts/notebook_tutorials/runnable_notebooks/executed/${directory}/*.ipynb --output-dir example_scripts/notebook_tutorials/markdown/${directory}/ echo "starting clean up (rm)" From 8657c70eb63e07a8e284b66252b04d68d86984fa Mon Sep 17 00:00:00 2001 From: tobfer Date: Mon, 11 Dec 2023 14:32:55 +0000 Subject: [PATCH 2/5] correct zarr installation on workflows --- .github/workflows/verify_package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/verify_package.yml b/.github/workflows/verify_package.yml index cb78a262..86ad4cdf 100644 --- a/.github/workflows/verify_package.yml +++ b/.github/workflows/verify_package.yml @@ -34,4 +34,5 @@ jobs: sudo apt-get update sudo apt-get install -y libgeos-dev pip install cartopy==0.21.0 + pip install zarr pytest tests From e8961ec93005e55a9757031df7cfec99b7be9873 Mon Sep 17 00:00:00 2001 From: tobfer Date: Mon, 11 Dec 2023 14:54:49 +0000 Subject: [PATCH 3/5] correct zarr installation on workflows --- .github/workflows/markdown-general.yml | 1 + .../general/zarr_files.ipynb | 33 +++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/markdown-general.yml b/.github/workflows/markdown-general.yml index 5bc147e9..bac5f0d6 100644 --- a/.github/workflows/markdown-general.yml +++ b/.github/workflows/markdown-general.yml @@ -42,5 +42,6 @@ jobs: sudo apt-get install -y libgeos-dev pip install cartopy==0.21.0 pip install zarr + pip install xarray[complete] mv config ./example_scripts/notebook_tutorials/runnable_notebooks/general/config bash notebook_to_md.sh general diff --git a/example_scripts/notebook_tutorials/runnable_notebooks/general/zarr_files.ipynb b/example_scripts/notebook_tutorials/runnable_notebooks/general/zarr_files.ipynb index 57d9806b..d779108f 100644 --- a/example_scripts/notebook_tutorials/runnable_notebooks/general/zarr_files.ipynb +++ b/example_scripts/notebook_tutorials/runnable_notebooks/general/zarr_files.ipynb @@ -73,16 +73,26 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 1, "id": "582e5ae5-e61d-409d-aa33-206f6ff3c0bd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/mnt/code/.pyenv/versions/3.10.12/envs/coast-10/lib/python3.10/site-packages/utide/harmonics.py:16: RuntimeWarning: invalid value encountered in cast\n", + "/mnt/code/.pyenv/versions/3.10.12/envs/coast-10/lib/python3.10/site-packages/utide/harmonics.py:17: RuntimeWarning: invalid value encountered in cast\n" + ] + } + ], "source": [ "import coast\n", "import matplotlib.pyplot as plt\n", "import datetime\n", "import numpy as np\n", "import xarray as xr\n", + "import zarr\n", "\n", "root = \"./\"\n", "fn_config_t_grid = root + \"./config/example_nemo_monthly_climate.json\"\n", @@ -114,10 +124,21 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 2, "id": "289bbdfd-991c-41ed-9acd-7e9bb35f157c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_221474/396929176.py:1: RuntimeWarning: Failed to open Zarr store with consolidated metadata, but successfully read with non-consolidated metadata. This is typically much slower for opening a dataset. To silence this warning, consider:\n", + "1. Consolidating metadata in this existing store with zarr.consolidate_metadata().\n", + "2. Explicitly setting consolidated=False, to avoid trying to read consolidate metadata, or\n", + "3. Explicitly setting consolidated=True, to raise an error in this case instead of falling back to try reading non-consolidated metadata.\n" + ] + } + ], "source": [ "dom = xr.open_zarr(fn_nemo_dom_mask)\n", "mesh_zgr = xr.open_zarr(fn_nemo_dom_mesh_zgr)\n", @@ -126,7 +147,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 3, "id": "e1f12c20-4076-4ae0-9135-c95218347805", "metadata": {}, "outputs": [], @@ -139,7 +160,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 4, "id": "50b2cbb7-a1ad-4ebc-aebb-42a1b8286e58", "metadata": {}, "outputs": [], From b56350c300f06ea2ffcc4cf4eac60681b53641c1 Mon Sep 17 00:00:00 2001 From: tobfer Date: Mon, 11 Dec 2023 15:12:13 +0000 Subject: [PATCH 4/5] correct zarr installation on workflows --- .github/workflows/markdown-general.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/markdown-general.yml b/.github/workflows/markdown-general.yml index bac5f0d6..5a71e00d 100644 --- a/.github/workflows/markdown-general.yml +++ b/.github/workflows/markdown-general.yml @@ -43,5 +43,6 @@ jobs: pip install cartopy==0.21.0 pip install zarr pip install xarray[complete] + pip install aiohttp requests mv config ./example_scripts/notebook_tutorials/runnable_notebooks/general/config bash notebook_to_md.sh general From 8d84563e4bc4129a3e53488d4b0c5a302c39be4a Mon Sep 17 00:00:00 2001 From: tobfer Date: Mon, 11 Dec 2023 15:29:53 +0000 Subject: [PATCH 5/5] correct dependencies to use zarr --- .../runnable_notebooks/general/zarr_files.ipynb | 2 +- notebook_to_md.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example_scripts/notebook_tutorials/runnable_notebooks/general/zarr_files.ipynb b/example_scripts/notebook_tutorials/runnable_notebooks/general/zarr_files.ipynb index d779108f..a45e128d 100644 --- a/example_scripts/notebook_tutorials/runnable_notebooks/general/zarr_files.ipynb +++ b/example_scripts/notebook_tutorials/runnable_notebooks/general/zarr_files.ipynb @@ -44,7 +44,7 @@ "id": "761fde27-8ef4-4805-887c-d795da34902e", "metadata": {}, "source": [ - "`pip install zarr`" + "`pip install zarr xarray[complete] aiohttp requests`" ] }, { diff --git a/notebook_to_md.sh b/notebook_to_md.sh index 312c760c..b1952dfa 100644 --- a/notebook_to_md.sh +++ b/notebook_to_md.sh @@ -5,7 +5,7 @@ directory=$1 # Convert ipynb files to markdown. echo "starting notebook execute for ${directory}" -jupyter nbconvert --to notebook --execute example_scripts/notebook_tutorials/runnable_notebooks/${directory}/*.ipynb --log-level=DEBUG --allow-errors --output-dir example_scripts/notebook_tutorials/runnable_notebooks/executed/${directory}/ +jupyter nbconvert --to notebook --execute example_scripts/notebook_tutorials/runnable_notebooks/${directory}/*.ipynb --allow-errors --output-dir example_scripts/notebook_tutorials/runnable_notebooks/executed/${directory}/ echo "starting notebook convert" jupyter nbconvert --to markdown example_scripts/notebook_tutorials/runnable_notebooks/executed/${directory}/*.ipynb --output-dir example_scripts/notebook_tutorials/markdown/${directory}/ echo "starting clean up (rm)"