diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..72e09583c --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,64 @@ +## devcontainer-focused Rocker +FROM ghcr.io/rocker-org/devcontainer/tidyverse:4.3 + +## latest version of geospatial libs +RUN /rocker_scripts/experimental/install_dev_osgeo.sh + +## install vim for convenience +## texlive installed for install_jupyter.sh because it tries to install some basic tex libraries +RUN apt-get update -qq && apt-get -y install vim texlive + +# standard python/jupyter setup +ENV NB_USER=rstudio +ENV VIRTUAL_ENV=/opt/venv +ENV PATH=${VIRTUAL_ENV}/bin:${PATH} +RUN wget https://github.com/rocker-org/rocker-versioned2/raw/master/scripts/install_jupyter.sh && \ + bash -e install_jupyter.sh && \ + rm install_jupyter.sh && \ + chown ${NB_USER}:staff -R ${VIRTUAL_ENV} + +# Set up conda +ENV CONDA_ENV=/opt/miniforge3 +ENV PATH=${PATH}:$CONDA_ENV/bin +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \ + bash Miniforge3-$(uname)-$(uname -m).sh -b -p ${CONDA_ENV} && \ + chown ${NB_USER}:staff -R ${CONDA_ENV} + +# podman doesn't not understand group permissions +# RUN chown ${NB_USER}:staff -R ${R_HOME}/site-library + +# some teaching preferences; might replace with gh-creds +RUN git config --system pull.rebase false && \ + git config --system credential.helper 'cache --timeout=36000' + +## codeserver; Add VSCode button +RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf .cache + +## Install the R packages into site library +COPY install.R install.R +RUN Rscript install.R && rm install.R + +## Openscapes-specific configs +USER rstudio +WORKDIR /home/rstudio +# make bash the user default +RUN usermod -s /bin/bash rstudio + +# install into the default venv environment +COPY nasa-requirements.txt requirements.txt +RUN python -m pip install -r requirements.txt && rm requirements.txt + +# Create a conda-based env and install into it without using conda init/conda activate +# (this yaml file doesn't include everything from pangeo, consider a different one...) +# pangeo uses the name 'notebook' +ENV ENV_NAME=notebook +ENV MY_ENV=${CONDA_ENV}/envs/${ENV_NAME} +RUN wget https://github.com/NASA-Openscapes/corn/raw/main/ci/environment.yml && \ + conda env create -p ${MY_ENV} -f environment.yml + +# This won't be the default environment but we register it +RUN ${MY_ENV}/bin/python -m pip install ipykernel && \ + ${MY_ENV}/bin/python -m ipykernel install --prefix /opt/venv --name=${ENV_NAME} + + + diff --git a/.devcontainer/conda.Dockerfile b/.devcontainer/conda.Dockerfile new file mode 100644 index 000000000..9e03c16cb --- /dev/null +++ b/.devcontainer/conda.Dockerfile @@ -0,0 +1,59 @@ +# devcontainer-focused Rocker +FROM ghcr.io/rocker-org/devcontainer/tidyverse:4.3 + +## Set the default shell +ENV SHELL=/bin/bash + +# latest version of geospatial libs -- big! +# RUN /rocker_scripts/experimental/install_dev_osgeo.sh && rm -rf /build_* +# install vim for convenience + +RUN apt-get update -qq && apt-get -y install vim + +# some teaching preferences +RUN git config --system pull.rebase false && \ + git config --system credential.helper 'cache --timeout=36000' + +# codeserver; Adds the VSCode button +RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf .cache + +# Rename the rstudio user as jovyan +RUN usermod -l jovyan rstudio && \ + usermod -d /home/jovyan -m jovyan && \ + groupmod -n jovyan rstudio + + +# Set up conda +ENV NB_USER=jovyan +ENV CONDA_DIR=/opt/miniforge3 +ENV CONDA_ENV=openscapes +ENV NB_PYTHON_PREFIX=${CONDA_DIR}/envs/${CONDA_ENV} +ENV PATH=${NB_PYTHON_PREFIX}/bin:/${CONDA_DIR}/bin:${PATH} + +# install conda +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \ + bash Miniforge3-$(uname)-$(uname -m).sh -b -p ${CONDA_DIR} && \ + chown ${NB_USER}:staff -R ${CONDA_DIR} && \ + rm -f Miniforge3*.sh *.deb + +# conda decorators +RUN conda init --system +RUN echo ". ${CONDA_DIR}/etc/profile.d/conda.sh ; conda activate ${CONDA_ENV}" > /etc/profile.d/init_conda.sh + +# Tell RStudio how to find Python +RUN echo "PATH=${PATH}" >>"${R_HOME}/etc/Renviron.site" + +# install R packages into the site library +COPY install.R install.R +RUN Rscript install.R && rm install.R + +# Standard user setup here +USER ${NB_USER} +WORKDIR /home/${NB_USER} +# make bash default shell +RUN usermod -s /bin/bash ${NB_USER} + +COPY environment.yml environment.yml +RUN conda env update -f environment.yml && conda clean --all +RUN rm environment.yml + diff --git a/.devcontainer/conda2.Dockerfile b/.devcontainer/conda2.Dockerfile new file mode 100644 index 000000000..d4306bfe1 --- /dev/null +++ b/.devcontainer/conda2.Dockerfile @@ -0,0 +1,47 @@ +# devcontainer-focused Rocker +FROM ghcr.io/rocker-org/devcontainer/tidyverse:4.3 + +# latest version of geospatial libs +RUN /rocker_scripts/experimental/install_dev_osgeo.sh && rm -rf /build_* +RUN apt-get update -qq && apt-get -y install vim + +# podman doesn't understand group permissions +RUN chown rstudio:staff ${R_HOME}/site-library + +# some teaching preferences +RUN git config --system pull.rebase false && \ + git config --system credential.helper 'cache --timeout=36000' + +# codeserver +RUN curl -fsSL https://code-server.dev/install.sh | sh + +# Set up conda +ENV NB_USER=rstudio +ENV CONDA_ENV=/opt/miniforge3 +ENV PATH=${CONDA_ENV}/bin:${PATH} +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \ + bash Miniforge3-$(uname)-$(uname -m).sh -b -p ${CONDA_ENV} && \ + chown ${NB_USER}:staff -R ${CONDA_ENV} && \ + rm Miniforge3*.sh *.deb + +# Set up a primary conda environment distinct from (base) +ENV MY_ENV=${CONDA_ENV}/envs/openscapes +ENV PATH=${MY_ENV}/bin:${PATH} +RUN echo "PATH=${PATH}" >>"${R_HOME}/etc/Renviron.site" + +# Initialize conda by default for all users: +# RUN conda init --system # will cause terminal to start in base + +# Standard user setup here +USER ${NB_USER} +WORKDIR /home/${NB_USER} +RUN usermod -s /bin/bash ${NB_USER} + +# install into the default environment +COPY install.R install.R +RUN Rscript install.R && rm install.R +RUN rm install.R + +COPY environment.yml environment.yml +RUN conda env create -p ${MY_ENV} -f environment.yml && conda clean --all + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..c76206d67 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,44 @@ +{ + "image": "ghcr.io/eeholmes/earthdata-cloud-cookbook/cookbook:venv", + // container run args for gdal netcdf vsi access + "runArgs": [ "--security-opt", "seccomp=unconfined" ], + "name": "NASA TOPS Environment", + "waitFor": "onCreateCommand", + "features": { + "ghcr.io/rocker-org/devcontainer-features/quarto-cli:1": {} + }, + "customizations": { + "codespaces": { + "openFiles": ["README.md"] + }, + "vscode": { + "settings": { + "r.rterm.linux": "/usr/local/bin/radian", + "r.bracketedPaste": true, + "r.plot.useHttpgd": true, + "[r]": { + "editor.wordSeparators": "`~!@#%$^&*()-=+[{]}\\|;:'\",<>/?" + } + }, + "extensions": [ + "reditorsupport.r", + "rdebugger.r-debugger", + "ms-toolsai.jupyter", + "ms-python.python" + ] + } + }, + // Forward the RStudio ports + "forwardPorts": [8787], + "portsAttributes": { + "8787": { + "label": "Rstudio", + "requireLocalPort": true, + "onAutoForward": "ignore" + } + }, + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "bash .devcontainer/setup.sh", + "postAttachCommand": "sudo rstudio-server start &> /dev/null && bash .devcontainer/welcome.sh", + "remoteUser": "rstudio" + } diff --git a/.devcontainer/environment.yml b/.devcontainer/environment.yml new file mode 100644 index 000000000..05e2e4dfc --- /dev/null +++ b/.devcontainer/environment.yml @@ -0,0 +1,88 @@ +name: openscapes +channels: + - conda-forge +dependencies: + - python=3.10 + - dask + - dask-gateway + - graphviz + - distributed + - rioxarray + - xarray>=2022.12 + - h5netcdf + - netcdf4>=1.6.4 + - h5py + - gdal~=3.8 + - geoviews + - holoviews=1.16.2 + - matplotlib-base + - seaborn + - intake + - hvplot + - datashader + - pyproj + - bqplot + - geopandas + - zarr + - cartopy + - pynco + - rasterio + - shapely + - pyresample + - icepyx + - joblib + - pystac-client + - odc-stac + - pydap + - lxml + - ipyleaflet + # when jlab 4 gets released we can upgrade dask-labextension + # - dask-labextension + # - dask-labextension==6.2.0 + - dask-labextension + - jupyter-server-proxy + - jupyter-vscode-proxy + - jupyter-rsession-proxy + - ipywidgets + - jupyter-book + - jupyterlab>4 + - jupyterlab-myst + - jupyterhub-singleuser + - jupyterlab-geojson + - jupyterlab-favorites + - jupyterlab-git + - jupyter-resource-usage + - jupyterlab-h5web + - hdf5plugin + - ipympl + - conda-lock + - pooch + - earthaccess>=0.8.2 + - sliderule>=3.6.0 + # QGIS + - qgis~=3.28 + - pyopencl + - ocl-icd-system + - websockify>=0.10 + # /QGIS + - h5coro + - git >=2.39 + - itslive + - rasterstats + - kerchunk + - rechunker + - pqdm + - spectral + - scikit-image + - coiled + - s3fs + - nbgitpuller + - awscliv2 + - jupyter-server-proxy + - jupyter-vscode-proxy + # https://github.com/jupyterlab/jupyter-collaboration/issues/202 + # - jupyter-collaboration + - code-server >=3.2 + - pip +platforms: + - linux-64 diff --git a/.devcontainer/install.R b/.devcontainer/install.R new file mode 100755 index 000000000..5bf6b8c13 --- /dev/null +++ b/.devcontainer/install.R @@ -0,0 +1,14 @@ +#! /usr/local/bin/Rscript +# install R dependencies + +# We could use renv.lock approach here instead, but will force re-creation of environment from scratch +# Does not provide a good way to ensure that sf/terra/gdalcubes are installed from source while other packages can be binary +# Likewise, pak insists on installing old gdal from apt instead of respecting system library source builds +install.packages("pak") +pak::pkg_install(c("rstac", "spData", "earthdatalogin", "quarto", "aws.s3", "tmap", "reticulate")) +pak::pkg_install('r-tmap/tmap') + +#pak::pkg_install("httpgd") +#pak::pkg_install(c("IRkernel", "languageserver")) +#IRkernel::installspec() + diff --git a/.devcontainer/jupyter-environment.yml b/.devcontainer/jupyter-environment.yml new file mode 100644 index 000000000..82f71375b --- /dev/null +++ b/.devcontainer/jupyter-environment.yml @@ -0,0 +1,21 @@ +name: openscapes +channels: + - conda-forge +dependencies: + - python=3.10 + - dask-labextension + - jupyter-server-proxy + - jupyter-vscode-proxy + - jupyter-rsession-proxy + - ipywidgets + - jupyter-book + - jupyterlab>4 + - jupyterlab-myst + - jupyterhub-singleuser + - jupyterlab-geojson + - jupyterlab-favorites + - jupyterlab-git + - jupyter-resource-usage + - jupyterlab-h5web + - code-server >=3.2 + - pip diff --git a/.devcontainer/nasa-requirements.txt b/.devcontainer/nasa-requirements.txt new file mode 100644 index 000000000..0bbe7a5b6 --- /dev/null +++ b/.devcontainer/nasa-requirements.txt @@ -0,0 +1,41 @@ +jupyterhub +jupyterlab +notebook +jupyterlab-quarto +jupyter-rsession-proxy +jupyter-vscode-proxy +jupyter-console +jupyter-events +jupyter-lsp +jupyter_client +jupyter_core +jupyter_server +jupyter_server_terminals +jupyterlab-pygments +jupyterlab-widgets +jupyterlab_server +gh-scoped-creds +nbconvert +nbformat +odc-stac +planetary-computer +polars +pyarrow +pystac-client +scikit-image +hvplot +geoviews +earthaccess +pyresample +contextily +rioxarray +geopandas +rasterstats +dask +distributed +h5netcdf +h5py +netCDF4 +zarr +lxml + diff --git a/.devcontainer/python-only-devcontainer.json b/.devcontainer/python-only-devcontainer.json new file mode 100644 index 000000000..b113da367 --- /dev/null +++ b/.devcontainer/python-only-devcontainer.json @@ -0,0 +1,21 @@ +{ + "image": "mcr.microsoft.com/devcontainers/universal:2", + // "hostRequirements": { + // "cpus": 4 + // }, + "waitFor": "onCreateCommand", + "updateContentCommand": "python3 -m pip install -r requirements.txt", + "postCreateCommand": "", + "customizations": { + "codespaces": { + "openFiles": ["README.md"] + }, + "vscode": { + "extensions": [ + "ms-toolsai.jupyter", + "ms-python.python" + ] + } + } + } + \ No newline at end of file diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100755 index 000000000..b89dcf16d --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo cp /etc/rstudio/disable_auth_rserver.conf /etc/rstudio/rserver.conf +sudo sudo bash -c 'echo "USER=rstudio" >>/etc/environment' +sudo /init &> /dev/null & + diff --git a/.devcontainer/venv.Dockerfile b/.devcontainer/venv.Dockerfile new file mode 100644 index 000000000..85f6ac30a --- /dev/null +++ b/.devcontainer/venv.Dockerfile @@ -0,0 +1,42 @@ +## devcontainer-focused Rocker +FROM ghcr.io/rocker-org/devcontainer/tidyverse:4.3 + +## Set the default shell +ENV SHELL=/bin/bash + +## latest version of geospatial libs +RUN /rocker_scripts/experimental/install_dev_osgeo.sh && rm -rf build_* + +## install vim +RUN apt-get update -qq && apt-get -y install vim + +# standard python/jupyter setup +# install python that is with the ubuntu install +ENV NB_USER=rstudio +ENV VIRTUAL_ENV=/opt/venv +ENV PATH=${VIRTUAL_ENV}/bin:${PATH} +RUN /rocker_scripts/install_python.sh && \ + chown ${NB_USER}:staff -R ${VIRTUAL_ENV} + +# some teaching preferences +RUN git config --system pull.rebase false && \ + git config --system credential.helper 'cache --timeout=36000' + +## codeserver +RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf .cache + +## Install R packages into the site library +COPY install.R install.R +RUN Rscript install.R && rm install.R +RUN rm install.R + +## Openscapes-specific configs +USER rstudio +WORKDIR /home/rstudio +RUN usermod -s /bin/bash rstudio + +# install into the default venv environment +COPY nasa-requirements.txt requirements.txt +RUN python -m pip install --no-cache-dir -r requirements.txt && rm requirements.txt +RUN rm requirements.txt + diff --git a/.devcontainer/welcome.sh b/.devcontainer/welcome.sh new file mode 100755 index 000000000..e749423a1 --- /dev/null +++ b/.devcontainer/welcome.sh @@ -0,0 +1,21 @@ +# Need to add this so that RStudio knows the project it is in +mkdir -p ~/.local/share/rstudio/projects_settings + +export RPROJ"=$(ls ${CODESPACE_VSCODE_FOLDER}/*.Rproj)" +echo ${RPROJ} > ~/.local/share/rstudio/projects_settings/last-project-path + +# Construct the message +message="## [Open in RStudio](https://$CODESPACE_NAME-8787.app.github.dev) +" +# Echo the message to the terminal +echo " +👋 Welcome to Codespaces! You are on our custom image. + - It includes runtimes and tools for Python & R using Jupyter, Quarto, or RStudio + +🌐 Open the RStudio editor here: https://$CODESPACE_NAME-8787.app.github.dev + - (This may take a few seconds to load, retry if necessary) + +🌐 Open the JupyterLab editor here: https://$CODESPACE_NAME.app.github.dev?editor=jupyter + - (This may take a few seconds to load, retry if necessary) + +" diff --git a/.github/workflows/docker-image-conda.yml b/.github/workflows/docker-image-conda.yml new file mode 100644 index 000000000..d2835361b --- /dev/null +++ b/.github/workflows/docker-image-conda.yml @@ -0,0 +1,30 @@ +name: Docker Image CI conda +on: + workflow_dispatch: null + push: + paths: + - '.devcontainer/conda.Dockerfile' + - '.devcontainer/environment.yml' + - '.github/workflows/docker-build-conda.yml' + branches: main +jobs: + build: + runs-on: ubuntu-latest + permissions: write-all + steps: + - uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + if: github.repository == 'eeholmes/earthdata-cloud-cookbook' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + - name: Build the Docker image + if: github.repository == 'eeholmes/earthdata-cloud-cookbook' + run: | + docker build .devcontainer -f .devcontainer/conda.Dockerfile --tag ghcr.io/eeholmes/earthdata-cloud-cookbook/cookbook-conda:latest + - name: Publish + if: github.repository == 'eeholmes/earthdata-cloud-cookbook' + run: | + docker push ghcr.io/eeholmes/earthdata-cloud-cookbook/cookbook-conda:latest diff --git a/.github/workflows/docker-image-venv.yml b/.github/workflows/docker-image-venv.yml new file mode 100644 index 000000000..9fa4ceb2a --- /dev/null +++ b/.github/workflows/docker-image-venv.yml @@ -0,0 +1,31 @@ +name: Docker Image CI venv +on: + workflow_dispatch: null + push: + paths: + - '.devcontainer/venv.Dockerfile' + - '.devcontainer/nasa-requirements.txt' + - '.github/workflows/docker-image-venv.yml' + branches: main +jobs: + build: + runs-on: ubuntu-latest + permissions: write-all + steps: + - uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + if: github.repository == 'eeholmes/earthdata-cloud-cookbook' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + - name: Build the Docker image + if: github.repository == 'eeholmes/earthdata-cloud-cookbook' + run: | + docker build .devcontainer -f .devcontainer/venv.Dockerfile --tag ghcr.io/eeholmes/earthdata-cloud-cookbook/cookbook-venv:latest + - name: Publish + if: github.repository == 'eeholmes/earthdata-cloud-cookbook' + run: | + docker push ghcr.io/eeholmes/earthdata-cloud-cookbook/cookbook-venv:latest + diff --git a/.github/workflows/quarto-publish.yml b/.github/workflows/quarto-publish.yml index 90cec110a..dbcd7caaf 100644 --- a/.github/workflows/quarto-publish.yml +++ b/.github/workflows/quarto-publish.yml @@ -1,44 +1,29 @@ -name: Render and deploy quarto files on: + workflow_dispatch: push: - branches: - - main + branches: main + +name: Quarto Publish jobs: - quarto-render-and-deploy: + build-deploy: runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} + container: + image: ghcr.io/boettiger-lab/nasa-tops:latest + options: --user root --security-opt seccomp=unconfined + permissions: + contents: write steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: 3.8 - channels: conda-forge - miniforge-variant: Mambaforge - activate-environment: quarto-import - environment-file: _import/environment.yml - - - name: "Import external notebooks" - working-directory: ./_import - run: | - conda info - pytest - python quarto_import.py -f assets.json - - - name: Set up Quarto - uses: quarto-dev/quarto-actions/setup@v2 - with: - # To install LaTeX to build PDF book - tinytex: true - # uncomment below and fill to pin a version - # version: 0.9.600 + - name: Quarto-inside-docker permissions + run: git config --system --add safe.directory '*' + + - name: Check out repository + uses: actions/checkout@v3 - - name: Publish to GitHub Pages (and render) - uses: quarto-dev/quarto-actions/publish@v2 - with: - target: gh-pages - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # this secret is always available for github actions - + - name: Render and Publish + if: github.repository == 'NASA-Openscapes/earthdata-cloud-cookbook' + uses: quarto-dev/quarto-actions/publish@v2 + with: + target: gh-pages + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml new file mode 100644 index 000000000..0dc65ed9e --- /dev/null +++ b/.github/workflows/test-build.yaml @@ -0,0 +1,27 @@ +on: + workflow_dispatch: + push: + branches: main + schedule: + - cron: "0 0 1 * *" # monthly + +name: Quarto Test Build + +jobs: + build-deploy: + runs-on: ubuntu-latest + container: + image: ghcr.io/eeholmes/earthdata-cloud-cookbook/cookbook:latest + options: --user root --security-opt seccomp=unconfined + permissions: + contents: write + steps: + - name: Quarto-inside-docker permissions + run: git config --system --add safe.directory '*' + + - name: Check out repository + uses: actions/checkout@v3 + + - name: Render + run: | + quarto render --no-cache diff --git a/README.md b/README.md index 614ec6d0d..1fdeeab4f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.7786710.svg)](https://zenodo.org/record/7786710) - +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/eeholmes/earthdata-cloud-cookbook?quickstart=1&editor=jupyter) # Hello!