From 8a02b516e8281dbecbd47b6cf61d18c059b0ec30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20S=2E=20Cofi=C3=B1o?= Date: Mon, 9 Aug 2021 09:10:54 +0200 Subject: [PATCH] Devel 2.5.3 (#20) * Fix docker build command * Include conda env from antonio * Added mamba doc for recipe * Update dockerfile to 1.5.0 * Update README (conda as preferred install method) * upgraded docker to 1.5.1 (added python packages) * Added python dependencies and using mambabuild * Change channel order in conda install command Setting conda-forge as primary channel avoids conflicts with the default channel * reduced meta.yml dependencies * Update README.md Include additional aid for jupyter notebook installation in new climate4r dedicated environments * Update README.md * Generated Dockerfile for 1.5.0 * updated conda to 1.5.1 * Updated README * using r- packages instead of install.R * The first release based only on conda packages * Fix build string * Relaxing R and Python version specification * Preparing release v2.5.3 Co-authored-by: zequihg50 Co-authored-by: Jesus Fernandez Co-authored-by: Jesus Fernandez Co-authored-by: Joaquin Bedia --- Dockerfile | 14 +++++++-- README.md | 46 +++++++++++++++------------- conda-base/README.md | 26 ---------------- conda-base/build.sh | 18 ----------- conda-base/get-last-release.sh | 52 -------------------------------- conda-base/meta.yaml | 29 ------------------ conda-full/README.md | 24 --------------- conda-full/build.sh | 36 ---------------------- conda-full/meta.yaml | 55 ---------------------------------- conda/README.md | 43 ++++++++++++++++++++++++++ conda/meta.yaml | 44 +++++++++++++++++++++++++++ docker/Dockerfile | 35 ++++++++-------------- docker/README.md | 2 +- 13 files changed, 139 insertions(+), 285 deletions(-) delete mode 100644 conda-base/README.md delete mode 100644 conda-base/build.sh delete mode 100755 conda-base/get-last-release.sh delete mode 100644 conda-base/meta.yaml delete mode 100644 conda-full/README.md delete mode 100644 conda-full/build.sh delete mode 100644 conda-full/meta.yaml create mode 100644 conda/README.md create mode 100644 conda/meta.yaml diff --git a/Dockerfile b/Dockerfile index 9c8823b..86a646b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,13 @@ -FROM santandermetgroup/climate4r:1.4.1 +FROM jupyter/base-notebook:latest -COPY --chown=jovyan:users notebooks /home/$NB_USER/notebooks +USER root +RUN apt-get update && apt-get install -y xorg git + +USER $NB_USER +RUN conda install mamba -y -n base -c conda-forge && \ + mamba install -y -c conda-forge -c r -c defaults -c santandermetgroup \ + r-climate4r=2.5.3 \ + xarray pandas matplotlib cartopy netcdf4 cftime dask iris \ + jupyter_contrib_nbextensions jupyter_nbextensions_configurator r-irkernel && \ + chown -R jovyan:users /home/jovyan && \ + R --vanilla -e 'IRkernel::installspec()' diff --git a/README.md b/README.md index ac6e3f2..e01b4ca 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,33 @@ Moreover, there is a [notebook repository](https://github.com/SantanderMetGroup/ ## Installation +The `climate4R` framework relies on a wealth of other R packages and bindings to third-party libraries. Therefore, the recommended installation is through the conda package below, which ensures the proper installation of all dependencies. + +### Installation using miniconda: + +[Miniconda](https://docs.conda.io/en/latest/miniconda.html) is a free minimal installer for conda. The [conda recipe](conda) installs an up-to-date version of the different packages composing the `climate4R` framework, along with the associated library dependencies (udunits, openjdk, netcdf Java etc.), avoiding potential problems like the R-java configuration etc. Note that the appropriate miniconda distribution must be installed (go to the [miniconda installers page](https://docs.conda.io/en/latest/miniconda.html)) before running the following commands. We recommend starting from a clean environment (named climate4R in this example): + +```bash +conda create --name climate4R +conda activate climate4R +``` + +In this environment, install `climate4R` by issuing: + +```bash +conda install -c conda-forge -c r -c defaults -c santandermetgroup climate4r +``` + +Activate the conda environment to work with climate4R. Deactivate the environment with: + +```bash +conda deactivate +``` + ### Direct package installation from github: +Individual packages can be installed directly from the github sources. + ``` r > library(devtools) > install_github(c("SantanderMetGroup/loadeR.java", @@ -60,26 +85,7 @@ Moreover, there is a [notebook repository](https://github.com/SantanderMetGroup/ "SantanderMetGroup/downscaleR")) ``` -### Installation using miniconda: - -[Miniconda](https://docs.conda.io/en/latest/miniconda.html) is a free minimal installer for conda. Using the [conda recipe](https://github.com/SantanderMetGroup/climate4R/tree/master/conda) it is possible to install an up-to-date version of the different packages composing the `climate4R` framework, and all the associated library dependencies (udunits, openjdk, netcdf Java etc.), avoiding some possible problems like the R-java configuration etc. Note that the appropriate miniconda distribution must be installed before running the command (go to the [miniconda installers page](https://docs.conda.io/en/latest/miniconda.html)): - -```bash -# create a conda environment -conda create --name nameofmycondaenvironment -# activate the environment -conda activate nameofmycondaenvironment -# install climate4R -conda install -c defaults -c r -c conda-forge -c santandermetgroup climate4r -``` -Activate the conda environment to work with climate4R. To deactivate the environment run the following: - -```bash -# deactivate the environment -conda deactivate nameofmycondaenvironment -``` - -### NOTE: installation of specific package versions +#### NOTE: installation of specific package versions In case a particular paper notebook is to be replicated, the installation of specific version tags can be done by just explicitly indicating the tag number in the repo name. For example: diff --git a/conda-base/README.md b/conda-base/README.md deleted file mode 100644 index 41af9a9..0000000 --- a/conda-base/README.md +++ /dev/null @@ -1,26 +0,0 @@ -## conda recipe for the climate4R core packages - -To install the whole framework go to [conda-full](https://github.com/SantanderMetGroup/climate4R/tree/devel/conda-full) - -```bash -# create a conda environment -conda create --name nameofmycondaenvironment -# activate the environment -conda activate nameofmycondaenvironment -# install climate4R -conda install -c defaults -c r -c conda-forge -c santandermetgroup climate4r-base=1.0.1 -``` -Activate the conda environment to work with climate4R. To deactivate the environment run the following: - -```bash -# deactivate the environment -conda deactivate -``` - -### Build (only for the admin) - -```bash -conda build -c defaults -c r -c conda-forge . -anaconda login -anaconda upload -u SantanderMetGroup TAR_BZ2 -``` diff --git a/conda-base/build.sh b/conda-base/build.sh deleted file mode 100644 index ed20fdd..0000000 --- a/conda-base/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -# for convertR -export UDUNITS2_LIB=${PREFIX}/lib -export UDUNITS2_INCLUDE=${PREFIX}/include -export CPATH=${PREFIX}/include - -# for rjava -export LD_LIBRARY_PATH=${PREFIX}/lib:${PREFIX}/lib/server:$PREFIX/lib/R/library/rJava/libs:$PREFIX/jre/lib/amd64/server - -R --vanilla -e 'install.packages("devtools", repos="https://cloud.r-project.org/")' -R --vanilla -e "install.packages('udunits2', repos='https://cloud.r-project.org/', configure.args='--with-udunits2-lib=${UDUNITS2_LIB}')" -R --vanilla -e 'install.packages("ncdf4", repos="https://cloud.r-project.org/")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/loadeR.java@v1.1.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/climate4R.UDG@devel")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/loadeR@devel")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/loadeR.2nc@v0.1.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/transformeR@v2.0.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/downscaleR@v3.3.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/visualizeR@v1.6.0")' diff --git a/conda-base/get-last-release.sh b/conda-base/get-last-release.sh deleted file mode 100755 index e118456..0000000 --- a/conda-base/get-last-release.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -u - -trap exit SIGINT - -repos=' -SantanderMetGroup/climate4R.UDG -SantanderMetGroup/loadeR.java -SantanderMetGroup/loadeR -SantanderMetGroup/loadeR.ECOMS -SantanderMetGroup/transformeR -SantanderMetGroup/downscaleR.keras -SantanderMetGroup/downscaleR -SantanderMetGroup/visualizeR -SantanderMetGroup/convertR -SantanderMetGroup/geoprocessoR -SantanderMetGroup/climate4R.climdex -SantanderMetGroup/climate4R.indices -SantanderMetGroup/loadeR.2nc -SantanderMetGroup/fireDanger -SantanderMetGroup/VALUE -SantanderMetGroup/climate4R.value -SantanderMetGroup/mopa -SantanderMetGroup/drought4R -SantanderMetGroup/climate4R.datasets' - -cat >build.sh <> build.sh -done diff --git a/conda-base/meta.yaml b/conda-base/meta.yaml deleted file mode 100644 index fda4539..0000000 --- a/conda-base/meta.yaml +++ /dev/null @@ -1,29 +0,0 @@ -package: - name: climate4r-base - version: 1.0.1 - -build: - script_env: - - GITHUB_PAT - rpaths: - - lib/R/lib/ - - lib/ - -requirements: - build: - - r-base=3.6.1 - - r-rjava - - openjdk - - curl - - udunits2 - - netcdf4 - run: - - r-base=3.6.1 - - r-rjava - - openjdk - - curl - - udunits2 - - netcdf4 - -about: - home: https://github.com/SantanderMetGroup/climate4R diff --git a/conda-full/README.md b/conda-full/README.md deleted file mode 100644 index 374aee9..0000000 --- a/conda-full/README.md +++ /dev/null @@ -1,24 +0,0 @@ -## conda recipe for climate4R - -```bash -# create a conda environment -conda create --name nameofmycondaenvironment -# activate the environment -conda activate nameofmycondaenvironment -# install climate4R -conda install -c defaults -c r -c conda-forge -c santandermetgroup climate4r=1.4.0 -``` -Activate the conda environment to work with climate4R. To deactivate the environment run the following: - -```bash -# deactivate the environment -conda deactivate -``` - -### Build (only for the admin) - -```bash -conda build -c defaults -c r -c conda-forge . -anaconda login -anaconda upload -u SantanderMetGroup TAR_BZ2 -``` diff --git a/conda-full/build.sh b/conda-full/build.sh deleted file mode 100644 index 72ace5e..0000000 --- a/conda-full/build.sh +++ /dev/null @@ -1,36 +0,0 @@ -# for convertR -export UDUNITS2_LIB=${PREFIX}/lib -export UDUNITS2_INCLUDE=${PREFIX}/include -export CPATH=${PREFIX}/include - -# for rjava -export LD_LIBRARY_PATH=${PREFIX}/lib:${PREFIX}/lib/server:$PREFIX/lib/R/library/rJava/libs:$PREFIX/jre/lib/amd64/server - -R --vanilla -e 'install.packages("devtools", repos="https://cloud.r-project.org/")' -R --vanilla -e "install.packages('udunits2', repos='https://cloud.r-project.org/', configure.args='--with-udunits2-lib=${UDUNITS2_LIB}')" -#R --vanilla -e 'install.packages("rJava", repos="https://cloud.r-project.org/")' -#R --vanilla -e 'install.packages("ncdf4", repos="https://cloud.r-project.org/")' -#R --vanilla -e 'install.packages("gh", repos="https://cloud.r-project.org/")' -#R --vanilla -e 'install.packages("matrix", repos="https://cloud.r-project.org/")' -#R --vanilla -e 'install.packages("geojsonio", repos="https://cloud.r-project.org/")' -#R --vanilla -e 'install.packages("rgdal", repos="https://cloud.r-project.org/")' -R --vanilla -e 'install.packages("https://cran.r-project.org/src/contrib/Archive/tree/tree_1.0-38.tar.gz")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/climate4R.UDG@v0.2.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/loadeR.java@v1.1.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/loadeR@v1.7.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/loadeR.ECOMS@v1.4.6")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/transformeR@v2.0.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/downscaleR.keras@v0.0.2")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/downscaleR@v3.3.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/visualizeR@v1.6.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/convertR@v0.2.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/geoprocessoR@v0.2.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/climate4R.climdex@v0.2.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/climate4R.indices@v0.1.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/loadeR.2nc@v0.1.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/fireDanger@v1.1.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/VALUE@v2.2.1")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/climate4R.value@v0.0.2")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/mopa@v1.0.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/drought4R@v0.2.0")' -R --vanilla -e 'library(devtools);install_github("SantanderMetGroup/climate4R.datasets@v0.0.1")' diff --git a/conda-full/meta.yaml b/conda-full/meta.yaml deleted file mode 100644 index 539d75d..0000000 --- a/conda-full/meta.yaml +++ /dev/null @@ -1,55 +0,0 @@ -package: - name: climate4r - version: 1.4.1 - -build: - script_env: - - GITHUB_PAT - rpaths: - - lib/R/lib/ - - lib/ - -requirements: - build: - - r-base=3.6.1 - - r-rjava - - openjdk - - gdal>=3 - - poppler - - xorg-libxt - - udunits2 - - curl - - zstd - - libv8 - - jq - - protobuf - - geojson - - libiconv - - geojsonio - - proj4 - - pkg-config - - libiconv - - libgomp - run: - - r-base=3.6.1 - - r-rjava - - openjdk - - gdal>=3 - - poppler - - xorg-libxt - - udunits2 - - curl - - zstd - - libv8 - - jq - - protobuf - - geojson - - libiconv - - geojsonio - - proj4 - - pkg-config - - libiconv - - libgomp - -about: - home: https://github.com/SantanderMetGroup/climate4R diff --git a/conda/README.md b/conda/README.md new file mode 100644 index 0000000..09a6cf3 --- /dev/null +++ b/conda/README.md @@ -0,0 +1,43 @@ +# Conda recipe for climate4R metapackage + +## Installing climate4R conda metapackage + +```bash +conda install -c conda-forge -c r -c defaults -c santandermetgroup r-climate4r +``` + +*** +_Note for jupyter notebook users_: From a dedicated environment, to run notebooks using climate4R include it as: + +```bash +conda install -c conda-forge -c r -c defaults -c santandermetgroup r-climate4r + jupyter r-irkernel +``` + +in order to include jupyter notebooks in your newly created environment +*** + +## Building climate4R conda metapackage + +```bash +conda build -c conda-forge -c r -c defaults -c santandermetgroup . +anaconda login +anaconda upload -u SantanderMetGroup PATH_TO_TAR_BZ2 +``` + +## ToDo, explore mamba and boa + +[mamba](https://github.com/mamba-org/mamba) and [boa](https://github.com/mamba-org/boa) + +Install: + +```bash +conda install mamba -n base -c conda-forge +``` + +Build: + +```bash +rm -rf ${HOME}/miniconda3/conda-bld/src_cache +conda mambabuild -c conda-forge -c r -c defaults -c santandermetgroup . &>log & +``` diff --git a/conda/meta.yaml b/conda/meta.yaml new file mode 100644 index 0000000..a41bee8 --- /dev/null +++ b/conda/meta.yaml @@ -0,0 +1,44 @@ +package: + name: r-climate4r + version: 2.5.3 +build: + string: r36_1 + noarch: generic +requirements: + build: [] + run: + #r dependencies + - r-base >=3.6.3,<3.7 + - r-climate4r.udg ==0.2.3 + - r-loader.java ==1.1.1 + - r-loader ==1.7.1 + - r-loader.ecoms ==1.4.6 + - r-transformer ==2.1.3 + - r-downscaler ==3.3.3 + - r-downscaler.keras ==1.0.0 + - r-visualizer ==1.6.1 + - r-convertr ==0.2.0 + - r-geoprocessor ==0.2.0 + - r-climate4r.climdex ==0.2.2 + - r-climate4r.indices ==0.2.0 + - r-loader.2nc ==0.1.2 + - r-firedanger ==1.1.0 + - r-value ==2.2.1 + - r-climate4r.value ==0.0.2 + - r-mopa ==1.0.0 + - r-drought4r ==0.2.0 + - r-climate4r.datasets ==0.0.1 + #python dependencies + - python >=3.7,<3.8 + - iris ==3.0.4 + +about: + home: http://www.meteo.unican.es/climate4r + license: GPL-3.0-or-later + summary: Climate4R Framework + license_family: GPL3 + +extra: + recipe-maintainers: + - conda-forge/r + - cofinoa diff --git a/docker/Dockerfile b/docker/Dockerfile index a79ceb6..21c9f29 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,26 +1,17 @@ FROM jupyter/base-notebook:latest -USER root -RUN apt-get update && apt-get install -y xorg git && ln -s /bin/tar /bin/gtar - -USER $NB_USER -RUN conda install -y -c defaults -c conda-forge -c santandermetgroup python=3.6 climate4r=1.4.1 \ - jupyter_contrib_nbextensions jupyter_nbextensions_configurator && \ - cp /opt/conda/lib/libgomp.so.1.0.0 /opt/conda/lib/libgomp.so - -ENV PROJ_LIB /opt/conda/share/proj -RUN R --vanilla -e 'install.packages("https://cran.r-project.org/src/contrib/Archive/foreign/foreign_0.8-64.tar.gz")' && \ - R --vanilla -e 'install.packages(c("crayon", "pbdZMQ", "classInt", "geojsonio"), repos="https://cloud.r-project.org/")' && \ - R --vanilla -e 'library(devtools);install_github("IRkernel/repr")' && \ - R --vanilla -e 'library(devtools);install_github("IRkernel/IRdisplay")' && \ - R --vanilla -e 'library(devtools);install_github("IRkernel/IRkernel")' && \ - chown -R jovyan:users /home/jovyan && \ - R --vanilla -e 'IRkernel::installspec()' - -USER root -RUN R --vanilla -e 'IRkernel::installspec(user = FALSE)' && \ - mkdir -p /gpfs/ces/share-7c11c2a4-9d9f-40f5-b95e-396bcbf3f608 && \ - mkdir -p /home/$NB_USER/shared && \ +USER root +RUN apt-get update && apt-get install -y xorg git && \ + mkdir -p /gpfs/ces/share-7c11c2a4-9d9f-40f5-b95e-396bcbf3f608 /home/$NB_USER/shared /gpfs/projects/meteo/IPCC/ATLAS && \ chown -R $NB_USER:users /gpfs /home/$NB_USER/shared && \ - ln -s /home/$NB_USER/shared /gpfs/ces/share-7c11c2a4-9d9f-40f5-b95e-396bcbf3f608/HUB + ln -s /home/$NB_USER/shared /gpfs/ces/share-7c11c2a4-9d9f-40f5-b95e-396bcbf3f608/HUB && \ + ln -s /home/$NB_USER/shared /gpfs/projects/meteo/IPCC/ATLAS/hub + USER $NB_USER +RUN conda install mamba -y -n base -c conda-forge && \ + mamba install -y -c conda-forge -c r -c defaults -c santandermetgroup \ + r-climate4r=2.5.3 \ + xarray pandas matplotlib cartopy netcdf4 cftime dask iris \ + jupyter_contrib_nbextensions jupyter_nbextensions_configurator r-irkernel && \ + chown -R jovyan:users /home/jovyan && \ + R --vanilla -e 'IRkernel::installspec()' diff --git a/docker/README.md b/docker/README.md index cf6cdab..1c5c71a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -36,7 +36,7 @@ Write http://firstchoice:8888/ or http://secondchoice:8888/ in your browser and ### Build (only for the admin) ```bash -docker build -t santandermetgroup/climate4r:latest . < Dockerfile +docker build -t santandermetgroup/climate4r:latest -f Dockerfile . docker login docker push santandermetgroup/climate4r:latest ```