From 318d1a37a6d7b0edbf2cee66d9340bfb0ed59334 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Thu, 21 Sep 2023 13:16:08 +0000 Subject: [PATCH 01/14] update requirements for ubuntu --- actinia-ubuntu/requirements_ubuntu22.txt | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 actinia-ubuntu/requirements_ubuntu22.txt diff --git a/actinia-ubuntu/requirements_ubuntu22.txt b/actinia-ubuntu/requirements_ubuntu22.txt new file mode 100644 index 0000000..d648f44 --- /dev/null +++ b/actinia-ubuntu/requirements_ubuntu22.txt @@ -0,0 +1,44 @@ +boto3>=1.6.6 +colorlog>=4.2.1 +docutils>=0.14 +Flask>=0.12.3 +Flask-HTTPAuth>=3.2.3 +Flask-RESTful>=0.3.6 +flask-restful-swagger-2 +Flask-Cors>=3.0.3 +fluent-logger>=0.9.4 +geopandas>=0.8.1 +google-cloud +google-cloud-bigquery +google-cloud-storage +gunicorn>=19.9.0 +matplotlib +numpy>=1.15.4 +pandas +passlib>=1.7.1 +ply>=3.11 +psutil>=5.7.0 +pyproj>=2.6.1 +python-json-logger +python-magic>=0.4.15 +psycopg2>=2.9.7 +scikit-learn +Shapely +six>=1.13.0 +Sphinx>=1.7.1 +redis>=2.10.6 +requests>=2.20.0 +rq>=0.10.0 +pystac>=0.5.6 +rasterio>=1.2.10 +## omitting very large packages +torch +torchvision +uWSGI>=2.0.17 +wheel + +#actinia-api@https://github.com/actinia-org/actinia-api/releases/download/3.4.0/actinia_api-3.4.0-py3-none-any.whl + +# currently break things - TODO - needed? +# pkg-resources==0.0.0 +# pycurl==7.43.0.1 From 237b0ef82e16dfdcef87795b20c59bb38aa23ec8 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Thu, 21 Sep 2023 13:17:30 +0000 Subject: [PATCH 02/14] update installation script for ESA SNAP --- actinia-ubuntu/snap_install.sh | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 actinia-ubuntu/snap_install.sh diff --git a/actinia-ubuntu/snap_install.sh b/actinia-ubuntu/snap_install.sh new file mode 100644 index 0000000..c996656 --- /dev/null +++ b/actinia-ubuntu/snap_install.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +# https://senbox.atlassian.net/wiki/spaces/SNAP/pages/30539778/Install+SNAP+on+the+command+line +# https://senbox.atlassian.net/wiki/spaces/SNAP/pages/30539785/Update+SNAP+from+the+command+line +# http://step.esa.int/main/download/snap-download/ + +SNAPVER=9 +# avoid NullPointer crash during S-1 processing +java_max_mem=80G + +# install module 'jpy' (A bi-directional Python-Java bridge) +git clone https://github.com/jpy-consortium/jpy.git /src/snap/jpy +python3 -m ensurepip +pip3 install --upgrade pip wheel +(cd /src/snap/jpy && python3 setup.py build maven bdist_wheel) +echo "done with jpy" +# hack because ./snappy-conf will create this dir but also needs *.whl files... +mkdir -p /root/.snap/snap-python/snappy +cp /src/snap/jpy/dist/*.whl "/root/.snap/snap-python/snappy" + + +# install and update snap +wget -q -O /src/snap/esa-snap_all_unix_${SNAPVER}_0_0.sh \ + "http://step.esa.int/downloads/${SNAPVER}.0/installers/esa-snap_all_unix_${SNAPVER}_0_0.sh" + +# # hack to make it run on alpine +sh /src/snap/esa-snap_all_unix_${SNAPVER}_0_0.sh -q -varfile /src/snap_response.varfile + +# one more hack to keep using system java +sed -i 's+jdkhome="./jre"+jdkhome="$JAVA_HOME"+g' /usr/local/snap/etc/snap.conf +# freezing, when no updates available. Now there are updates, so reactivating. +snap --nosplash --nogui --modules --update-all 2>&1 | while read -r line; do + echo "$line" + [ "$line" = "updates=0" ] && + (ps -u + # ps -u | grep snap && + #ls /usr/local/snap/ + ls /usr/local/snap/jre + pgrep -f -a "/usr/local/snap/bin" + echo "" + pgrep -f -a "snap") + [ "$line" = "updates=0" ] && sleep 2 && pkill -f "/usr/local/snap/bin" +done +# /usr/local/snap/bin/snap --nosplash --nogui --modules --update-all + +rm -rf /usr/local/snap/jre + +# Maybe remove more unnecessary files from /root/.snap which is later copied to the final image + +# create snappy and python binding with snappy +/usr/local/snap/bin/snappy-conf /usr/bin/python3 +(cd /root/.snap/snap-python/snappy && python3 setup.py install) + +# increase the JAVA VM size to avoid NullPointer exception in Snappy during S-1 processing +(cd /root/.snap/snap-python/snappy && sed -i "s/^java_max_mem:.*/java_max_mem: $java_max_mem/" snappy.ini) + +# test +#/usr/bin/python3 -c 'from snappy import ProductIO' +#if [ -f /src/snap/about.py ] +#then +# /usr/bin/python3 /src/snap/about.py +# cp /src/snap/about.py /root/.snap/ +#fi + + +# cleanup installer +rm -f /src/snap/esa-snap_all_unix_${SNAPVER}_0.sh + +################################################################################ +# keep for debugging +# export INSTALL4J_KEEP_TEMP=yes From 789fdc713e15f899f988a03c2d85995c27b06726 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Thu, 21 Sep 2023 13:17:58 +0000 Subject: [PATCH 03/14] choose ESA SNAP components --- actinia-ubuntu/snap_response.varfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 actinia-ubuntu/snap_response.varfile diff --git a/actinia-ubuntu/snap_response.varfile b/actinia-ubuntu/snap_response.varfile new file mode 100644 index 0000000..f3c37a5 --- /dev/null +++ b/actinia-ubuntu/snap_response.varfile @@ -0,0 +1,14 @@ +# install4j response file for ESA SNAP 9.0 +deleteSnapDir=DESKTOP +executeLauncherWithPythonAction$Boolean=true +forcePython$Boolean=true +pythonExecutable=/usr/bin/python3 +sys.adminRights$Boolean=true +sys.component.RSTB$Boolean=true +sys.component.S1TBX$Boolean=true +sys.component.S2TBX$Boolean=true +sys.component.S3TBX$Boolean=true +sys.component.SNAP$Boolean=true +sys.installationDir=/usr/local/snap +sys.languageId=en +sys.programGroupDisabled$Boolean=true From b55e1a1056741fc653119d5ca72692929307c870 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 25 Sep 2023 10:28:55 +0000 Subject: [PATCH 04/14] deactivate installed or too large packages --- actinia-ubuntu/requirements_ubuntu22.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/actinia-ubuntu/requirements_ubuntu22.txt b/actinia-ubuntu/requirements_ubuntu22.txt index d648f44..006993e 100644 --- a/actinia-ubuntu/requirements_ubuntu22.txt +++ b/actinia-ubuntu/requirements_ubuntu22.txt @@ -12,16 +12,16 @@ google-cloud google-cloud-bigquery google-cloud-storage gunicorn>=19.9.0 -matplotlib -numpy>=1.15.4 +# matplotlib +# numpy>=1.15.4 pandas passlib>=1.7.1 -ply>=3.11 +#ply>=3.11 psutil>=5.7.0 pyproj>=2.6.1 python-json-logger -python-magic>=0.4.15 -psycopg2>=2.9.7 +# python-magic>=0.4.15 +# psycopg2>=2.9.7 scikit-learn Shapely six>=1.13.0 @@ -32,10 +32,10 @@ rq>=0.10.0 pystac>=0.5.6 rasterio>=1.2.10 ## omitting very large packages -torch -torchvision +# torch +# torchvision uWSGI>=2.0.17 -wheel +#wheel #actinia-api@https://github.com/actinia-org/actinia-api/releases/download/3.4.0/actinia_api-3.4.0-py3-none-any.whl From abccc8211a85d6f2ffa7daf521dd193b50459617 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 25 Sep 2023 10:30:52 +0000 Subject: [PATCH 05/14] cleanup --- actinia-ubuntu/Dockerfile | 157 ++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 66 deletions(-) diff --git a/actinia-ubuntu/Dockerfile b/actinia-ubuntu/Dockerfile index ac0b93a..f8cc501 100644 --- a/actinia-ubuntu/Dockerfile +++ b/actinia-ubuntu/Dockerfile @@ -1,48 +1,41 @@ -FROM osgeo/grass-gis:stable-ubuntu +FROM FROM osgeo/grass-gis:stable-ubuntu as grass # based on GRASS GIS releasebranch_8_2 (grass) # for most recent GRASS GIS, change base image to # FROM osgeo/grass-gis:latest-ubuntu -LABEL authors="Carmen Tawalika,Anika Weinmann,Markus Neteler,Sören Gebbert" +LABEL authors="Carmen Tawalika,Anika Weinmann,Markus Neteler,Sören Gebbert,Stefan Blumentrath" LABEL maintainer="tawalika@mundialis.de,weinmann@mundialis.de,neteler@mundialis.de,soerengebbert@gmail.com" +ENV LC_ALL "en_US.UTF-8" ENV GDAL_CACHEMAX=2000 ENV GRASS_COMPRESSOR=ZSTD +#ENV GRASS_SKIP_MAPSET_OWNER_CHECK 1 +#ENV GISBASE "" +ENV ACTINIA_CORE_VERSION=4.10.0 USER root # Install dependencies and tools RUN apt-get update && apt-get upgrade -y && \ apt-get install -y --no-install-recommends --no-install-suggests \ - # gnutls-bin \ - # libgnutls28-dev \ - # ruby \ build-essential \ - bzip2 \ curl \ git \ language-pack-en-base \ libcurl4-gnutls-dev \ - libpython3-all-dev \ moreutils \ - python3 \ - python3-dateutil \ - python3-dev \ - python3-gdal \ - python3-magic \ - python3-pip \ - python3-ply \ + openjdk-11-jre \ redis-server \ redis-tools \ rsync \ - subversion \ unzip \ vim \ wget \ zip \ && apt-get autoremove -y \ - && apt-get clean -y + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* ENV LC_ALL "en_US.UTF-8" ENV GRASS_SKIP_MAPSET_OWNER_CHECK 1 @@ -56,68 +49,100 @@ RUN mkdir -p /actinia_core/grassdb && \ mkdir -p /actinia_core/workspace/download_cache && \ mkdir -p /actinia_core/userdata -# Create generic GRASS GIS binary name regardless of version number -RUN ln -sf `find /usr/local/bin -name "grass??" | sort | tail -n 1` /usr/local/bin/grass -# Create a generic link to GRASS GIS libs (simplifies actinia configuration) -RUN ln -s `grass --config path` /usr/local/grass -# show installed versions -RUN grass --tmp-location EPSG:4326 --exec g.version -rge && \ - pdal --version && \ - python3 --version - # Install selected GRASS GIS addons. Done with -s, so it can be used by actinia # https://grass.osgeo.org/grass-stable/manuals/g.extension.html # -s uses $GISBASE instead of $GRASS_ADDON_BASE -WORKDIR /src -# git clone various openEO plugins (local checkout due to non-standard repo structure) -RUN git clone https://github.com/mundialis/openeo-addons.git +FROM grass as build +WORKDIR /src # loop over addon list COPY grass_addons_list.csv /src/grass_addons_list.csv -RUN while IFS=, read -r ADDON SERVER; do unset URL; test -z $SERVER || URL="url=$SERVER"; grass --tmp-location EPSG:4326 --exec g.extension -s extension=$ADDON $URL;done < /src/grass_addons_list.csv -# install SNAPPY +# git clone various openEO plugins (local checkout due to non-standard repo structure) +RUN git clone https://github.com/mundialis/openeo-addons.git \ + && mkdir /src/grass_addons \ + # python3 -m pip install --no-cache-dir GDAL==$(gdal-config --version) && \ + && while IFS=, read -r ADDON SERVER; do unset URL; test -z $SERVER || URL="url=$SERVER"; grass --tmp-location EPSG:4326 --exec g.extension prefix=/src/grass_addons extension=$ADDON $URL;done < /src/grass_addons_list.csv && \ + rm -r openeo-addons + +# Install SNAP and SNAPPY # SNAP wants the current folder '.' included in LD_LIBRARY_PATH ENV LD_LIBRARY_PATH ".:$LD_LIBRARY_PATH" -RUN apt-get install default-jdk maven -y +RUN apt-get update \ + && apt-get install -y --no-install-recommends --no-install-suggests \ + default-jdk \ + maven \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + ENV JAVA_HOME "/usr/lib/jvm/java-11-openjdk-amd64" -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 -COPY actinia-ubuntu/snap /src/snap -RUN bash /src/snap/install.sh -RUN update-alternatives --remove python /usr/bin/python3 +#RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 +COPY actinia-ubuntu/snap_* /src/ +RUN bash /src/snap_install.sh +#RUN update-alternatives --remove python /usr/bin/python3 # set link to match actinia default config (needed for tests) RUN ln -s /actinia_core /root/actinia -# Install actinia-core -RUN git clone https://github.com/actinia-org/actinia-core.git /src/actinia_core -WORKDIR /src/actinia_core -RUN python3 -m pip install --user --upgrade setuptools wheel -RUN python3 setup.py sdist bdist_wheel -d /build -RUN pip3 install /build/* - -## install (Cython must be installed before requirements.txt) -#RUN pip3 install Cython cython-setuptools setuptools_cython -RUN pip3 install -r requirements_ubuntu19.txt && python3 setup.py install -## TODO: fix tests -#\ -# && python3 setup.py test - -# Install actinia-core plugins -WORKDIR /src -RUN git config --global http.sslVerify false -RUN git clone https://github.com/actinia-org/actinia-statistic-plugin.git /src/actinia_statistic_plugin -WORKDIR /src/actinia_statistic_plugin -RUN pip3 install -r requirements.txt && python3 setup.py install -## TODO: fix tests -#\ -# && python3 setup.py test - -# Reduce the image size -RUN apt-get autoremove -y -RUN apt-get clean -y -RUN rm -rf /src/snap +COPY actinia-ubuntu/requirements_ubuntu22.txt requirements_ubuntu22.txt + +RUN mkdir /src/site-packages \ + && mkdir /src/pip_cache \ + && wget https://raw.githubusercontent.com/actinia-org/actinia_core/${ACTINIA_CORE_VERSION}/requirements.txt \ + && sed -i 's/==/>=/' requirements.txt \ + && python3 -m pip install \ + --cache-dir /src/pip_cache \ + -r requirements.txt \ + && python3 -m pip install \ + --cache-dir /src/pip_cache \ + -r requirements.txt \ + -r requirements_ubuntu22.txt + + +RUN mkdir /src/build && \ + PLUGIN_VERSION=0.2.1; WHEEL_NAME=actinia_statistic_plugin-${PLUGIN_VERSION}-py2.py3-none-any.whl; \ + curl -L --output /src/build/${WHEEL_NAME} \ + https://github.com/actinia-org/actinia_statistic_plugin/releases/download/${PLUGIN_VERSION}/${WHEEL_NAME} && \ + PLUGIN_VERSION=0.1.0; WHEEL_NAME=actinia_satellite_plugin-${PLUGIN_VERSION}-py2.py3-none-any.whl; \ + curl -L --output /src/build/${WHEEL_NAME} \ + https://github.com/actinia-org/actinia_satellite_plugin/releases/download/${PLUGIN_VERSION}/${WHEEL_NAME} && \ + PLUGIN_VERSION=1.0.2; WHEEL_NAME=actinia_metadata_plugin.wsgi-${PLUGIN_VERSION}-py2.py3-none-any.whl; \ + curl -L --output /src/build/${WHEEL_NAME} \ + https://github.com/actinia-org/actinia-metadata-plugin/releases/download/${PLUGIN_VERSION}/${WHEEL_NAME} && \ + PLUGIN_VERSION=2.5.0; WHEEL_NAME=actinia_module_plugin.wsgi-${PLUGIN_VERSION}-py2.py3-none-any.whl; \ + curl -L --output /src/build/${WHEEL_NAME} \ + https://github.com/actinia-org/actinia-module-plugin/releases/download/${PLUGIN_VERSION}/${WHEEL_NAME} && \ + PLUGIN_VERSION=0.1.1; WHEEL_NAME=actinia_stac_plugin.wsgi-${PLUGIN_VERSION}-py2.py3-none-any.whl; \ + curl -L --output /src/build/${WHEEL_NAME} \ + https://github.com/actinia-org/actinia-stac-plugin/releases/download/${PLUGIN_VERSION}/${WHEEL_NAME} && \ + PLUGIN_VERSION=1.0.0; WHEEL_NAME=actinia_tiling_plugin.wsgi-${PLUGIN_VERSION}-py2.py3-none-any.whl; \ + curl -L --output /src/build/${WHEEL_NAME} \ + https://github.com/actinia-org/actinia-tiling-plugin/releases/download/${PLUGIN_VERSION}/${WHEEL_NAME} && \ + ACTINIA_API_VERSION=3.4.0; WHEEL_NAME=actinia_api-${ACTINIA_API_VERSION}-py3-none-any.whl; \ + curl -L --output /src/build/${WHEEL_NAME} \ + https://github.com/actinia-org/actinia-api/releases/download/${ACTINIA_API_VERSION}/${WHEEL_NAME} && \ + WHEEL_NAME=actinia_core-${ACTINIA_CORE_VERSION}-py3-none-any.whl; \ + curl -L --output /src/build/${WHEEL_NAME} \ + https://github.com/actinia-org/actinia-core/releases/download/${ACTINIA_CORE_VERSION}/${WHEEL_NAME} && \ + for i in /src/build/*; do python3 -m pip install \ + --cache-dir /src/pip_cache \ + # Dependencies should have been included in the requirements + --no-deps \ + $i; done && \ + rm -r /src/build + +RUN python3 -m pip install \ + --cache-dir /src/pip_cache \ + Cython cython-setuptools setuptools_cython + +FROM grass as actinia + +COPY --from=build /usr/local/snap /usr/local/snap +COPY --from=build /src/grass_addons /usr/local/grass +COPY --from=build /root/.snap /root/.snap +COPY --from=build /usr/local/lib/python3.10/ /usr/local/lib/python3.10/ # Data directory WORKDIR /grassdb @@ -127,13 +152,13 @@ VOLUME /grassdb COPY actinia-ubuntu/actinia.cfg /etc/default/actinia COPY actinia-ubuntu/start.sh /src/start.sh +ENV PATH=${PATH}:/usr/local/snap/bin + # Set actinia config file for testing ENV ACTINIA_CUSTOM_TEST_CFG /etc/default/actinia -ENTRYPOINT ["/bin/bash"] +# ENTRYPOINT ["/bin/bash"] CMD ["/src/start.sh"] -ENV GISBASE "" - EXPOSE 8088 EXPOSE 9191 From e4a51a42f45eed541bb6e39f4f0a2930c2736c4a Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 25 Sep 2023 10:31:31 +0000 Subject: [PATCH 06/14] fix syntax --- actinia-ubuntu/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actinia-ubuntu/Dockerfile b/actinia-ubuntu/Dockerfile index f8cc501..646fd28 100644 --- a/actinia-ubuntu/Dockerfile +++ b/actinia-ubuntu/Dockerfile @@ -1,4 +1,4 @@ -FROM FROM osgeo/grass-gis:stable-ubuntu as grass +FROM osgeo/grass-gis:stable-ubuntu as grass # based on GRASS GIS releasebranch_8_2 (grass) # for most recent GRASS GIS, change base image to From 1acea593ac0e53d087a3d60d9dee28d0cb5109b6 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Tue, 26 Sep 2023 08:37:11 +0000 Subject: [PATCH 07/14] add libmagic --- actinia-ubuntu/Dockerfile | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/actinia-ubuntu/Dockerfile b/actinia-ubuntu/Dockerfile index 646fd28..7e6b00e 100644 --- a/actinia-ubuntu/Dockerfile +++ b/actinia-ubuntu/Dockerfile @@ -24,6 +24,8 @@ RUN apt-get update && apt-get upgrade -y && \ git \ language-pack-en-base \ libcurl4-gnutls-dev \ + libmagic1 \ + libmagic-mgc \ moreutils \ openjdk-11-jre \ redis-server \ @@ -95,11 +97,19 @@ RUN mkdir /src/site-packages \ && python3 -m pip install \ --cache-dir /src/pip_cache \ -r requirements.txt \ - && python3 -m pip install \ + && python3 -m pip install \ --cache-dir /src/pip_cache \ -r requirements.txt \ - -r requirements_ubuntu22.txt + -r requirements_ubuntu22.txt \ + && rm -rf /tmp/pip-* + +FROM grass as actinia +COPY --from=build /usr/local/snap /usr/local/snap +COPY --from=build /src/grass_addons /usr/local/grass +COPY --from=build /root/.snap /root/.snap +COPY --from=build /usr/local/lib/python3.10/ /usr/local/lib/python3.10/ +COPY --from=build /usr/local/bin/ /usr/local/bin RUN mkdir /src/build && \ PLUGIN_VERSION=0.2.1; WHEEL_NAME=actinia_statistic_plugin-${PLUGIN_VERSION}-py2.py3-none-any.whl; \ @@ -137,13 +147,6 @@ RUN python3 -m pip install \ --cache-dir /src/pip_cache \ Cython cython-setuptools setuptools_cython -FROM grass as actinia - -COPY --from=build /usr/local/snap /usr/local/snap -COPY --from=build /src/grass_addons /usr/local/grass -COPY --from=build /root/.snap /root/.snap -COPY --from=build /usr/local/lib/python3.10/ /usr/local/lib/python3.10/ - # Data directory WORKDIR /grassdb VOLUME /grassdb From edd6001a021b662f6b2fe4b6b3dcc3110520d5ac Mon Sep 17 00:00:00 2001 From: ninsbl Date: Tue, 26 Sep 2023 08:37:48 +0000 Subject: [PATCH 08/14] add missing dependencies --- actinia-ubuntu/requirements_ubuntu22.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/actinia-ubuntu/requirements_ubuntu22.txt b/actinia-ubuntu/requirements_ubuntu22.txt index 006993e..e0edac8 100644 --- a/actinia-ubuntu/requirements_ubuntu22.txt +++ b/actinia-ubuntu/requirements_ubuntu22.txt @@ -23,7 +23,7 @@ python-json-logger # python-magic>=0.4.15 # psycopg2>=2.9.7 scikit-learn -Shapely +Shapely<2.0 six>=1.13.0 Sphinx>=1.7.1 redis>=2.10.6 @@ -31,6 +31,8 @@ requests>=2.20.0 rq>=0.10.0 pystac>=0.5.6 rasterio>=1.2.10 +jsonmodels +stac_validator ## omitting very large packages # torch # torchvision From f69d47c31f898da9ac71919f2abc21f8d3f46190 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 27 Sep 2023 11:50:05 +0000 Subject: [PATCH 09/14] add s2/3 toolbox --- actinia-ubuntu/snap/response.varfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/actinia-ubuntu/snap/response.varfile b/actinia-ubuntu/snap/response.varfile index 43388b1..f3c37a5 100644 --- a/actinia-ubuntu/snap/response.varfile +++ b/actinia-ubuntu/snap/response.varfile @@ -1,4 +1,4 @@ -# install4j response file for ESA SNAP 7.0 +# install4j response file for ESA SNAP 9.0 deleteSnapDir=DESKTOP executeLauncherWithPythonAction$Boolean=true forcePython$Boolean=true @@ -6,8 +6,8 @@ pythonExecutable=/usr/bin/python3 sys.adminRights$Boolean=true sys.component.RSTB$Boolean=true sys.component.S1TBX$Boolean=true -sys.component.S2TBX$Boolean=false -sys.component.S3TBX$Boolean=false +sys.component.S2TBX$Boolean=true +sys.component.S3TBX$Boolean=true sys.component.SNAP$Boolean=true sys.installationDir=/usr/local/snap sys.languageId=en From 562445ac6483d172ae32bfc801b059a872cd9f6e Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 27 Sep 2023 11:52:21 +0000 Subject: [PATCH 10/14] remove --- actinia-ubuntu/snap_install.sh | 71 ---------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 actinia-ubuntu/snap_install.sh diff --git a/actinia-ubuntu/snap_install.sh b/actinia-ubuntu/snap_install.sh deleted file mode 100644 index c996656..0000000 --- a/actinia-ubuntu/snap_install.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh - -# https://senbox.atlassian.net/wiki/spaces/SNAP/pages/30539778/Install+SNAP+on+the+command+line -# https://senbox.atlassian.net/wiki/spaces/SNAP/pages/30539785/Update+SNAP+from+the+command+line -# http://step.esa.int/main/download/snap-download/ - -SNAPVER=9 -# avoid NullPointer crash during S-1 processing -java_max_mem=80G - -# install module 'jpy' (A bi-directional Python-Java bridge) -git clone https://github.com/jpy-consortium/jpy.git /src/snap/jpy -python3 -m ensurepip -pip3 install --upgrade pip wheel -(cd /src/snap/jpy && python3 setup.py build maven bdist_wheel) -echo "done with jpy" -# hack because ./snappy-conf will create this dir but also needs *.whl files... -mkdir -p /root/.snap/snap-python/snappy -cp /src/snap/jpy/dist/*.whl "/root/.snap/snap-python/snappy" - - -# install and update snap -wget -q -O /src/snap/esa-snap_all_unix_${SNAPVER}_0_0.sh \ - "http://step.esa.int/downloads/${SNAPVER}.0/installers/esa-snap_all_unix_${SNAPVER}_0_0.sh" - -# # hack to make it run on alpine -sh /src/snap/esa-snap_all_unix_${SNAPVER}_0_0.sh -q -varfile /src/snap_response.varfile - -# one more hack to keep using system java -sed -i 's+jdkhome="./jre"+jdkhome="$JAVA_HOME"+g' /usr/local/snap/etc/snap.conf -# freezing, when no updates available. Now there are updates, so reactivating. -snap --nosplash --nogui --modules --update-all 2>&1 | while read -r line; do - echo "$line" - [ "$line" = "updates=0" ] && - (ps -u - # ps -u | grep snap && - #ls /usr/local/snap/ - ls /usr/local/snap/jre - pgrep -f -a "/usr/local/snap/bin" - echo "" - pgrep -f -a "snap") - [ "$line" = "updates=0" ] && sleep 2 && pkill -f "/usr/local/snap/bin" -done -# /usr/local/snap/bin/snap --nosplash --nogui --modules --update-all - -rm -rf /usr/local/snap/jre - -# Maybe remove more unnecessary files from /root/.snap which is later copied to the final image - -# create snappy and python binding with snappy -/usr/local/snap/bin/snappy-conf /usr/bin/python3 -(cd /root/.snap/snap-python/snappy && python3 setup.py install) - -# increase the JAVA VM size to avoid NullPointer exception in Snappy during S-1 processing -(cd /root/.snap/snap-python/snappy && sed -i "s/^java_max_mem:.*/java_max_mem: $java_max_mem/" snappy.ini) - -# test -#/usr/bin/python3 -c 'from snappy import ProductIO' -#if [ -f /src/snap/about.py ] -#then -# /usr/bin/python3 /src/snap/about.py -# cp /src/snap/about.py /root/.snap/ -#fi - - -# cleanup installer -rm -f /src/snap/esa-snap_all_unix_${SNAPVER}_0.sh - -################################################################################ -# keep for debugging -# export INSTALL4J_KEEP_TEMP=yes From d6c7b5aab81dee3e575bd44a29a3af32624c3b24 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 27 Sep 2023 11:52:31 +0000 Subject: [PATCH 11/14] remove --- actinia-ubuntu/snap_response.varfile | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 actinia-ubuntu/snap_response.varfile diff --git a/actinia-ubuntu/snap_response.varfile b/actinia-ubuntu/snap_response.varfile deleted file mode 100644 index f3c37a5..0000000 --- a/actinia-ubuntu/snap_response.varfile +++ /dev/null @@ -1,14 +0,0 @@ -# install4j response file for ESA SNAP 9.0 -deleteSnapDir=DESKTOP -executeLauncherWithPythonAction$Boolean=true -forcePython$Boolean=true -pythonExecutable=/usr/bin/python3 -sys.adminRights$Boolean=true -sys.component.RSTB$Boolean=true -sys.component.S1TBX$Boolean=true -sys.component.S2TBX$Boolean=true -sys.component.S3TBX$Boolean=true -sys.component.SNAP$Boolean=true -sys.installationDir=/usr/local/snap -sys.languageId=en -sys.programGroupDisabled$Boolean=true From 976e922177bcb2d07fc44b52c9fd4b206f8084f1 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 27 Sep 2023 12:39:02 +0000 Subject: [PATCH 12/14] SNAP 9 without snappy --- actinia-ubuntu/snap/install.sh | 75 +++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/actinia-ubuntu/snap/install.sh b/actinia-ubuntu/snap/install.sh index 909ed06..9064947 100644 --- a/actinia-ubuntu/snap/install.sh +++ b/actinia-ubuntu/snap/install.sh @@ -1,43 +1,70 @@ -#!/bin/bash +#!/bin/sh # https://senbox.atlassian.net/wiki/spaces/SNAP/pages/30539778/Install+SNAP+on+the+command+line # https://senbox.atlassian.net/wiki/spaces/SNAP/pages/30539785/Update+SNAP+from+the+command+line - # http://step.esa.int/main/download/snap-download/ -SNAPVER=7 -# avoid NullPointer exception during S-1 processing -java_max_mem=10G -# set JAVA_HOME (done in Docker as well) -export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 +SNAPVER=9 +# avoid NullPointer crash during S-1 processing +java_max_mem=80G # install module 'jpy' (A bi-directional Python-Java bridge) -git clone https://github.com/bcdev/jpy.git /src/snap/jpy -pip3 install wheel -(cd /src/snap/jpy && python3 setup.py bdist_wheel) +#git clone https://github.com/jpy-consortium/jpy.git /src/snap/jpy +#python3 -m pip install wheel +#git clone https://github.com/bcdev/jpy.git /src/snap/jpy +#(cd /src/snap/jpy && python3 setup.py bdist_wheel) +#python3 -m pip install /src/snap/jpy/dist/*.whl + +#/usr/bin/python3 -c 'import jpy' +#echo "done with jpy" # hack because ./snappy-conf will create this dir but also needs *.whl files... -mkdir -p /root/.snap/snap-python/snappy -cp /src/snap/jpy/dist/*.whl "/root/.snap/snap-python/snappy" +#mkdir -p /root/.snap/snap-python +#cp /src/snap/jpy/dist/*.whl "/root/.snap/snap-python/" # install and update snap -wget -q -O /src/snap/esa-snap_all_unix_${SNAPVER}_0.sh \ - "http://step.esa.int/downloads/${SNAPVER}.0/installers/esa-snap_all_unix_${SNAPVER}_0.sh" -sh /src/snap/esa-snap_all_unix_${SNAPVER}_0.sh -q -varfile /src/snap/response.varfile -/usr/local/snap/bin/snap --nosplash --nogui --modules --update-all +wget -q -O /src/snap/esa-snap_all_unix_${SNAPVER}_0_0.sh \ + "http://step.esa.int/downloads/${SNAPVER}.0/installers/esa-snap_all_unix_${SNAPVER}_0_0.sh" + +# # hack to make it run on alpine +sh /src/snap/esa-snap_all_unix_${SNAPVER}_0_0.sh -q -varfile /src/snap/response.varfile + +# one more hack to keep using system java +sed -i 's+jdkhome="./jre"+jdkhome="$JAVA_HOME"+g' /usr/local/snap/etc/snap.conf +# freezing, when no updates available. Now there are updates, so reactivating. +snap --nosplash --nogui --modules --update-all 2>&1 | while read -r line; do + echo "$line" + [ "$line" = "updates=0" ] && + (ps -u + pgrep -f -a "/usr/local/snap/bin" + echo "" + pgrep -f -a "snap") + [ "$line" = "updates=0" ] && sleep 2 && pkill -f "/usr/local/snap/bin" +done +# /usr/local/snap/bin/snap --nosplash --nogui --modules --update-all + +rm -rf /usr/local/snap/jre + +# Maybe remove more unnecessary files from /root/.snap which is later copied to the final image # create snappy and python binding with snappy -/usr/local/snap/bin/snappy-conf /usr/bin/python3 -(cd /root/.snap/snap-python/snappy && python3 setup.py install) +# cd /usr/local/snap/bin/snappy-conf /usr/bin/python3 /root/.snap/snap-python/snappy +# (cd /root/.snap/snap-python/snappy && python3 setup.py install) # increase the JAVA VM size to avoid NullPointer exception in Snappy during S-1 processing -(cd /root/.snap/snap-python/snappy && sed -i "s/^java_max_mem:.*/java_max_mem: $java_max_mem/" snappy.ini) - -# get minor python version -PYMINOR=$(python3 -c 'import platform; major, minor, patch = platform.python_version_tuple(); print(minor)') -(cd /usr/local/lib/python3.$PYMINOR/dist-packages/snappy/ && sed -i "s/^java_max_mem:.*/java_max_mem: $java_max_mem/" snappy.ini) +# (cd /root/.snap/snap-python/snappy && sed -i "s/^java_max_mem:.*/java_max_mem: $java_max_mem/" snappy.ini) # test -/usr/bin/python3 -c 'from snappy import ProductIO' +# /usr/bin/python3 -c 'from esa_snappy import ProductIO' +# if [ -f /src/snap/about.py ] +# then +# /usr/bin/python3 /src/snap/about.py +# cp /src/snap/about.py /root/.snap/ +# fi + # cleanup installer rm -f /src/snap/esa-snap_all_unix_${SNAPVER}_0.sh + +################################################################################ +# keep for debugging +# export INSTALL4J_KEEP_TEMP=yes From 104bd2ee8f56d3c0c65524158f01491339f1a0ff Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 27 Sep 2023 13:10:32 +0000 Subject: [PATCH 13/14] more stages, grass version as argument --- actinia-ubuntu/Dockerfile | 70 +++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/actinia-ubuntu/Dockerfile b/actinia-ubuntu/Dockerfile index 7e6b00e..9028653 100644 --- a/actinia-ubuntu/Dockerfile +++ b/actinia-ubuntu/Dockerfile @@ -1,8 +1,8 @@ -FROM osgeo/grass-gis:stable-ubuntu as grass +ARG GRASS_VERSION=current +# current = GRASS GIS docker image from latest release branch +# for development verions use "main" or "releasebranch_8_2" for a specific GRASS GIS version -# based on GRASS GIS releasebranch_8_2 (grass) -# for most recent GRASS GIS, change base image to -# FROM osgeo/grass-gis:latest-ubuntu +FROM osgeo/grass-gis:${GRASS_VERSION}-ubuntu as grass LABEL authors="Carmen Tawalika,Anika Weinmann,Markus Neteler,Sören Gebbert,Stefan Blumentrath" LABEL maintainer="tawalika@mundialis.de,weinmann@mundialis.de,neteler@mundialis.de,soerengebbert@gmail.com" @@ -12,7 +12,17 @@ ENV GDAL_CACHEMAX=2000 ENV GRASS_COMPRESSOR=ZSTD #ENV GRASS_SKIP_MAPSET_OWNER_CHECK 1 #ENV GISBASE "" + +ARG JAVA_VERSION=11 +ARG PYTHON_VERSION=3.10 + ENV ACTINIA_CORE_VERSION=4.10.0 +ENV PATH=${PATH}:/usr/local/snap/bin +ENV SNAP_HOME=/usr/local/snap +ENV JAVA_HOME "/usr/lib/jvm/java-${JAVA_VERSION}-openjdk-amd64" + +# SNAP wants the current folder '.' included in LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH ".:$LD_LIBRARY_PATH" USER root @@ -24,10 +34,8 @@ RUN apt-get update && apt-get upgrade -y && \ git \ language-pack-en-base \ libcurl4-gnutls-dev \ - libmagic1 \ - libmagic-mgc \ moreutils \ - openjdk-11-jre \ + openjdk-${JAVA_VERSION}-jre \ redis-server \ redis-tools \ rsync \ @@ -55,7 +63,7 @@ RUN mkdir -p /actinia_core/grassdb && \ # https://grass.osgeo.org/grass-stable/manuals/g.extension.html # -s uses $GISBASE instead of $GRASS_ADDON_BASE -FROM grass as build +FROM grass as addons WORKDIR /src # loop over addon list @@ -68,22 +76,28 @@ RUN git clone https://github.com/mundialis/openeo-addons.git \ && while IFS=, read -r ADDON SERVER; do unset URL; test -z $SERVER || URL="url=$SERVER"; grass --tmp-location EPSG:4326 --exec g.extension prefix=/src/grass_addons extension=$ADDON $URL;done < /src/grass_addons_list.csv && \ rm -r openeo-addons +FROM grass as snap + # Install SNAP and SNAPPY -# SNAP wants the current folder '.' included in LD_LIBRARY_PATH -ENV LD_LIBRARY_PATH ".:$LD_LIBRARY_PATH" RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ - default-jdk \ + openjdk-${JAVA_VERSION}-jdk \ maven \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* -ENV JAVA_HOME "/usr/lib/jvm/java-11-openjdk-amd64" #RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 -COPY actinia-ubuntu/snap_* /src/ -RUN bash /src/snap_install.sh +COPY actinia-ubuntu/snap /src/snap + +RUN bash /src/snap/install.sh #RUN update-alternatives --remove python /usr/bin/python3 +#RUN /usr/bin/python3 ./snappyutil.py --snap_home /usr/local/snap --java_module /usr/local/snap/snap/modules/org-esa-snap-snap-python.jar --force --log_file ./snappyutil.log --jvm_max_mem 88G --java_home /usr/lib/jvm/java-11-openjdk-amd64 --req_arch amd64 +#RUN cat /root/.snap/snap-python/snappy/snappyutil.log +#RUN /usr/local/snap/bin/snappy-conf /usr/bin/python3 || cat /root/.snap/snap-python/snappy/snappyutil.log +#RUN /usr/local/snap/bin/snap --nosplash --nogui --modules --update-all + +FROM grass as actinia_python # set link to match actinia default config (needed for tests) RUN ln -s /actinia_core /root/actinia @@ -94,23 +108,12 @@ RUN mkdir /src/site-packages \ && mkdir /src/pip_cache \ && wget https://raw.githubusercontent.com/actinia-org/actinia_core/${ACTINIA_CORE_VERSION}/requirements.txt \ && sed -i 's/==/>=/' requirements.txt \ - && python3 -m pip install \ - --cache-dir /src/pip_cache \ - -r requirements.txt \ && python3 -m pip install \ --cache-dir /src/pip_cache \ -r requirements.txt \ -r requirements_ubuntu22.txt \ && rm -rf /tmp/pip-* -FROM grass as actinia - -COPY --from=build /usr/local/snap /usr/local/snap -COPY --from=build /src/grass_addons /usr/local/grass -COPY --from=build /root/.snap /root/.snap -COPY --from=build /usr/local/lib/python3.10/ /usr/local/lib/python3.10/ -COPY --from=build /usr/local/bin/ /usr/local/bin - RUN mkdir /src/build && \ PLUGIN_VERSION=0.2.1; WHEEL_NAME=actinia_statistic_plugin-${PLUGIN_VERSION}-py2.py3-none-any.whl; \ curl -L --output /src/build/${WHEEL_NAME} \ @@ -143,8 +146,18 @@ RUN mkdir /src/build && \ $i; done && \ rm -r /src/build +FROM grass as actinia + +COPY --link --from=snap /usr/local/snap /usr/local/snap +COPY --link --from=snap /root/.snap /root/.snap + +COPY --link --from=addons /src/grass_addons /usr/local/grass + +COPY --link --from=actinia_python /usr/local/lib/python${PYTHON_VERSION}/ /usr/local/lib/python${PYTHON_VERSION}/ +COPY --link --from=actinia_python /usr/local/bin/ /usr/local/bin + RUN python3 -m pip install \ - --cache-dir /src/pip_cache \ + --no-cache-dir \ Cython cython-setuptools setuptools_cython # Data directory @@ -154,13 +167,12 @@ VOLUME /grassdb # Copy actinia config file and start scripts COPY actinia-ubuntu/actinia.cfg /etc/default/actinia COPY actinia-ubuntu/start.sh /src/start.sh - -ENV PATH=${PATH}:/usr/local/snap/bin +RUN chmod ugo+x /src/start.sh # Set actinia config file for testing ENV ACTINIA_CUSTOM_TEST_CFG /etc/default/actinia -# ENTRYPOINT ["/bin/bash"] +# ENTRYPOINT ["/bin/bash", "-c", "$@"] CMD ["/src/start.sh"] EXPOSE 8088 From 8e73210d291aea0afeff6d9398aa952c514f1683 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 27 Sep 2023 13:20:11 +0000 Subject: [PATCH 14/14] document build-arg --- actinia-ubuntu/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/actinia-ubuntu/README.md b/actinia-ubuntu/README.md index 60284d4..fa6c53b 100644 --- a/actinia-ubuntu/README.md +++ b/actinia-ubuntu/README.md @@ -6,7 +6,8 @@ https://hub.docker.com/r/mundialis/actinia ## Background info -This docker image is based on https://hub.docker.com/r/osgeo/grass-gis (tag: stable-ubuntu) which provides the most recent stable version of GRASS GIS (releasebranch) with Python3 and PDAL support. For most recent version of GRASS GIS, change base image to grass-gis:latest-ubuntu. +This docker image is based on https://hub.docker.com/r/osgeo/grass-gis (tag: current-ubuntu) which provides the most recent stable version of GRASS GIS (releasebranch) with Python3 and PDAL support. For most recent version of GRASS GIS, change base image to grass-gis:main-ubuntu. +This can be done using the build argument `--build-arg="GRASS_VERSION=main"` (see below) The Dockerfile contained in this folder is used to build via pipeline. If you want to build manually... @@ -27,6 +28,15 @@ $ docker build \ --tag actinia:latest-ubuntu . ``` +In order to build an image based on the GRASS GIS development version run: + +```bash +$ docker build \ + --build-arg="GRASS_VERSION=main" \ + --file actinia-ubuntu/Dockerfile \ + --tag actinia:latest-ubuntu . +``` + View the images available using `sudo docker images` and open a bash terminal with: ```bash