Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix ngen image for building t-route #474

Merged
merged 3 commits into from
Jan 24, 2024
Merged

Fix ngen image for building t-route #474

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9e8329f
Fix ngen image for building t-route.
robertbartel Jan 4, 2024
c758ba5
Fix apply build type for ngen code cmake builds.
robertbartel Jan 24, 2024
59ff764
Add gdal-devel as image dnf dep for ARM support.
robertbartel Jan 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 79 additions & 48 deletions docker/main/ngen/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ ARG ROCKY_BASE_REQUIRED="sudo openssh openssh-server bash git which"

ARG ROCKY_NGEN_DEPS_REQUIRED="sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran \
python3 python3-devel python3-pip \
bzip2 expat expat-devel flex bison udunits2 udunits2-devel zlib-devel sqlite-devel sqlite-libs"
bzip2 expat expat-devel flex bison udunits2 udunits2-devel zlib-devel sqlite-devel sqlite-libs gdal-devel"
# TODO: removed texinfo from list because it couldn't be found; make sure this doesn't lead to issues

ARG BOOST_VERSION=1.82.0
Expand All @@ -64,7 +64,7 @@ ARG HD5_VERSION=1.10.9
# Work around Fortran MPI issue
ARG FCFLAGS="-w -fallow-argument-mismatch -O2"
ARG FFLAGS="-w -fallow-argument-mismatch -O2"
ARG MPICH_CONFIGURE_OPTIONS=""
ARG MPICH_CONFIGURE_OPTIONS="--enable-shared"
ARG MPICH_MAKE_OPTIONS

ARG BUILD_PARALLEL_JOBS
Expand Down Expand Up @@ -275,6 +275,23 @@ RUN dnf update -y && dnf install -y ${ROCKY_NGEN_DEPS_REQUIRED} && dnf clean -y
&& if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install --no-cache-dir numpy; fi
USER ${USER}

################################################################################################################
################################################################################################################
##### Create intermediate Docker build stage for building t-route in Rocky Linux environment
FROM rocky-base as rocky_init_troute_repo
ARG TROUTE_REPO_URL
ARG TROUTE_BRANCH
ARG TROUTE_COMMIT
ARG WORKDIR

WORKDIR ${WORKDIR}

RUN cd ${WORKDIR} \
&& git clone --single-branch --branch $TROUTE_BRANCH $TROUTE_REPO_URL \
&& cd ./t-route \
&& if [ "x$TROUTE_COMMIT" != "x" ]; then git checkout $TROUTE_COMMIT; fi \
&& git submodule update --init

################################################################################################################
################################################################################################################
##### Create intermediate Docker build stage for Rocky-Linux-based "ngen-deps"
Expand Down Expand Up @@ -308,6 +325,8 @@ ENV NETCDF_C_VERSION=${NETCDF_C_VERSION}
ENV NETCDF_CXX_VERSION=${NETCDF_CXX_VERSION}
ENV NETCDF_FORTRAN_VERSION=${NETCDF_FORTRAN_VERSION}

ARG NGEN_WITH_ROUTING

COPY --from=download_mpich /tmp/mpich-${MPICH_VERSION}.tar.gz /tmp/ngen-deps/mpich-${MPICH_VERSION}.tar.gz
COPY --from=download_hd5 /tmp/hdf5-${HD5_VERSION}.tar.gz /tmp/ngen-deps/hdf5-${HD5_VERSION}.tar.gz
COPY --from=download_netcdf /tmp/netcdf-${NETCDF_C_VERSION}.tar.gz /tmp/ngen-deps/netcdf-${NETCDF_C_VERSION}.tar.gz
Expand Down Expand Up @@ -343,7 +362,8 @@ RUN echo "export HYDRA_HOST_FILE=${HYDRA_HOST_FILE}" >> /etc/profile \
&& mkdir netcdf \
&& tar -xzf netcdf-${NETCDF_C_VERSION}.tar.gz -C netcdf --strip 1 \
&& cd netcdf \
&& LIBS=curl && ./configure --prefix=/usr \
# Make sure the netcdf prefix and install dirs align with t-route env var NETCDFINC set a little further below \
&& LIBS=curl && ./configure --enable-shared --prefix=/usr \
&& make -j ${BUILD_PARALLEL_JOBS} && make install \
# TODO: if we run into any problem, might need to reactivate this \
#&& make check \
Expand Down Expand Up @@ -374,37 +394,43 @@ RUN echo "export HYDRA_HOST_FILE=${HYDRA_HOST_FILE}" >> /etc/profile \
&& chown -R ${USER}:${USER} ${BOOST_ROOT} \
&& rm -rf /tmp/ngen-deps

# Install required Python packages for troute build/run that needed one or more of the source dependencies
RUN pip install --no-cache-dir numpy pandas pyyaml bmipy Cython blosc2==${BLOSC2_VERSION:-2.0.0} netCDF4 wheel packaging \
&& env HDF5_DIR=/usr pip install --no-cache-dir --no-build-isolation tables

ENV PATH=${PATH}:/usr/lib64/mpich/bin
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib:/usr/local/lib64
# This next value is eventually needed for the t-route build ...
# ... make sure it stays in sync with configure step for netcdf above
ENV NETCDFINC=/usr/include
ENV CC=gcc

# Get this for the times we need to install t-route
COPY --from=rocky_init_troute_repo ${WORKDIR}/t-route/requirements.txt ${WORKDIR}/t-route-requirements.txt

# Install troute and required Python packages, with some deps needing some special attention
RUN if [ "${NGEN_WITH_ROUTING}" == "ON" ]; then \
# Get subset of requirements needed first (including version specifiers if present) because of netCDF4 handling \
cat ${WORKDIR}/t-route-requirements.txt | grep -i numpy >> first_reqs.txt ; \
cat ${WORKDIR}/t-route-requirements.txt | grep -i pandas | grep -v geo >> first_reqs.txt ; \
cat ${WORKDIR}/t-route-requirements.txt | grep -i pyyaml >> first_reqs.txt ; \
cat ${WORKDIR}/t-route-requirements.txt | grep -i cython >> first_reqs.txt ; \
cat ${WORKDIR}/t-route-requirements.txt | grep -i pyarrow >> first_reqs.txt ; \
pip3 install --no-cache-dir -r first_reqs.txt ; \
rm first_reqs.txt ; \
# Then install a few more things, including a particular blosc2 version and an adjusted build of tables \
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like fiona is the culprit. geopandas depends on fiona and fiona by default will install from pip with a pre-built libgdal, if it can. It seems like the issue is fiona does not have a distribution with pre-build libgdal for the particular target.

Fiona has several extension modules which link against libgdal. This complicates installation. Binary distributions (wheels) containing libgdal and its own dependencies are available from the Python Package Index and can be installed using pip.

source

Suggested change
# Then install a few more things, including a particular blosc2 version and an adjusted build of tables \
# Then install a few more things, including a particular blosc2 version and an adjusted build of tables \
dnf install -y gdal-devel ; \

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I fixed this differently, by including gdal-devel in the main step where the list of packages to be installed via dnf is defined. It'll be installed now in an ancestor build stage of the one the suggestion would have been in.

pip3 install --no-cache-dir bmipy blosc2==${BLOSC2_VERSION:-2.0.0} build wheel packaging pyproj fiona geopandas ; \
env HDF5_DIR=/usr pip3 install --no-cache-dir --no-build-isolation tables ; \
env HDF5_DIR=/usr pip3 install --no-cache-dir netCDF4==1.6.3 ; \
# Then install remaining t-route requirements
pip3 install --no-cache-dir -r ${WORKDIR}/t-route-requirements.txt ; \
fi \
&& rm ${WORKDIR}/t-route-requirements.txt

USER ${USER}

################################################################################################################
################################################################################################################
##### Create intermediate Docker build stage for building t-route in Rocky Linux environment
FROM rocky-base as rocky_init_troute_repo
ARG TROUTE_REPO_URL
ARG TROUTE_BRANCH
ARG TROUTE_COMMIT
ARG WORKDIR

WORKDIR ${WORKDIR}

RUN cd ${WORKDIR} \
&& git clone --single-branch --branch $TROUTE_BRANCH $TROUTE_REPO_URL \
&& cd ./t-route \
&& if [ "x$TROUTE_COMMIT" != "x" ]; then git checkout $TROUTE_COMMIT; fi \
&& git submodule update --init

################################################################################################################
################################################################################################################
##### Create intermediate Docker build stage for building framework in Rocky Linux environment
FROM rocky-base as rocky_init_repo
ARG REPO_URL
ARG NGEN_BUILD_CONFIG_TYPE
ARG BRANCH
ARG COMMIT
ARG WORKDIR
Expand All @@ -430,33 +456,39 @@ RUN cd ${WORKDIR} \

################################################################################################################
################################################################################################################
##### Create intermediate Docker build stage for building t-route in Rocky Linux environment
###### Create intermediate Docker build stage for building t-route in Rocky Linux environment
FROM rocky-ngen-deps as rocky_build_troute

ARG TROUTE_REPO_URL
ARG TROUTE_BRANCH
ARG TROUTE_COMMIT
ARG BUILD_PARALLEL_JOBS
ARG NGEN_WITH_ROUTING

COPY --chown=${USER} --from=rocky_init_troute_repo ${WORKDIR}/t-route ${WORKDIR}/t-route

# && python(){ /usr/bin/python3 \$@; } && export -f python \
RUN cd ${WORKDIR}/t-route \
&& mkdir wheels \
&& pip3 install -r ./requirements.txt \
&& pip3 install wheel deprecated dask pyarrow geopandas Cython==3.0.3 \
&& export FC=gfortran \
&& cd ${WORKDIR}/t-route \
&& ./compiler.sh \
&& cd ./src/troute-network \
&& python3 setup.py --use-cython bdist_wheel \
&& cp dist/*.whl ${WORKDIR}/t-route/wheels/ \
&& cd ../troute-routing \
&& python3 setup.py --use-cython bdist_wheel \
&& cp dist/*.whl ${WORKDIR}/t-route/wheels/ \
&& cd ../troute-nwm \
&& python3 setup.py bdist_wheel \
&& cp dist/*.whl ${WORKDIR}/t-route/wheels/
RUN if [ "${NGEN_WITH_ROUTING}" == "ON" ]; then \
export FC=gfortran ; \
export F90=gfortran ; \
mkdir ${WORKDIR}/t-route/wheels \
&& cd ${WORKDIR}/t-route \
&& ./compiler.sh \
&& cd ${WORKDIR}/t-route/src/troute-network \
&& python3 setup.py --use-cython bdist_wheel \
&& cp dist/*.whl ${WORKDIR}/t-route/wheels/ \
&& cd ${WORKDIR}/t-route/src/troute-routing \
&& python3 setup.py --use-cython bdist_wheel \
&& cp dist/*.whl ${WORKDIR}/t-route/wheels/ \
&& cd ${WORKDIR}/t-route/src/troute-config \
# troute-config doesn't use setup.py ... use build to make the wheel \
#&& python3 setup.py --use-cython bdist_wheel \
&& python3 -m build . \
&& cp dist/*.whl ${WORKDIR}/t-route/wheels/ \
&& cd ${WORKDIR}/t-route/src/troute-nwm \
&& python3 setup.py bdist_wheel \
&& cp dist/*.whl ${WORKDIR}/t-route/wheels/ ; \
fi

USER root
RUN rm /usr/bin/python
Expand Down Expand Up @@ -494,17 +526,14 @@ ARG BUILD_SLOTH

COPY --chown=${USER} --from=rocky_init_repo ${WORKDIR}/ngen ${WORKDIR}/ngen
COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/wheels /tmp/t-route-wheels
COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/src/kernel /tmp/t-route-kernels
COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/requirements.txt /tmp/t-route-requirements.txt
ENV BOOST_ROOT=${WORKDIR}/boost

USER root
RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then \
chgrp -R ${USER} /usr/local/lib*/python3.* ; \
chmod -R g+sw /usr/local/lib*/python3.* ; \
fi \
&& if [ "${NGEN_WITH_ROUTING}" == "ON" ]; then \
# These packages install command line tools, which try to go in /usr/local/bin \
pip3 install --no-cache-dir pyarrow pyproj fiona; \
fi
USER ${USER}

Expand All @@ -521,9 +550,7 @@ RUN cd ${WORKDIR}/ngen \
&& if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then \
pip3 install --no-cache-dir -r extern/test_bmi_py/requirements.txt; \
if [ "${NGEN_WITH_ROUTING}" == "ON" ] ; then \
pip3 install --no-cache-dir /tmp/t-route-wheels/*.whl; \
pip3 install --no-cache-dir -r /tmp/t-route-requirements.txt; \
pip3 install --no-cache-dir deprecated geopandas ; \
pip3 install --no-cache-dir /tmp/t-route-wheels/*.whl ; \
fi; \
fi \
&& if [ "${NGEN_WITH_BMI_FORTRAN}" == "ON" ]; then \
Expand All @@ -538,6 +565,7 @@ RUN cd ${WORKDIR}/ngen \
&& if [ "${BUILD_SLOTH}" == "true" ] ; then ./build_sub extern/sloth; fi \
&& if [ "${BUILD_NGEN_SERIAL}" == "true" ]; then \
cmake -B cmake_build_serial -S . \
-DCMAKE_BUILD_TYPE=${NGEN_BUILD_CONFIG_TYPE} \
-DNGEN_WITH_MPI:BOOL=OFF \
-DNGEN_WITH_NETCDF:BOOL=${NGEN_WITH_NETCDF} \
-DNGEN_WITH_BMI_C:BOOL=${NGEN_WITH_BMI_C} \
Expand All @@ -555,6 +583,7 @@ RUN cd ${WORKDIR}/ngen \
fi \
&& if [ "${BUILD_NGEN_PARALLEL}" == "true" ]; then \
cmake -B cmake_build_parallel -S . \
-DCMAKE_BUILD_TYPE=${NGEN_BUILD_CONFIG_TYPE} \
-DNGEN_WITH_MPI:BOOL=ON \
-DNGEN_WITH_NETCDF:BOOL=${NGEN_WITH_NETCDF} \
-DNGEN_WITH_BMI_C:BOOL=${NGEN_WITH_BMI_C} \
Expand Down Expand Up @@ -632,6 +661,7 @@ ARG NGEN_WITH_NETCDF
ARG NGEN_WITH_UDUNITS
ARG NGEN_UDUNITS_QUIET
ARG NGEN_WITH_SQLITE
ARG NGEN_BUILD_CONFIG_TYPE
ARG PARTITIONER_EXECUTABLE

COPY --chown=${USER} partitioner_entrypoint.sh ${WORKDIR}/entrypoint.sh
Expand All @@ -646,6 +676,7 @@ RUN cd ${WORKDIR}/ngen \
fi \
&& if [ -z "${PARTITIONER_EXECUTABLE:-}" ]; then echo "Error: target/executable name not set" 2>&1 ; exit 1; fi \
&& cmake -B cmake_build -S . \
-DCMAKE_BUILD_TYPE=${NGEN_BUILD_CONFIG_TYPE} \
-DNGEN_WITH_MPI:BOOL=OFF \
-DNGEN_WITH_NETCDF:BOOL=${NGEN_WITH_NETCDF} \
#-DNGEN_WITH_BMI_C:BOOL=${NGEN_WITH_BMI_C} \
Expand Down
Loading