Support for TChem in PartMC (gas phase) (#198)

* first working version * add spec file reading for tchem. pass value of config to tchem * update tchem_interface * minor clean up * fix bug in mapping * fix test specfiles * first step to cleaner CMakeList.txt. consider TChem install path variables * update set and get statevector tchem. add simple test * fix missing precompile for tchem * simplify CMakeList * accounting for water vapor * stub out aerosols to subroutine calls * resolve most of CMakeList.txt * improve tchem testcases. add chapman mechanism * fix aero_data issues. add aero and numerics yaml file name reading * clean up aerosols in testcase * improve testcase * add solver yaml passing. passing of time step * fix missing declarations * unignore tchem clone * add c++ library * add testing conditions to tchem * fix build of non tchem partmc * fix typo * update tchem tests * closer to building * clean up of tchem interface code * add more to cmakelist.txt * add dockerfile for tchem and add it to CI * fix missing line continuation in dockerfile * improve CMakeLists.txt * clean up tchem_interface * fix filename for tchem test * make cmake behave. -Wimplicit-interface issue * experiment with parallel testing * update scenario spec files * pass batch and nbatch to tchem * specify solving one system * change timing to use system_clock * comment out currently unused aero_data related code * fix logic for reading of gas_data * add comment in spec files for tchem. fix camp comment in tests * update dockerfile * add equation reference * add search for compiler dependent openmp flags * disable serial requirement for tchem tests * fix tchem dockerfile to use main * another try at tchem dockerfile * testing removal of ssh for submodule * remove serial requirement for tchem tests * move getting time to a util function * move tchem needed libraries out of compiler flags * do unit conversions in place * change dockerfile.tchem back to main branch * less aggressive cmakelist * change mole_dens subroutines to molar_conc * remove unneeded subroutine * Fix missing space Co-authored-by: Matthew West <[email protected]> * add function to convert rh to water vapor mixing ratio * change gas_state_set_camp_conc to use partmc env_state instead of camp env_state * test checking for interfaces in CMakelists.txt * remove cmake interface checking --------- Co-authored-by: Matthew West <[email protected]>
compdyn · Oct 31, 2024 · 2341ef4 · 2341ef4
1 parent ecc26d3
commit 2341ef4
Show file tree

Hide file tree

Showing 78 changed files with 7,610 additions and 60 deletions.
diff --git a/.github/workflows/tchem.yml b/.github/workflows/tchem.yml
@@ -0,0 +1,15 @@
+name: TChem
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Build the testing docker image
+      run: docker build -f Dockerfile.tchem -t partmc-test .
+    - name: Run the tests in the docker container
+      run: docker run -t partmc-test bash -c 'export CTEST_PARALLEL_LEVEL=4; cd /build; make test'
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,7 +3,17 @@ cmake_minimum_required(VERSION 2.6.4)
 # http://www.cmake.org/Bug/view.php?id=8598
 # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=35707
 # http://www.cmake.org/Bug/view.php?id=8408
-project(PARTMC Fortran C)
+if (ENABLE_TCHEM)
+  # Use the wrapper to nvcc for the CXX compiler for GPU
+  if (ENABLE_TCHEM_GPU)
+    set(ENV{CXX} $ENV{KOKKOS_PATH}/bin/nvcc_wrapper)
+    project(PARTMC Fortran C CXX)
+  else()
+    project(PARTMC Fortran C)
+  endif()
+else()
+  project(PARTMC Fortran C)
+endif()
 
 set(PACKAGE_BUGREPORT "[email protected]")
 set(PACKAGE_NAME "PartMC")
@@ -17,6 +27,7 @@ set(PACKAGE_VERSION "2.8.0")
 option(ENABLE_GSL "Enable GSL library for random number generation" OFF)
 option(ENABLE_MOSAIC "Enable MOSAIC chemistry support" OFF)
 option(ENABLE_CAMP "Enable CAMP chemistry support" OFF)
+option(ENABLE_TCHEM "Enable TChem chemistry support" OFF)
 option(ENABLE_MPI "Enable MPI parallel support" OFF)
 option(ENABLE_SUNDIALS "Enable SUNDIALS solver for condensation support" OFF)
 option(ENABLE_C_SORT "Enable C sorting routines" OFF)
@@ -98,6 +109,50 @@ if(ENABLE_CAMP)
   add_definitions(-DPMC_USE_CAMP)
 endif()
 
+######################################################################
+
+if(ENABLE_TCHEM)
+  find_path(TCHEM_INCLUDE_DIR tchem_atm/TChem_Driver.hpp
+    DOC "TChem include directory"
+    PATHS $ENV{TCHEM_HOME}/include/) 
+  find_library(TCHEM_LIB tchem_atm
+    DOC "TChem library"
+    PATHS $ENV{TCHEM_HOME}/lib64 $ENV{TCHEM_HOME}/lib)
+  add_definitions(-DPMC_USE_TCHEM)
+  include_directories(${TCHEM_INCLUDE_DIR})
+  find_path(YAML_INCLUDE_DIR yaml-cpp/yaml.h 
+    DOC "TChem include directory"
+    PATHS $ENV{YAML_PATH}/include/)
+  find_library(YAML_LIB yaml-cpp
+    PATHS $ENV{YAML_PATH}/lib)
+  include_directories(${YAML_INCLUDE_DIR})
+  find_path(KOKKOS_INCLUDE_DIR Kokkos_Core.hpp
+    DOC "Kokkos include directory"
+    PATHS $ENV{KOKKOS_PATH}/include)
+  find_library(KOKKOS_LIB kokkoscore
+    PATHS $ENV{KOKKOS_PATH}/lib64 $ENV{KOKKOS_PATH}/lib)
+  include_directories(${KOKKOS_INCLUDE_DIR})
+  find_path(KOKKOSKERNEL_INCLUDE_DIR KokkosKernels_Utils.hpp
+    DOC "Kokkos include directory"
+    PATHS $ENV{KOKKOSKERNEL_PATH}/include)
+  find_library(KOKKOSKERNEL_LIB kokkoskernels
+    PATHS $ENV{KOKKOSKERNEL_PATH}/lib64 $ENV{KOKKOSKERNEL_PATH}/lib)
+  find_path(TINES_INCLUDE_DIR tines/Tines.hpp
+    DOC "Tines include directory"
+    PATHS $ENV{TINES_PATH}/include)
+  find_library(TINES_LIB tines
+    PATHS $ENV{TINES_PATH}/lib64 $ENV{TINES_PATH}/lib)
+  include_directories(${TINES_INCLUDE_DIR})
+  set(CPP_LIB -lstdc++)
+#  find_path(LAPACK_INCLUDE_DIR lapack.h
+#    DOC "LAPACK include directory"
+#    PATHS $ENV{LAPACK_PATH}/include)
+  find_library(LAPACK_LIB openblas
+    PATHS $ENV{LAPACK_PATH}/lib)
+#  include_directories(${LAPACK_INCLUDE_DIR})
+  option(ENABLE_TCHEM_GPU "Enable TChem GPU chemistry support" OFF)
+endif()
+
 ######################################################################
 # MPI
 
@@ -154,6 +209,9 @@ if(ENABLE_MPI)
 endif()
 add_test(test_rand ${CMAKE_BINARY_DIR}/test_run/run_test_directory.sh rand)
 add_test(test_sedi ${CMAKE_BINARY_DIR}/test_run/run_test_directory.sh sedi)
+if (ENABLE_TCHEM)
+  add_test(test_tchem ${CMAKE_BINARY_DIR}/test_run/run_test_directory.sh tchem)
+endif()
 add_test(test_weighting ${CMAKE_BINARY_DIR}/test_run/run_test_directory.sh weighting)
 
 ######################################################################
@@ -176,21 +234,41 @@ add_library(partmclib src/aero_state.F90 src/integer_varray.F90
   src/netcdf.F90 src/aero_info.F90 src/aero_info_array.F90
   src/nucleate.F90 src/condense.F90 src/fractal.F90 src/chamber.F90
   src/camp_interface.F90 src/photolysis.F90 src/sys.F90
+  src/tchem_interface.F90
   src/aero_component.F90
-  ${SUNDIALS_SRC} ${GSL_SRC}
+  ${SUNDIALS_SRC} ${GSL_SRC} ${TCHEM_SRC}
   ${C_SORT_SRC})
 
 target_link_libraries(partmclib ${NETCDF_LIBS} ${SUNDIALS_LIBS}
-  ${MOSAIC_LIB} ${GSL_LIBS} ${CAMP_LIB})
+  ${MOSAIC_LIB} ${GSL_LIBS} ${CAMP_LIB} ${TCHEM_LIB} ${YAML_LIB}
+  ${KOKKOS_LIB} ${KOKKOSKERNEL_LIB} ${TINES_LIB} ${CPP_LIB} ${LAPACK_LIB})
+
+if (ENABLE_TCHEM)
+   find_package(OpenMP REQUIRED)
+   if (ENABLE_TCHEM_GPU)
+      add_link_options(${OpenMP_CXX_FLAGS})
+   else()
+      add_link_options(${OpenMP_Fortran_FLAGS})
+   endif()
+   include(FindThreads)
+   target_link_libraries(partmclib ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
+endif()
 
 set_target_properties(partmclib PROPERTIES OUTPUT_NAME partmc)
+if (ENABLE_TCHEM_GPU)
+   add_link_options(${OpenMP_CXX_FLAGS})
+   set_target_properties(partmclib PROPERTIES LINKER_LANGUAGE CXX)
+endif()
 
 ######################################################################
 # partmc executable
 
 add_executable(partmc src/partmc.F90)
 
 target_link_libraries(partmc partmclib)
+if (ENABLE_TCHEM_GPU)
+  set_property(TARGET partmc PROPERTY LINKER_LANGUAGE CXX)
+endif()
 
 ######################################################################
 # test_bidisperse_ode

diff --git a/Dockerfile.tchem b/Dockerfile.tchem
@@ -0,0 +1,177 @@
+FROM ubuntu:22.04
+
+ARG BUILD_TYPE=RELEASE
+ARG SACADO=ON
+RUN echo "BUILD TYPE:" ${BUILD_TYPE}
+RUN echo "SACADO:" ${SACADO}
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        autoconf \
+        clang-format \
+        cmake \
+        gcc \
+        g++ \
+        gfortran \
+        git \
+        lcov \
+        make \
+        libopenblas-dev \
+        pkg-config \
+        ca-certificates \
+        less \
+        tmux \
+        libnetcdf-dev \
+        libnetcdff-dev \
+        libgsl-dev \
+        openssh-client
+
+RUN git clone https://github.com/PCLAeroParams/TChem-atm.git /tchem_dir/
+WORKDIR /tchem_dir/
+RUN git submodule update --init --recursive
+
+RUN cmake -S /tchem_dir/external/Tines/ext/kokkos -B /build/kokkos_build \
+          -DCMAKE_INSTALL_PREFIX="/install/kokkos_install" \
+          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_CXX_FLAGS="-fopenmp -g" \
+          -DKokkos_ENABLE_SERIAL=ON \
+          -DKokkos_ENABLE_OPENMP=ON \
+          -DKokkos_ENABLE_CUDA=OFF \
+          -DKokkos_ENABLE_CUDA_CONSTEXPR=OFF \
+          -DKokkos_ENABLE_CUDA_LAMBDA=OFF
+WORKDIR /build/kokkos_build/
+RUN make -j \
+    && make install
+
+RUN cmake -S /tchem_dir/external/Tines/ext/gtest -B /build/gtest_build \
+          -DCMAKE_INSTALL_PREFIX="/install/gtest_install" \
+          -DCMAKE_CXX_COMPILER=g++
+WORKDIR /build/gtest_build
+RUN make -j \
+    && make install
+
+RUN cmake -S /tchem_dir/external/Tines/ext/yaml -B /build//yaml_build \
+          -DCMAKE_INSTALL_PREFIX="/install/yaml_install" \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_C_COMPILER=gcc \
+          -DCMAKE_CXX_FLAGS="-g -c" \
+          -DCMAKE_EXE_LINKER_FLAGS="" \
+          -DCMAKE_BUILD_TYPE=RELEASE
+WORKDIR /build/yaml_build
+RUN make -j \
+    && make install
+
+RUN cmake -S /tchem_dir/external/Sundials -B /build/sundials_build \
+          -DCMAKE_INSTALL_PREFIX="/install/sundials_install" \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_C_COMPILER=gcc \
+          -DCMAKE_CXX_FLAGS="-g" \
+          -DCMAKE_C_FLAGS="-g" \
+          -DCMAKE_EXE_LINKER_FLAGS="" \
+          -DENABLE_CALIPER:BOOL=OFF \
+          -DCMAKE_BUILD_TYPE=RELEASE
+WORKDIR /build/sundials_build
+RUN make -j \
+    && make install
+
+RUN cmake -S /tchem_dir/external/Skywalker -B /build/skywalker_build \
+          -DCMAKE_INSTALL_PREFIX="/install/skywalker_install" \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_C_COMPILER=gcc \
+          -DSKYWALKER_PRECISION=double \
+          -DCMAKE_BUILD_TYPE=RELEASE
+WORKDIR /build/skywalker_build
+RUN make -j \
+    && make install
+
+RUN cmake -S /tchem_dir/external/Tines/src -B /build/tines_build \
+          -DCMAKE_INSTALL_PREFIX="/install/tines_install" \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_CXX_FLAGS="-g" \
+          -DCMAKE_C_COMPILER=gcc \
+          -DCMAKE_EXE_LINKER_FLAGS="-lgfortran" \
+          -DTINES_ENABLE_DEBUG=OFF \
+          -DTINES_ENABLE_VERBOSE=OFF \
+          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+          -DTINES_ENABLE_TEST=OFF \
+          -DTINES_ENABLE_EXAMPLE=OFF \
+          -DSUNDIALS_INSTALL_PATH=/install/sundials_install \
+          -DYAML_INSTALL_PATH=/install/yaml_install \
+          -DKOKKOS_INSTALL_PATH=/install/kokkos_install \
+          -DOPENBLAS_INSTALL_PATH=`/usr/lib64` \
+          -DGTEST_INSTALL_PATH=/install/gtest_install
+WORKDIR /build/tines_build
+RUN make -j \
+    && make install
+
+RUN cmake -S /tchem_dir/external/kokkos-kernels -B /build/kokkoskernels_build \
+          -DCMAKE_INSTALL_PREFIX="/install/kokkoskernels_install" \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_CXX_FLAGS="-g" \
+          -DCMAKE_EXE_LINKER_FLAGS="-lgfortran" \
+          -DKokkosKernels_ENABLE_EXAMPLES=OFF \
+          -DKokkosKernels_ENABLE_EXPERIMENTAL=OFF \
+          -DKokkosKernels_ENABLE_TESTS=OFF \
+          -DKokkosKernels_ENABLE_COMPONENT_BLAS=OFF \
+          -DKokkosKernels_ENABLE_COMPONENT_GRAPH=OFF \
+          -DKokkosKernels_ENABLE_COMPONENT_LAPACK=OFF \
+          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+          -DKokkos_ROOT=/install/kokkos_install
+WORKDIR /build/kokkoskernels_build
+RUN make -j4 \
+    && make install
+
+
+
+RUN cmake -S /tchem_dir/src -B /tchem_build \
+          -DCMAKE_INSTALL_PREFIX=/tchem_install \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_CXX_FLAGS="-g" \
+          -DCMAKE_C_COMPILER=gcc \
+          -DCMAKE_EXE_LINKER_FLAGS="-lgfortran" \
+          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+          -DTCHEM_ATM_ENABLE_VERBOSE=OFF \
+          -DTCHEM_ATM_ENABLE_TEST=ON \
+          -DTCHEM_ATM_ENABLE_EXAMPLE=ON \
+          -DTCHEM_ATM_ENABLE_SACADO_JACOBIAN_ATMOSPHERIC_CHEMISTRY=${SACADO} \
+          -DKOKKOS_INSTALL_PATH=/install/kokkos_install \
+          -DTINES_INSTALL_PATH=/install/tines_install \
+          -DTCHEM_ATM_ENABLE_SKYWALKER=ON \
+          -DTCHEM_ATM_ENABLE_REAL_TYPE="double" \
+          -DSKYWALKER_INSTALL_PATH=/install/skywalker_install \
+          -DTCHEM_ATM_ENABLE_KOKKOSKERNELS=ON \
+          -DKOKKOSKERNELS_INSTALL_PATH=/install/kokkoskernels_install \
+          -DTCHEM_ATM_ENABLE_SKYWALKER=ON \
+          -DGTEST_INSTALL_PATH=/install/gtest_install
+WORKDIR /tchem_build
+RUN make -j \
+    && make install
+
+#RUN export FC=gfortran \
+#    export CC=gcc \
+#    export CXX=g++
+
+COPY . /partmc/
+RUN cd /build \
+    && export TCHEM_HOME="/tchem_install" \
+    && export KOKKOS_PATH="/install/kokkos_install" \
+    && export KOKKOSKERNEL_PATH="/install/kokkoskernels_install" \
+    && export YAML_PATH="/install/yaml_install" \
+    && export TINES_PATH="/install/tines_install" \
+    && export LAPACK_PATH="/usr/" \
+    && export SUNDIALS_HOME="/install/sundials_install/" \
+    && export LD_LIBRARY_PATH="/usr/lib:/usr/lib64:$LD_LIBRARY_PATH" \
+    && cmake -DCMAKE_BUILD_TYPE=release \
+             -DCMAKE_C_COMPILER=gcc \
+             -DCMAKE_Fortran_COMPILER=gfortran \ 
+             -DCMAKE_C_FLAGS_RELEASE="-O2 -g -Werror -Wall -Wextra" \
+             -DCMAKE_Fortran_FLAGS_RELEASE="-O2 -g -Werror -fimplicit-none -Wall -Wextra -Wconversion -Wunderflow -Wimplicit-interface -Wno-compare-reals -Wno-unused -Wno-unused-parameter -Wno-unused-dummy-argument -fbounds-check" \
+             -D ENABLE_GSL:BOOL=TRUE \
+             -D ENABLE_TCHEM:BOOL=TRUE  \
+             -D ENABLE_SUNDIALS:BOOL=TRUE \
+             -DCMAKE_C_FLAGS="-lpthread -ldl -fopenmp" \
+             -DCMAKE_Fortran_FLAGS="-lpthread -ldl -fopenmp" \
+             /partmc \
+    && make
+WORKDIR /build
diff --git a/scenarios/1_urban_plume/urban_plume.spec b/scenarios/1_urban_plume/urban_plume.spec
@@ -11,6 +11,7 @@ t_output 3600                   # output interval (0 disables) (s)
 t_progress 600                  # progress printing interval (0 disables) (s)
 
 do_camp_chem no                 # whether to use CAMP for chemistry
+do_tchem no                     # whether to use TChem for chemistry
 
 gas_data gas_data.dat           # file containing gas data
 gas_init gas_init.dat           # initial gas concentrations

diff --git a/scenarios/2_urban_plume2/urban_plume2.spec b/scenarios/2_urban_plume2/urban_plume2.spec
@@ -11,6 +11,7 @@ t_output 3600                   # output interval (0 disables) (s)
 t_progress 600                  # progress printing interval (0 disables) (s)
 
 do_camp_chem no                 # whether to use CAMP for chemistry
+do_tchem no                     # whether to use TChem for chemistry 
 
 gas_data gas_data.dat           # file containing gas data
 gas_init gas_init.dat           # initial gas concentrations

diff --git a/scenarios/3_condense/cond_template.spec b/scenarios/3_condense/cond_template.spec
@@ -11,6 +11,7 @@ t_output 1                      # output interval (0 disables) (s)
 t_progress 60                   # progress printing interval (0 disables) (s)
 
 do_camp_chem no                 # whether to use CAMP for chemistry
+do_tchem no                     # whether to use TChem for chemistry 
 
 temp_profile %%TEMP_PROFILE%%   # temperature profile file
 pressure_profile pressure.dat   # pressure profile file

diff --git a/scenarios/4_chamber/chamber.spec b/scenarios/4_chamber/chamber.spec
@@ -11,6 +11,7 @@ t_output 420                    # output interval (0 disables) (s)
 t_progress 2520                 # progress printing interval (0 disables) (s)
 
 do_camp_chem no                 # whether to use CAMP for chemistry
+do_tchem no                     # whether to use TChem for chemistry 
 
 gas_data gas_data.dat           # file containing gas data
 gas_init gas_init.dat           # initial gas mixing ratios

diff --git a/scenarios/5_coag_brownian/example.spec b/scenarios/5_coag_brownian/example.spec
@@ -11,6 +11,7 @@ t_output 3600                   # output interval (0 disables) (s)
 t_progress 600                  # progress printing interval (0 disables) (s)
 
 do_camp_chem no                 # whether to use CAMP for chemistry
+do_tchem no                     # whether to use TChem for chemistry 
 
 gas_data gas_data.dat           # file containing gas data
 gas_init gas_init.dat           # initial gas concentrations

diff --git a/scenarios/6_camp/camp.spec b/scenarios/6_camp/camp.spec
@@ -12,6 +12,7 @@ t_progress 600                  # progress printing interval (0 disables) (s)
 
 do_camp_chem yes                # whether to use CAMP for chemistry
 camp_config config.json
+do_tchem no                     # whether to use TChem for chemistry 
 
 gas_init gas_init.dat           # initial gas concentrations
 

diff --git a/src/camp_interface.F90 b/src/camp_interface.F90
@@ -68,7 +68,7 @@ subroutine pmc_camp_interface_solve(camp_core, camp_state, &
     call env_state_set_camp_env_state(env_state, camp_state)
 
     ! Set the CAMP gas-phase species concentrations
-    call gas_state_set_camp_conc(gas_state, camp_state, gas_data)
+    call gas_state_set_camp_conc(gas_state, env_state, camp_state, gas_data)
 
     ! Recalculate the photolysis rate constants
     call photolysis%update_rate_constants()