diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0ff3fac..666a6b9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,13 +13,15 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Set system to non-interactive mode run: export DEBIAN_FRONTEND=noninteractive - name: install dependencies - run: sudo apt-get install -y --force-yes -qq build-essential libhdf5-serial-dev + run: | + sudo apt-get update -y -qq + sudo apt-get install -y --force-yes -qq build-essential libhdf5-serial-dev - name: build and run tests run: | mkdir -p bin diff --git a/.gitmodules b/.gitmodules index 0c70533..d73a6c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "utils/kokkos"] path = utils/kokkos url = https://github.com/kokkos/kokkos.git +[submodule "utils/ports-of-call"] + path = utils/ports-of-call + url = git@github.com:lanl/ports-of-call.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e95875..b34b014 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2023. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract # 89233218CNA000001 for Los Alamos National Laboratory (LANL), which # is operated by Triad National Security, LLC for the U.S. Department @@ -53,11 +53,21 @@ target_include_directories(singularity-opac::flags INTERFACE $) -include (SetupDeps) include (SetupOptions) +include (SetupDeps) include (SetupCompilers) include (SetupFlags) +if (SINGULARITY_USE_HDF5) + message(STATUS "HDF5 enabled. Requesting it in spiner") + set(SPINER_USE_HDF ON CACHE BOOL "") +else() + message(STATUS "HDF5 not enabled. Turning it off in spiner") + set(SPINER_USE_HDF OFF CACHE BOOL "") +endif() +add_subdirectory(utils/spiner) +target_link_libraries(singularity-opac::flags INTERFACE spiner) + include(GNUInstallDirs) include(CTest) diff --git a/cmake/SetupCompilers.cmake b/cmake/SetupCompilers.cmake index 70259ad..ebab8e2 100644 --- a/cmake/SetupCompilers.cmake +++ b/cmake/SetupCompilers.cmake @@ -6,7 +6,7 @@ enable_language(CXX) include(CMakeDetermineCXXCompiler) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/cmake/SetupDeps.cmake b/cmake/SetupDeps.cmake index 81d8153..d7782b1 100644 --- a/cmake/SetupDeps.cmake +++ b/cmake/SetupDeps.cmake @@ -33,7 +33,9 @@ endif() #======================================= # Can't play the target games above unless using # cmake 3.20+, so leave this one for now. -find_package(HDF5 COMPONENTS C HL QUIET) +# JMM: DO NOT SEARCH FOR HDF5 IF WE DON'T NEED IT +if (SINGULARITY_USE_HDF5) +find_package(HDF5 COMPONENTS C HL) # findpackage doesnt export an interface for HDF5, # so create one @@ -64,9 +66,22 @@ if (HDF5_FOUND) else() message("MPI::MPI_CXX provided by parent package") endif() + else() + message(status "HDF5 is not parallel") endif() +else() + message(STATUS "No HDF5") +endif() endif() +# JMM: I'm putting this here, as it depends on what happens with HDF5, +# and I want that to be set AFTER we set all our options, so we don't +# set HDF5 unnecessarily. +cmake_dependent_option(SINGULARITY_USE_MPI + "Link to MPI" + ON "${SINGULARITY_USE_MPI};${HDF5_IS_PARALLEL}" + OFF) + #======================================= # Setup Catch2 # - provides Catch2::Catch2 diff --git a/singularity-opac/base/indexers.hpp b/singularity-opac/base/indexers.hpp index e48f63f..4f4657c 100644 --- a/singularity-opac/base/indexers.hpp +++ b/singularity-opac/base/indexers.hpp @@ -46,6 +46,7 @@ struct Identity { class Linear { public: + using DataBox = Spiner::DataBox; Linear() = default; PORTABLE_INLINE_FUNCTION @@ -58,7 +59,7 @@ class Linear { SetRange_(numin, numax, N); } - PORTABLE_INLINE_FUNCTION Linear(const Spiner::DataBox &data, Real numin, + PORTABLE_INLINE_FUNCTION Linear(const DataBox &data, Real numin, Real numax, int N) : data_(data) { SetRange_(numin, numax, N); @@ -79,11 +80,12 @@ class Linear { void SetRange_(Real numin, Real numax, int N) { data_.setRange(0, numin, numax, N); } - Spiner::DataBox data_; + DataBox data_; }; class LogLinear { public: + using DataBox = Spiner::DataBox; LogLinear() = default; PORTABLE_INLINE_FUNCTION @@ -97,7 +99,7 @@ class LogLinear { } PORTABLE_INLINE_FUNCTION - LogLinear(const Spiner::DataBox &data, Real numin, Real numax, int N) + LogLinear(const DataBox &data, Real numin, Real numax, int N) : data_(data) { SetRange_(numin, numax, N); } @@ -119,7 +121,7 @@ class LogLinear { void SetRange_(Real numin, Real numax, int N) { data_.setRange(0, BDMath::log10(numin), BDMath::log10(numax), N); } - Spiner::DataBox data_; + DataBox data_; }; template diff --git a/singularity-opac/neutrinos/mean_opacity_neutrinos.hpp b/singularity-opac/neutrinos/mean_opacity_neutrinos.hpp index 99c9956..61a6396 100644 --- a/singularity-opac/neutrinos/mean_opacity_neutrinos.hpp +++ b/singularity-opac/neutrinos/mean_opacity_neutrinos.hpp @@ -210,8 +210,8 @@ class MeanOpacity { PORTABLE_INLINE_FUNCTION Real fromLog_(const Real lx) const { return std::pow(10., lx); } - Spiner::DataBox lkappaPlanck_; - Spiner::DataBox lkappaRosseland_; + Spiner::DataBox lkappaPlanck_; + Spiner::DataBox lkappaRosseland_; const char *filename_; }; diff --git a/singularity-opac/neutrinos/mean_s_opacity_neutrinos.hpp b/singularity-opac/neutrinos/mean_s_opacity_neutrinos.hpp index f378190..8665531 100644 --- a/singularity-opac/neutrinos/mean_s_opacity_neutrinos.hpp +++ b/singularity-opac/neutrinos/mean_s_opacity_neutrinos.hpp @@ -41,6 +41,7 @@ namespace impl { template class MeanSOpacity { public: + using DataBox = Spiner::DataBox; MeanSOpacity() = default; template MeanSOpacity(const SOpacity &s_opac, const Real lRhoMin, const Real lRhoMax, @@ -211,8 +212,8 @@ class MeanSOpacity { PORTABLE_INLINE_FUNCTION Real fromLog_(const Real lx) const { return std::pow(10., lx); } - Spiner::DataBox lkappaPlanck_; - Spiner::DataBox lkappaRosseland_; + DataBox lkappaPlanck_; + DataBox lkappaRosseland_; const char *filename_; }; diff --git a/singularity-opac/neutrinos/spiner_opac_neutrinos.hpp b/singularity-opac/neutrinos/spiner_opac_neutrinos.hpp index 84ecabd..07c6a7b 100644 --- a/singularity-opac/neutrinos/spiner_opac_neutrinos.hpp +++ b/singularity-opac/neutrinos/spiner_opac_neutrinos.hpp @@ -61,6 +61,7 @@ enum class DataStatus { Deallocated, OnDevice, OnHost }; template class SpinerOpacity { public: + using DataBox = Spiner::DataBox; static constexpr Real EPS = 10.0 * std::numeric_limits::min(); static constexpr Real Hz2MeV = pc::h / (1e6 * pc::eV); static constexpr Real MeV2Hz = 1 / Hz2MeV; @@ -130,8 +131,8 @@ class SpinerOpacity { // DataBox constructor. Note that this constructor *shallow* copies // the databoxes, so they must be managed externally. - SpinerOpacity(const Spiner::DataBox &lalphanu, const Spiner::DataBox ljnu, - const Spiner::DataBox lJ, const Spiner::DataBox lJYe) + SpinerOpacity(const DataBox &lalphanu, const DataBox ljnu, + const DataBox lJ, const DataBox lJYe) : memoryStatus_(impl::DataStatus::OnHost), lalphanu_(lalphanu), ljnu_(ljnu), lJ_(lJ), lJYe_(lJYe) {} @@ -391,7 +392,7 @@ class SpinerOpacity { impl::DataStatus memoryStatus_ = impl::DataStatus::Deallocated; // TODO(JMM): Integrating J and JYe seems wise. // We can add more things here as needed. - Spiner::DataBox lalphanu_, ljnu_, lJ_, lJYe_; + DataBox lalphanu_, ljnu_, lJ_, lJYe_; // TODO(JMM): Should we add table bounds? Given they're recorded in // each spiner table, I lean towards no, but could be convinced // otherwise if we need to do extrapolation, etc. diff --git a/singularity-opac/photons/mean_opacity_photons.hpp b/singularity-opac/photons/mean_opacity_photons.hpp index 2cc42a7..8a58859 100644 --- a/singularity-opac/photons/mean_opacity_photons.hpp +++ b/singularity-opac/photons/mean_opacity_photons.hpp @@ -41,6 +41,7 @@ template class MeanOpacity { public: + using DataBox = Spiner::DataBox; MeanOpacity() = default; template MeanOpacity(const Opacity &opac, const Real lRhoMin, const Real lRhoMax, @@ -186,8 +187,8 @@ class MeanOpacity { PORTABLE_INLINE_FUNCTION Real fromLog_(const Real lx) const { return std::pow(10., lx); } - Spiner::DataBox lkappaPlanck_; - Spiner::DataBox lkappaRosseland_; + DataBox lkappaPlanck_; + DataBox lkappaRosseland_; const char *filename_; }; diff --git a/singularity-opac/photons/mean_s_opacity_photons.hpp b/singularity-opac/photons/mean_s_opacity_photons.hpp index 8497fc4..7fb6ff3 100644 --- a/singularity-opac/photons/mean_s_opacity_photons.hpp +++ b/singularity-opac/photons/mean_s_opacity_photons.hpp @@ -42,6 +42,8 @@ template class MeanSOpacity { public: + using DataBox = Spiner::DataBox; + MeanSOpacity() = default; template MeanSOpacity(const SOpacity &s_opac, const Real lRhoMin, const Real lRhoMax, @@ -192,8 +194,8 @@ class MeanSOpacity { PORTABLE_INLINE_FUNCTION Real fromLog_(const Real lx) const { return std::pow(10., lx); } - Spiner::DataBox lkappaPlanck_; - Spiner::DataBox lkappaRosseland_; + DataBox lkappaPlanck_; + DataBox lkappaRosseland_; const char *filename_; }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ae8cabb..06f6a86 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,6 +29,8 @@ message(STATUS "Configuring unit tests") add_library(catch2_define) target_sources(catch2_define PUBLIC catch2_define.cpp) target_link_libraries(catch2_define PUBLIC + ${PROJECT_NAME} + singularity-opac::flags Catch2::Catch2) add_executable(${PROJECT_NAME}_unit_tests) @@ -50,7 +52,8 @@ PRIVATE target_link_libraries(${PROJECT_NAME}_unit_tests PRIVATE catch2_define - ${PROJECT_NAME}) + ${PROJECT_NAME} + singularity-opac::flags) # Ensure code works with C++11 and earlier # TODO(MM): Remove this later when it's not needed. diff --git a/test/catch2_define.cpp b/test/catch2_define.cpp index 1fb99a8..a48068f 100644 --- a/test/catch2_define.cpp +++ b/test/catch2_define.cpp @@ -13,7 +13,7 @@ // publicly, and to permit others to do so. // ====================================================================== -#include "../utils/ports-of-call/portability.hpp" +#include #define CATCH_CONFIG_RUNNER #include diff --git a/test/test_brt_opacities.cpp b/test/test_brt_opacities.cpp index b0d3628..0720fdb 100644 --- a/test/test_brt_opacities.cpp +++ b/test/test_brt_opacities.cpp @@ -32,6 +32,7 @@ using namespace singularity; using pc = PhysicalConstantsCGS; +using DataBox = Spiner::DataBox; #ifdef PORTABILITY_STRATEGY_KOKKOS using atomic_view = Kokkos::MemoryTraits; @@ -169,7 +170,7 @@ TEST_CASE("BRT neutrino opacities", "[BRTNeutrinos]") { nu_coeffs, nu_min, nu_max); opac.EmissivityPerNu(rho, temp, Ye, type, nu_bins, J_cheb, nbins); Real Jtrue = opac.EmissivityPerNu(rho, temp, Ye, type, nu); - J_cheb.SetInterpCoeffs(Spiner::DataBox(vm9, 9, 9)); + J_cheb.SetInterpCoeffs(DataBox(vm9, 9, 9)); if (std::isnan(J_cheb(nu)) || ((std::abs(Jtrue) >= 1e-14 || J_cheb(nu) >= 1e-14) && FractionalDifference(J_cheb(nu), Jtrue) > EPS_TEST)) { diff --git a/test/test_chebyshev.cpp b/test/test_chebyshev.cpp index 71c4dea..71039a9 100644 --- a/test/test_chebyshev.cpp +++ b/test/test_chebyshev.cpp @@ -23,6 +23,8 @@ #include using namespace singularity::chebyshev; +using DataBox = Spiner::DataBox; + #ifdef PORTABILITY_STRATEGY_KOKKOS using atomic_view = Kokkos::MemoryTraits; #endif @@ -62,7 +64,7 @@ TEST_CASE("Chebyshev Polynomials", "[Chebyshev]") { Real *ycoeffs = (Real *)PORTABLE_MALLOC(sizeof(Real) * N); portableFor( "Compute Cheb polynomial", 0, 1, PORTABLE_LAMBDA(const int &i) { - MatMultiply(Spiner::DataBox(vm9, 9, 9), y, ycoeffs, N); + MatMultiply(DataBox(vm9, 9, 9), y, ycoeffs, N); }); AND_THEN("The chebyshev polynomials fit") { int n_wrong_h = 0; diff --git a/test/test_epbremsstrahlung_opacities.cpp b/test/test_epbremsstrahlung_opacities.cpp index 004d054..95b8a83 100644 --- a/test/test_epbremsstrahlung_opacities.cpp +++ b/test/test_epbremsstrahlung_opacities.cpp @@ -31,6 +31,7 @@ using namespace singularity; using pc = PhysicalConstantsCGS; +using DataBox = Spiner::DataBox; #ifdef PORTABILITY_STRATEGY_KOKKOS using atomic_view = Kokkos::MemoryTraits; @@ -132,7 +133,7 @@ TEST_CASE("E-P bremsstrahlung photon opacities", "[EPBremsstrahlungPhotons]") { Real *nu_bins = (Real *)PORTABLE_MALLOC(nbins * sizeof(Real)); Real *temp_bins = (Real *)PORTABLE_MALLOC(ntemps * sizeof(Real)); - Spiner::DataBox loglin_bins(Spiner::AllocationTarget::Device, ntemps, + DataBox loglin_bins(Spiner::AllocationTarget::Device, ntemps, nbins); portableFor( diff --git a/test/test_gray_opacities.cpp b/test/test_gray_opacities.cpp index 496cef2..464dbd1 100644 --- a/test/test_gray_opacities.cpp +++ b/test/test_gray_opacities.cpp @@ -32,6 +32,7 @@ using namespace singularity; using pc = PhysicalConstantsCGS; +using DataBox = Spiner::DataBox; #ifdef PORTABILITY_STRATEGY_KOKKOS using atomic_view = Kokkos::MemoryTraits; @@ -167,7 +168,7 @@ TEST_CASE("Gray neutrino opacities", "[GrayNeutrinos]") { nu_coeffs, nu_min, nu_max); opac.EmissivityPerNu(rho, temp, Ye, type, nu_bins, J_cheb, nbins); Real Jtrue = opac.EmissivityPerNu(rho, temp, Ye, type, nu); - J_cheb.SetInterpCoeffs(Spiner::DataBox(vm9, 9, 9)); + J_cheb.SetInterpCoeffs(DataBox(vm9, 9, 9)); if (std::isnan(J_cheb(nu)) || ((std::abs(Jtrue) >= 1e-14 || J_cheb(nu) >= 1e-14) && FractionalDifference(J_cheb(nu), Jtrue) > EPS_TEST)) { @@ -282,7 +283,7 @@ TEST_CASE("Gray photon opacities", "[GrayPhotons]") { Real *nu_bins = (Real *)PORTABLE_MALLOC(nbins * sizeof(Real)); Real *temp_bins = (Real *)PORTABLE_MALLOC(ntemps * sizeof(Real)); - Spiner::DataBox loglin_bins(Spiner::AllocationTarget::Device, ntemps, + DataBox loglin_bins(Spiner::AllocationTarget::Device, ntemps, nbins); portableFor( diff --git a/test/test_spiner_opac_neutrinos.cpp b/test/test_spiner_opac_neutrinos.cpp index 9390c6d..a927459 100644 --- a/test/test_spiner_opac_neutrinos.cpp +++ b/test/test_spiner_opac_neutrinos.cpp @@ -73,7 +73,7 @@ TEST_CASE("Spiner opacities, filled with gray data", const std::string grayname = "gray.sp5"; WHEN("We initialize a gray neutrino opacity and tabulate it") { - using Grid_t = Spiner::RegularGrid1D; + using Grid_t = Spiner::RegularGrid1D; // Offset the frequencies to hopefully get over the point // where the power laws transition diff --git a/utils/ports-of-call b/utils/ports-of-call deleted file mode 120000 index 2ec1e74..0000000 --- a/utils/ports-of-call +++ /dev/null @@ -1 +0,0 @@ -spiner/ports-of-call/ \ No newline at end of file diff --git a/utils/ports-of-call b/utils/ports-of-call new file mode 160000 index 0000000..993d820 --- /dev/null +++ b/utils/ports-of-call @@ -0,0 +1 @@ +Subproject commit 993d8209de98da186c5e0b445a4d6c359afa8c81 diff --git a/utils/spiner b/utils/spiner index cadaf1e..33862f8 160000 --- a/utils/spiner +++ b/utils/spiner @@ -1 +1 @@ -Subproject commit cadaf1e3f759251176f58cec0961093ae9019a18 +Subproject commit 33862f831be65e2dd2284b42f05c477b6fe7367a