From 864810f568a1f58cc1cec7e4d08d188d8b20131f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 14 Apr 2021 11:59:03 +0200 Subject: [PATCH 001/282] include only necessary parts of odeint --- include/libcloudph++/blk_1m/extincl.hpp | 5 ++++- src/impl/particles_impl.ipp | 4 ++-- src/particles_ctor.ipp | 5 ----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/libcloudph++/blk_1m/extincl.hpp b/include/libcloudph++/blk_1m/extincl.hpp index 955557c77..76f3dba49 100644 --- a/include/libcloudph++/blk_1m/extincl.hpp +++ b/include/libcloudph++/blk_1m/extincl.hpp @@ -2,7 +2,10 @@ #include -#include +#include +#include +#include +#include #include "../common/const_cp.hpp" #include "../common/theta_dry.hpp" diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 409fb074f..d03ccca57 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -10,10 +10,10 @@ #include #include -#include +#include #include #include -#include +#include #include #include diff --git a/src/particles_ctor.ipp b/src/particles_ctor.ipp index 2e02b299f..d99580831 100644 --- a/src/particles_ctor.ipp +++ b/src/particles_ctor.ipp @@ -9,11 +9,6 @@ #include #include -#include -#include -#include -#include - #include namespace libcloudphxx From c7272ff2216b3be6794351a81939c20fb80b74cd Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 15 Apr 2021 17:10:12 +0200 Subject: [PATCH 002/282] include iomanip in debug runs --- src/particles.tpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/particles.tpp b/src/particles.tpp index 4483c8829..0a38f6e33 100644 --- a/src/particles.tpp +++ b/src/particles.tpp @@ -7,6 +7,9 @@ */ #include +#if !defined(NDEBUG) + #include +#endif #include From c46fcc1ac07a07b4099de7a91d1b26e279319e7c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 11 May 2021 12:40:34 +0200 Subject: [PATCH 003/282] relaxation: opts, opts_init and step --- include/libcloudph++/lgrngn/opts.hpp | 4 +- include/libcloudph++/lgrngn/opts_init.hpp | 65 +++++++++++++++++------ src/impl/particles_impl.ipp | 5 +- src/particles_step.ipp | 27 ++++++++-- 4 files changed, 75 insertions(+), 26 deletions(-) diff --git a/include/libcloudph++/lgrngn/opts.hpp b/include/libcloudph++/lgrngn/opts.hpp index 3f54557d4..8a32a7e25 100644 --- a/include/libcloudph++/lgrngn/opts.hpp +++ b/include/libcloudph++/lgrngn/opts.hpp @@ -19,7 +19,7 @@ namespace libcloudphxx struct opts_t { // process toggling - bool adve, sedi, subs, cond, coal, src, rcyc, turb_adve, turb_cond, turb_coal; + bool adve, sedi, subs, cond, coal, src, rlx, rcyc, turb_adve, turb_cond, turb_coal; // RH limit for drop growth real_t RH_max; @@ -33,7 +33,7 @@ namespace libcloudphxx // ctor with defaults (C++03 compliant) ... opts_t() : - adve(true), sedi(true), subs(false), cond(true), coal(true), src(false), rcyc(false), + adve(true), sedi(true), subs(false), cond(true), coal(true), src(false), rlx(false), rcyc(false), chem_dsl(false), chem_dsc(false), chem_rct(false), turb_adve(false), turb_cond(false), turb_coal(false), RH_max(44), // :) (anything greater than 1.1 would be enough diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index fa57407bc..e3183564e 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -48,9 +48,6 @@ namespace libcloudphxx // no. of substeps int sstp_cond, sstp_coal; - - // timestep interval at which source will be applied - int supstp_src; // Lagrangian domain extents real_t x0, y0, z0, x1, y1, z1; @@ -74,19 +71,6 @@ namespace libcloudphxx // should be enough to store particles from sources unsigned long long n_sd_max; - // source distro per unit time - dry_distros_t src_dry_distros; - - // number of SDs created from src_dry_distros per cell per source iteration - unsigned long long src_sd_conc; - - // dry sizes of droplets added from the source, STP_concentration created per unit time instead of the STP_concentration - dry_sizes_t src_dry_sizes; - - // box in which aerosol from source will be created - // will be rounded to cell number - cells are supposed to be uniform - real_t src_x0, src_y0, src_z0, src_x1, src_y1, src_z1; - // coalescence Kernel type kernel_t::kernel_t kernel; @@ -108,6 +92,7 @@ namespace libcloudphxx sedi_switch, // if false no sedimentation throughout the whole simulation subs_switch, // if false no subsidence throughout the whole simulation src_switch, // if false no source throughout the whole simulation + rlx_switch, // if false no relaxation throughout the whole simulation turb_adve_switch, // if true, turbulent motion of SDs is modeled turb_cond_switch, // if true, turbulent condensation of SDs is modeled turb_coal_switch, // if true, turbulent coalescence kernels can be used @@ -146,6 +131,50 @@ namespace libcloudphxx periodic_topbot_walls; // if true, top and bot walls are periodic. Open otherwise + // --- aerosol source stuff --- + + // source distro per unit time + dry_distros_t src_dry_distros; + + // number of SDs created from src_dry_distros per cell per source iteration + unsigned long long src_sd_conc; + + // dry sizes of droplets added from the source, STP_concentration created per unit time instead of the STP_concentration + dry_sizes_t src_dry_sizes; + + // box in which aerosol from source will be created + // will be rounded to cell number - cells are supposed to be uniform + real_t src_x0, src_y0, src_z0, src_x1, src_y1, src_z1; + + // timestep interval at which source will be applied + int supstp_src; + + + // --- aerosol relaxation stuff --- + + // initial dry sizes of aerosol + // defined with a distribution + // uses shared_ptr to make opts_init copyable + typedef std::unordered_map< + real_t, // kappa + std::tuple< + std::shared_ptr>, // n(ln(rd)) @ STP; alternatively it's n(ln(rd)) independent of rhod if aerosol_independent_of_rhod=true + std::pair, // kappa range of CCN considered to belong to this distribution, ranges of different members of the map need to be exclusive (TODO: add a check of this) + std::pair // range of altitudes at which this relaxation acts + > + > rlx_dry_distros_t; + + rlx_dry_distros_t rlx_dry_distros; + + // number of bins into which the relaxation distro is divided, up to one SD is created per bin + unsigned long long rlx_bins; + + // timestep interval at which relaxation will be applied + int supstp_rlx; + + + // -- ctors --- + // ctor with defaults (C++03 compliant) ... opts_init_t() : nx(0), ny(0), nz(0), @@ -158,7 +187,6 @@ namespace libcloudphxx sd_const_multi(0), dt(0), sstp_cond(1), sstp_coal(1), sstp_chem(1), - supstp_src(1), chem_switch(false), // chemical reactions turned off by default sedi_switch(true), // sedimentation turned on by default subs_switch(false), // subsidence turned off by default @@ -186,6 +214,9 @@ namespace libcloudphxx src_y1(0), src_z0(0), src_z1(0), + supstp_src(1), + rlx_bins(0), + supstp_rlx(1), rd_min(0.), diag_incloud_time(false), no_ccn_at_init(false), diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 409fb074f..8afcb8683 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -172,7 +172,7 @@ namespace libcloudphxx allow_sstp_chem; // timestep counter - n_t stp_ctr; + n_t src_stp_ctr, rlx_stp_ctr; // maps linear Lagrangian component indices into Eulerian component linear indices // the map key is the address of the Thrust vector @@ -310,7 +310,8 @@ namespace libcloudphxx n_user_params(_opts_init.kernel_parameters.size()), un(tmp_device_n_part), rng(_opts_init.rng_seed), - stp_ctr(0), + src_stp_ctr(0), + rlx_stp_ctr(0), bcond(bcond), n_x_bfr(0), n_cell_bfr(0), diff --git a/src/particles_step.ipp b/src/particles_step.ipp index 0fa2f925e..2c3805254 100644 --- a/src/particles_step.ipp +++ b/src/particles_step.ipp @@ -261,6 +261,8 @@ namespace libcloudphxx } } + // TODO: move source and relax to async? th/rv changes are minimal + // aerosol source, in sync since it changes th/rv if (opts.src && !(pimpl->opts_init.src_x0 == 0 && pimpl->opts_init.src_x1 == 0)) // src_x0=0 and src_x1=0 is a way of disabling source in some domains in distmem simulations { @@ -268,22 +270,37 @@ namespace libcloudphxx if (pimpl->opts_init.src_switch == false) throw std::runtime_error("aerosol source was switched off in opts_init"); // introduce new particles with the given time interval - if(pimpl->stp_ctr % pimpl->opts_init.supstp_src == 0) + if(pimpl->src_stp_ctr % pimpl->opts_init.supstp_src == 0) { pimpl->src(pimpl->opts_init.supstp_src * pimpl->dt); } } - if(opts.cond || (opts.src && pimpl->stp_ctr % pimpl->opts_init.supstp_src == 0)) + // aerosol relaxation, in sync since it changes th/rv + if (opts.rlx) + { + // sanity check + if (pimpl->opts_init.rlx_switch == false) throw std::runtime_error("aerosol relaxation was switched off in opts_init"); + + // introduce new particles with the given time interval + if(pimpl->rlx_stp_ctr % pimpl->opts_init.supstp_rlx == 0) + { + pimpl->rlx(pimpl->opts_init.supstp_rlx * pimpl->dt); + } + } + + if(opts.cond || (opts.src && pimpl->src_stp_ctr % pimpl->opts_init.supstp_src == 0) || (opts.rlx && pimpl->rlx_stp_ctr % pimpl->opts_init.supstp_rlx == 0)) { // syncing out // TODO: this is not necesarry in off-line mode (see coupling with DALES) pimpl->sync(pimpl->th, th); pimpl->sync(pimpl->rv, rv); } - // update the step counter since src was turned on - if (opts.src) ++pimpl->stp_ctr; - else pimpl->stp_ctr = 0; //reset the counter if source was turned off + // update the step counter since src/rlx was turned on + if (opts.src) ++pimpl->src_stp_ctr; + else pimpl->src_stp_ctr = 0; //reset the counter if source was turned off + if (opts.rlx) ++pimpl->rlx_stp_ctr; + else pimpl->rlx_stp_ctr = 0; //reset the counter if source was turned off if (opts.chem_dsl == true) { From 9d567a1fa3c8ae0544d5c2eeaf0b54a888a0fbed Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 11 May 2021 12:59:31 +0200 Subject: [PATCH 004/282] separate ante adding sd and post adding sd and reuse in relaxation --- src/impl/particles_impl.ipp | 3 + src/impl/particles_impl_ante_adding_SD.ipp | 42 ++++++++++++++ src/impl/particles_impl_post_adding_SD.ipp | 55 ++++++++++++++++++ src/impl/particles_impl_src.ipp | 65 +--------------------- 4 files changed, 102 insertions(+), 63 deletions(-) create mode 100644 src/impl/particles_impl_ante_adding_SD.ipp create mode 100644 src/impl/particles_impl_post_adding_SD.ipp diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 8afcb8683..c8293cba8 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -576,6 +576,9 @@ namespace libcloudphxx void post_copy(const opts_t&); + void ante_adding_SD(); + void post_adding_SD(); + // distmem stuff void xchng_domains(); void xchng_courants(); diff --git a/src/impl/particles_impl_ante_adding_SD.ipp b/src/impl/particles_impl_ante_adding_SD.ipp new file mode 100644 index 000000000..b77502a11 --- /dev/null +++ b/src/impl/particles_impl_ante_adding_SD.ipp @@ -0,0 +1,42 @@ +// vim:filetype=cpp +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ + +namespace libcloudphxx +{ + namespace lgrngn + { + template + void particles_t::impl::ante_adding_SD() + { + // --- calc liquid water content before src --- + hskpng_sort(); + thrust_device::vector &drv(tmp_device_real_cell1); // NOTE: this can't be changed by any function called before a call to after_adding_SD... + thrust::fill(drv.begin(), drv.end(), real_t(0.)); + + moms_all(); + moms_calc(rw2.begin(), real_t(3./2.)); + + // drv = - tot_vol_bfr + thrust::transform( + count_mom.begin(), count_mom.begin() + count_n, // input - 1st arg + thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // output + thrust::negate() + ); + + // drv = -tot_vol_bfr + dry_vol_bfr +/* + moms_calc(rd3.begin(), 1); + thrust::transform( + count_mom.begin(), count_mom.begin() + count_n, // input - 1st arg + thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // 2nd arg + thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // output + thrust::plus() + ); +*/ + } + }; +}; diff --git a/src/impl/particles_impl_post_adding_SD.ipp b/src/impl/particles_impl_post_adding_SD.ipp new file mode 100644 index 000000000..8384a7a57 --- /dev/null +++ b/src/impl/particles_impl_post_adding_SD.ipp @@ -0,0 +1,55 @@ +// vim:filetype=cpp +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ + +namespace libcloudphxx +{ + namespace lgrngn + { + template + void particles_t::impl::post_adding_SD() + { + // --- after source particles are no longer sorted --- + sorted = false; + + // --- calc liquid water content after src --- + hskpng_sort(); + moms_all(); + moms_calc(rw2.begin(), real_t(3./2.)); + + // drv = tot_vol_after -tot_vol_bfr + dry_vol_bfr + thrust::transform( + count_mom.begin(), count_mom.begin() + count_n, // input - 1st arg + thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // 2nd arg + thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // output + thrust::plus() + ); + + // drv = tot_vol_after - dry_vol_after - tot_vol_bfr + dry_vol_bfr +/* + moms_calc(rd3.begin(), 1); + thrust::transform( + count_mom.begin(), count_mom.begin() + count_n, // input - 1st arg + thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // 2nd arg + thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // output + thrust::minus() + ); +*/ + + // update th and rv + update_th_rv(drv); + + // update count_ijk and count_num + hskpng_count(); + + // update vt of new SD + hskpng_vterm_invalid(); + + // init _old values in per-particle substepping + init_sstp(); + } + }; +}; diff --git a/src/impl/particles_impl_src.ipp b/src/impl/particles_impl_src.ipp index 71d53add0..2c18390f2 100644 --- a/src/impl/particles_impl_src.ipp +++ b/src/impl/particles_impl_src.ipp @@ -12,76 +12,15 @@ namespace libcloudphxx template void particles_t::impl::src(const real_t &dt) { - // --- calc liquid water content before src --- - hskpng_sort(); - thrust_device::vector &drv(tmp_device_real_cell1); // NOTE: this can't be changed by any function called in src..... - thrust::fill(drv.begin(), drv.end(), real_t(0.)); - - moms_all(); - moms_calc(rw2.begin(), real_t(3./2.)); - - // drv = - tot_vol_bfr - thrust::transform( - count_mom.begin(), count_mom.begin() + count_n, // input - 1st arg - thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // output - thrust::negate() - ); - - // drv = -tot_vol_bfr + dry_vol_bfr -/* - moms_calc(rd3.begin(), 1); - thrust::transform( - count_mom.begin(), count_mom.begin() + count_n, // input - 1st arg - thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // 2nd arg - thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // output - thrust::plus() - ); -*/ + ante_adding_SD(); if(!opts_init.src_dry_distros.empty()) src_dry_distros(dt); if(!opts_init.src_dry_sizes.empty()) src_dry_sizes(dt); - - // --- after source particles are no longer sorted --- - sorted = false; - - // --- calc liquid water content after src --- - hskpng_sort(); - moms_all(); - moms_calc(rw2.begin(), real_t(3./2.)); - - // drv = tot_vol_after -tot_vol_bfr + dry_vol_bfr - thrust::transform( - count_mom.begin(), count_mom.begin() + count_n, // input - 1st arg - thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // 2nd arg - thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // output - thrust::plus() - ); - - // drv = tot_vol_after - dry_vol_after - tot_vol_bfr + dry_vol_bfr -/* - moms_calc(rd3.begin(), 1); - thrust::transform( - count_mom.begin(), count_mom.begin() + count_n, // input - 1st arg - thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // 2nd arg - thrust::make_permutation_iterator(drv.begin(), count_ijk.begin()), // output - thrust::minus() - ); -*/ - - // update th and rv - update_th_rv(drv); - - // update count_ijk and count_num - hskpng_count(); - - // update vt of new SD - hskpng_vterm_invalid(); - // init _old values in per-particle substepping - init_sstp(); + post_adding_SD(); } }; }; From 93393b7153cde1df6668a3677a37e4cc4aa8ab5f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 11 May 2021 16:53:01 +0200 Subject: [PATCH 005/282] rlx wip: done horizontal avg --- src/impl/particles_impl.ipp | 3 + src/impl/particles_impl_rlx.ipp | 23 ++ src/impl/particles_impl_rlx_dry_distros.ipp | 395 ++++++++++++++++++++ src/particles.tpp | 2 + src/particles_step.ipp | 1 + 5 files changed, 424 insertions(+) create mode 100644 src/impl/particles_impl_rlx.ipp create mode 100644 src/impl/particles_impl_rlx_dry_distros.ipp diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index c8293cba8..e97367c0d 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -567,6 +567,9 @@ namespace libcloudphxx void src_dry_distros(const real_t &dt); void src_dry_sizes(const real_t &dt); + void rlx(const real_t &dt); + void rlx_dry_distros(const real_t &dt); + void sstp_step(const int &step); void sstp_step_exact(const int &step); void sstp_step_ssp(const real_t &dt); diff --git a/src/impl/particles_impl_rlx.ipp b/src/impl/particles_impl_rlx.ipp new file mode 100644 index 000000000..9d6991806 --- /dev/null +++ b/src/impl/particles_impl_rlx.ipp @@ -0,0 +1,23 @@ +// vim:filetype=cpp +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ + +namespace libcloudphxx +{ + namespace lgrngn + { + template + void particles_t::impl::rlx(const real_t &dt) + { + ante_adding_SD(); + + if(!opts_init.rlx_dry_distros.empty()) + rlx_dry_distros(dt); + + post_adding_SD(); + } + }; +}; diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp new file mode 100644 index 000000000..70a552f72 --- /dev/null +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -0,0 +1,395 @@ +// vim:filetype=cpp +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ +// #include +#include +#include + +namespace libcloudphxx +{ + namespace lgrngn + { + // create new aerosol particles to relax towards a size distribution + template + void particles_t::impl::rlx_dry_distros(const real_t &dt) + { + namespace arg = thrust::placeholders; + + // vectors of size nz used in calculation of horizontal averages, TODO: allocate them at init + thrust::device_vector hor_avg(opts_init.nz); + thrust::device_vector hor_avg_count(opts_init.nz); + thrust::device_vector hor_avg_k(opts_init.nz); + + // calc sum of ln(rd) ranges of all relax distributions + real_t tot_lnrd_rng = 0.; + for (typename opts_init_t::rlx_dry_distros_t::const_iterator ddi = opts_init.rlx_dry_distros.begin(); ddi != opts_init.rlx_dry_distros.end(); ++ddi) + { + dist_analysis_sd_conc( + *(std::get<0>(ddi->second)), + opts_init.rlx_bins, + dt + ); + tot_lnrd_rng += log_rd_max - log_rd_min; + } + + // initialize SDs of each kappa-type + for (typename opts_init_t::rlx_dry_distros_t::const_iterator ddi = opts_init.rlx_dry_distros.begin(); ddi != opts_init.rlx_dry_distros.end(); ++ddi) + { + // analyze distribution to get rd_min and max needed for bin sizes + // TODO: this was done a moment ago! + // TODO2: this could be done once at the start of the simulation! + dist_analysis_sd_conc( + *(std::get<0>(ddi->second)), + opts_init.rlx_bins, + dt + ); + const real_t lnrd_rng = log_rd_max - log_rd_min; + + + + // calculate bin edges (in rd3) + // tmp vector with bin edges, probably could be allocated once in init + const int n_bins = opts_init.rlx_bins * lnrd_rng / tot_lnrd_rng; + const real_t lnrd_bin_size = lnrd_rng / n_bins; + std::vector bin_rd3_left_edges(n_bins+1); // on CPU because of small number of edges + std::iota(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), 0); // fill with a 0,1,2,... sequence + std::transform(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), bin_rd3_left_edges.begin(), [log_rd_min_val=log_rd_min, lnrd_bin_size] (real_t bin_number) { return std::exp( 3 * (log_rd_min_val + bin_number * lnrd_bin_size)) ; }); // calculate left edges + + // loop over the bins + for(int i=0; i(ddi->second)).first, (std::get<1>(ddi->second)).second, kpa.begin(), false); + // out of those, select droplets within the desired rd3 range + moms_rng(rd3_min, rd3_max, rd3.begin(), true); + // calculate 0-th non-specific moment of rd3 (number of droplets in a cell) of droplets in this rd3 and kappa range + moms_calc(rd3.begin(), 0, false); + // horizontal average of this moment + thrust::fill(hor_avg.begin(), hor_avg.end(), 0); + thrust_device::vector &count_k(tmp_device_size_cell); // NOTE: tmp_device_size_cell is also used in some other inits, careful not to overwrite it! + thrust::transform(count_ijk.begin(), count_ijk.begin() + count_n, count_k.begin(), arg::_1 % opts_init.nz); + thrust::sort_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin()); + auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_avg_k.begin(), hor_avg_count.begin()); + int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); + thrust::copy(hor_avg_count.begin(), hor_avg_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_avg.begin(), hor_avg_k.begin())); + // divide sum by volume (uppermost and lowermost cells are half the height + thrust::transform(hor_avg.begin()+1, hor_avg.end()-1, hor_avg.begin()+1, arg::_1 / ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz)); + hor_avg[0] /= 0.5 * ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz); + hor_avg[opts_init.nz-1] /= 0.5 * ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz); + + + // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... + + } + } + + + // + +// // set number of SDs to init; use count_num as storage +// init_count_num_src(opts_init.src_sd_conc); + +/* + + // --- see how many already existing SDs match size bins --- + { + namespace arg = thrust::placeholders; + + // set no of particles to init + n_part_old = n_part; + n_part_to_init = thrust::reduce(count_num.begin(), count_num.end()); + n_part = n_part_old + n_part_to_init; + hskpng_resize_npart(); + + thrust_size_t n_part_bfr_src = n_part_old, + n_part_tot_in_src = n_part_to_init; + + // tmp vector with bin number of existing SDs + thrust_device::vector bin_no(n_part); + + const thrust_size_t out_of_bins = 4444444444; // would cause an error for src_sd_conc > out_of_bins + // calc bin no + thrust::transform( + sorted_rd3.begin(), + sorted_rd3.end(), + thrust::make_permutation_iterator(count_num.begin(), sorted_ijk.begin()), + bin_no.begin(), + detail::get_bin_no(log_rd_min, log_rd_max) + ); + + // init ijk and rd3 of new particles + init_ijk(); + init_dry_sd_conc(); + + // translate these new rd3 into bin no; bin_no just got resized + thrust::transform( + rd3.begin() + n_part_old, + rd3.end(), + thrust::make_permutation_iterator(count_num.begin(), ijk.begin() + n_part_old), + bin_no.begin() + n_part_old, + detail::get_bin_no(log_rd_min, log_rd_max) + ); + + // -- init new SDs that didnt have a match -- + { + thrust_device::vector tmp_bin_no(n_part_old); + thrust::copy(bin_no.begin(), bin_no.begin() + n_part_old, tmp_bin_no.begin()); + + thrust_size_t n_out_of_bins = thrust::count(tmp_bin_no.begin(), tmp_bin_no.end(), out_of_bins); + + // remove reference to those outside of bins from tmp_bin_no and sorted_ijk + thrust::remove_if( + sorted_ijk.begin(), + sorted_ijk.begin() + n_part_old, + tmp_bin_no.begin(), + arg::_1 == out_of_bins + ); + + thrust::remove( + tmp_bin_no.begin(), + tmp_bin_no.begin() + n_part_old, + out_of_bins + ); // if these two removes are done in a single step with a tuple, it fails on CUDA; TODO: report this? + + thrust_size_t count_bins; + { + // remove duplicates from tmp_bin_no + thrust::pair< + thrust_device::vector::iterator, + typename thrust_device::vector::iterator + > np = thrust::unique_by_key(tmp_bin_no.begin(), tmp_bin_no.begin() + n_part_old - n_out_of_bins, sorted_ijk.begin()); + count_bins = np.first - tmp_bin_no.begin(); // total no of bins with a match + } + + // --- remove rd3 and ijk of newly added SDs that have counterparts --- + thrust_device::vector have_match(n_part_to_init); + // find those with a match + thrust::binary_search( + thrust::make_zip_iterator(thrust::make_tuple( + sorted_ijk.begin(), + tmp_bin_no.begin() + )), + thrust::make_zip_iterator(thrust::make_tuple( + sorted_ijk.begin(), + tmp_bin_no.begin() + )) + count_bins, + thrust::make_zip_iterator(thrust::make_tuple( + ijk.begin() + n_part_old, + bin_no.begin() + n_part_old + )), + thrust::make_zip_iterator(thrust::make_tuple( + ijk.begin() + n_part_old, + bin_no.begin() + n_part_old + )) + n_part_to_init, + have_match.begin(), + detail::two_keys_sort() + ); + // remove those with a match + thrust::remove_if( + thrust::make_zip_iterator(thrust::make_tuple( + rd3.begin() + n_part_old, + ijk.begin() + n_part_old + )), + thrust::make_zip_iterator(thrust::make_tuple( + rd3.begin() + n_part_old, + ijk.begin() + n_part_old + )) + n_part_to_init, + have_match.begin(), + thrust::identity() + ); + + n_part_to_init -= count_bins; + n_part -= count_bins; + hskpng_resize_npart(); + + // init other peoperties of SDs that didnt have a match + init_kappa( + opts_init.src_dry_distros.begin()->first + ); + + if(opts_init.diag_incloud_time) + init_incloud_time(); + + init_n_sd_conc( + *(opts_init.src_dry_distros.begin()->second) + ); // TODO: document that n_of_lnrd_stp is expected! + + // init rw + init_wet(); + + // init x, y, z, i, j, k + init_xyz(); + + // TODO: init chem + + { + // count number of matched bins per cell + thrust::pair< + thrust_device::vector::iterator, + typename thrust_device::vector::iterator + > it_pair = thrust::reduce_by_key( + sorted_ijk.begin(), sorted_ijk.begin() + count_bins, + thrust::make_constant_iterator(1), + sorted_ijk.begin(), + tmp_bin_no.begin() + ); + count_bins = it_pair.first - sorted_ijk.begin(); // now it counts no of cells that have any bins matched + } + + // set count_num to the number of SDs matched per cell + // they still need to be initialized + thrust::copy( + tmp_bin_no.begin(), + tmp_bin_no.begin() + count_bins, + thrust::make_permutation_iterator(count_num.begin(), sorted_ijk.begin()) + ); + // sorted_ijk no longer valid + } + + // tmp vector to hold number of particles in a given size bin in a given cell + thrust_device::vector bin_cell_count(n_part_tot_in_src + n_cell + 1); // needs space for out_of_bins + // tmp vector for number of particles in bins up to this one + thrust_device::vector bin_cell_count_ptr(n_part_tot_in_src + n_cell + 1); + + thrust_size_t count_bins; + { + thrust_device::vector &out(bin_cell_count_ptr); // use it temporarily + // calc no of SDs in bins/cells + thrust::pair< + thrust_device::vector::iterator, + typename thrust_device::vector::iterator + > np = thrust::reduce_by_key( + bin_no.begin(), + bin_no.begin() + n_part_bfr_src, + thrust::make_constant_iterator(1), + out.begin(),// output bin no - in place didn't work well, why? + bin_cell_count.begin()// output number of SDs + ); + count_bins = np.second - bin_cell_count.begin(); // number of bins with SDs inside, includes the out_of_bins + thrust::copy(out.begin(), out.begin() + count_bins, bin_no.begin()); + } + + // number of SDs (incl. out_of_bins) in bins up to (i-1) + thrust::exclusive_scan( + bin_cell_count.begin(), + bin_cell_count.begin() + count_bins, + bin_cell_count_ptr.begin() + ); + + // remove out of bins from bin_cell_count, bins_no and bin cell_count_ptr + count_bins = + thrust::remove_if( + thrust::make_zip_iterator(thrust::make_tuple( + bin_no.begin(), + bin_cell_count.begin(), + bin_cell_count_ptr.begin() + )), + thrust::make_zip_iterator(thrust::make_tuple( + bin_no.begin(), + bin_cell_count.begin(), + bin_cell_count_ptr.begin() + )) + count_bins, + bin_no.begin(), + arg::_1 == out_of_bins + ) - + thrust::make_zip_iterator(thrust::make_tuple( + bin_no.begin(), + bin_cell_count.begin(), + bin_cell_count_ptr.begin() + )); // count_bins now does not count out_of_bins + + // randomly select which old SD will be increased + // overwrites sorted_rd3 + rand_u01(count_bins); + + // TODO: merge the following transforms into one + + // randomly choose one SD per bin + thrust::transform( + u01.begin(), + u01.begin() + count_bins, + bin_cell_count.begin(), + bin_cell_count.begin(), + thrust::multiplies() + ); + // translate no in bin to the total id + thrust::transform( + bin_cell_count_ptr.begin(), + bin_cell_count_ptr.begin() + count_bins, + bin_cell_count.begin(), + bin_cell_count.begin(), + thrust::plus() + ); + + // --- increase multiplicity of existing SDs --- + + n_part_old = n_part; // number of those before src + no of those w/o match + n_part_to_init = count_bins; // number of matched SDs + n_part = n_part_old + n_part_to_init; + hskpng_resize_npart(); + + // copy rd3 and ijk of the selected SDs to the end of the respective vectors + thrust::copy( + thrust::make_permutation_iterator( + rd3.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id + ) + ), + thrust::make_permutation_iterator( + rd3.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() + ) + ) + count_bins, + rd3.begin() + n_part_old // output + ); + thrust::copy( + thrust::make_permutation_iterator( + ijk.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id + ) + ), + thrust::make_permutation_iterator( + ijk.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() + ) + ) + count_bins, + ijk.begin() + n_part_old // output + ); + + // init n of the copied SDs, but using the src distribution + init_n_sd_conc( + *(opts_init.src_dry_distros.begin()->second) + ); // TODO: document that n_of_lnrd_stp is expected! + + // add the just-initialized multiplicities to the old ones + thrust::transform( + n.begin() + n_part_old, + n.end(), + thrust::make_permutation_iterator( + n.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id + ) + ), + thrust::make_permutation_iterator( + n.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id + ) + ), //in-place + thrust::plus() + ); + // TODO: check for overflows of na after addition + + // --- properly reduce size of the vectors back to no before src + no w/o match --- + n_part = n_part_old; + hskpng_resize_npart(); + } + */ + } + }; +}; diff --git a/src/particles.tpp b/src/particles.tpp index 4483c8829..c825572cb 100644 --- a/src/particles.tpp +++ b/src/particles.tpp @@ -114,6 +114,8 @@ #include "impl/particles_impl_src.ipp" #include "impl/particles_impl_src_dry_distros.ipp" #include "impl/particles_impl_src_dry_sizes.ipp" +#include "impl/particles_impl_rlx.ipp" +#include "impl/particles_impl_rlx_dry_distros.ipp" #include "impl/particles_impl_update_incloud_time.ipp" #include "impl/particles_impl_adjust_timesteps.ipp" diff --git a/src/particles_step.ipp b/src/particles_step.ipp index 2c3805254..a02dd69a7 100644 --- a/src/particles_step.ipp +++ b/src/particles_step.ipp @@ -277,6 +277,7 @@ namespace libcloudphxx } // aerosol relaxation, in sync since it changes th/rv + // TODO: more sanity checks for rlx! 3D only, values of rlx_bins etc. check that appa ranges are exclusive if (opts.rlx) { // sanity check From 631bec014646ce3ec895a68d44b6803f8d366257 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 12 May 2021 10:54:17 +0200 Subject: [PATCH 006/282] relaxation: calc expected concentration --- src/impl/particles_impl.ipp | 4 +- src/impl/particles_impl_rlx.ipp | 4 +- src/impl/particles_impl_rlx_dry_distros.ipp | 45 ++++++++++++++++----- src/particles_step.ipp | 2 +- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index e97367c0d..010da54df 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -567,8 +567,8 @@ namespace libcloudphxx void src_dry_distros(const real_t &dt); void src_dry_sizes(const real_t &dt); - void rlx(const real_t &dt); - void rlx_dry_distros(const real_t &dt); + void rlx(); + void rlx_dry_distros(); void sstp_step(const int &step); void sstp_step_exact(const int &step); diff --git a/src/impl/particles_impl_rlx.ipp b/src/impl/particles_impl_rlx.ipp index 9d6991806..2f37040b4 100644 --- a/src/impl/particles_impl_rlx.ipp +++ b/src/impl/particles_impl_rlx.ipp @@ -10,12 +10,12 @@ namespace libcloudphxx namespace lgrngn { template - void particles_t::impl::rlx(const real_t &dt) + void particles_t::impl::rlx() { ante_adding_SD(); if(!opts_init.rlx_dry_distros.empty()) - rlx_dry_distros(dt); + rlx_dry_distros(); post_adding_SD(); } diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 70a552f72..a740e77f6 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -14,7 +14,7 @@ namespace libcloudphxx { // create new aerosol particles to relax towards a size distribution template - void particles_t::impl::rlx_dry_distros(const real_t &dt) + void particles_t::impl::rlx_dry_distros() { namespace arg = thrust::placeholders; @@ -22,6 +22,7 @@ namespace libcloudphxx thrust::device_vector hor_avg(opts_init.nz); thrust::device_vector hor_avg_count(opts_init.nz); thrust::device_vector hor_avg_k(opts_init.nz); + thrust::device_vector expected_hor_avg(opts_init.nz); // calc sum of ln(rd) ranges of all relax distributions real_t tot_lnrd_rng = 0.; @@ -29,8 +30,7 @@ namespace libcloudphxx { dist_analysis_sd_conc( *(std::get<0>(ddi->second)), - opts_init.rlx_bins, - dt + opts_init.rlx_bins ); tot_lnrd_rng += log_rd_max - log_rd_min; } @@ -38,13 +38,14 @@ namespace libcloudphxx // initialize SDs of each kappa-type for (typename opts_init_t::rlx_dry_distros_t::const_iterator ddi = opts_init.rlx_dry_distros.begin(); ddi != opts_init.rlx_dry_distros.end(); ++ddi) { + const auto &n_of_lnrd_stp(*(std::get<0>(ddi->second))); + // analyze distribution to get rd_min and max needed for bin sizes // TODO: this was done a moment ago! // TODO2: this could be done once at the start of the simulation! dist_analysis_sd_conc( - *(std::get<0>(ddi->second)), - opts_init.rlx_bins, - dt + n_of_lnrd_stp, + opts_init.rlx_bins ); const real_t lnrd_rng = log_rd_max - log_rd_min; @@ -59,10 +60,10 @@ namespace libcloudphxx std::transform(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), bin_rd3_left_edges.begin(), [log_rd_min_val=log_rd_min, lnrd_bin_size] (real_t bin_number) { return std::exp( 3 * (log_rd_min_val + bin_number * lnrd_bin_size)) ; }); // calculate left edges // loop over the bins - for(int i=0; i(ddi->second)).first, (std::get<1>(ddi->second)).second, kpa.begin(), false); @@ -78,12 +79,36 @@ namespace libcloudphxx auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_avg_k.begin(), hor_avg_count.begin()); int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); thrust::copy(hor_avg_count.begin(), hor_avg_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_avg.begin(), hor_avg_k.begin())); - // divide sum by volume (uppermost and lowermost cells are half the height + // divide sum by volume (uppermost and lowermost cells are half the height) [1/m^3] thrust::transform(hor_avg.begin()+1, hor_avg.end()-1, hor_avg.begin()+1, arg::_1 / ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz)); hor_avg[0] /= 0.5 * ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz); hor_avg[opts_init.nz-1] /= 0.5 * ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz); + // calculate how many CCN are missing + const real_t bin_lnrd_center = log_rd_min + (bin_number + 0.5) * lnrd_bin_size; + const real_t expected_STP_concentration = n_of_lnrd_stp(bin_lnrd_center) * lnrd_bin_size; + thrust::fill(expected_hor_avg.begin(), expected_hor_avg.end(), expected_STP_concentration); + + // correcting STP -> actual ambient conditions + if(!opts_init.aerosol_independent_of_rhod) + { + using common::earth::rho_stp; + thrust::transform( + expected_hor_avg.begin(), + expected_hor_avg.begin() + opts_init.nz, + rhod.begin(), // rhod has size ncell, but vertical cooridnate varies first, so rhod.begin() to rhod.begin()+nz should be the vertical profile? + expected_hor_avg.begin(), + arg::_1 * arg::_2 / real_t(rho_stp() / si::kilograms * si::cubic_metres) + ); + } + + // account for minimum and maximum relaxation heights (zero-out expected value) + const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.nz + 0.5, + z_max_index = (std::get<2>(ddi->second)).second / opts_init.nz + 0.5; + + thrust::transform_if(thrust::make_counting_iterator(0), thrust::make_counting_iterator(opts_init.nz), hor_avg.begin(), arg::_1 = 0, arg::_1 < z_min_index || arg::_1 > z_max_index); + // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } diff --git a/src/particles_step.ipp b/src/particles_step.ipp index a02dd69a7..f30756b4c 100644 --- a/src/particles_step.ipp +++ b/src/particles_step.ipp @@ -286,7 +286,7 @@ namespace libcloudphxx // introduce new particles with the given time interval if(pimpl->rlx_stp_ctr % pimpl->opts_init.supstp_rlx == 0) { - pimpl->rlx(pimpl->opts_init.supstp_rlx * pimpl->dt); + pimpl->rlx(); } } From 938c8d66e7ef3a014ec3e7049cbe522b074d645d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 12 May 2021 11:50:49 +0200 Subject: [PATCH 007/282] python bindings for relaxation --- bindings/python/lgrngn.hpp | 33 +++++++++++++++++++++ bindings/python/lib.cpp | 5 ++++ src/impl/particles_impl_rlx_dry_distros.ipp | 9 +++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/bindings/python/lgrngn.hpp b/bindings/python/lgrngn.hpp index 458eeea67..58c1fe84e 100644 --- a/bindings/python/lgrngn.hpp +++ b/bindings/python/lgrngn.hpp @@ -319,6 +319,31 @@ namespace libcloudphxx } } + template + void set_rdd( // rlx_dry_distros + lgr::opts_init_t *arg, + const bp::dict &kappa_func + ) + { + arg->rlx_dry_distros.clear(); + if(len(kappa_func.keys()) == 0) + return; + + // loop over kappas + for (int i = 0; i < len(kappa_func.keys()); ++i) + { + const real_t kappa = bp::extract(kappa_func.keys()[i]); + const bp::list nlnrd_kparange_zrange_list = bp::extract(kappa_func.values()[i]); + assert(len(nlnrd_kparange_zrange_list) == 3); + auto nlnrd = std::make_shared>(nlnrd_kparange_zrange_list[0]); + const bp::list kparange_list = bp::extract(nlnrd_kparange_zrange_list[1]); + std::pair kparange{bp::extract(kparange_list[0]), bp::extract(kparange_list[1])}; + const bp::list zrange_list = bp::extract(nlnrd_kparange_zrange_list[2]); + std::pair zrange{bp::extract(zrange_list[0]), bp::extract(zrange_list[1])}; + arg->rlx_dry_distros[kappa] = std::make_tuple(nlnrd, kparange, zrange); + } + } + template void get_ds( lgr::opts_init_t *arg @@ -351,6 +376,14 @@ namespace libcloudphxx throw std::runtime_error("source_dry_distros does not feature a getter yet - TODO"); } + template + void get_rdd( + lgr::opts_init_t *arg + ) + { + throw std::runtime_error("relax_dry_distros does not feature a getter yet - TODO"); + } + template void set_kp( lgr::opts_init_t *arg, diff --git a/bindings/python/lib.cpp b/bindings/python/lib.cpp index e285596c4..5a0b3de70 100644 --- a/bindings/python/lib.cpp +++ b/bindings/python/lib.cpp @@ -243,6 +243,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("coal", &lgr::opts_t::coal) .def_readwrite("src", &lgr::opts_t::src) .def_readwrite("rcyc", &lgr::opts_t::rcyc) + .def_readwrite("rlx", &lgr::opts_t::rlx) .def_readwrite("chem_dsl", &lgr::opts_t::chem_dsl) .def_readwrite("chem_dsc", &lgr::opts_t::chem_dsc) .def_readwrite("chem_rct", &lgr::opts_t::chem_rct) @@ -257,6 +258,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .add_property("dry_sizes", &lgrngn::get_ds, &lgrngn::set_ds) .add_property("src_dry_distros", &lgrngn::get_sdd, &lgrngn::set_sdd) .add_property("src_dry_sizes", &lgrngn::get_ds, &lgrngn::set_sds) + .add_property("rlx_dry_distros", &lgrngn::get_rdd, &lgrngn::set_rdd) .def_readwrite("nx", &lgr::opts_init_t::nx) .def_readwrite("ny", &lgr::opts_init_t::ny) .def_readwrite("nz", &lgr::opts_init_t::nz) @@ -285,6 +287,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("sedi_switch", &lgr::opts_init_t::sedi_switch) .def_readwrite("subs_switch", &lgr::opts_init_t::subs_switch) .def_readwrite("src_switch", &lgr::opts_init_t::src_switch) + .def_readwrite("rlx_switch", &lgr::opts_init_t::rlx_switch) .def_readwrite("turb_adve_switch", &lgr::opts_init_t::turb_adve_switch) .def_readwrite("turb_cond_switch", &lgr::opts_init_t::turb_cond_switch) .def_readwrite("turb_coal_switch", &lgr::opts_init_t::turb_coal_switch) @@ -293,6 +296,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("sstp_coal", &lgr::opts_init_t::sstp_coal) .def_readwrite("sstp_chem", &lgr::opts_init_t::sstp_chem) .def_readwrite("supstp_src", &lgr::opts_init_t::supstp_src) + .def_readwrite("supstp_rlx", &lgr::opts_init_t::supstp_rlx) .def_readwrite("kernel", &lgr::opts_init_t::kernel) .def_readwrite("adve_scheme", &lgr::opts_init_t::adve_scheme) .def_readwrite("sd_conc", &lgr::opts_init_t::sd_conc) @@ -300,6 +304,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("aerosol_independent_of_rhod", &lgr::opts_init_t::aerosol_independent_of_rhod) .def_readwrite("sd_const_multi", &lgr::opts_init_t::sd_const_multi) .def_readwrite("src_sd_conc", &lgr::opts_init_t::src_sd_conc) + .def_readwrite("rlx_bins", &lgr::opts_init_t::rlx_bins) .def_readwrite("n_sd_max", &lgr::opts_init_t::n_sd_max) .def_readwrite("terminal_velocity", &lgr::opts_init_t::terminal_velocity) .def_readwrite("RH_formula", &lgr::opts_init_t::RH_formula) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index a740e77f6..cf44fd5f6 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -84,7 +84,7 @@ namespace libcloudphxx hor_avg[0] /= 0.5 * ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz); hor_avg[opts_init.nz-1] /= 0.5 * ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz); - // calculate how many CCN are missing + // calculate expected CCN number const real_t bin_lnrd_center = log_rd_min + (bin_number + 0.5) * lnrd_bin_size; const real_t expected_STP_concentration = n_of_lnrd_stp(bin_lnrd_center) * lnrd_bin_size; thrust::fill(expected_hor_avg.begin(), expected_hor_avg.end(), expected_STP_concentration); @@ -108,6 +108,13 @@ namespace libcloudphxx thrust::transform_if(thrust::make_counting_iterator(0), thrust::make_counting_iterator(opts_init.nz), hor_avg.begin(), arg::_1 = 0, arg::_1 < z_min_index || arg::_1 > z_max_index); + // calculate how many CCN are missing + thrust::device_vector &hor_missing(expected_hor_avg); + thrust::transform(expected_hor_avg.begin(), expected_hor_avg.end(), hor_avg.begin(), hor_missing.begin(), arg::_1 - arg::_2); + thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); + + std::cerr << "bin number: " << bin_number << std::endl; + debug::print(hor_missing); // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... From 6a6b3b36c9facc28da8593a13cb87ec7a3c646d2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 12 May 2021 11:58:00 +0200 Subject: [PATCH 008/282] api lgrngn test: add rlx dry distros --- tests/python/unit/api_lgrngn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/unit/api_lgrngn.py b/tests/python/unit/api_lgrngn.py index 7f233d7cc..a30fe876e 100644 --- a/tests/python/unit/api_lgrngn.py +++ b/tests/python/unit/api_lgrngn.py @@ -35,6 +35,7 @@ def lognormal(lnr): opts_init.rng_seed = 396 opts_init.rng_seed_init = 456 opts_init.src_dry_distros = {kappa1:lognormal} +opts_init.rlx_dry_distros = {kappa1:[lognormal, [0,2], [0,opts_init.dz]]} opts_init.src_sd_conc = 64 opts_init.src_z1 = opts_init.dz opts_init.diag_incloud_time = True From 7f2245850571408f0207d6819dea220db0ff3e63 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 12 May 2021 17:27:06 +0200 Subject: [PATCH 009/282] fix dv in hor avg --- bindings/python/lgrngn.hpp | 4 ++ include/libcloudph++/lgrngn/opts_init.hpp | 3 +- src/impl/particles_impl_post_adding_SD.ipp | 2 + src/impl/particles_impl_rlx_dry_distros.ipp | 48 +++++++++++++++------ src/particles.tpp | 2 + 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/bindings/python/lgrngn.hpp b/bindings/python/lgrngn.hpp index 58c1fe84e..08a93883e 100644 --- a/bindings/python/lgrngn.hpp +++ b/bindings/python/lgrngn.hpp @@ -332,14 +332,18 @@ namespace libcloudphxx // loop over kappas for (int i = 0; i < len(kappa_func.keys()); ++i) { + std::cerr << i << std::endl; const real_t kappa = bp::extract(kappa_func.keys()[i]); + std::cerr << kappa << std::endl; const bp::list nlnrd_kparange_zrange_list = bp::extract(kappa_func.values()[i]); assert(len(nlnrd_kparange_zrange_list) == 3); auto nlnrd = std::make_shared>(nlnrd_kparange_zrange_list[0]); const bp::list kparange_list = bp::extract(nlnrd_kparange_zrange_list[1]); std::pair kparange{bp::extract(kparange_list[0]), bp::extract(kparange_list[1])}; + std::cerr << kparange.first << " " << kparange.second << std::endl; const bp::list zrange_list = bp::extract(nlnrd_kparange_zrange_list[2]); std::pair zrange{bp::extract(zrange_list[0]), bp::extract(zrange_list[1])}; + std::cerr << zrange.first << " " << zrange.second << std::endl; arg->rlx_dry_distros[kappa] = std::make_tuple(nlnrd, kparange, zrange); } } diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index e3183564e..e9c5083c6 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -151,7 +151,6 @@ namespace libcloudphxx // --- aerosol relaxation stuff --- - // initial dry sizes of aerosol // defined with a distribution // uses shared_ptr to make opts_init copyable @@ -172,7 +171,6 @@ namespace libcloudphxx // timestep interval at which relaxation will be applied int supstp_rlx; - // -- ctors --- // ctor with defaults (C++03 compliant) ... @@ -192,6 +190,7 @@ namespace libcloudphxx subs_switch(false), // subsidence turned off by default coal_switch(true), // coalescence turned on by default src_switch(false), // source turned off by default + rlx_switch(false), exact_sstp_cond(false), turb_cond_switch(false), turb_adve_switch(false), diff --git a/src/impl/particles_impl_post_adding_SD.ipp b/src/impl/particles_impl_post_adding_SD.ipp index 8384a7a57..9bebe5352 100644 --- a/src/impl/particles_impl_post_adding_SD.ipp +++ b/src/impl/particles_impl_post_adding_SD.ipp @@ -12,6 +12,8 @@ namespace libcloudphxx template void particles_t::impl::post_adding_SD() { + thrust_device::vector &drv(tmp_device_real_cell1); // NOTE: this can't be changed by any function called since ante_adding_SD() was called..... + // --- after source particles are no longer sorted --- sorted = false; diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index cf44fd5f6..4514608aa 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -59,6 +59,10 @@ namespace libcloudphxx std::iota(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), 0); // fill with a 0,1,2,... sequence std::transform(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), bin_rd3_left_edges.begin(), [log_rd_min_val=log_rd_min, lnrd_bin_size] (real_t bin_number) { return std::exp( 3 * (log_rd_min_val + bin_number * lnrd_bin_size)) ; }); // calculate left edges + // minimum and maximum cell indices + const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.nz + 0.5, + z_max_index = (std::get<2>(ddi->second)).second / opts_init.nz + 0.5; + // loop over the bins for(int bin_number=0; bin_number() + ); + // horizontal average of this moment thrust::fill(hor_avg.begin(), hor_avg.end(), 0); thrust_device::vector &count_k(tmp_device_size_cell); // NOTE: tmp_device_size_cell is also used in some other inits, careful not to overwrite it! @@ -79,12 +94,11 @@ namespace libcloudphxx auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_avg_k.begin(), hor_avg_count.begin()); int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); thrust::copy(hor_avg_count.begin(), hor_avg_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_avg.begin(), hor_avg_k.begin())); - // divide sum by volume (uppermost and lowermost cells are half the height) [1/m^3] - thrust::transform(hor_avg.begin()+1, hor_avg.end()-1, hor_avg.begin()+1, arg::_1 / ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz)); - hor_avg[0] /= 0.5 * ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz); - hor_avg[opts_init.nz-1] /= 0.5 * ((opts_init.x1 - opts_init.x0) * (opts_init.y1 - opts_init.y0) * opts_init.dz); + // divide sum by the number of cells at this level + thrust::transform(hor_avg.begin(), hor_avg.end(), hor_avg.begin(), arg::_1 / (opts_init.nx * m1(opts_init.ny))); + - // calculate expected CCN number + // calculate expected CCN conc const real_t bin_lnrd_center = log_rd_min + (bin_number + 0.5) * lnrd_bin_size; const real_t expected_STP_concentration = n_of_lnrd_stp(bin_lnrd_center) * lnrd_bin_size; thrust::fill(expected_hor_avg.begin(), expected_hor_avg.end(), expected_STP_concentration); @@ -102,20 +116,30 @@ namespace libcloudphxx ); } - // account for minimum and maximum relaxation heights (zero-out expected value) - const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.nz + 0.5, - z_max_index = (std::get<2>(ddi->second)).second / opts_init.nz + 0.5; + // set to zero outside of the defined range of altitudes + thrust::transform_if(thrust::make_counting_iterator(0), thrust::make_counting_iterator(opts_init.nz), expected_hor_avg.begin(), arg::_1 = 0, arg::_1 < z_min_index || arg::_1 >= z_max_index); - thrust::transform_if(thrust::make_counting_iterator(0), thrust::make_counting_iterator(opts_init.nz), hor_avg.begin(), arg::_1 = 0, arg::_1 < z_min_index || arg::_1 > z_max_index); + std::cerr << "bin number: " << bin_number + << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) + << " r_center: " << std::exp(bin_lnrd_center) + << " z_indices: (" << z_min_index << ", " << z_max_index << "), " + << " expected STD concentration: " << expected_STP_concentration + << std::endl; + + std::cerr << "hor_avg:" << std::endl; + debug::print(hor_avg); + + std::cerr << "expected_hor_avg:" << std::endl; + debug::print(expected_hor_avg); // calculate how many CCN are missing thrust::device_vector &hor_missing(expected_hor_avg); thrust::transform(expected_hor_avg.begin(), expected_hor_avg.end(), hor_avg.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); - - std::cerr << "bin number: " << bin_number << std::endl; - debug::print(hor_missing); + std::cerr << "hor_missing:" << std::endl; + debug::print(hor_missing); + // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } diff --git a/src/particles.tpp b/src/particles.tpp index c825572cb..3b193d239 100644 --- a/src/particles.tpp +++ b/src/particles.tpp @@ -111,6 +111,8 @@ #include "impl/particles_impl_rcyc.ipp" #include "impl/particles_impl_sstp.ipp" #include "impl/particles_impl_sstp_chem.ipp" +#include "impl/particles_impl_ante_adding_SD.ipp" +#include "impl/particles_impl_post_adding_SD.ipp" #include "impl/particles_impl_src.ipp" #include "impl/particles_impl_src_dry_distros.ipp" #include "impl/particles_impl_src_dry_sizes.ipp" From 44771aabcd809d6598019f0fb27cda251ccbdc4d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 12 May 2021 18:33:29 +0200 Subject: [PATCH 010/282] relaxation: create_SD --- src/impl/particles_impl_init_count_num.ipp | 56 +++++++++++++++++++++ src/impl/particles_impl_rlx_dry_distros.ipp | 14 +++--- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/impl/particles_impl_init_count_num.ipp b/src/impl/particles_impl_init_count_num.ipp index ad3dcb414..b675a1522 100644 --- a/src/impl/particles_impl_init_count_num.ipp +++ b/src/impl/particles_impl_init_count_num.ipp @@ -154,5 +154,61 @@ namespace libcloudphxx break; } } + +/* + // 1 SD to be created per level in which concentration < tolerance * expected_concentration + + template + void particles_t::impl::init_count_num_rlx() + { + // init count_num to number, but only in cells within the source area + namespace arg = thrust::placeholders; + + // indices of cells on edges of the box in which aerosol is created + thrust_size_t i0 = opts_init.src_x0 / opts_init.dx + 0.5; + thrust_size_t i1 = opts_init.src_x1 / opts_init.dx + 0.5; + thrust_size_t j0 = opts_init.src_y0 / opts_init.dy + 0.5; + thrust_size_t j1 = opts_init.src_y1 / opts_init.dy + 0.5; + thrust_size_t k0 = opts_init.src_z0 / opts_init.dz + 0.5; + thrust_size_t k1 = opts_init.src_z1 / opts_init.dz + 0.5; + + // NOTE: some cells may be used only partially in thr super-droplet method + // e.g. when Lagrangian domain (x0, x1, etc...) is smaller than the + // Eulerian domain (0, nx*dx, etc...) + // TODO: fix for the case when x0>dx or x1<(n-1)*dx (same in the y direction, z maybe too) + + switch(n_dims) + { + case 0 : throw std::runtime_error("init_count_num_src called in 0D"); + case 1 : throw std::runtime_error("init_count_num_src called in 1D"); + case 2: + thrust::transform( + zero, + zero + n_cell, + count_num.begin(), + real_t(number) // no of SDs to create + * ((arg::_1 % opts_init.nz) < k1) + * ((arg::_1 % opts_init.nz) >= k0) + * ((arg::_1 / opts_init.nz) < i1) + * ((arg::_1 / opts_init.nz) >= i0) + ); + break; + case 3: + thrust::transform( + zero, + zero + n_cell, + count_num.begin(), + real_t(number) // no of SDs to create + * ((arg::_1 % opts_init.nz) < k1) + * ((arg::_1 % opts_init.nz) >= k0) + * ((arg::_1 / (opts_init.nz * opts_init.ny)) < i1) + * ((arg::_1 / (opts_init.nz * opts_init.ny)) >= i0) + * (((arg::_1 / opts_init.nz) % opts_init.ny) < j1) + * (((arg::_1 / opts_init.nz) % opts_init.ny) >= j0) + ); + break; + } + } + */ }; }; diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 4514608aa..6618b1482 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -23,6 +23,7 @@ namespace libcloudphxx thrust::device_vector hor_avg_count(opts_init.nz); thrust::device_vector hor_avg_k(opts_init.nz); thrust::device_vector expected_hor_avg(opts_init.nz); + thrust::device_vector create_SD(opts_init.nz); // calc sum of ln(rd) ranges of all relax distributions real_t tot_lnrd_rng = 0.; @@ -139,17 +140,18 @@ namespace libcloudphxx std::cerr << "hor_missing:" << std::endl; debug::print(hor_missing); + + // set number of SDs to init + thrust::transform(hor_missing.begin(), hor_missing.end(), create_SD.begin(), arg::_1 > 0); + + std::cerr << "create_SD:" << std::endl; + debug::print(create_SD); + // init_count_num_rlx(); // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } } - - - // - -// // set number of SDs to init; use count_num as storage -// init_count_num_src(opts_init.src_sd_conc); /* From 7bb2c497ab9285091048cc487ce8c87705184e0b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 12:01:15 +0200 Subject: [PATCH 011/282] wip --- src/detail/config.hpp | 2 + src/impl/particles_impl.ipp | 1 + src/impl/particles_impl_hskpng_ijk.ipp | 70 ++-- src/impl/particles_impl_init_count_num.ipp | 56 ---- src/impl/particles_impl_rlx_dry_distros.ipp | 333 +++----------------- 5 files changed, 83 insertions(+), 379 deletions(-) diff --git a/src/detail/config.hpp b/src/detail/config.hpp index 25c36950f..5c396b9df 100644 --- a/src/detail/config.hpp +++ b/src/detail/config.hpp @@ -30,6 +30,8 @@ namespace libcloudphxx const real_t bcond_tolerance = 2e-4; // [m]; error tolerance for position near bcond after distmem copy + const real_t rlx_conc_tolerance = 0.1; // tolerance of the relaxation scheme; new SD will be created if missing_conc/expected_conc > tolerance + // ctor config(): vt0_ln_r_min(log(5e-7)), diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 010da54df..b4c1f7e21 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -487,6 +487,7 @@ namespace libcloudphxx void hskpng_sort(); void hskpng_shuffle_and_sort(); void hskpng_count(); + void ravel_ijk(const thrust_size_t begin_shift = 0); void hskpng_ijk(); void hskpng_Tpr(); void hskpng_mfp(); diff --git a/src/impl/particles_impl_hskpng_ijk.ipp b/src/impl/particles_impl_hskpng_ijk.ipp index a06b67a3c..816b25947 100644 --- a/src/impl/particles_impl_hskpng_ijk.ipp +++ b/src/impl/particles_impl_hskpng_ijk.ipp @@ -29,57 +29,37 @@ namespace libcloudphxx }; template - void particles_t::impl::hskpng_ijk() - { - // helper functor - struct { - void operator()( - thrust_device::vector &vx, - thrust_device::vector &vi, - const real_t &vd - ) { - thrust::transform( - vx.begin(), vx.end(), // input - vi.begin(), // output - detail::divide_by_constant_and_cast(vd) // has to be done on doubles to avoid i==nx due to low precision of nvcc math - ); - } - } helper; - - if (opts_init.nx != 0) helper(x, i, opts_init.dx); - if (opts_init.ny != 0) helper(y, j, opts_init.dy); - if (opts_init.nz != 0) helper(z, k, opts_init.dz); - - // raveling i, j & k into ijk + void particles_t::impl::ravel_ijk(const thrust_size_t begin_shift) // default = 0 + { switch (n_dims) { case 0: break; case 1: - thrust::copy(i.begin(), i.end(), ijk.begin()); + thrust::copy(i.begin()+begin_shift, i.end(), ijk.begin()+begin_shift); break; case 2: namespace arg = thrust::placeholders; thrust::transform( - i.begin(), i.end(), // input - first arg - k.begin(), // input - second arg - ijk.begin(), // output + i.begin()+begin_shift, i.end(), // input - first arg + k.begin()+begin_shift, // input - second arg + ijk.begin()+begin_shift, // output arg::_1 * opts_init.nz + arg::_2 // assuming z varies first ); break; case 3: namespace arg = thrust::placeholders; thrust::transform( - i.begin(), i.end(), // input - first arg - j.begin(), // input - second arg - ijk.begin(), // output + i.begin()+begin_shift, i.end(), // input - first arg + j.begin()+begin_shift, // input - second arg + ijk.begin()+begin_shift, // output arg::_1 * (opts_init.nz * opts_init.ny) + arg::_2 * opts_init.nz ); thrust::transform( - ijk.begin(), ijk.end(), - k.begin(), - ijk.begin(), // in-place! + ijk.begin()+begin_shift, ijk.end(), + k.begin()+begin_shift, + ijk.begin()+begin_shift, // in-place! arg::_1 + arg::_2 ); // TODO: replace these two transforms with single one @@ -87,6 +67,32 @@ namespace libcloudphxx default: assert(false); } + } + + template + void particles_t::impl::hskpng_ijk() + { + // helper functor + struct { + void operator()( + thrust_device::vector &vx, + thrust_device::vector &vi, + const real_t &vd + ) { + thrust::transform( + vx.begin(), vx.end(), // input + vi.begin(), // output + detail::divide_by_constant_and_cast(vd) // has to be done on doubles to avoid i==nx due to low precision of nvcc math + ); + } + } helper; + + if (opts_init.nx != 0) helper(x, i, opts_init.dx); + if (opts_init.ny != 0) helper(y, j, opts_init.dy); + if (opts_init.nz != 0) helper(z, k, opts_init.dz); + + // raveling i, j & k into ijk + ravel_ijk(); // flagging that particles are no longer sorted sorted = false; diff --git a/src/impl/particles_impl_init_count_num.ipp b/src/impl/particles_impl_init_count_num.ipp index b675a1522..ad3dcb414 100644 --- a/src/impl/particles_impl_init_count_num.ipp +++ b/src/impl/particles_impl_init_count_num.ipp @@ -154,61 +154,5 @@ namespace libcloudphxx break; } } - -/* - // 1 SD to be created per level in which concentration < tolerance * expected_concentration - - template - void particles_t::impl::init_count_num_rlx() - { - // init count_num to number, but only in cells within the source area - namespace arg = thrust::placeholders; - - // indices of cells on edges of the box in which aerosol is created - thrust_size_t i0 = opts_init.src_x0 / opts_init.dx + 0.5; - thrust_size_t i1 = opts_init.src_x1 / opts_init.dx + 0.5; - thrust_size_t j0 = opts_init.src_y0 / opts_init.dy + 0.5; - thrust_size_t j1 = opts_init.src_y1 / opts_init.dy + 0.5; - thrust_size_t k0 = opts_init.src_z0 / opts_init.dz + 0.5; - thrust_size_t k1 = opts_init.src_z1 / opts_init.dz + 0.5; - - // NOTE: some cells may be used only partially in thr super-droplet method - // e.g. when Lagrangian domain (x0, x1, etc...) is smaller than the - // Eulerian domain (0, nx*dx, etc...) - // TODO: fix for the case when x0>dx or x1<(n-1)*dx (same in the y direction, z maybe too) - - switch(n_dims) - { - case 0 : throw std::runtime_error("init_count_num_src called in 0D"); - case 1 : throw std::runtime_error("init_count_num_src called in 1D"); - case 2: - thrust::transform( - zero, - zero + n_cell, - count_num.begin(), - real_t(number) // no of SDs to create - * ((arg::_1 % opts_init.nz) < k1) - * ((arg::_1 % opts_init.nz) >= k0) - * ((arg::_1 / opts_init.nz) < i1) - * ((arg::_1 / opts_init.nz) >= i0) - ); - break; - case 3: - thrust::transform( - zero, - zero + n_cell, - count_num.begin(), - real_t(number) // no of SDs to create - * ((arg::_1 % opts_init.nz) < k1) - * ((arg::_1 % opts_init.nz) >= k0) - * ((arg::_1 / (opts_init.nz * opts_init.ny)) < i1) - * ((arg::_1 / (opts_init.nz * opts_init.ny)) >= i0) - * (((arg::_1 / opts_init.nz) % opts_init.ny) < j1) - * (((arg::_1 / opts_init.nz) % opts_init.ny) >= j0) - ); - break; - } - } - */ }; }; diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 6618b1482..e66d76ad1 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -61,8 +61,8 @@ namespace libcloudphxx std::transform(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), bin_rd3_left_edges.begin(), [log_rd_min_val=log_rd_min, lnrd_bin_size] (real_t bin_number) { return std::exp( 3 * (log_rd_min_val + bin_number * lnrd_bin_size)) ; }); // calculate left edges // minimum and maximum cell indices - const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.nz + 0.5, - z_max_index = (std::get<2>(ddi->second)).second / opts_init.nz + 0.5; + const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.nz, + z_max_index = (std::get<2>(ddi->second)).second / opts_init.nz; // loop over the bins for(int bin_number=0; bin_number(0), thrust::make_counting_iterator(opts_init.nz), expected_hor_avg.begin(), arg::_1 = 0, arg::_1 < z_min_index || arg::_1 >= z_max_index); + thrust::transform_if(zero, zero+opts_init.nz, expected_hor_avg.begin(), arg::_1 = 0, arg::_1 < z_min_index || arg::_1 > z_max_index); std::cerr << "bin number: " << bin_number << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) @@ -141,313 +141,64 @@ namespace libcloudphxx std::cerr << "hor_missing:" << std::endl; debug::print(hor_missing); - // set number of SDs to init - thrust::transform(hor_missing.begin(), hor_missing.end(), create_SD.begin(), arg::_1 > 0); + // set number of SDs to init; create only if concentration is lower than expected with a tolerance + thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_avg.begin(), create_SD.begin(), arg::_2 > 0 && arg::_1 / arg::_2 > config.rlx_conc_tolerance); // WARNING: watch out for div by 0 std::cerr << "create_SD:" << std::endl; debug::print(create_SD); - // init_count_num_rlx(); - - // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... - - } - } - -/* - - // --- see how many already existing SDs match size bins --- - { - namespace arg = thrust::placeholders; - - // set no of particles to init - n_part_old = n_part; - n_part_to_init = thrust::reduce(count_num.begin(), count_num.end()); - n_part = n_part_old + n_part_to_init; - hskpng_resize_npart(); - - thrust_size_t n_part_bfr_src = n_part_old, - n_part_tot_in_src = n_part_to_init; - - // tmp vector with bin number of existing SDs - thrust_device::vector bin_no(n_part); - - const thrust_size_t out_of_bins = 4444444444; // would cause an error for src_sd_conc > out_of_bins - // calc bin no - thrust::transform( - sorted_rd3.begin(), - sorted_rd3.end(), - thrust::make_permutation_iterator(count_num.begin(), sorted_ijk.begin()), - bin_no.begin(), - detail::get_bin_no(log_rd_min, log_rd_max) - ); - - // init ijk and rd3 of new particles - init_ijk(); - init_dry_sd_conc(); - // translate these new rd3 into bin no; bin_no just got resized - thrust::transform( - rd3.begin() + n_part_old, - rd3.end(), - thrust::make_permutation_iterator(count_num.begin(), ijk.begin() + n_part_old), - bin_no.begin() + n_part_old, - detail::get_bin_no(log_rd_min, log_rd_max) - ); + n_part_old = n_part; + n_part_to_init = thrust::reduce(create_SD.begin(), create_SD.end()); + n_part = n_part_old + n_part_to_init; - // -- init new SDs that didnt have a match -- - { - thrust_device::vector tmp_bin_no(n_part_old); - thrust::copy(bin_no.begin(), bin_no.begin() + n_part_old, tmp_bin_no.begin()); + // resize cell indices, resize should be cheap, because we allocate a large chunk of memory at the start + ijk.resize(n_part); + i.resize(n_part); + k.resize(n_part); + if(n_dims==3) j.resize(n_part); // we dont check in i and k because relax works only in 2D and 3D - thrust_size_t n_out_of_bins = thrust::count(tmp_bin_no.begin(), tmp_bin_no.end(), out_of_bins); - - // remove reference to those outside of bins from tmp_bin_no and sorted_ijk - thrust::remove_if( - sorted_ijk.begin(), - sorted_ijk.begin() + n_part_old, - tmp_bin_no.begin(), - arg::_1 == out_of_bins - ); + // k index based on create_SD + thrust::copy_if(zero, zero+opts_init.nz, create_SD.begin(), k.begin()+n_part_old, arg::_1 == 1); + // i and j random + // tossing random numbers [0,1) (or is it [0,1]??) TODO: do it once for all bins + rand_u01(n_part_to_init * (n_dims-1)); + // WARNING: might crash if u01 can have value 1, double check if it can + thrust::transform(u01.begin(), u01.begin() + n_part_to_init, i.begin(), arg::_1 * opts_init.nx); + if(n_dims==3) thrust::transform(u01.begin() + n_part_to_init, u01.begin() + 2*n_part_to_init, j.begin(), arg::_1 * opts_init.ny); - thrust::remove( - tmp_bin_no.begin(), - tmp_bin_no.begin() + n_part_old, - out_of_bins - ); // if these two removes are done in a single step with a tuple, it fails on CUDA; TODO: report this? + // raveling i, j & k into ijk; only of the new SD + ravel_ijk(n_part_old); - thrust_size_t count_bins; - { - // remove duplicates from tmp_bin_no - thrust::pair< - thrust_device::vector::iterator, - typename thrust_device::vector::iterator - > np = thrust::unique_by_key(tmp_bin_no.begin(), tmp_bin_no.begin() + n_part_old - n_out_of_bins, sorted_ijk.begin()); - count_bins = np.first - tmp_bin_no.begin(); // total no of bins with a match - } + // set count_num to the number of SD to init per cell + thrust::fill(count_num.begin(), count_num.end(), 0); + thrust::scatter(thrust::make_constant_iterator(1), thrust::make_constant_iterator(1) + n_part_to_init, ijk.begin() + n_part_old, count_num.begin()); - // --- remove rd3 and ijk of newly added SDs that have counterparts --- - thrust_device::vector have_match(n_part_to_init); - // find those with a match - thrust::binary_search( - thrust::make_zip_iterator(thrust::make_tuple( - sorted_ijk.begin(), - tmp_bin_no.begin() - )), - thrust::make_zip_iterator(thrust::make_tuple( - sorted_ijk.begin(), - tmp_bin_no.begin() - )) + count_bins, - thrust::make_zip_iterator(thrust::make_tuple( - ijk.begin() + n_part_old, - bin_no.begin() + n_part_old - )), - thrust::make_zip_iterator(thrust::make_tuple( - ijk.begin() + n_part_old, - bin_no.begin() + n_part_old - )) + n_part_to_init, - have_match.begin(), - detail::two_keys_sort() - ); - // remove those with a match - thrust::remove_if( - thrust::make_zip_iterator(thrust::make_tuple( - rd3.begin() + n_part_old, - ijk.begin() + n_part_old - )), - thrust::make_zip_iterator(thrust::make_tuple( - rd3.begin() + n_part_old, - ijk.begin() + n_part_old - )) + n_part_to_init, - have_match.begin(), - thrust::identity() - ); - - n_part_to_init -= count_bins; - n_part -= count_bins; - hskpng_resize_npart(); + std::cerr << "i:" << std::endl; + debug::print(i.begin()+n_part_old, i.end()); - // init other peoperties of SDs that didnt have a match - init_kappa( - opts_init.src_dry_distros.begin()->first - ); + std::cerr << "k:" << std::endl; + debug::print(k.begin()+n_part_old, k.end()); - if(opts_init.diag_incloud_time) - init_incloud_time(); + std::cerr << "ijk:" << std::endl; + debug::print(ijk.begin()+n_part_old, ijk.end()); - init_n_sd_conc( - *(opts_init.src_dry_distros.begin()->second) - ); // TODO: document that n_of_lnrd_stp is expected! + std::cerr << "count_num:" << std::endl; + debug::print(count_num); - // init rw - init_wet(); + // init i,j,k, (random i and j, k from create SD) + // init count num + // init ijk + // other inits, TODO: how to make these init not in the bins loop? - // init x, y, z, i, j, k - init_xyz(); - - // TODO: init chem - - { - // count number of matched bins per cell - thrust::pair< - thrust_device::vector::iterator, - typename thrust_device::vector::iterator - > it_pair = thrust::reduce_by_key( - sorted_ijk.begin(), sorted_ijk.begin() + count_bins, - thrust::make_constant_iterator(1), - sorted_ijk.begin(), - tmp_bin_no.begin() - ); - count_bins = it_pair.first - sorted_ijk.begin(); // now it counts no of cells that have any bins matched - } + // init_count_num_rlx(); - // set count_num to the number of SDs matched per cell - // they still need to be initialized - thrust::copy( - tmp_bin_no.begin(), - tmp_bin_no.begin() + count_bins, - thrust::make_permutation_iterator(count_num.begin(), sorted_ijk.begin()) - ); - // sorted_ijk no longer valid - } + // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... - // tmp vector to hold number of particles in a given size bin in a given cell - thrust_device::vector bin_cell_count(n_part_tot_in_src + n_cell + 1); // needs space for out_of_bins - // tmp vector for number of particles in bins up to this one - thrust_device::vector bin_cell_count_ptr(n_part_tot_in_src + n_cell + 1); + // at the end we need to set sorting=false - thrust_size_t count_bins; - { - thrust_device::vector &out(bin_cell_count_ptr); // use it temporarily - // calc no of SDs in bins/cells - thrust::pair< - thrust_device::vector::iterator, - typename thrust_device::vector::iterator - > np = thrust::reduce_by_key( - bin_no.begin(), - bin_no.begin() + n_part_bfr_src, - thrust::make_constant_iterator(1), - out.begin(),// output bin no - in place didn't work well, why? - bin_cell_count.begin()// output number of SDs - ); - count_bins = np.second - bin_cell_count.begin(); // number of bins with SDs inside, includes the out_of_bins - thrust::copy(out.begin(), out.begin() + count_bins, bin_no.begin()); } - - // number of SDs (incl. out_of_bins) in bins up to (i-1) - thrust::exclusive_scan( - bin_cell_count.begin(), - bin_cell_count.begin() + count_bins, - bin_cell_count_ptr.begin() - ); - - // remove out of bins from bin_cell_count, bins_no and bin cell_count_ptr - count_bins = - thrust::remove_if( - thrust::make_zip_iterator(thrust::make_tuple( - bin_no.begin(), - bin_cell_count.begin(), - bin_cell_count_ptr.begin() - )), - thrust::make_zip_iterator(thrust::make_tuple( - bin_no.begin(), - bin_cell_count.begin(), - bin_cell_count_ptr.begin() - )) + count_bins, - bin_no.begin(), - arg::_1 == out_of_bins - ) - - thrust::make_zip_iterator(thrust::make_tuple( - bin_no.begin(), - bin_cell_count.begin(), - bin_cell_count_ptr.begin() - )); // count_bins now does not count out_of_bins - - // randomly select which old SD will be increased - // overwrites sorted_rd3 - rand_u01(count_bins); - - // TODO: merge the following transforms into one - - // randomly choose one SD per bin - thrust::transform( - u01.begin(), - u01.begin() + count_bins, - bin_cell_count.begin(), - bin_cell_count.begin(), - thrust::multiplies() - ); - // translate no in bin to the total id - thrust::transform( - bin_cell_count_ptr.begin(), - bin_cell_count_ptr.begin() + count_bins, - bin_cell_count.begin(), - bin_cell_count.begin(), - thrust::plus() - ); - - // --- increase multiplicity of existing SDs --- - - n_part_old = n_part; // number of those before src + no of those w/o match - n_part_to_init = count_bins; // number of matched SDs - n_part = n_part_old + n_part_to_init; - hskpng_resize_npart(); - - // copy rd3 and ijk of the selected SDs to the end of the respective vectors - thrust::copy( - thrust::make_permutation_iterator( - rd3.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id - ) - ), - thrust::make_permutation_iterator( - rd3.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() - ) - ) + count_bins, - rd3.begin() + n_part_old // output - ); - thrust::copy( - thrust::make_permutation_iterator( - ijk.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id - ) - ), - thrust::make_permutation_iterator( - ijk.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() - ) - ) + count_bins, - ijk.begin() + n_part_old // output - ); - - // init n of the copied SDs, but using the src distribution - init_n_sd_conc( - *(opts_init.src_dry_distros.begin()->second) - ); // TODO: document that n_of_lnrd_stp is expected! - - // add the just-initialized multiplicities to the old ones - thrust::transform( - n.begin() + n_part_old, - n.end(), - thrust::make_permutation_iterator( - n.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id - ) - ), - thrust::make_permutation_iterator( - n.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id - ) - ), //in-place - thrust::plus() - ); - // TODO: check for overflows of na after addition - - // --- properly reduce size of the vectors back to no before src + no w/o match --- - n_part = n_part_old; - hskpng_resize_npart(); } - */ + } }; }; From e726024bee2fc157e34b2b22e456454c62a213c5 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 12:01:27 +0200 Subject: [PATCH 012/282] add relaxation test --- tests/python/unit/relax.py | 128 +++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/python/unit/relax.py diff --git a/tests/python/unit/relax.py b/tests/python/unit/relax.py new file mode 100644 index 000000000..e988b44f0 --- /dev/null +++ b/tests/python/unit/relax.py @@ -0,0 +1,128 @@ +import sys +#try: +# import boost.mpi +#except: +# pass + +sys.path.insert(0, "../../bindings/python/") +sys.path.insert(0, "../../../build/bindings/python/") + +from libcloudphxx import lgrngn + +from numpy import array as arr_t, frombuffer + +from math import exp, log, sqrt, pi +from time import time + +def lognormal(lnr): + mean_r = .04e-6 / 2 + stdev = 1.4 + n_tot = 60e6 + return n_tot * exp( + -pow((lnr - log(mean_r)), 2) / 2 / pow(log(stdev),2) + ) / log(stdev) / sqrt(2*pi); + +#def lognormal_rlx(lnr): +# mean_r = .10e-6 / 2 +# stdev = 1.4 +# n_tot = 60e4 +# return n_tot * exp( +# -pow((lnr - log(mean_r)), 2) / 2 / pow(log(stdev),2) +# ) / log(stdev) / sqrt(2*pi); + +def test(opts_init): + opts_init.supstp_rlx = 50 + opts_init.rng_seed = int(time()) + opts_init.rng_seed_init = int(time()) + print('rng_seed:', opts_init.rng_seed) + print('rng_seed_init:', opts_init.rng_seed_init) + opts_init.dt = 1 + opts_init.nx = 2; + opts_init.nz = 2; + opts_init.dx=1.; + opts_init.dz=1.; + opts_init.x0=0.; + opts_init.z0=0.; + opts_init.x1=opts_init.nx * opts_init.dx; + opts_init.z1=opts_init.nz * opts_init.dz; + opts_init.aerosol_independent_of_rhod=1; + + opts_init.chem_switch = 0; + opts_init.coal_switch = 0; + opts_init.adve_switch = 0; + opts_init.cond_switch = 0; + opts_init.sedi_switch = 0; + opts_init.src_switch = 0; + opts_init.rlx_switch = 1; + + opts_init.src_z0 = 0; + opts_init.src_z1 = opts_init.dz; #create aerosol only in the lower cells + opts_init.src_x0 = 0; + opts_init.src_x1 = opts_init.dx*opts_init.nx; + + opts = lgrngn.opts_t() + + opts.adve = 0; + opts.chem = 0; + opts.sedi = 0; + opts.coal = 0; + opts.cond = 0; + + rhod = arr_t([[ 1., 1. ],[ 1., 1. ]]) + th = arr_t([[300., 300. ],[ 300., 300. ]]) + rv = arr_t([[ .01, .01],[ .01, .01]]) + + try: + prtcls = lgrngn.factory(lgrngn.backend_t.OpenMP, opts_init) + except: + prtcls = lgrngn.factory(lgrngn.backend_t.serial, opts_init) + + prtcls.init(th, rv, rhod) + + # 100 steps during which number of droplets should be doubled in two calls to src + opts.rlx = 1 + for i in range(100): + prtcls.step_sync(opts,th,rv,rhod) + prtcls.step_async(opts) + + prtcls.diag_all() + prtcls.diag_sd_conc() + sd_conc = frombuffer(prtcls.outbuf()).copy() + + prtcls.diag_all() + prtcls.diag_wet_mom(0) + wet_mom0 = frombuffer(prtcls.outbuf()).copy() + + prtcls.diag_all() + prtcls.diag_wet_mom(1) + wet_mom1 = frombuffer(prtcls.outbuf()).copy() + + return sd_conc, wet_mom0, wet_mom1 + +# test source with dry_distros +kappa = .61 +opts_init = lgrngn.opts_init_t() +opts_init.dry_distros = {kappa:lognormal} +opts_init.rlx_dry_distros = {kappa: [lognormal, [0,2],[0,2*opts_init.dz]]} +opts_init.sd_conc = 2#100#24 +opts_init.rlx_bins = 2# 512 +opts_init.n_sd_max = int((opts_init.sd_conc * 2 + opts_init.rlx_bins * 2) * 2) # assuming nx=nz=2 + +print(' --- dry_distros rlx ---') + +sd_conc, wet_mom0, wet_mom1 = test(opts_init) + +print('diag_sd_conc', sd_conc) +#if not((sd_conc[0] == 1164 or sd_conc[0] == 1165) and (sd_conc[2] == 1164 or sd_conc[2] == 1165)): +# raise Exception("wrong amount of SDs were added") +#if not(sd_conc[1] == 1024 and sd_conc[3] == 1024): +# raise Exception("SDs were added in wrong cells") + +print(('wet mom0', wet_mom0)) +if (abs( 2 - (wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) ) > 0.015): + raise Exception("incorrect multiplicity after source") + +print(('wet mom1', wet_mom1)) +if (abs( (7.84 / 2.12) - (wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) ) > 0.015): + raise Exception("incorrect radius after source") + From 9b010702c212379cf9bce0850f3eb220a93873cd Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 12:10:21 +0200 Subject: [PATCH 013/282] rng_seed_init used only if rng_seed_init_switch is true --- bindings/python/lib.cpp | 1 + include/libcloudph++/lgrngn/opts_init.hpp | 8 ++++++-- src/particles_init.ipp | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bindings/python/lib.cpp b/bindings/python/lib.cpp index e285596c4..60e23a980 100644 --- a/bindings/python/lib.cpp +++ b/bindings/python/lib.cpp @@ -307,6 +307,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("RH_max", &lgr::opts_init_t::RH_max) .def_readwrite("rng_seed", &lgr::opts_init_t::rng_seed) .def_readwrite("rng_seed_init", &lgr::opts_init_t::rng_seed_init) + .def_readwrite("rng_seed_init_switch", &lgr::opts_init_t::rng_seed_init_switch) .def_readwrite("diag_incloud_time", &lgr::opts_init_t::diag_incloud_time) .add_property("w_LS", &lgrngn::get_w_LS, &lgrngn::set_w_LS) .add_property("SGS_mix_len", &lgrngn::get_SGS_mix_len, &lgrngn::set_SGS_mix_len) diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index fa57407bc..3408fb588 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -126,6 +126,9 @@ namespace libcloudphxx int rng_seed, rng_seed_init; // seed used to init SD (positions and dry sizes) + // should the separate rng seed for initialization be used? + bool rng_seed_init_switch; + // no of GPUs per MPI node to use, 0 for all available int dev_count; @@ -171,7 +174,7 @@ namespace libcloudphxx RH_max(.95), // value seggested in Lebo and Seinfeld 2011 chem_rho(0), // dry particle density //TODO add checking if the user gave a different value (np w init) (was 1.8e-3) rng_seed(44), - rng_seed_init(rng_seed), + rng_seed_init(44), terminal_velocity(vt_t::undefined), kernel(kernel_t::undefined), adve_scheme(as_t::implicit), @@ -191,7 +194,8 @@ namespace libcloudphxx no_ccn_at_init(false), open_side_walls(false), periodic_topbot_walls(false), - variable_dt_switch(false) + variable_dt_switch(false), + rng_seed_init_switch(false) {} // dtor (just to silence -Winline warnings) diff --git a/src/particles_init.ipp b/src/particles_init.ipp index 672682453..5fcf9554c 100644 --- a/src/particles_init.ipp +++ b/src/particles_init.ipp @@ -28,7 +28,8 @@ namespace libcloudphxx pimpl->init_sanity_check(th, rv, rhod, p, courant_x, courant_y, courant_z, ambient_chem); // set rng seed to be used during init - pimpl->rng.reseed(pimpl->opts_init.rng_seed_init); + if(pimpl->opts_init.rng_seed_init_switch) + pimpl->rng.reseed(pimpl->opts_init.rng_seed_init); // is a constant pressure profile used? pimpl->const_p = !p.is_null(); From 4382efa866299167c17e5fd06fbf9b17f497dd2c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 12:11:13 +0200 Subject: [PATCH 014/282] rng_seed_init_switch in api_lgrngn --- tests/python/unit/api_lgrngn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/unit/api_lgrngn.py b/tests/python/unit/api_lgrngn.py index 7f233d7cc..75b96c815 100644 --- a/tests/python/unit/api_lgrngn.py +++ b/tests/python/unit/api_lgrngn.py @@ -34,6 +34,7 @@ def lognormal(lnr): opts_init.n_sd_max = int(1e6) # some space for tail SDs opts_init.rng_seed = 396 opts_init.rng_seed_init = 456 +opts_init.rng_seed_init_switch = True opts_init.src_dry_distros = {kappa1:lognormal} opts_init.src_sd_conc = 64 opts_init.src_z1 = opts_init.dz From 4550b003010bc170f06c881b4bf5a00bbdf6da23 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 13:05:43 +0200 Subject: [PATCH 015/282] clarify random number generation comments --- src/impl/particles_impl.ipp | 4 ++-- src/impl/particles_impl_init_dry_const_multi.ipp | 2 +- src/impl/particles_impl_init_dry_sd_conc.ipp | 2 +- src/impl/particles_impl_init_xyz.ipp | 2 +- tests/python/unit/relax.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index b4c1f7e21..d5c261727 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -276,10 +276,10 @@ namespace libcloudphxx // methods - // fills u01[0:n] with random numbers + // fills u01 with n random real numbers uniformly distributed in range [0,1) on CPU or (0,1] on GPU (!) void rand_u01(thrust_size_t n) { rng.generate_n(u01, n); } - // fills un[0:n] with random numbers + // fills un with n random integers uniformly distributed on the whole integer range void rand_un(thrust_size_t n) { rng.generate_n(un, n); } // max(1, n) diff --git a/src/impl/particles_impl_init_dry_const_multi.ipp b/src/impl/particles_impl_init_dry_const_multi.ipp index 7528051a0..28f86cf44 100644 --- a/src/impl/particles_impl_init_dry_const_multi.ipp +++ b/src/impl/particles_impl_init_dry_const_multi.ipp @@ -54,7 +54,7 @@ namespace libcloudphxx detail::calc_CDF(n_of_lnrd_stp, log_rd_min, log_rd_max, config.bin_precision, cdf); - // tossing random numbers [0,1] for dry radii + // tossing random numbers rand_u01(n_part_to_init); // rd3 temporarily means logarithm of radius! diff --git a/src/impl/particles_impl_init_dry_sd_conc.ipp b/src/impl/particles_impl_init_dry_sd_conc.ipp index ee2c461ca..fa4ebb2fe 100644 --- a/src/impl/particles_impl_init_dry_sd_conc.ipp +++ b/src/impl/particles_impl_init_dry_sd_conc.ipp @@ -42,7 +42,7 @@ namespace libcloudphxx template void particles_t::impl::init_dry_sd_conc() { - // tossing random numbers [0,1] for dry radii + // tossing random numbers rand_u01(n_part_to_init); // rd3 temporarily means logarithm of radius! diff --git a/src/impl/particles_impl_init_xyz.ipp b/src/impl/particles_impl_init_xyz.ipp index 36399b634..1fc6b6782 100644 --- a/src/impl/particles_impl_init_xyz.ipp +++ b/src/impl/particles_impl_init_xyz.ipp @@ -99,7 +99,7 @@ namespace libcloudphxx { if (n[ix] == 0) continue; - // tossing random numbers [0,1] + // tossing random numbers rand_u01(n_part_to_init); // shifting from [0,1] to random position within respective cell diff --git a/tests/python/unit/relax.py b/tests/python/unit/relax.py index e988b44f0..dd5da350d 100644 --- a/tests/python/unit/relax.py +++ b/tests/python/unit/relax.py @@ -33,9 +33,9 @@ def lognormal(lnr): def test(opts_init): opts_init.supstp_rlx = 50 opts_init.rng_seed = int(time()) - opts_init.rng_seed_init = int(time()) +# opts_init.rng_seed_init = int(time()) print('rng_seed:', opts_init.rng_seed) - print('rng_seed_init:', opts_init.rng_seed_init) +# print('rng_seed_init:', opts_init.rng_seed_init) opts_init.dt = 1 opts_init.nx = 2; opts_init.nz = 2; From 944eb4af06ced417baa6e63fac52b5b114aea824 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 13:14:25 +0200 Subject: [PATCH 016/282] rand_u01: [0,1) range also on GPU --- src/detail/urand.hpp | 13 +++++++++---- src/impl/particles_impl.ipp | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/detail/urand.hpp b/src/detail/urand.hpp index 6e8213be5..02ee2ac2b 100644 --- a/src/detail/urand.hpp +++ b/src/detail/urand.hpp @@ -66,7 +66,7 @@ namespace libcloudphxx const thrust_size_t n ) { // note: generate_n copies the third argument!!! - std::generate_n(u01.begin(), n, fnctr_u01({engine, dist_u01})); + std::generate_n(u01.begin(), n, fnctr_u01({engine, dist_u01})); // [0,1) range } void generate_normal_n( @@ -131,10 +131,12 @@ namespace libcloudphxx const thrust_size_t n ) { - int status = curandGenerateUniform(gen, thrust::raw_pointer_cast(v.data()), n); + int status = curandGenerateUniform(gen, thrust::raw_pointer_cast(v.data()), n); // (0,1] range assert(status == CURAND_STATUS_SUCCESS /* && "curandGenerateUniform failed"*/); _unused(status); - + // shift into the expected [0,1) range + namespace arg = thrust::placeholders; + thrust::transform(v.begin(), v.begin() + n, float(1) - arg::_1); } void generate_n( @@ -142,9 +144,12 @@ namespace libcloudphxx const thrust_size_t n ) { - int status = curandGenerateUniformDouble(gen, thrust::raw_pointer_cast(v.data()), n); + int status = curandGenerateUniformDouble(gen, thrust::raw_pointer_cast(v.data()), n); // (0,1] range assert(status == CURAND_STATUS_SUCCESS /* && "curandGenerateUniform failed"*/); _unused(status); + // shift into the expected [0,1) range + namespace arg = thrust::placeholders; + thrust::transform(v.begin(), v.begin() + n, double(1) - arg::_1); } void generate_normal_n( diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index d5c261727..ef7929f5a 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -276,7 +276,7 @@ namespace libcloudphxx // methods - // fills u01 with n random real numbers uniformly distributed in range [0,1) on CPU or (0,1] on GPU (!) + // fills u01 with n random real numbers uniformly distributed in range [0,1) void rand_u01(thrust_size_t n) { rng.generate_n(u01, n); } // fills un with n random integers uniformly distributed on the whole integer range From ec9f2f806466292e6e3c5d82d04509299f77e439 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 13:41:56 +0200 Subject: [PATCH 017/282] relaxation: fix i and j init --- src/impl/particles_impl_hskpng_ijk.ipp | 2 +- src/impl/particles_impl_init_xyz.ipp | 1 + src/impl/particles_impl_rlx_dry_distros.ipp | 30 ++++++++++++++++----- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/impl/particles_impl_hskpng_ijk.ipp b/src/impl/particles_impl_hskpng_ijk.ipp index 816b25947..eee8467ce 100644 --- a/src/impl/particles_impl_hskpng_ijk.ipp +++ b/src/impl/particles_impl_hskpng_ijk.ipp @@ -82,7 +82,7 @@ namespace libcloudphxx thrust::transform( vx.begin(), vx.end(), // input vi.begin(), // output - detail::divide_by_constant_and_cast(vd) // has to be done on doubles to avoid i==nx due to low precision of nvcc math + detail::divide_by_constant_and_cast(vd) // has to be done on doubles to avoid i==nx due to low precision of nvcc math; TODO: now that rand uniform has range [0,1), float might be good here? ); } } helper; diff --git a/src/impl/particles_impl_init_xyz.ipp b/src/impl/particles_impl_init_xyz.ipp index 1fc6b6782..f34f0c137 100644 --- a/src/impl/particles_impl_init_xyz.ipp +++ b/src/impl/particles_impl_init_xyz.ipp @@ -103,6 +103,7 @@ namespace libcloudphxx rand_u01(n_part_to_init); // shifting from [0,1] to random position within respective cell + // TODO: now the rand range is [0,1), include this here { namespace arg = thrust::placeholders; thrust::transform( diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index e66d76ad1..e956554a8 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -12,6 +12,23 @@ namespace libcloudphxx { namespace lgrngn { + namespace detail + { + /// @brief returns ret_t(x*c) + template + struct multiply_by_constant_and_cast + { + arg_t c; + multiply_by_constant_and_cast(arg_t c) : c(c) {} + + BOOST_GPU_ENABLED + ret_t operator()(arg_t x) + { + return ret_t(x*c); + } + }; + }; + // create new aerosol particles to relax towards a size distribution template void particles_t::impl::rlx_dry_distros() @@ -160,11 +177,14 @@ namespace libcloudphxx // k index based on create_SD thrust::copy_if(zero, zero+opts_init.nz, create_SD.begin(), k.begin()+n_part_old, arg::_1 == 1); // i and j random - // tossing random numbers [0,1) (or is it [0,1]??) TODO: do it once for all bins + // tossing random numbers [0,1) TODO: do it once for all bins rand_u01(n_part_to_init * (n_dims-1)); - // WARNING: might crash if u01 can have value 1, double check if it can - thrust::transform(u01.begin(), u01.begin() + n_part_to_init, i.begin(), arg::_1 * opts_init.nx); - if(n_dims==3) thrust::transform(u01.begin() + n_part_to_init, u01.begin() + 2*n_part_to_init, j.begin(), arg::_1 * opts_init.ny); + + std::cerr << "u01:" << std::endl; + debug::print(u01.begin(), u01.begin()+n_part_to_init); + + thrust::transform(u01.begin(), u01.begin() + n_part_to_init, i.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.nx)); + if(n_dims==3) thrust::transform(u01.begin() + n_part_to_init, u01.begin() + 2*n_part_to_init, j.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.ny)); // raveling i, j & k into ijk; only of the new SD ravel_ijk(n_part_old); @@ -190,8 +210,6 @@ namespace libcloudphxx // init ijk // other inits, TODO: how to make these init not in the bins loop? - // init_count_num_rlx(); - // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... // at the end we need to set sorting=false From b5c13bff0b4edec1147ada666f9c03b868aa4934 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 15:28:38 +0200 Subject: [PATCH 018/282] moms functions with an option to select on how many particles to include --- src/impl/particles_impl.ipp | 12 ++++++++++ src/impl/particles_impl_moms.ipp | 38 +++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index ef7929f5a..2c1543f3e 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -512,8 +512,20 @@ namespace libcloudphxx void moms_rng( const real_t &min, const real_t &max, const typename thrust_device::vector::iterator &vec_bgn, + const thrust_size_t npart, const bool cons ); + void moms_rng( + const real_t &min, const real_t &max, + const typename thrust_device::vector::iterator &vec_bgn, + const bool cons + ); + void moms_calc( + const typename thrust_device::vector::iterator &vec_bgn, + const thrust_size_t npart, + const real_t power, + const bool specific = true + ); void moms_calc( const typename thrust_device::vector::iterator &vec_bgn, const real_t power, diff --git a/src/impl/particles_impl_moms.ipp b/src/impl/particles_impl_moms.ipp index 5f52c5d6b..4baac9cd8 100644 --- a/src/impl/particles_impl_moms.ipp +++ b/src/impl/particles_impl_moms.ipp @@ -65,6 +65,7 @@ namespace libcloudphxx void particles_t::impl::moms_rng( const real_t &min, const real_t &max, const typename thrust_device::vector::iterator &vec_bgn, + const thrust_size_t npart, const bool cons // is it a consecutive selection after previous one ) { @@ -75,21 +76,31 @@ namespace libcloudphxx if(!cons) thrust::transform( - n.begin(), n.end(), // input - 1st arg - vec_bgn, // input - 2nd arg - n_filtered.begin(), // output + n.begin(), n.begin() + npart, // input - 1st arg + vec_bgn, // input - 2nd arg + n_filtered.begin(), // output detail::range_filter(min, max) ); else thrust::transform( - n_filtered.begin(), n_filtered.end(), // input - 1st arg - vec_bgn, // input - 2nd arg - n_filtered.begin(), // output + n_filtered.begin(), n_filtered.begin() + npart, // input - 1st arg + vec_bgn, // input - 2nd arg + n_filtered.begin(), // output detail::range_filter(min, max) ); selected_before_counting = true; } + + template + void particles_t::impl::moms_rng( + const real_t &min, const real_t &max, + const typename thrust_device::vector::iterator &vec_bgn, + const bool cons // is it a consecutive selection after previous one + ) + { + moms_rng(min, max, vec_bgn, n_part, cons); + } // selects particles for which vec1[i] >= vec2[i] template @@ -179,6 +190,7 @@ namespace libcloudphxx template void particles_t::impl::moms_calc( const typename thrust_device::vector::iterator &vec_bgn, + const thrust_size_t npart, const real_t power, const bool specific ) @@ -199,7 +211,7 @@ namespace libcloudphxx typename thrust_device::vector::iterator > it_pair = thrust::reduce_by_key( // input - keys - sorted_ijk.begin(), sorted_ijk.end(), + sorted_ijk.begin(), sorted_ijk.begin()+npart, // input - values thrust::make_transform_iterator( zip_it_t(thrust::make_tuple( @@ -285,7 +297,7 @@ namespace libcloudphxx std::cout << "sorted_id:" << std::endl; debug::print(sorted_id); std::cout << "vec:" << std::endl; - debug::print(vec_bgn, vec_bgn + n_part); + debug::print(vec_bgn, vec_bgn + npart); std::cout << "dv:" << std::endl; debug::print(dv); std::cout << "rhod:" << std::endl; @@ -294,5 +306,15 @@ namespace libcloudphxx } #endif } + + template + void particles_t::impl::moms_calc( + const typename thrust_device::vector::iterator &vec_bgn, + const real_t power, + const bool specific + ) + { + moms_calc(vec_bgn, n_part, power, specific); + } }; }; From 5704c84fa40b2677248672051b1670ff62369ede Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 15:28:53 +0200 Subject: [PATCH 019/282] relaxation: asserts and all inits --- src/impl/particles_impl_rlx_dry_distros.ipp | 55 ++++++++++++++++----- src/particles_step.ipp | 2 +- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index e956554a8..fd915eae1 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -56,6 +56,8 @@ namespace libcloudphxx // initialize SDs of each kappa-type for (typename opts_init_t::rlx_dry_distros_t::const_iterator ddi = opts_init.rlx_dry_distros.begin(); ddi != opts_init.rlx_dry_distros.end(); ++ddi) { + const auto &kappa(ddi->first); + assert(kappa >= 0); const auto &n_of_lnrd_stp(*(std::get<0>(ddi->second))); // analyze distribution to get rd_min and max needed for bin sizes @@ -66,13 +68,14 @@ namespace libcloudphxx opts_init.rlx_bins ); const real_t lnrd_rng = log_rd_max - log_rd_min; - - + assert(lnrd_rng > 0); // calculate bin edges (in rd3) // tmp vector with bin edges, probably could be allocated once in init const int n_bins = opts_init.rlx_bins * lnrd_rng / tot_lnrd_rng; + assert(n_bins>0); const real_t lnrd_bin_size = lnrd_rng / n_bins; + assert(lnrd_bin_size > 0); std::vector bin_rd3_left_edges(n_bins+1); // on CPU because of small number of edges std::iota(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), 0); // fill with a 0,1,2,... sequence std::transform(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), bin_rd3_left_edges.begin(), [log_rd_min_val=log_rd_min, lnrd_bin_size] (real_t bin_number) { return std::exp( 3 * (log_rd_min_val + bin_number * lnrd_bin_size)) ; }); // calculate left edges @@ -81,11 +84,18 @@ namespace libcloudphxx const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.nz, z_max_index = (std::get<2>(ddi->second)).second / opts_init.nz; + assert(z_max_index > z_min_index); + assert(z_min_index >= 0); + assert(z_max_index < opts_init.nz); + + const auto n_part_pre_bins_loop = n_part; + // loop over the bins for(int bin_number=0; bin_number(ddi->second)).first, (std::get<1>(ddi->second)).second, kpa.begin(), false); @@ -111,6 +121,7 @@ namespace libcloudphxx thrust::sort_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin()); auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_avg_k.begin(), hor_avg_count.begin()); int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); + assert(number_of_levels_with_droplets <= opts_init.nz); thrust::copy(hor_avg_count.begin(), hor_avg_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_avg.begin(), hor_avg_k.begin())); // divide sum by the number of cells at this level thrust::transform(hor_avg.begin(), hor_avg.end(), hor_avg.begin(), arg::_1 / (opts_init.nx * m1(opts_init.ny))); @@ -119,6 +130,7 @@ namespace libcloudphxx // calculate expected CCN conc const real_t bin_lnrd_center = log_rd_min + (bin_number + 0.5) * lnrd_bin_size; const real_t expected_STP_concentration = n_of_lnrd_stp(bin_lnrd_center) * lnrd_bin_size; + assert(expected_STP_concentration >= 0); thrust::fill(expected_hor_avg.begin(), expected_hor_avg.end(), expected_STP_concentration); // correcting STP -> actual ambient conditions @@ -168,23 +180,24 @@ namespace libcloudphxx n_part_to_init = thrust::reduce(create_SD.begin(), create_SD.end()); n_part = n_part_old + n_part_to_init; - // resize cell indices, resize should be cheap, because we allocate a large chunk of memory at the start + // resize arrays set in the bins loop: cell indices and rd3, resize should be cheap, because we allocate a large chunk of memory at the start ijk.resize(n_part); i.resize(n_part); k.resize(n_part); if(n_dims==3) j.resize(n_part); // we dont check in i and k because relax works only in 2D and 3D + rd3.resize(n_part); // k index based on create_SD thrust::copy_if(zero, zero+opts_init.nz, create_SD.begin(), k.begin()+n_part_old, arg::_1 == 1); // i and j random // tossing random numbers [0,1) TODO: do it once for all bins - rand_u01(n_part_to_init * (n_dims-1)); + rand_u01(n_part_to_init * (n_dims)); // random numbers for: i, rd, j (j only in 3D) std::cerr << "u01:" << std::endl; debug::print(u01.begin(), u01.begin()+n_part_to_init); thrust::transform(u01.begin(), u01.begin() + n_part_to_init, i.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.nx)); - if(n_dims==3) thrust::transform(u01.begin() + n_part_to_init, u01.begin() + 2*n_part_to_init, j.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.ny)); + if(n_dims==3) thrust::transform(u01.begin() + 2*n_part_to_init, u01.begin() + 3*n_part_to_init, j.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.ny)); // raveling i, j & k into ijk; only of the new SD ravel_ijk(n_part_old); @@ -205,18 +218,38 @@ namespace libcloudphxx std::cerr << "count_num:" << std::endl; debug::print(count_num); - // init i,j,k, (random i and j, k from create SD) - // init count num - // init ijk - // other inits, TODO: how to make these init not in the bins loop? + // init dry radius + // set rd3 randomized within the bin, uniformly distributed on the log(rd) axis + const real_t lnrd_min = std::log(std::pow(rd3_min, 1./3.)); + thrust::transform(u01.begin() + 1*n_part_to_init, u01.begin() + 2*n_part_to_init, rd3.begin()+n_part_old, lnrd_min + arg::_1 * lnrd_bin_size); + + // converting from lnrd to rd3 + thrust::transform( + rd3.begin() + n_part_old, + rd3.end(), + rd3.begin() + n_part_old, + detail::exp3x() + ); + + std::cerr << "rd3:" << std::endl; + debug::print(rd3.begin()+n_part_old, rd3.end()); // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... // at the end we need to set sorting=false - } - } + } // end of the bins loop + + // init other SD characteristics that don't have to be initialized in the bins loop + n_part_old = n_part_pre_bins_loop; + n_part_to_init = n_part - n_part_old; + hskpng_resize_npart(); + + // init multiplicities + init_n_sd_conc(n_of_lnrd_stp); + init_SD_with_distros_finalize(kappa); + } // end of the distros loop } }; }; diff --git a/src/particles_step.ipp b/src/particles_step.ipp index f30756b4c..7b3657278 100644 --- a/src/particles_step.ipp +++ b/src/particles_step.ipp @@ -277,7 +277,7 @@ namespace libcloudphxx } // aerosol relaxation, in sync since it changes th/rv - // TODO: more sanity checks for rlx! 3D only, values of rlx_bins etc. check that appa ranges are exclusive + // TODO: more sanity checks for rlx! 3D only, values of rlx_bins etc. check that appa ranges are exclusive, zmin Date: Thu, 13 May 2021 15:31:53 +0200 Subject: [PATCH 020/282] relaxation: calculate moments only on the SD exisitng before relaxation --- src/impl/particles_impl_rlx_dry_distros.ipp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index fd915eae1..caadb946d 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -53,6 +53,8 @@ namespace libcloudphxx tot_lnrd_rng += log_rd_max - log_rd_min; } + const auto n_part_pre_relax = n_part; + // initialize SDs of each kappa-type for (typename opts_init_t::rlx_dry_distros_t::const_iterator ddi = opts_init.rlx_dry_distros.begin(); ddi != opts_init.rlx_dry_distros.end(); ++ddi) { @@ -97,12 +99,12 @@ namespace libcloudphxx rd3_max = bin_rd3_left_edges.at(bin_number+1); assert(rd3_min < rd3_max); // TODO: these selections could be optimised - // select droplets within the desired kappa range - moms_rng((std::get<1>(ddi->second)).first, (std::get<1>(ddi->second)).second, kpa.begin(), false); + // select droplets within the desired kappa range; only from the droplets existing before relaxation began, because relaxed ones are not sorted + moms_rng((std::get<1>(ddi->second)).first, (std::get<1>(ddi->second)).second, kpa.begin(), n_part_pre_relax, false); // out of those, select droplets within the desired rd3 range - moms_rng(rd3_min, rd3_max, rd3.begin(), true); + moms_rng(rd3_min, rd3_max, rd3.begin(), n_part_pre_relax, true); // calculate 0-th non-specific moment of rd3 (number of droplets in a cell) of droplets in this rd3 and kappa range - moms_calc(rd3.begin(), 0, false); + moms_calc(rd3.begin(), n_part_pre_relax, 0, false); // divide by volume thrust::transform( count_mom.begin(), count_mom.begin() + count_n, // input - first arg @@ -234,10 +236,7 @@ namespace libcloudphxx std::cerr << "rd3:" << std::endl; debug::print(rd3.begin()+n_part_old, rd3.end()); - // TODO: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... - - // at the end we need to set sorting=false - + // NOTE: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } // end of the bins loop // init other SD characteristics that don't have to be initialized in the bins loop @@ -250,6 +249,7 @@ namespace libcloudphxx init_SD_with_distros_finalize(kappa); } // end of the distros loop + sorted = false; } }; }; From d18362c9abe573f4b3509e55191f24dea3ebc22b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 16:01:39 +0200 Subject: [PATCH 021/282] relaxation test: working version --- src/impl/particles_impl_rlx_dry_distros.ipp | 16 ++++++-- tests/python/unit/CMakeLists.txt | 2 +- tests/python/unit/relax.py | 42 +++++++++++++-------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index caadb946d..fda968a7e 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -86,7 +86,7 @@ namespace libcloudphxx const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.nz, z_max_index = (std::get<2>(ddi->second)).second / opts_init.nz; - assert(z_max_index > z_min_index); + assert(z_max_index >= z_min_index); assert(z_min_index >= 0); assert(z_max_index < opts_init.nz); @@ -151,6 +151,7 @@ namespace libcloudphxx // set to zero outside of the defined range of altitudes thrust::transform_if(zero, zero+opts_init.nz, expected_hor_avg.begin(), arg::_1 = 0, arg::_1 < z_min_index || arg::_1 > z_max_index); +/* std::cerr << "bin number: " << bin_number << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) << " r_center: " << std::exp(bin_lnrd_center) @@ -163,20 +164,24 @@ namespace libcloudphxx std::cerr << "expected_hor_avg:" << std::endl; debug::print(expected_hor_avg); - +*/ // calculate how many CCN are missing thrust::device_vector &hor_missing(expected_hor_avg); thrust::transform(expected_hor_avg.begin(), expected_hor_avg.end(), hor_avg.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); + /* std::cerr << "hor_missing:" << std::endl; debug::print(hor_missing); + */ // set number of SDs to init; create only if concentration is lower than expected with a tolerance thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_avg.begin(), create_SD.begin(), arg::_2 > 0 && arg::_1 / arg::_2 > config.rlx_conc_tolerance); // WARNING: watch out for div by 0 + /* std::cerr << "create_SD:" << std::endl; debug::print(create_SD); + */ n_part_old = n_part; n_part_to_init = thrust::reduce(create_SD.begin(), create_SD.end()); @@ -195,8 +200,10 @@ namespace libcloudphxx // tossing random numbers [0,1) TODO: do it once for all bins rand_u01(n_part_to_init * (n_dims)); // random numbers for: i, rd, j (j only in 3D) +/* std::cerr << "u01:" << std::endl; debug::print(u01.begin(), u01.begin()+n_part_to_init); + */ thrust::transform(u01.begin(), u01.begin() + n_part_to_init, i.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.nx)); if(n_dims==3) thrust::transform(u01.begin() + 2*n_part_to_init, u01.begin() + 3*n_part_to_init, j.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.ny)); @@ -208,6 +215,7 @@ namespace libcloudphxx thrust::fill(count_num.begin(), count_num.end(), 0); thrust::scatter(thrust::make_constant_iterator(1), thrust::make_constant_iterator(1) + n_part_to_init, ijk.begin() + n_part_old, count_num.begin()); +/* std::cerr << "i:" << std::endl; debug::print(i.begin()+n_part_old, i.end()); @@ -219,6 +227,7 @@ namespace libcloudphxx std::cerr << "count_num:" << std::endl; debug::print(count_num); + */ // init dry radius // set rd3 randomized within the bin, uniformly distributed on the log(rd) axis @@ -233,9 +242,10 @@ namespace libcloudphxx detail::exp3x() ); +/* std::cerr << "rd3:" << std::endl; debug::print(rd3.begin()+n_part_old, rd3.end()); - +*/ // NOTE: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } // end of the bins loop diff --git a/tests/python/unit/CMakeLists.txt b/tests/python/unit/CMakeLists.txt index c3d337546..d7a14aac2 100644 --- a/tests/python/unit/CMakeLists.txt +++ b/tests/python/unit/CMakeLists.txt @@ -1,5 +1,5 @@ # non-pytest tests -foreach(test api_blk_1m api_blk_2m api_lgrngn api_common segfault_20150216 col_kernels terminal_velocities SD_removal uniform_init source chem_coal sstp_cond multiple_kappas adve_scheme lgrngn_subsidence sat_adj_blk_1m diag_incloud_time) +foreach(test api_blk_1m api_blk_2m api_lgrngn api_common segfault_20150216 col_kernels terminal_velocities SD_removal uniform_init source chem_coal sstp_cond multiple_kappas adve_scheme lgrngn_subsidence sat_adj_blk_1m diag_incloud_time relax) #TODO: indicate that tests depend on the lib add_test( NAME ${test} diff --git a/tests/python/unit/relax.py b/tests/python/unit/relax.py index dd5da350d..9f59e43cc 100644 --- a/tests/python/unit/relax.py +++ b/tests/python/unit/relax.py @@ -22,6 +22,14 @@ def lognormal(lnr): -pow((lnr - log(mean_r)), 2) / 2 / pow(log(stdev),2) ) / log(stdev) / sqrt(2*pi); +def lognormal(lnr): + mean_r = .04e-6 / 2 + stdev = 1.4 + n_tot = 120e6 + return n_tot * exp( + -pow((lnr - log(mean_r)), 2) / 2 / pow(log(stdev),2) + ) / log(stdev) / sqrt(2*pi); + #def lognormal_rlx(lnr): # mean_r = .10e-6 / 2 # stdev = 1.4 @@ -31,11 +39,8 @@ def lognormal(lnr): # ) / log(stdev) / sqrt(2*pi); def test(opts_init): - opts_init.supstp_rlx = 50 + opts_init.supstp_rlx = 2 opts_init.rng_seed = int(time()) -# opts_init.rng_seed_init = int(time()) - print('rng_seed:', opts_init.rng_seed) -# print('rng_seed_init:', opts_init.rng_seed_init) opts_init.dt = 1 opts_init.nx = 2; opts_init.nz = 2; @@ -79,9 +84,9 @@ def test(opts_init): prtcls.init(th, rv, rhod) - # 100 steps during which number of droplets should be doubled in two calls to src + # 4 steps during which relaxation should be done twice opts.rlx = 1 - for i in range(100): + for i in range(4): prtcls.step_sync(opts,th,rv,rhod) prtcls.step_async(opts) @@ -103,9 +108,9 @@ def test(opts_init): kappa = .61 opts_init = lgrngn.opts_init_t() opts_init.dry_distros = {kappa:lognormal} -opts_init.rlx_dry_distros = {kappa: [lognormal, [0,2],[0,2*opts_init.dz]]} -opts_init.sd_conc = 2#100#24 -opts_init.rlx_bins = 2# 512 +opts_init.rlx_dry_distros = {kappa: [lognormal, [0,2],[0,opts_init.dz]]} +opts_init.sd_conc = 1024#100#24 +opts_init.rlx_bins = 1024#100# 512 opts_init.n_sd_max = int((opts_init.sd_conc * 2 + opts_init.rlx_bins * 2) * 2) # assuming nx=nz=2 print(' --- dry_distros rlx ---') @@ -113,16 +118,23 @@ def test(opts_init): sd_conc, wet_mom0, wet_mom1 = test(opts_init) print('diag_sd_conc', sd_conc) -#if not((sd_conc[0] == 1164 or sd_conc[0] == 1165) and (sd_conc[2] == 1164 or sd_conc[2] == 1165)): -# raise Exception("wrong amount of SDs were added") -#if not(sd_conc[1] == 1024 and sd_conc[3] == 1024): -# raise Exception("SDs were added in wrong cells") +# relaxation should add SD only in the lower cells +# there is some randomness in the number of SD that should be added +exp_sd_min = 1370 +exp_sd_max = 1470 +if sd_conc[0] < exp_sd_min or sd_conc[0] > exp_sd_max or sd_conc[2] < exp_sd_min or sd_conc[2] > exp_sd_max: + raise Exception("wrong amount of SDs were added") +if not(sd_conc[1] == 1024 and sd_conc[3] == 1024): + raise Exception("SDs were added in wrong cells") print(('wet mom0', wet_mom0)) -if (abs( 2 - (wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) ) > 0.015): +print(abs( (sd_conc[0] + sd_conc[2]) / (sd_conc[1] + sd_conc[3]) - (wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) )) +if (abs( (sd_conc[0] + sd_conc[2]) / (sd_conc[1] + sd_conc[3]) - (wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) ) > 0.1): raise Exception("incorrect multiplicity after source") print(('wet mom1', wet_mom1)) -if (abs( (7.84 / 2.12) - (wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) ) > 0.015): +print(abs( (sd_conc[0] + sd_conc[2]) / (sd_conc[1] + sd_conc[3]) - (wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) )) +if (abs( (sd_conc[0] + sd_conc[2]) / (sd_conc[1] + sd_conc[3]) - (wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) ) > 0.1): raise Exception("incorrect radius after source") + From 0469fb9eed555e5c22c14cce35545023692235d0 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 May 2021 16:15:15 +0200 Subject: [PATCH 022/282] distmem warning about relaxations --- src/impl/particles_impl_init_sanity_check.ipp | 4 ++++ src/particles_multi_gpu_ctor.ipp | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/impl/particles_impl_init_sanity_check.ipp b/src/impl/particles_impl_init_sanity_check.ipp index 40a68b371..4fb657b9e 100644 --- a/src/impl/particles_impl_init_sanity_check.ipp +++ b/src/impl/particles_impl_init_sanity_check.ipp @@ -127,6 +127,10 @@ namespace libcloudphxx throw std::runtime_error("at least one of opts_init.turb_adve_switch, opts_init.turb_cond_switch is true, but SGS mixing length profile size != nz"); if(opts_init.SGS_mix_len.size() > 0 && *std::min(opts_init.SGS_mix_len.begin(), opts_init.SGS_mix_len.end()) <= 0) throw std::runtime_error("SGS_mix_len <= 0"); + #if defined(USE_MPI) + if(opts_init.rlx_switch) + std::cerr << "libcloudph++ WARNING: relaxation is not fully supported in MPI runs. Mean calculation and addition of SD will be done locally on each node." << std::endl; + #endif } }; }; diff --git a/src/particles_multi_gpu_ctor.ipp b/src/particles_multi_gpu_ctor.ipp index e51acbf57..84f3ce7a6 100644 --- a/src/particles_multi_gpu_ctor.ipp +++ b/src/particles_multi_gpu_ctor.ipp @@ -37,6 +37,9 @@ namespace libcloudphxx const std::map > ambient_chem ) { + if(opts_init.rlx_switch) + std::cerr << "libcloudph++ WARNING: relaxation is not fully supported in the multi_CUDA backend. Mean calculation and addition of SD will be done locally on each GPU." << std::endl; + pimpl->mcuda_run( &particles_t::init, th, rv, rhod, p, courant_1, courant_2, courant_3, ambient_chem From fd31e1faaf803ebfc98454ba0f7e80f0bb2a2592 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 13 May 2021 16:23:57 +0200 Subject: [PATCH 023/282] urand: fix shift on CUDA from (0,1] to [0,1) --- CMakeLists.txt | 4 ++-- src/detail/urand.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03bd41a7a..f5ada93ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,9 +355,9 @@ endif() enable_testing() -add_subdirectory(tests) +#add_subdirectory(tests) add_subdirectory(include) -add_subdirectory(bindings) +#add_subdirectory(bindings) ############################################################################################ # installation diff --git a/src/detail/urand.hpp b/src/detail/urand.hpp index 02ee2ac2b..a785bcb53 100644 --- a/src/detail/urand.hpp +++ b/src/detail/urand.hpp @@ -136,7 +136,7 @@ namespace libcloudphxx _unused(status); // shift into the expected [0,1) range namespace arg = thrust::placeholders; - thrust::transform(v.begin(), v.begin() + n, float(1) - arg::_1); + thrust::transform(v.begin(), v.begin() + n, v.begin(), float(1) - arg::_1); } void generate_n( @@ -149,7 +149,7 @@ namespace libcloudphxx _unused(status); // shift into the expected [0,1) range namespace arg = thrust::placeholders; - thrust::transform(v.begin(), v.begin() + n, double(1) - arg::_1); + thrust::transform(v.begin(), v.begin() + n, v.begin(), double(1) - arg::_1); } void generate_normal_n( From 41246468f3ad7fa82ae3153affaa4a908d1812b9 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Mon, 17 May 2021 10:27:56 +0200 Subject: [PATCH 024/282] CUDA relax fixes --- src/impl/particles_impl_rlx_dry_distros.ipp | 2 +- src/particles_multi_gpu_ctor.ipp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index fda968a7e..3f93501b7 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -149,7 +149,7 @@ namespace libcloudphxx } // set to zero outside of the defined range of altitudes - thrust::transform_if(zero, zero+opts_init.nz, expected_hor_avg.begin(), arg::_1 = 0, arg::_1 < z_min_index || arg::_1 > z_max_index); + thrust::replace_if(expected_hor_avg.begin(), expected_hor_avg.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 > z_max_index, real_t(0)); /* std::cerr << "bin number: " << bin_number diff --git a/src/particles_multi_gpu_ctor.ipp b/src/particles_multi_gpu_ctor.ipp index 84f3ce7a6..f5d1905e9 100644 --- a/src/particles_multi_gpu_ctor.ipp +++ b/src/particles_multi_gpu_ctor.ipp @@ -37,7 +37,7 @@ namespace libcloudphxx const std::map > ambient_chem ) { - if(opts_init.rlx_switch) + if(pimpl->glob_opts_init.rlx_switch) std::cerr << "libcloudph++ WARNING: relaxation is not fully supported in the multi_CUDA backend. Mean calculation and addition of SD will be done locally on each GPU." << std::endl; pimpl->mcuda_run( From 105c3c9d775c49be255d357f752a15514babc1be Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 18 May 2021 12:30:47 +0200 Subject: [PATCH 025/282] relax: temp vectors properly in device memory --- src/impl/particles_impl_rlx_dry_distros.ipp | 86 ++++++++++----------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 3f93501b7..d0e7fa3d9 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -36,11 +36,11 @@ namespace libcloudphxx namespace arg = thrust::placeholders; // vectors of size nz used in calculation of horizontal averages, TODO: allocate them at init - thrust::device_vector hor_avg(opts_init.nz); - thrust::device_vector hor_avg_count(opts_init.nz); - thrust::device_vector hor_avg_k(opts_init.nz); - thrust::device_vector expected_hor_avg(opts_init.nz); - thrust::device_vector create_SD(opts_init.nz); + thrust_device::vector hor_avg(opts_init.nz); + thrust_device::vector hor_avg_count(opts_init.nz); + thrust_device::vector hor_avg_k(opts_init.nz); + thrust_device::vector expected_hor_avg(opts_init.nz); + thrust_device::vector create_SD(opts_init.nz); // calc sum of ln(rd) ranges of all relax distributions real_t tot_lnrd_rng = 0.; @@ -121,8 +121,18 @@ namespace libcloudphxx thrust_device::vector &count_k(tmp_device_size_cell); // NOTE: tmp_device_size_cell is also used in some other inits, careful not to overwrite it! thrust::transform(count_ijk.begin(), count_ijk.begin() + count_n, count_k.begin(), arg::_1 % opts_init.nz); thrust::sort_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin()); + + // std::cerr << "count_n: " << count_n << std::endl; + // std::cerr << "count_k:" << std::endl; + // debug::print(count_k.begin(), count_k.end()); + // std::cerr << "count_mom:" << std::endl; + // debug::print(count_mom.begin(), count_mom.end()); + auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_avg_k.begin(), hor_avg_count.begin()); int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); + + // std::cerr << "number_of_levels_with_droplets: " << number_of_levels_with_droplets << std::endl; + assert(number_of_levels_with_droplets <= opts_init.nz); thrust::copy(hor_avg_count.begin(), hor_avg_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_avg.begin(), hor_avg_k.begin())); // divide sum by the number of cells at this level @@ -151,37 +161,31 @@ namespace libcloudphxx // set to zero outside of the defined range of altitudes thrust::replace_if(expected_hor_avg.begin(), expected_hor_avg.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 > z_max_index, real_t(0)); -/* - std::cerr << "bin number: " << bin_number - << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) - << " r_center: " << std::exp(bin_lnrd_center) - << " z_indices: (" << z_min_index << ", " << z_max_index << "), " - << " expected STD concentration: " << expected_STP_concentration - << std::endl; + //std::cerr << "bin number: " << bin_number + // << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) + // << " r_center: " << std::exp(bin_lnrd_center) + // << " z_indices: (" << z_min_index << ", " << z_max_index << "), " + // << " expected STD concentration: " << expected_STP_concentration + // << std::endl; - std::cerr << "hor_avg:" << std::endl; - debug::print(hor_avg); + //std::cerr << "hor_avg:" << std::endl; + //debug::print(hor_avg); - std::cerr << "expected_hor_avg:" << std::endl; - debug::print(expected_hor_avg); -*/ + //std::cerr << "expected_hor_avg:" << std::endl; + //debug::print(expected_hor_avg); // calculate how many CCN are missing - thrust::device_vector &hor_missing(expected_hor_avg); + thrust_device::vector &hor_missing(expected_hor_avg); thrust::transform(expected_hor_avg.begin(), expected_hor_avg.end(), hor_avg.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); - /* - std::cerr << "hor_missing:" << std::endl; - debug::print(hor_missing); - */ + //std::cerr << "hor_missing:" << std::endl; + //debug::print(hor_missing); // set number of SDs to init; create only if concentration is lower than expected with a tolerance thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_avg.begin(), create_SD.begin(), arg::_2 > 0 && arg::_1 / arg::_2 > config.rlx_conc_tolerance); // WARNING: watch out for div by 0 - /* - std::cerr << "create_SD:" << std::endl; - debug::print(create_SD); - */ + //std::cerr << "create_SD:" << std::endl; + //debug::print(create_SD); n_part_old = n_part; n_part_to_init = thrust::reduce(create_SD.begin(), create_SD.end()); @@ -200,10 +204,8 @@ namespace libcloudphxx // tossing random numbers [0,1) TODO: do it once for all bins rand_u01(n_part_to_init * (n_dims)); // random numbers for: i, rd, j (j only in 3D) -/* - std::cerr << "u01:" << std::endl; - debug::print(u01.begin(), u01.begin()+n_part_to_init); - */ + //std::cerr << "u01:" << std::endl; + //debug::print(u01.begin(), u01.begin()+n_part_to_init); thrust::transform(u01.begin(), u01.begin() + n_part_to_init, i.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.nx)); if(n_dims==3) thrust::transform(u01.begin() + 2*n_part_to_init, u01.begin() + 3*n_part_to_init, j.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.ny)); @@ -215,19 +217,17 @@ namespace libcloudphxx thrust::fill(count_num.begin(), count_num.end(), 0); thrust::scatter(thrust::make_constant_iterator(1), thrust::make_constant_iterator(1) + n_part_to_init, ijk.begin() + n_part_old, count_num.begin()); -/* - std::cerr << "i:" << std::endl; - debug::print(i.begin()+n_part_old, i.end()); + // std::cerr << "i:" << std::endl; + // debug::print(i.begin()+n_part_old, i.end()); - std::cerr << "k:" << std::endl; - debug::print(k.begin()+n_part_old, k.end()); + // std::cerr << "k:" << std::endl; + // debug::print(k.begin()+n_part_old, k.end()); - std::cerr << "ijk:" << std::endl; - debug::print(ijk.begin()+n_part_old, ijk.end()); + // std::cerr << "ijk:" << std::endl; + // debug::print(ijk.begin()+n_part_old, ijk.end()); - std::cerr << "count_num:" << std::endl; - debug::print(count_num); - */ + // std::cerr << "count_num:" << std::endl; + // debug::print(count_num); // init dry radius // set rd3 randomized within the bin, uniformly distributed on the log(rd) axis @@ -242,10 +242,8 @@ namespace libcloudphxx detail::exp3x() ); -/* - std::cerr << "rd3:" << std::endl; - debug::print(rd3.begin()+n_part_old, rd3.end()); -*/ + //std::cerr << "rd3:" << std::endl; + //debug::print(rd3.begin()+n_part_old, rd3.end()); // NOTE: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } // end of the bins loop From 0f19086dfaa30d9febc25f504872d1d3f2d54612 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 18 May 2021 16:15:04 +0200 Subject: [PATCH 026/282] throw error when relaxation is turned on in 0d and 1d --- src/impl/particles_impl_init_sanity_check.ipp | 2 ++ src/impl/particles_impl_rlx_dry_distros.ipp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/impl/particles_impl_init_sanity_check.ipp b/src/impl/particles_impl_init_sanity_check.ipp index 4fb657b9e..97972aebc 100644 --- a/src/impl/particles_impl_init_sanity_check.ipp +++ b/src/impl/particles_impl_init_sanity_check.ipp @@ -131,6 +131,8 @@ namespace libcloudphxx if(opts_init.rlx_switch) std::cerr << "libcloudph++ WARNING: relaxation is not fully supported in MPI runs. Mean calculation and addition of SD will be done locally on each node." << std::endl; #endif + if(n_dims < 2 && opts_init.rlx_switch) + throw std::runtime_error("CCN relaxation works only in 2D and 3D, set rlx_switch to false"); } }; }; diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index d0e7fa3d9..4ed00807c 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -256,6 +256,8 @@ namespace libcloudphxx init_n_sd_conc(n_of_lnrd_stp); init_SD_with_distros_finalize(kappa); + + // TODO: asserts of newly added SD parameters? e.g. how many SD, how big is multiplicity etc. } // end of the distros loop sorted = false; } From 0f0040a2bb83cf43a8316917edea2305887153a4 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 18 May 2021 21:16:03 +0200 Subject: [PATCH 027/282] create SD type to thrust_Size_t --- src/impl/particles_impl_rlx_dry_distros.ipp | 32 +++++++++++---------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index d0e7fa3d9..74b440581 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -40,7 +40,7 @@ namespace libcloudphxx thrust_device::vector hor_avg_count(opts_init.nz); thrust_device::vector hor_avg_k(opts_init.nz); thrust_device::vector expected_hor_avg(opts_init.nz); - thrust_device::vector create_SD(opts_init.nz); + thrust_device::vector create_SD(opts_init.nz); // could be bool, but then thrust::reduce does not add bools as expected // calc sum of ln(rd) ranges of all relax distributions real_t tot_lnrd_rng = 0.; @@ -122,16 +122,16 @@ namespace libcloudphxx thrust::transform(count_ijk.begin(), count_ijk.begin() + count_n, count_k.begin(), arg::_1 % opts_init.nz); thrust::sort_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin()); - // std::cerr << "count_n: " << count_n << std::endl; - // std::cerr << "count_k:" << std::endl; - // debug::print(count_k.begin(), count_k.end()); - // std::cerr << "count_mom:" << std::endl; - // debug::print(count_mom.begin(), count_mom.end()); + //std::cerr << "count_n: " << count_n << std::endl; + //std::cerr << "count_k:" << std::endl; + //debug::print(count_k.begin(), count_k.end()); + //std::cerr << "count_mom:" << std::endl; + //debug::print(count_mom.begin(), count_mom.end()); auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_avg_k.begin(), hor_avg_count.begin()); int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); - // std::cerr << "number_of_levels_with_droplets: " << number_of_levels_with_droplets << std::endl; + //std::cerr << "number_of_levels_with_droplets: " << number_of_levels_with_droplets << std::endl; assert(number_of_levels_with_droplets <= opts_init.nz); thrust::copy(hor_avg_count.begin(), hor_avg_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_avg.begin(), hor_avg_k.begin())); @@ -191,6 +191,8 @@ namespace libcloudphxx n_part_to_init = thrust::reduce(create_SD.begin(), create_SD.end()); n_part = n_part_old + n_part_to_init; + //std::cerr << "n_part_to_init: " << n_part_to_init << std::endl; + // resize arrays set in the bins loop: cell indices and rd3, resize should be cheap, because we allocate a large chunk of memory at the start ijk.resize(n_part); i.resize(n_part); @@ -217,17 +219,17 @@ namespace libcloudphxx thrust::fill(count_num.begin(), count_num.end(), 0); thrust::scatter(thrust::make_constant_iterator(1), thrust::make_constant_iterator(1) + n_part_to_init, ijk.begin() + n_part_old, count_num.begin()); - // std::cerr << "i:" << std::endl; - // debug::print(i.begin()+n_part_old, i.end()); + //std::cerr << "i:" << std::endl; + //debug::print(i.begin()+n_part_old, i.end()); - // std::cerr << "k:" << std::endl; - // debug::print(k.begin()+n_part_old, k.end()); + //std::cerr << "k:" << std::endl; + //debug::print(k.begin()+n_part_old, k.end()); - // std::cerr << "ijk:" << std::endl; - // debug::print(ijk.begin()+n_part_old, ijk.end()); + //std::cerr << "ijk:" << std::endl; + //debug::print(ijk.begin()+n_part_old, ijk.end()); - // std::cerr << "count_num:" << std::endl; - // debug::print(count_num); + //std::cerr << "count_num:" << std::endl; + //debug::print(count_num); // init dry radius // set rd3 randomized within the bin, uniformly distributed on the log(rd) axis From 8388dbb303006ee5fe9c820a64828c5d7b212cae Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 19 May 2021 10:56:32 +0200 Subject: [PATCH 028/282] relaxation: fix zmin and zmax indices --- src/impl/particles_impl_rlx_dry_distros.ipp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 2a134db69..2efa66441 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -59,6 +59,7 @@ namespace libcloudphxx for (typename opts_init_t::rlx_dry_distros_t::const_iterator ddi = opts_init.rlx_dry_distros.begin(); ddi != opts_init.rlx_dry_distros.end(); ++ddi) { const auto &kappa(ddi->first); + //std::cerr << "rlx kappa: " << kappa << std::endl; assert(kappa >= 0); const auto &n_of_lnrd_stp(*(std::get<0>(ddi->second))); @@ -83,8 +84,8 @@ namespace libcloudphxx std::transform(bin_rd3_left_edges.begin(), bin_rd3_left_edges.end(), bin_rd3_left_edges.begin(), [log_rd_min_val=log_rd_min, lnrd_bin_size] (real_t bin_number) { return std::exp( 3 * (log_rd_min_val + bin_number * lnrd_bin_size)) ; }); // calculate left edges // minimum and maximum cell indices - const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.nz, - z_max_index = (std::get<2>(ddi->second)).second / opts_init.nz; + const int z_min_index = (std::get<2>(ddi->second)).first / opts_init.dz, + z_max_index = (std::get<2>(ddi->second)).second / opts_init.dz; assert(z_max_index >= z_min_index); assert(z_min_index >= 0); @@ -129,8 +130,13 @@ namespace libcloudphxx //debug::print(count_mom.begin(), count_mom.end()); auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_avg_k.begin(), hor_avg_count.begin()); - int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); - + + //std::cerr << "hor_avg_k:" << std::endl; + //debug::print(hor_avg_k.begin(), hor_avg_k.end()); + //std::cerr << "hor_avg_count:" << std::endl; + //debug::print(hor_avg_count.begin(), hor_avg_count.end()); + + int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); // number of levels with any SD, not with SD in this size and kappa range //std::cerr << "number_of_levels_with_droplets: " << number_of_levels_with_droplets << std::endl; assert(number_of_levels_with_droplets <= opts_init.nz); @@ -252,11 +258,15 @@ namespace libcloudphxx // init other SD characteristics that don't have to be initialized in the bins loop n_part_old = n_part_pre_bins_loop; n_part_to_init = n_part - n_part_old; + //std::cerr << "n_part: " << n_part << " n_part_old: " << n_part_old << " n_part_to_init: " << n_part_to_init << std::endl; hskpng_resize_npart(); // init multiplicities init_n_sd_conc(n_of_lnrd_stp); + //std::cerr << "rlx, n of new particles:" << std::endl; + //debug::print(n.begin()+n_part_old, n.end()); + init_SD_with_distros_finalize(kappa); // TODO: asserts of newly added SD parameters? e.g. how many SD, how big is multiplicity etc. From 57e246aef4f1bf721bbee1138c0fa13828f9586c Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 19 May 2021 18:33:42 +0200 Subject: [PATCH 029/282] relax: init n properly baed on hor_missing --- src/impl/particles_impl_rlx_dry_distros.ipp | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 2efa66441..d6ab1fe49 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -250,6 +250,23 @@ namespace libcloudphxx detail::exp3x() ); + // init multiplicities (includes casting from real to n) + thrust::copy_if( + thrust::make_transform_iterator(hor_missing.begin(), arg::_1 + real_t(0.5)), + thrust::make_transform_iterator(hor_missing.begin(), arg::_1 + real_t(0.5)) + opts_init.nz, + create_SD.begin(), + n.begin()+n_part_old, + arg::_1 == 1 + ); + + //std::cerr << "rlx, n of new particles:" << std::endl; + //debug::print(n.begin()+n_part_old, n.end()); + + // detecting possible overflows of n type + thrust_size_t ix = thrust::max_element(n.begin() + n_part_old, n.end()) - (n.begin() + n_part_old); + assert(n[ix] < (typename impl::n_t)(-1) / 10000); + + //std::cerr << "rd3:" << std::endl; //debug::print(rd3.begin()+n_part_old, rd3.end()); // NOTE: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... @@ -261,12 +278,6 @@ namespace libcloudphxx //std::cerr << "n_part: " << n_part << " n_part_old: " << n_part_old << " n_part_to_init: " << n_part_to_init << std::endl; hskpng_resize_npart(); - // init multiplicities - init_n_sd_conc(n_of_lnrd_stp); - - //std::cerr << "rlx, n of new particles:" << std::endl; - //debug::print(n.begin()+n_part_old, n.end()); - init_SD_with_distros_finalize(kappa); // TODO: asserts of newly added SD parameters? e.g. how many SD, how big is multiplicity etc. From 38567605ad60d0ea30e10659ae0c6b1f701b75ad Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 20 May 2021 00:26:35 +0200 Subject: [PATCH 030/282] relax: separate array for hor_missing --- src/impl/particles_impl_rlx_dry_distros.ipp | 41 ++++++++++++++++----- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index d6ab1fe49..026f752e7 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -27,6 +27,28 @@ namespace libcloudphxx return ret_t(x*c); } }; + + template + struct relax_conc_tolerance + { + const real_t tolerance; + + relax_conc_tolerance(const real_t tol): + tolerance(tol) + {} + + BOOST_GPU_ENABLED + int operator()(const real_t &a1, const real_t &a2) + { +// printf("missing %g expected %g tolerance %g result %d\n", a1, a2, tolerance, a2 > real_t(0) ? +// a1 / a2 > tolerance ? 1 : 0 +// : 0); +// //std::cerr << "a1: " << a1 << " a2: " << a2 << " tol: " << tolerance << " result: " << bool(a1 / a2 > tolerance) << std::endl; + return a2 > real_t(0) ? + a1 / a2 > tolerance ? 1 : 0 + : 0; + } + }; }; // create new aerosol particles to relax towards a size distribution @@ -38,6 +60,7 @@ namespace libcloudphxx // vectors of size nz used in calculation of horizontal averages, TODO: allocate them at init thrust_device::vector hor_avg(opts_init.nz); thrust_device::vector hor_avg_count(opts_init.nz); + thrust_device::vector hor_missing(opts_init.nz); thrust_device::vector hor_avg_k(opts_init.nz); thrust_device::vector expected_hor_avg(opts_init.nz); thrust_device::vector create_SD(opts_init.nz); // could be bool, but then thrust::reduce does not add bools as expected @@ -167,12 +190,12 @@ namespace libcloudphxx // set to zero outside of the defined range of altitudes thrust::replace_if(expected_hor_avg.begin(), expected_hor_avg.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 > z_max_index, real_t(0)); - //std::cerr << "bin number: " << bin_number - // << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) - // << " r_center: " << std::exp(bin_lnrd_center) - // << " z_indices: (" << z_min_index << ", " << z_max_index << "), " - // << " expected STD concentration: " << expected_STP_concentration - // << std::endl; + //std::cerr << "bin number: " << bin_number ; + //std::cerr << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) ; + //std::cerr << " r_center: " << std::exp(bin_lnrd_center) ; + //std::cerr << " z_indices: (" << z_min_index << ", " << z_max_index << "), " ; + //std::cerr << " expected STD concentration: " << expected_STP_concentration ; + //std::cerr << std::endl; //std::cerr << "hor_avg:" << std::endl; //debug::print(hor_avg); @@ -180,7 +203,6 @@ namespace libcloudphxx //std::cerr << "expected_hor_avg:" << std::endl; //debug::print(expected_hor_avg); // calculate how many CCN are missing - thrust_device::vector &hor_missing(expected_hor_avg); thrust::transform(expected_hor_avg.begin(), expected_hor_avg.end(), hor_avg.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); @@ -188,7 +210,7 @@ namespace libcloudphxx //debug::print(hor_missing); // set number of SDs to init; create only if concentration is lower than expected with a tolerance - thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_avg.begin(), create_SD.begin(), arg::_2 > 0 && arg::_1 / arg::_2 > config.rlx_conc_tolerance); // WARNING: watch out for div by 0 + thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_avg.begin(), create_SD.begin(), detail::relax_conc_tolerance(config.rlx_conc_tolerance)); //std::cerr << "create_SD:" << std::endl; //debug::print(create_SD); @@ -205,6 +227,7 @@ namespace libcloudphxx k.resize(n_part); if(n_dims==3) j.resize(n_part); // we dont check in i and k because relax works only in 2D and 3D rd3.resize(n_part); + n.resize(n_part); // k index based on create_SD thrust::copy_if(zero, zero+opts_init.nz, create_SD.begin(), k.begin()+n_part_old, arg::_1 == 1); @@ -259,7 +282,7 @@ namespace libcloudphxx arg::_1 == 1 ); - //std::cerr << "rlx, n of new particles:" << std::endl; + //std::cerr << "n:" << std::endl; //debug::print(n.begin()+n_part_old, n.end()); // detecting possible overflows of n type From 1b7a4c82c6c8ba5323983f24f84806517c7191c6 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 20 May 2021 11:39:51 +0200 Subject: [PATCH 031/282] separate unravel ijk, dont do it twice in relax --- src/impl/particles_impl.ipp | 3 +- src/impl/particles_impl_hskpng_ijk.ipp | 53 +++++++++++++++++++ .../particles_impl_init_SD_with_distros.ipp | 6 ++- .../particles_impl_init_SD_with_sizes.ipp | 3 ++ src/impl/particles_impl_init_xyz.ipp | 47 ---------------- src/impl/particles_impl_rlx_dry_distros.ipp | 2 +- src/impl/particles_impl_src_dry_distros.ipp | 3 ++ src/impl/particles_impl_src_dry_sizes.ipp | 3 ++ 8 files changed, 70 insertions(+), 50 deletions(-) diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 2c1543f3e..a5958315d 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -425,7 +425,7 @@ namespace libcloudphxx void init_SD_with_distros_sd_conc(const common::unary_function &, const real_t &); void init_SD_with_distros_tail(const common::unary_function &, const real_t); void init_SD_with_distros_const_multi(const common::unary_function &); - void init_SD_with_distros_finalize(const real_t &); + void init_SD_with_distros_finalize(const real_t &, const bool unravel_ijk = true); void init_SD_with_sizes(); void init_sanity_check( const arrinfo_t, const arrinfo_t, const arrinfo_t, @@ -488,6 +488,7 @@ namespace libcloudphxx void hskpng_shuffle_and_sort(); void hskpng_count(); void ravel_ijk(const thrust_size_t begin_shift = 0); + void unravel_ijk(const thrust_size_t begin_shift = 0); void hskpng_ijk(); void hskpng_Tpr(); void hskpng_mfp(); diff --git a/src/impl/particles_impl_hskpng_ijk.ipp b/src/impl/particles_impl_hskpng_ijk.ipp index eee8467ce..2320462f2 100644 --- a/src/impl/particles_impl_hskpng_ijk.ipp +++ b/src/impl/particles_impl_hskpng_ijk.ipp @@ -28,6 +28,7 @@ namespace libcloudphxx }; }; + // calc ijk from i, j and k template void particles_t::impl::ravel_ijk(const thrust_size_t begin_shift) // default = 0 { @@ -69,6 +70,58 @@ namespace libcloudphxx } } + + // calc i, j and k from ijk + template + void particles_t::impl::unravel_ijk(const thrust_size_t begin_shift) // default = 0 + { + switch(n_dims) + { + case 3: + namespace arg = thrust::placeholders; + // y + thrust::transform( + ijk.begin() + begin_shift, ijk.end(), // input - first arg + j.begin() + begin_shift, // output + (arg::_1 / opts_init.nz) % (opts_init.ny) // z varies first + ); + // z + thrust::transform( + ijk.begin() + begin_shift, ijk.end(), // input - first arg + k.begin() + begin_shift, // output + arg::_1 % (opts_init.nz) // z varies first + ); + // x + thrust::transform( + ijk.begin() + begin_shift, ijk.end(), // input - first arg + i.begin() + begin_shift, // output + arg::_1 / (opts_init.nz * opts_init.ny) // z and y vary first + ); + break; + case 2: + // z + thrust::transform( + ijk.begin() + begin_shift, ijk.end(), // input - first arg + k.begin() + begin_shift, // output + arg::_1 % (opts_init.nz) // z varies first + ); + // x + thrust::transform( + ijk.begin() + begin_shift, ijk.end(), // input - first arg + i.begin() + begin_shift, // output + arg::_1 / (opts_init.nz) + ); + break; + case 1: + thrust::copy(ijk.begin() + begin_shift, ijk.end(), i.begin() + begin_shift); // only x + case 0: + break; + default: + assert(false); + break; + } + } + template void particles_t::impl::hskpng_ijk() { diff --git a/src/impl/particles_impl_init_SD_with_distros.ipp b/src/impl/particles_impl_init_SD_with_distros.ipp index 48f813196..a6a8e94c4 100644 --- a/src/impl/particles_impl_init_SD_with_distros.ipp +++ b/src/impl/particles_impl_init_SD_with_distros.ipp @@ -50,7 +50,7 @@ namespace libcloudphxx // final inits common for tail/sd_conc/const_multi template - void particles_t::impl::init_SD_with_distros_finalize(const real_t &kappa) + void particles_t::impl::init_SD_with_distros_finalize(const real_t &kappa, const bool unravel_ijk_switch) { // init kappa init_kappa(kappa); @@ -78,6 +78,10 @@ namespace libcloudphxx chem_vol_ante(); } + // ijk -> i, j, k + if(unravel_ijk_switch) + unravel_ijk(n_part_old); + // initialising particle positions init_xyz(); } diff --git a/src/impl/particles_impl_init_SD_with_sizes.ipp b/src/impl/particles_impl_init_SD_with_sizes.ipp index cc14cf238..b7e89cccb 100644 --- a/src/impl/particles_impl_init_SD_with_sizes.ipp +++ b/src/impl/particles_impl_init_SD_with_sizes.ipp @@ -73,6 +73,9 @@ namespace libcloudphxx if (opts_init.chem_switch){ chem_vol_ante(); } + + // ijk -> i, j, k + unravel_ijk(n_part_old); // initialising particle positions init_xyz(); diff --git a/src/impl/particles_impl_init_xyz.ipp b/src/impl/particles_impl_init_xyz.ipp index f34f0c137..23978d31e 100644 --- a/src/impl/particles_impl_init_xyz.ipp +++ b/src/impl/particles_impl_init_xyz.ipp @@ -39,53 +39,6 @@ namespace libcloudphxx template void particles_t::impl::init_xyz() { - // get i, j, k from ijk - switch(n_dims) - { - case 3: - namespace arg = thrust::placeholders; - // y - thrust::transform( - ijk.begin() + n_part_old, ijk.end(), // input - first arg - j.begin() + n_part_old, // output - (arg::_1 / opts_init.nz) % (opts_init.ny) // z varies first - ); - // z - thrust::transform( - ijk.begin() + n_part_old, ijk.end(), // input - first arg - k.begin() + n_part_old, // output - arg::_1 % (opts_init.nz) // z varies first - ); - // x - thrust::transform( - ijk.begin() + n_part_old, ijk.end(), // input - first arg - i.begin() + n_part_old, // output - arg::_1 / (opts_init.nz * opts_init.ny) // z and y vary first - ); - break; - case 2: - // z - thrust::transform( - ijk.begin() + n_part_old, ijk.end(), // input - first arg - k.begin() + n_part_old, // output - arg::_1 % (opts_init.nz) // z varies first - ); - // x - thrust::transform( - ijk.begin() + n_part_old, ijk.end(), // input - first arg - i.begin() + n_part_old, // output - arg::_1 / (opts_init.nz) - ); - break; - case 1: - thrust::copy(ijk.begin() + n_part_old, ijk.end(), i.begin() + n_part_old); // only x - case 0: - break; - default: - assert(false); - break; - } - thrust_device::vector *v[3] = { &x, &y, &z }; const int n[3] = { opts_init.nx, opts_init.ny, opts_init.nz }; diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 026f752e7..73d414753 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -301,7 +301,7 @@ namespace libcloudphxx //std::cerr << "n_part: " << n_part << " n_part_old: " << n_part_old << " n_part_to_init: " << n_part_to_init << std::endl; hskpng_resize_npart(); - init_SD_with_distros_finalize(kappa); + init_SD_with_distros_finalize(kappa, false); // no need to unravel ijk there, becaues i j k are already initialized // TODO: asserts of newly added SD parameters? e.g. how many SD, how big is multiplicity etc. } // end of the distros loop diff --git a/src/impl/particles_impl_src_dry_distros.ipp b/src/impl/particles_impl_src_dry_distros.ipp index c6a89e551..c22f59e67 100644 --- a/src/impl/particles_impl_src_dry_distros.ipp +++ b/src/impl/particles_impl_src_dry_distros.ipp @@ -221,6 +221,9 @@ namespace libcloudphxx // init rw init_wet(); + + // ijk -> i, j, k + unravel_ijk(n_part_old); // init x, y, z, i, j, k init_xyz(); diff --git a/src/impl/particles_impl_src_dry_sizes.ipp b/src/impl/particles_impl_src_dry_sizes.ipp index 21c975300..8957d7d5c 100644 --- a/src/impl/particles_impl_src_dry_sizes.ipp +++ b/src/impl/particles_impl_src_dry_sizes.ipp @@ -73,6 +73,9 @@ namespace libcloudphxx if (opts_init.chem_switch){ chem_vol_ante(); } + + // ijk -> i, j, k + unravel_ijk(n_part_old); // initialising particle positions init_xyz(); From c37410713db5e48c68e501861a3201abe56fcc35 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 20 May 2021 12:33:27 +0200 Subject: [PATCH 032/282] relaxation: horizontal sum of population instead of the aveage --- src/impl/particles_impl_rlx_dry_distros.ipp | 115 +++++++++++++------- 1 file changed, 76 insertions(+), 39 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 73d414753..9fadb4da7 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -49,6 +49,38 @@ namespace libcloudphxx : 0; } }; + + // domain volume at this height level + template + struct hor_dv_eval + { + // note: having a copy of opts_init here causes CUDA crashes (alignment problems?) + const real_t + dz, + x0, y0, z0, + x1, y1, z1; + + hor_dv_eval(const opts_init_t &o) : + dz(o.dz), + x0(o.x0), y0(o.y0), z0(o.z0), + x1(o.x1), y1(o.y1), z1(o.z1) + {} + + BOOST_GPU_ENABLED + real_t operator()(const int &k) + { +#if !defined(__NVCC__) + using std::min; + using std::max; +#endif + return + max(real_t(0), + (x1 - x0) * + (y1 - y0) * // NOTE: size in y is taken into account even in 2D! + (min((k + 1) * dz, z1) - max(k * dz, z0)) + ); + } + }; }; // create new aerosol particles to relax towards a size distribution @@ -58,11 +90,11 @@ namespace libcloudphxx namespace arg = thrust::placeholders; // vectors of size nz used in calculation of horizontal averages, TODO: allocate them at init - thrust_device::vector hor_avg(opts_init.nz); - thrust_device::vector hor_avg_count(opts_init.nz); + thrust_device::vector hor_sum(opts_init.nz); + thrust_device::vector hor_sum_count(opts_init.nz); thrust_device::vector hor_missing(opts_init.nz); - thrust_device::vector hor_avg_k(opts_init.nz); - thrust_device::vector expected_hor_avg(opts_init.nz); + thrust_device::vector hor_sum_k(opts_init.nz); + thrust_device::vector expected_hor_sum(opts_init.nz); thrust_device::vector create_SD(opts_init.nz); // could be bool, but then thrust::reduce does not add bools as expected // calc sum of ln(rd) ranges of all relax distributions @@ -129,19 +161,20 @@ namespace libcloudphxx moms_rng(rd3_min, rd3_max, rd3.begin(), n_part_pre_relax, true); // calculate 0-th non-specific moment of rd3 (number of droplets in a cell) of droplets in this rd3 and kappa range moms_calc(rd3.begin(), n_part_pre_relax, 0, false); - // divide by volume - thrust::transform( - count_mom.begin(), count_mom.begin() + count_n, // input - first arg - thrust::make_permutation_iterator( // input - second arg - dv.begin(), - count_ijk.begin() - ), - count_mom.begin(), // output (in place) - thrust::divides() - ); - // horizontal average of this moment - thrust::fill(hor_avg.begin(), hor_avg.end(), 0); +// // divide by volume +// thrust::transform( +// count_mom.begin(), count_mom.begin() + count_n, // input - first arg +// thrust::make_permutation_iterator( // input - second arg +// dv.begin(), +// count_ijk.begin() +// ), +// count_mom.begin(), // output (in place) +// thrust::divides() +// ); + + // horizontal sum of this moment + thrust::fill(hor_sum.begin(), hor_sum.end(), 0); thrust_device::vector &count_k(tmp_device_size_cell); // NOTE: tmp_device_size_cell is also used in some other inits, careful not to overwrite it! thrust::transform(count_ijk.begin(), count_ijk.begin() + count_n, count_k.begin(), arg::_1 % opts_init.nz); thrust::sort_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin()); @@ -152,43 +185,45 @@ namespace libcloudphxx //std::cerr << "count_mom:" << std::endl; //debug::print(count_mom.begin(), count_mom.end()); - auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_avg_k.begin(), hor_avg_count.begin()); + auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_sum_k.begin(), hor_sum_count.begin()); - //std::cerr << "hor_avg_k:" << std::endl; - //debug::print(hor_avg_k.begin(), hor_avg_k.end()); - //std::cerr << "hor_avg_count:" << std::endl; - //debug::print(hor_avg_count.begin(), hor_avg_count.end()); + //std::cerr << "hor_sum_k:" << std::endl; + //debug::print(hor_sum_k.begin(), hor_sum_k.end()); + //std::cerr << "hor_sum_count:" << std::endl; + //debug::print(hor_sum_count.begin(), hor_sum_count.end()); - int number_of_levels_with_droplets = new_end.first - hor_avg_k.begin(); // number of levels with any SD, not with SD in this size and kappa range + int number_of_levels_with_droplets = new_end.first - hor_sum_k.begin(); // number of levels with any SD, not with SD in this size and kappa range //std::cerr << "number_of_levels_with_droplets: " << number_of_levels_with_droplets << std::endl; assert(number_of_levels_with_droplets <= opts_init.nz); - thrust::copy(hor_avg_count.begin(), hor_avg_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_avg.begin(), hor_avg_k.begin())); + thrust::copy(hor_sum_count.begin(), hor_sum_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_sum.begin(), hor_sum_k.begin())); // divide sum by the number of cells at this level - thrust::transform(hor_avg.begin(), hor_avg.end(), hor_avg.begin(), arg::_1 / (opts_init.nx * m1(opts_init.ny))); - +// thrust::transform(hor_sum.begin(), hor_sum.end(), hor_sum.begin(), arg::_1 / (opts_init.nx * m1(opts_init.ny))); - // calculate expected CCN conc + // calculate expected CCN number const real_t bin_lnrd_center = log_rd_min + (bin_number + 0.5) * lnrd_bin_size; const real_t expected_STP_concentration = n_of_lnrd_stp(bin_lnrd_center) * lnrd_bin_size; assert(expected_STP_concentration >= 0); - thrust::fill(expected_hor_avg.begin(), expected_hor_avg.end(), expected_STP_concentration); + thrust::transform(zero, zero + opts_init.nz, expected_hor_sum.begin(), detail::hor_dv_eval(opts_init)); // fill with volume of the domain at this level + thrust::transform(expected_hor_sum.begin(), expected_hor_sum.end(), expected_hor_sum.begin(), expected_STP_concentration * arg::_1); // multiply by the expected concentration + + // TODO: check for overflows? // correcting STP -> actual ambient conditions if(!opts_init.aerosol_independent_of_rhod) { using common::earth::rho_stp; thrust::transform( - expected_hor_avg.begin(), - expected_hor_avg.begin() + opts_init.nz, + expected_hor_sum.begin(), + expected_hor_sum.begin() + opts_init.nz, rhod.begin(), // rhod has size ncell, but vertical cooridnate varies first, so rhod.begin() to rhod.begin()+nz should be the vertical profile? - expected_hor_avg.begin(), + expected_hor_sum.begin(), arg::_1 * arg::_2 / real_t(rho_stp() / si::kilograms * si::cubic_metres) ); } // set to zero outside of the defined range of altitudes - thrust::replace_if(expected_hor_avg.begin(), expected_hor_avg.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 > z_max_index, real_t(0)); + thrust::replace_if(expected_hor_sum.begin(), expected_hor_sum.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 > z_max_index, real_t(0)); //std::cerr << "bin number: " << bin_number ; //std::cerr << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) ; @@ -197,20 +232,20 @@ namespace libcloudphxx //std::cerr << " expected STD concentration: " << expected_STP_concentration ; //std::cerr << std::endl; - //std::cerr << "hor_avg:" << std::endl; - //debug::print(hor_avg); + //std::cerr << "hor_sum:" << std::endl; + //debug::print(hor_sum); - //std::cerr << "expected_hor_avg:" << std::endl; - //debug::print(expected_hor_avg); + //std::cerr << "expected_hor_sum:" << std::endl; + //debug::print(expected_hor_sum); // calculate how many CCN are missing - thrust::transform(expected_hor_avg.begin(), expected_hor_avg.end(), hor_avg.begin(), hor_missing.begin(), arg::_1 - arg::_2); + thrust::transform(expected_hor_sum.begin(), expected_hor_sum.end(), hor_sum.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); //std::cerr << "hor_missing:" << std::endl; //debug::print(hor_missing); // set number of SDs to init; create only if concentration is lower than expected with a tolerance - thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_avg.begin(), create_SD.begin(), detail::relax_conc_tolerance(config.rlx_conc_tolerance)); + thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_sum.begin(), create_SD.begin(), detail::relax_conc_tolerance(config.rlx_conc_tolerance)); //std::cerr << "create_SD:" << std::endl; //debug::print(create_SD); @@ -286,8 +321,10 @@ namespace libcloudphxx //debug::print(n.begin()+n_part_old, n.end()); // detecting possible overflows of n type - thrust_size_t ix = thrust::max_element(n.begin() + n_part_old, n.end()) - (n.begin() + n_part_old); - assert(n[ix] < (typename impl::n_t)(-1) / 10000); + { + thrust_size_t ix = thrust::max_element(n.begin() + n_part_old, n.end()) - (n.begin() + n_part_old); + assert(n[ix] < (typename impl::n_t)(-1) / 10000); + } //std::cerr << "rd3:" << std::endl; From f4067d8ae05e71aade55c096afbaf1294f54c256 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 20 May 2021 16:26:38 +0200 Subject: [PATCH 033/282] relaxation: sd_per_bin --- CMakeLists.txt | 4 +- bindings/python/lib.cpp | 1 + include/libcloudph++/lgrngn/opts_init.hpp | 6 +- src/impl/particles_impl_init_ijk.ipp | 7 +- src/impl/particles_impl_init_sanity_check.ipp | 8 +- src/impl/particles_impl_rlx_dry_distros.ipp | 124 +++++++++++------- tests/python/unit/relax.py | 70 +++++++--- 7 files changed, 146 insertions(+), 74 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f5ada93ea..03bd41a7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,9 +355,9 @@ endif() enable_testing() -#add_subdirectory(tests) +add_subdirectory(tests) add_subdirectory(include) -#add_subdirectory(bindings) +add_subdirectory(bindings) ############################################################################################ # installation diff --git a/bindings/python/lib.cpp b/bindings/python/lib.cpp index 3acd937f7..352b46677 100644 --- a/bindings/python/lib.cpp +++ b/bindings/python/lib.cpp @@ -305,6 +305,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("sd_const_multi", &lgr::opts_init_t::sd_const_multi) .def_readwrite("src_sd_conc", &lgr::opts_init_t::src_sd_conc) .def_readwrite("rlx_bins", &lgr::opts_init_t::rlx_bins) + .def_readwrite("rlx_sd_per_bin", &lgr::opts_init_t::rlx_sd_per_bin) .def_readwrite("n_sd_max", &lgr::opts_init_t::n_sd_max) .def_readwrite("terminal_velocity", &lgr::opts_init_t::terminal_velocity) .def_readwrite("RH_formula", &lgr::opts_init_t::RH_formula) diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index dc0d736c7..6d201ae46 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -168,8 +168,11 @@ namespace libcloudphxx rlx_dry_distros_t rlx_dry_distros; - // number of bins into which the relaxation distro is divided, up to one SD is created per bin + // number of bins into which the relaxation distro is divided unsigned long long rlx_bins; + + // number of SD created per bin + unsigned long long rlx_sd_per_bin; // timestep interval at which relaxation will be applied int supstp_rlx; @@ -218,6 +221,7 @@ namespace libcloudphxx src_z1(0), supstp_src(1), rlx_bins(0), + rlx_sd_per_bin(0), supstp_rlx(1), rd_min(0.), diag_incloud_time(false), diff --git a/src/impl/particles_impl_init_ijk.ipp b/src/impl/particles_impl_init_ijk.ipp index 709335f60..ee2282a33 100644 --- a/src/impl/particles_impl_init_ijk.ipp +++ b/src/impl/particles_impl_init_ijk.ipp @@ -12,10 +12,11 @@ namespace libcloudphxx { namespace detail { + template struct arbitrary_sequence //fill container with n 0s, m 1s, l 2s, etc... { - thrust_device::pointer res; - arbitrary_sequence(thrust_device::pointer res): res(res) {} + thrust_device::pointer res; + arbitrary_sequence(thrust_device::pointer res): res(res) {} template BOOST_GPU_ENABLED @@ -45,7 +46,7 @@ namespace libcloudphxx thrust::make_zip_iterator(thrust::make_tuple( count_num.end(), ptr.end(), zero + n_cell )), - detail::arbitrary_sequence(&(ijk[n_part_old])) + detail::arbitrary_sequence(&(ijk[n_part_old])) ); } }; diff --git a/src/impl/particles_impl_init_sanity_check.ipp b/src/impl/particles_impl_init_sanity_check.ipp index 97972aebc..02b825698 100644 --- a/src/impl/particles_impl_init_sanity_check.ipp +++ b/src/impl/particles_impl_init_sanity_check.ipp @@ -131,8 +131,12 @@ namespace libcloudphxx if(opts_init.rlx_switch) std::cerr << "libcloudph++ WARNING: relaxation is not fully supported in MPI runs. Mean calculation and addition of SD will be done locally on each node." << std::endl; #endif - if(n_dims < 2 && opts_init.rlx_switch) - throw std::runtime_error("CCN relaxation works only in 2D and 3D, set rlx_switch to false"); + if(n_dims < 2 && opts_init.rlx_switch) + throw std::runtime_error("CCN relaxation works only in 2D and 3D, set rlx_switch to false"); + if(opts_init.rlx_switch && opts_init.rlx_bins <= 0) + throw std::runtime_error("rlx_bins <= 0"); + if(opts_init.rlx_switch && opts_init.rlx_sd_per_bin <= 0) + throw std::runtime_error("n_sd_per_bin <= 0"); } }; }; diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 9fadb4da7..4b98ad305 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -29,12 +29,14 @@ namespace libcloudphxx }; template - struct relax_conc_tolerance + struct calc_n_sd_to_create { const real_t tolerance; + const int n_sd_per_bin; - relax_conc_tolerance(const real_t tol): - tolerance(tol) + calc_n_sd_to_create(const real_t tol, const int n_sd_per_bin): + tolerance(tol), + n_sd_per_bin(n_sd_per_bin) {} BOOST_GPU_ENABLED @@ -45,7 +47,7 @@ namespace libcloudphxx // : 0); // //std::cerr << "a1: " << a1 << " a2: " << a2 << " tol: " << tolerance << " result: " << bool(a1 / a2 > tolerance) << std::endl; return a2 > real_t(0) ? - a1 / a2 > tolerance ? 1 : 0 + a1 / a2 > tolerance ? n_sd_per_bin : 0 : 0; } }; @@ -95,7 +97,7 @@ namespace libcloudphxx thrust_device::vector hor_missing(opts_init.nz); thrust_device::vector hor_sum_k(opts_init.nz); thrust_device::vector expected_hor_sum(opts_init.nz); - thrust_device::vector create_SD(opts_init.nz); // could be bool, but then thrust::reduce does not add bools as expected + thrust_device::vector n_SD_to_create(opts_init.nz); // could be bool, but then thrust::reduce does not add bools as expected // calc sum of ln(rd) ranges of all relax distributions real_t tot_lnrd_rng = 0.; @@ -223,7 +225,7 @@ namespace libcloudphxx } // set to zero outside of the defined range of altitudes - thrust::replace_if(expected_hor_sum.begin(), expected_hor_sum.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 > z_max_index, real_t(0)); + thrust::replace_if(expected_hor_sum.begin(), expected_hor_sum.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 >= z_max_index, real_t(0)); //std::cerr << "bin number: " << bin_number ; //std::cerr << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) ; @@ -241,17 +243,17 @@ namespace libcloudphxx thrust::transform(expected_hor_sum.begin(), expected_hor_sum.end(), hor_sum.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); - //std::cerr << "hor_missing:" << std::endl; - //debug::print(hor_missing); + std::cerr << "hor_missing:" << std::endl; + debug::print(hor_missing); // set number of SDs to init; create only if concentration is lower than expected with a tolerance - thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_sum.begin(), create_SD.begin(), detail::relax_conc_tolerance(config.rlx_conc_tolerance)); + thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_sum.begin(), n_SD_to_create.begin(), detail::calc_n_sd_to_create(config.rlx_conc_tolerance, opts_init.rlx_sd_per_bin)); - //std::cerr << "create_SD:" << std::endl; - //debug::print(create_SD); + std::cerr << "n_SD_to_create:" << std::endl; + debug::print(n_SD_to_create); n_part_old = n_part; - n_part_to_init = thrust::reduce(create_SD.begin(), create_SD.end()); + n_part_to_init = thrust::reduce(n_SD_to_create.begin(), n_SD_to_create.end()); n_part = n_part_old + n_part_to_init; //std::cerr << "n_part_to_init: " << n_part_to_init << std::endl; @@ -264,9 +266,55 @@ namespace libcloudphxx rd3.resize(n_part); n.resize(n_part); - // k index based on create_SD - thrust::copy_if(zero, zero+opts_init.nz, create_SD.begin(), k.begin()+n_part_old, arg::_1 == 1); - // i and j random + // --- init k --- + thrust_device::vector &ptr(tmp_device_size_cell); + thrust::exclusive_scan(n_SD_to_create.begin(), n_SD_to_create.end(), ptr.begin()); // number of SDs in cells to init up to (i-1) + + thrust::for_each( + thrust::make_zip_iterator(thrust::make_tuple( + n_SD_to_create.begin(), ptr.begin(), zero + )), + thrust::make_zip_iterator(thrust::make_tuple( + n_SD_to_create.end(), ptr.end(), zero + opts_init.nz + )), + detail::arbitrary_sequence(&(k[n_part_old])) + ); + + + // --- init multiplicities (includes casting from real to n) --- + +// thrust::copy_if( +// thrust::make_transform_iterator(hor_missing.begin(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) + real_t(0.5)), +// thrust::make_transform_iterator(hor_missing.begin(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) + real_t(0.5)) + opts_init.nz, +// n_SD_to_create.begin(), +// n.begin()+n_part_old, +// arg::_1 > 0 +// ); + + thrust::for_each( + thrust::make_zip_iterator(thrust::make_tuple( + n_SD_to_create.begin(), ptr.begin(), + thrust::make_transform_iterator(hor_missing.begin(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) + real_t(0.5)) +// zero + )), + thrust::make_zip_iterator(thrust::make_tuple( + n_SD_to_create.end(), ptr.end(), + thrust::make_transform_iterator(hor_missing.end(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) + real_t(0.5)) + // zero + opts_init.nz + )), + detail::arbitrary_sequence(&(n[n_part_old])) + ); + + std::cerr << "n:" << std::endl; + debug::print(n.begin()+n_part_old, n.end()); + + // detecting possible overflows of n type + { + thrust_size_t ix = thrust::max_element(n.begin() + n_part_old, n.end()) - (n.begin() + n_part_old); + assert(n[ix] < (typename impl::n_t)(-1) / 10000); + } + + // --- init of i and j --- // tossing random numbers [0,1) TODO: do it once for all bins rand_u01(n_part_to_init * (n_dims)); // random numbers for: i, rd, j (j only in 3D) @@ -280,20 +328,20 @@ namespace libcloudphxx ravel_ijk(n_part_old); // set count_num to the number of SD to init per cell - thrust::fill(count_num.begin(), count_num.end(), 0); - thrust::scatter(thrust::make_constant_iterator(1), thrust::make_constant_iterator(1) + n_part_to_init, ijk.begin() + n_part_old, count_num.begin()); - - //std::cerr << "i:" << std::endl; - //debug::print(i.begin()+n_part_old, i.end()); +// thrust::fill(count_num.begin(), count_num.end(), 0); +// thrust::scatter(thrust::make_constant_iterator(opts_init.rlx_sd_per_bin), thrust::make_constant_iterator(opts_init.rlx_sd_per_bin) + n_part_to_init, ijk.begin() + n_part_old, count_num.begin()); - //std::cerr << "k:" << std::endl; - //debug::print(k.begin()+n_part_old, k.end()); + std::cerr << "i:" << std::endl; + debug::print(i.begin()+n_part_old, i.end()); - //std::cerr << "ijk:" << std::endl; - //debug::print(ijk.begin()+n_part_old, ijk.end()); + std::cerr << "k:" << std::endl; + debug::print(k.begin()+n_part_old, k.end()); - //std::cerr << "count_num:" << std::endl; - //debug::print(count_num); + std::cerr << "ijk:" << std::endl; + debug::print(ijk.begin()+n_part_old, ijk.end()); +// +// std::cerr << "count_num:" << std::endl; +// debug::print(count_num); // init dry radius // set rd3 randomized within the bin, uniformly distributed on the log(rd) axis @@ -308,34 +356,16 @@ namespace libcloudphxx detail::exp3x() ); - // init multiplicities (includes casting from real to n) - thrust::copy_if( - thrust::make_transform_iterator(hor_missing.begin(), arg::_1 + real_t(0.5)), - thrust::make_transform_iterator(hor_missing.begin(), arg::_1 + real_t(0.5)) + opts_init.nz, - create_SD.begin(), - n.begin()+n_part_old, - arg::_1 == 1 - ); - - //std::cerr << "n:" << std::endl; - //debug::print(n.begin()+n_part_old, n.end()); - - // detecting possible overflows of n type - { - thrust_size_t ix = thrust::max_element(n.begin() + n_part_old, n.end()) - (n.begin() + n_part_old); - assert(n[ix] < (typename impl::n_t)(-1) / 10000); - } - - //std::cerr << "rd3:" << std::endl; - //debug::print(rd3.begin()+n_part_old, rd3.end()); + std::cerr << "rd3:" << std::endl; + debug::print(rd3.begin()+n_part_old, rd3.end()); // NOTE: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } // end of the bins loop // init other SD characteristics that don't have to be initialized in the bins loop n_part_old = n_part_pre_bins_loop; n_part_to_init = n_part - n_part_old; - //std::cerr << "n_part: " << n_part << " n_part_old: " << n_part_old << " n_part_to_init: " << n_part_to_init << std::endl; + std::cerr << "n_part: " << n_part << " n_part_old: " << n_part_old << " n_part_to_init: " << n_part_to_init << std::endl; hskpng_resize_npart(); init_SD_with_distros_finalize(kappa, false); // no need to unravel ijk there, becaues i j k are already initialized diff --git a/tests/python/unit/relax.py b/tests/python/unit/relax.py index 9f59e43cc..c0eee6c36 100644 --- a/tests/python/unit/relax.py +++ b/tests/python/unit/relax.py @@ -22,7 +22,7 @@ def lognormal(lnr): -pow((lnr - log(mean_r)), 2) / 2 / pow(log(stdev),2) ) / log(stdev) / sqrt(2*pi); -def lognormal(lnr): +def lognormal_rlx(lnr): mean_r = .04e-6 / 2 stdev = 1.4 n_tot = 120e6 @@ -51,6 +51,9 @@ def test(opts_init): opts_init.x1=opts_init.nx * opts_init.dx; opts_init.z1=opts_init.nz * opts_init.dz; opts_init.aerosol_independent_of_rhod=1; + + opts_init.y0=0.; + opts_init.y1=1.; opts_init.chem_switch = 0; opts_init.coal_switch = 0; @@ -59,11 +62,6 @@ def test(opts_init): opts_init.sedi_switch = 0; opts_init.src_switch = 0; opts_init.rlx_switch = 1; - - opts_init.src_z0 = 0; - opts_init.src_z1 = opts_init.dz; #create aerosol only in the lower cells - opts_init.src_x0 = 0; - opts_init.src_x1 = opts_init.dx*opts_init.nx; opts = lgrngn.opts_t() @@ -84,9 +82,9 @@ def test(opts_init): prtcls.init(th, rv, rhod) - # 4 steps during which relaxation should be done twice + # 2 steps during which relaxation should be done once opts.rlx = 1 - for i in range(4): + for i in range(2): prtcls.step_sync(opts,th,rv,rhod) prtcls.step_async(opts) @@ -108,33 +106,67 @@ def test(opts_init): kappa = .61 opts_init = lgrngn.opts_init_t() opts_init.dry_distros = {kappa:lognormal} -opts_init.rlx_dry_distros = {kappa: [lognormal, [0,2],[0,opts_init.dz]]} -opts_init.sd_conc = 1024#100#24 -opts_init.rlx_bins = 1024#100# 512 -opts_init.n_sd_max = int((opts_init.sd_conc * 2 + opts_init.rlx_bins * 2) * 2) # assuming nx=nz=2 +opts_init.rlx_dry_distros = {kappa: [lognormal_rlx, [0,2],[0,opts_init.dz]]} +opts_init.sd_conc = 1024 +opts_init.rlx_bins = 1024 + + + + +print(' --- dry_distros rlx sd_per_bin = 1 ---') -print(' --- dry_distros rlx ---') +opts_init.rlx_sd_per_bin = 1 +opts_init.n_sd_max = int((opts_init.sd_conc * 2 + opts_init.rlx_bins * opts_init.rlx_sd_per_bin * 2) * 2) # assuming nx=nz=2 sd_conc, wet_mom0, wet_mom1 = test(opts_init) print('diag_sd_conc', sd_conc) # relaxation should add SD only in the lower cells # there is some randomness in the number of SD that should be added -exp_sd_min = 1370 -exp_sd_max = 1470 +exp_sd_min = 1024 + opts_init.rlx_sd_per_bin * 400 +exp_sd_max = 1024 + opts_init.rlx_sd_per_bin * 600 if sd_conc[0] < exp_sd_min or sd_conc[0] > exp_sd_max or sd_conc[2] < exp_sd_min or sd_conc[2] > exp_sd_max: raise Exception("wrong amount of SDs were added") if not(sd_conc[1] == 1024 and sd_conc[3] == 1024): raise Exception("SDs were added in wrong cells") print(('wet mom0', wet_mom0)) -print(abs( (sd_conc[0] + sd_conc[2]) / (sd_conc[1] + sd_conc[3]) - (wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) )) -if (abs( (sd_conc[0] + sd_conc[2]) / (sd_conc[1] + sd_conc[3]) - (wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) ) > 0.1): +print((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3])) # relax n_stp is two times bigger +if abs((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) - 2) > 0.01: raise Exception("incorrect multiplicity after source") print(('wet mom1', wet_mom1)) -print(abs( (sd_conc[0] + sd_conc[2]) / (sd_conc[1] + sd_conc[3]) - (wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) )) -if (abs( (sd_conc[0] + sd_conc[2]) / (sd_conc[1] + sd_conc[3]) - (wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) ) > 0.1): +print((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3])) # relax n_stp is two times bigger +if abs((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) - 2) > 0.01: raise Exception("incorrect radius after source") + + +print(' --- dry_distros rlx sd_per_bin = 10 ---') + +opts_init.rlx_sd_per_bin = 10 +opts_init.n_sd_max = int((opts_init.sd_conc * 2 + opts_init.rlx_bins * opts_init.rlx_sd_per_bin * 2) * 2) # assuming nx=nz=2 + +sd_conc, wet_mom0, wet_mom1 = test(opts_init) + +print('diag_sd_conc', sd_conc) +# relaxation should add SD only in the lower cells +# there is some randomness in the number of SD that should be added +exp_sd_min = 1024 + opts_init.rlx_sd_per_bin * 400 +exp_sd_max = 1024 + opts_init.rlx_sd_per_bin * 600 +if sd_conc[0] < exp_sd_min or sd_conc[0] > exp_sd_max or sd_conc[2] < exp_sd_min or sd_conc[2] > exp_sd_max: + raise Exception("wrong amount of SDs were added") +if not(sd_conc[1] == 1024 and sd_conc[3] == 1024): + raise Exception("SDs were added in wrong cells") + +print(('wet mom0', wet_mom0)) +print((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3])) # relax n_stp is two times bigger +if abs((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) - 2) > 0.01: + raise Exception("incorrect multiplicity after source") + +print(('wet mom1', wet_mom1)) +print((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3])) # relax n_stp is two times bigger +if abs((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) - 2) > 0.01: + raise Exception("incorrect radius after source") + From a1d509f62f323488ab1edb51ecf13319023bba6a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 20 May 2021 17:07:14 +0200 Subject: [PATCH 034/282] relaxation: include timescale --- bindings/python/lib.cpp | 1 + include/libcloudph++/lgrngn/opts_init.hpp | 4 ++++ src/impl/particles_impl.ipp | 4 ++-- src/impl/particles_impl_init_sanity_check.ipp | 2 ++ src/impl/particles_impl_rlx.ipp | 4 ++-- src/impl/particles_impl_rlx_dry_distros.ipp | 19 +++++++------------ src/particles_step.ipp | 2 +- tests/python/unit/relax.py | 9 +++++---- 8 files changed, 24 insertions(+), 21 deletions(-) diff --git a/bindings/python/lib.cpp b/bindings/python/lib.cpp index 352b46677..d2c4931ae 100644 --- a/bindings/python/lib.cpp +++ b/bindings/python/lib.cpp @@ -306,6 +306,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("src_sd_conc", &lgr::opts_init_t::src_sd_conc) .def_readwrite("rlx_bins", &lgr::opts_init_t::rlx_bins) .def_readwrite("rlx_sd_per_bin", &lgr::opts_init_t::rlx_sd_per_bin) + .def_readwrite("rlx_timescale", &lgr::opts_init_t::rlx_timescale) .def_readwrite("n_sd_max", &lgr::opts_init_t::n_sd_max) .def_readwrite("terminal_velocity", &lgr::opts_init_t::terminal_velocity) .def_readwrite("RH_formula", &lgr::opts_init_t::RH_formula) diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index 6d201ae46..ab1b790bd 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -177,6 +177,9 @@ namespace libcloudphxx // timestep interval at which relaxation will be applied int supstp_rlx; + // relaxation time scale [s] + real_t rlx_timescale; + // -- ctors --- // ctor with defaults (C++03 compliant) ... @@ -221,6 +224,7 @@ namespace libcloudphxx src_z1(0), supstp_src(1), rlx_bins(0), + rlx_timescale(1), rlx_sd_per_bin(0), supstp_rlx(1), rd_min(0.), diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index a5958315d..ff2a08879 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -581,8 +581,8 @@ namespace libcloudphxx void src_dry_distros(const real_t &dt); void src_dry_sizes(const real_t &dt); - void rlx(); - void rlx_dry_distros(); + void rlx(const real_t); + void rlx_dry_distros(const real_t); void sstp_step(const int &step); void sstp_step_exact(const int &step); diff --git a/src/impl/particles_impl_init_sanity_check.ipp b/src/impl/particles_impl_init_sanity_check.ipp index 02b825698..96c3be6e7 100644 --- a/src/impl/particles_impl_init_sanity_check.ipp +++ b/src/impl/particles_impl_init_sanity_check.ipp @@ -137,6 +137,8 @@ namespace libcloudphxx throw std::runtime_error("rlx_bins <= 0"); if(opts_init.rlx_switch && opts_init.rlx_sd_per_bin <= 0) throw std::runtime_error("n_sd_per_bin <= 0"); + if(opts_init.rlx_switch && opts_init.rlx_timescale <= 0) + throw std::runtime_error("rlx_timescale <= 0"); } }; }; diff --git a/src/impl/particles_impl_rlx.ipp b/src/impl/particles_impl_rlx.ipp index 2f37040b4..c8264d8f8 100644 --- a/src/impl/particles_impl_rlx.ipp +++ b/src/impl/particles_impl_rlx.ipp @@ -10,12 +10,12 @@ namespace libcloudphxx namespace lgrngn { template - void particles_t::impl::rlx() + void particles_t::impl::rlx(const real_t dt) { ante_adding_SD(); if(!opts_init.rlx_dry_distros.empty()) - rlx_dry_distros(); + rlx_dry_distros(dt); post_adding_SD(); } diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 4b98ad305..49457709f 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -87,7 +87,7 @@ namespace libcloudphxx // create new aerosol particles to relax towards a size distribution template - void particles_t::impl::rlx_dry_distros() + void particles_t::impl::rlx_dry_distros(const real_t dt) { namespace arg = thrust::placeholders; @@ -282,25 +282,20 @@ namespace libcloudphxx // --- init multiplicities (includes casting from real to n) --- +#if !defined(__NVCC__) + using std::min; +#endif -// thrust::copy_if( -// thrust::make_transform_iterator(hor_missing.begin(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) + real_t(0.5)), -// thrust::make_transform_iterator(hor_missing.begin(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) + real_t(0.5)) + opts_init.nz, -// n_SD_to_create.begin(), -// n.begin()+n_part_old, -// arg::_1 > 0 -// ); + std::cerr << "dt: " << dt << " rlx_timescale: " << opts_init.rlx_timescale << " hor_avg fraction added: " << min(dt / opts_init.rlx_timescale, real_t(1)) << std::endl; thrust::for_each( thrust::make_zip_iterator(thrust::make_tuple( n_SD_to_create.begin(), ptr.begin(), - thrust::make_transform_iterator(hor_missing.begin(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) + real_t(0.5)) -// zero + thrust::make_transform_iterator(hor_missing.begin(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) * min(dt / opts_init.rlx_timescale, real_t(1)) + real_t(0.5)) )), thrust::make_zip_iterator(thrust::make_tuple( n_SD_to_create.end(), ptr.end(), - thrust::make_transform_iterator(hor_missing.end(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) + real_t(0.5)) - // zero + opts_init.nz + thrust::make_transform_iterator(hor_missing.end(), arg::_1 / real_t(opts_init.rlx_sd_per_bin) * min(dt / opts_init.rlx_timescale, real_t(1)) + real_t(0.5)) )), detail::arbitrary_sequence(&(n[n_part_old])) ); diff --git a/src/particles_step.ipp b/src/particles_step.ipp index 7b3657278..45b0de07d 100644 --- a/src/particles_step.ipp +++ b/src/particles_step.ipp @@ -286,7 +286,7 @@ namespace libcloudphxx // introduce new particles with the given time interval if(pimpl->rlx_stp_ctr % pimpl->opts_init.supstp_rlx == 0) { - pimpl->rlx(); + pimpl->rlx(pimpl->opts_init.supstp_rlx * pimpl->dt); } } diff --git a/tests/python/unit/relax.py b/tests/python/unit/relax.py index c0eee6c36..6a1228f5a 100644 --- a/tests/python/unit/relax.py +++ b/tests/python/unit/relax.py @@ -109,6 +109,7 @@ def test(opts_init): opts_init.rlx_dry_distros = {kappa: [lognormal_rlx, [0,2],[0,opts_init.dz]]} opts_init.sd_conc = 1024 opts_init.rlx_bins = 1024 +opts_init.rlx_timescale = 4 # whole simulation time is 2, so this means we should get half of the droplets added @@ -132,12 +133,12 @@ def test(opts_init): print(('wet mom0', wet_mom0)) print((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3])) # relax n_stp is two times bigger -if abs((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) - 2) > 0.01: +if abs((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) - 1.5) > 0.01: # we expect 1.5, because n_rlx is two tmes bigger, but relaxation time scale is twice the simulation time raise Exception("incorrect multiplicity after source") print(('wet mom1', wet_mom1)) print((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3])) # relax n_stp is two times bigger -if abs((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) - 2) > 0.01: +if abs((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) - 1.5) > 0.01: raise Exception("incorrect radius after source") @@ -162,11 +163,11 @@ def test(opts_init): print(('wet mom0', wet_mom0)) print((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3])) # relax n_stp is two times bigger -if abs((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) - 2) > 0.01: +if abs((wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) - 1.5) > 0.01: raise Exception("incorrect multiplicity after source") print(('wet mom1', wet_mom1)) print((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3])) # relax n_stp is two times bigger -if abs((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) - 2) > 0.01: +if abs((wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) - 1.5) > 0.01: raise Exception("incorrect radius after source") From c6f7c1b3917d8124b5204a59747805c7800acdef Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 20 May 2021 17:08:06 +0200 Subject: [PATCH 035/282] comment debug output in relax --- src/impl/particles_impl_rlx_dry_distros.ipp | 36 ++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 49457709f..aa5bc3795 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -243,14 +243,14 @@ namespace libcloudphxx thrust::transform(expected_hor_sum.begin(), expected_hor_sum.end(), hor_sum.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); - std::cerr << "hor_missing:" << std::endl; - debug::print(hor_missing); + //std::cerr << "hor_missing:" << std::endl; + //debug::print(hor_missing); // set number of SDs to init; create only if concentration is lower than expected with a tolerance thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_sum.begin(), n_SD_to_create.begin(), detail::calc_n_sd_to_create(config.rlx_conc_tolerance, opts_init.rlx_sd_per_bin)); - std::cerr << "n_SD_to_create:" << std::endl; - debug::print(n_SD_to_create); + //std::cerr << "n_SD_to_create:" << std::endl; + //debug::print(n_SD_to_create); n_part_old = n_part; n_part_to_init = thrust::reduce(n_SD_to_create.begin(), n_SD_to_create.end()); @@ -286,7 +286,7 @@ namespace libcloudphxx using std::min; #endif - std::cerr << "dt: " << dt << " rlx_timescale: " << opts_init.rlx_timescale << " hor_avg fraction added: " << min(dt / opts_init.rlx_timescale, real_t(1)) << std::endl; + //std::cerr << "dt: " << dt << " rlx_timescale: " << opts_init.rlx_timescale << " hor_avg fraction added: " << min(dt / opts_init.rlx_timescale, real_t(1)) << std::endl; thrust::for_each( thrust::make_zip_iterator(thrust::make_tuple( @@ -300,8 +300,8 @@ namespace libcloudphxx detail::arbitrary_sequence(&(n[n_part_old])) ); - std::cerr << "n:" << std::endl; - debug::print(n.begin()+n_part_old, n.end()); + //std::cerr << "n:" << std::endl; + //debug::print(n.begin()+n_part_old, n.end()); // detecting possible overflows of n type { @@ -326,17 +326,17 @@ namespace libcloudphxx // thrust::fill(count_num.begin(), count_num.end(), 0); // thrust::scatter(thrust::make_constant_iterator(opts_init.rlx_sd_per_bin), thrust::make_constant_iterator(opts_init.rlx_sd_per_bin) + n_part_to_init, ijk.begin() + n_part_old, count_num.begin()); - std::cerr << "i:" << std::endl; - debug::print(i.begin()+n_part_old, i.end()); + //std::cerr << "i:" << std::endl; + //debug::print(i.begin()+n_part_old, i.end()); - std::cerr << "k:" << std::endl; - debug::print(k.begin()+n_part_old, k.end()); + //std::cerr << "k:" << std::endl; + //debug::print(k.begin()+n_part_old, k.end()); - std::cerr << "ijk:" << std::endl; - debug::print(ijk.begin()+n_part_old, ijk.end()); + //std::cerr << "ijk:" << std::endl; + //debug::print(ijk.begin()+n_part_old, ijk.end()); // -// std::cerr << "count_num:" << std::endl; -// debug::print(count_num); +// //std::cerr << "count_num:" << std::endl; +// //debug::print(count_num); // init dry radius // set rd3 randomized within the bin, uniformly distributed on the log(rd) axis @@ -352,15 +352,15 @@ namespace libcloudphxx ); - std::cerr << "rd3:" << std::endl; - debug::print(rd3.begin()+n_part_old, rd3.end()); + //std::cerr << "rd3:" << std::endl; + //debug::print(rd3.begin()+n_part_old, rd3.end()); // NOTE: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } // end of the bins loop // init other SD characteristics that don't have to be initialized in the bins loop n_part_old = n_part_pre_bins_loop; n_part_to_init = n_part - n_part_old; - std::cerr << "n_part: " << n_part << " n_part_old: " << n_part_old << " n_part_to_init: " << n_part_to_init << std::endl; + //std::cerr << "n_part: " << n_part << " n_part_old: " << n_part_old << " n_part_to_init: " << n_part_to_init << std::endl; hskpng_resize_npart(); init_SD_with_distros_finalize(kappa, false); // no need to unravel ijk there, becaues i j k are already initialized From 2c93d6769bb3a7536feb79228042ca6b8242ecfa Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 21 May 2021 12:05:17 +0200 Subject: [PATCH 036/282] do src in rlx in async, they dont affect th and rv now --- src/impl/particles_impl_rlx.ipp | 4 +- src/impl/particles_impl_src.ipp | 4 +- src/particles_step.ipp | 73 +++++++++++++++++---------------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/impl/particles_impl_rlx.ipp b/src/impl/particles_impl_rlx.ipp index c8264d8f8..3d83a4c2e 100644 --- a/src/impl/particles_impl_rlx.ipp +++ b/src/impl/particles_impl_rlx.ipp @@ -12,12 +12,12 @@ namespace libcloudphxx template void particles_t::impl::rlx(const real_t dt) { - ante_adding_SD(); + // ante_adding_SD(); if(!opts_init.rlx_dry_distros.empty()) rlx_dry_distros(dt); - post_adding_SD(); +// post_adding_SD(); } }; }; diff --git a/src/impl/particles_impl_src.ipp b/src/impl/particles_impl_src.ipp index 2c18390f2..4044b5cab 100644 --- a/src/impl/particles_impl_src.ipp +++ b/src/impl/particles_impl_src.ipp @@ -12,7 +12,7 @@ namespace libcloudphxx template void particles_t::impl::src(const real_t &dt) { - ante_adding_SD(); + // ante_adding_SD(); if(!opts_init.src_dry_distros.empty()) src_dry_distros(dt); @@ -20,7 +20,7 @@ namespace libcloudphxx if(!opts_init.src_dry_sizes.empty()) src_dry_sizes(dt); - post_adding_SD(); +// post_adding_SD(); } }; }; diff --git a/src/particles_step.ipp b/src/particles_step.ipp index 45b0de07d..8d46af8fa 100644 --- a/src/particles_step.ipp +++ b/src/particles_step.ipp @@ -261,48 +261,13 @@ namespace libcloudphxx } } - // TODO: move source and relax to async? th/rv changes are minimal - - // aerosol source, in sync since it changes th/rv - if (opts.src && !(pimpl->opts_init.src_x0 == 0 && pimpl->opts_init.src_x1 == 0)) // src_x0=0 and src_x1=0 is a way of disabling source in some domains in distmem simulations - { - // sanity check - if (pimpl->opts_init.src_switch == false) throw std::runtime_error("aerosol source was switched off in opts_init"); - - // introduce new particles with the given time interval - if(pimpl->src_stp_ctr % pimpl->opts_init.supstp_src == 0) - { - pimpl->src(pimpl->opts_init.supstp_src * pimpl->dt); - } - } - - // aerosol relaxation, in sync since it changes th/rv - // TODO: more sanity checks for rlx! 3D only, values of rlx_bins etc. check that appa ranges are exclusive, zminopts_init.rlx_switch == false) throw std::runtime_error("aerosol relaxation was switched off in opts_init"); - - // introduce new particles with the given time interval - if(pimpl->rlx_stp_ctr % pimpl->opts_init.supstp_rlx == 0) - { - pimpl->rlx(pimpl->opts_init.supstp_rlx * pimpl->dt); - } - } - - if(opts.cond || (opts.src && pimpl->src_stp_ctr % pimpl->opts_init.supstp_src == 0) || (opts.rlx && pimpl->rlx_stp_ctr % pimpl->opts_init.supstp_rlx == 0)) + if(opts.cond) // || (opts.src && pimpl->src_stp_ctr % pimpl->opts_init.supstp_src == 0) || (opts.rlx && pimpl->rlx_stp_ctr % pimpl->opts_init.supstp_rlx == 0)) { // syncing out // TODO: this is not necesarry in off-line mode (see coupling with DALES) pimpl->sync(pimpl->th, th); pimpl->sync(pimpl->rv, rv); } - // update the step counter since src/rlx was turned on - if (opts.src) ++pimpl->src_stp_ctr; - else pimpl->src_stp_ctr = 0; //reset the counter if source was turned off - if (opts.rlx) ++pimpl->rlx_stp_ctr; - else pimpl->rlx_stp_ctr = 0; //reset the counter if source was turned off - if (opts.chem_dsl == true) { // syncing out trace gases // TODO: this is not necesarry in off-line mode (see coupling with DALES) @@ -427,6 +392,42 @@ namespace libcloudphxx pimpl->subs(pimpl->dt); } + // NOTE: source and relax should affect th and rv (because we add humidifed aerosols), but these changes are minimal and we neglect them. + // otherwise src and rlx should be don in step_sync + + // aerosol source, in sync since it changes th/rv + if (opts.src && !(pimpl->opts_init.src_x0 == 0 && pimpl->opts_init.src_x1 == 0)) // src_x0=0 and src_x1=0 is a way of disabling source in some domains in distmem simulations + { + // sanity check + if (pimpl->opts_init.src_switch == false) throw std::runtime_error("aerosol source was switched off in opts_init"); + + // introduce new particles with the given time interval + if(pimpl->src_stp_ctr % pimpl->opts_init.supstp_src == 0) + { + pimpl->src(pimpl->opts_init.supstp_src * pimpl->dt); + } + } + + // aerosol relaxation, in sync since it changes th/rv + // TODO: more sanity checks for rlx! 3D only, values of rlx_bins etc. check that appa ranges are exclusive, zminopts_init.rlx_switch == false) throw std::runtime_error("aerosol relaxation was switched off in opts_init"); + + // introduce new particles with the given time interval + if(pimpl->rlx_stp_ctr % pimpl->opts_init.supstp_rlx == 0) + { + pimpl->rlx(pimpl->opts_init.supstp_rlx * pimpl->dt); + } + } + + // update the step counter since src/rlx was turned on + if (opts.src) ++pimpl->src_stp_ctr; + else pimpl->src_stp_ctr = 0; //reset the counter if source was turned off + if (opts.rlx) ++pimpl->rlx_stp_ctr; + else pimpl->rlx_stp_ctr = 0; //reset the counter if source was turned off + // boundary condition + accumulated rainfall to be returned // distmem version overwrites i and tmp_device_size_part // and they both need to be unchanged untill distmem copies From d022084d8f18c8ba9eda7e790253b1b42a4a3734 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 24 May 2021 14:51:43 +0200 Subject: [PATCH 037/282] multi CUDA: divide rlx_sd_per_bin by the number of GPUs, because SD are created separately on each GPU --- include/libcloudph++/lgrngn/opts_init.hpp | 2 +- src/detail/distmem_opts.hpp | 2 ++ src/impl/particles_impl_rlx_dry_distros.ipp | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index ab1b790bd..1348e0cc3 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -172,7 +172,7 @@ namespace libcloudphxx unsigned long long rlx_bins; // number of SD created per bin - unsigned long long rlx_sd_per_bin; + real_t rlx_sd_per_bin; // floating, because it gets divided by the number of GPUs per node * number of nodes // timestep interval at which relaxation will be applied int supstp_rlx; diff --git a/src/detail/distmem_opts.hpp b/src/detail/distmem_opts.hpp index dd9601b2d..79f400a5e 100644 --- a/src/detail/distmem_opts.hpp +++ b/src/detail/distmem_opts.hpp @@ -46,6 +46,8 @@ namespace libcloudphxx opts_init.n_sd_max = opts_init.n_sd_max / size + 1; + opts_init.rlx_sd_per_bin /= size; + return n_x_bfr; } } diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index aa5bc3795..c2677cbf7 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -34,9 +34,9 @@ namespace libcloudphxx const real_t tolerance; const int n_sd_per_bin; - calc_n_sd_to_create(const real_t tol, const int n_sd_per_bin): + calc_n_sd_to_create(const real_t tol, const real_t n_sd_per_bin_real): tolerance(tol), - n_sd_per_bin(n_sd_per_bin) + n_sd_per_bin(std::max(1, int(n_sd_per_bin_real + 0.5))) {} BOOST_GPU_ENABLED From 9a6fde323bfd01a1091795b2d4969a51670b29e5 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 26 May 2021 11:17:39 +0200 Subject: [PATCH 038/282] correct sanity check message --- src/impl/particles_impl_init_sanity_check.ipp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/impl/particles_impl_init_sanity_check.ipp b/src/impl/particles_impl_init_sanity_check.ipp index 96c3be6e7..30216f68a 100644 --- a/src/impl/particles_impl_init_sanity_check.ipp +++ b/src/impl/particles_impl_init_sanity_check.ipp @@ -136,7 +136,7 @@ namespace libcloudphxx if(opts_init.rlx_switch && opts_init.rlx_bins <= 0) throw std::runtime_error("rlx_bins <= 0"); if(opts_init.rlx_switch && opts_init.rlx_sd_per_bin <= 0) - throw std::runtime_error("n_sd_per_bin <= 0"); + throw std::runtime_error("rlx_sd_per_bin <= 0"); if(opts_init.rlx_switch && opts_init.rlx_timescale <= 0) throw std::runtime_error("rlx_timescale <= 0"); } From 480b7cb05961336e321bb0110cb1bc40affd6874 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 26 May 2021 11:57:35 +0200 Subject: [PATCH 039/282] rlx: include numeric dor iota --- src/impl/particles_impl_rlx_dry_distros.ipp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index c2677cbf7..6fd6487e9 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -7,6 +7,8 @@ // #include #include #include +#include + namespace libcloudphxx { From 8c9248ba511b8bb304ee871f6422f3b7ad706d5c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 9 Jun 2021 15:54:32 +0200 Subject: [PATCH 040/282] init incloud time in init_sd_with_distros and in src_dry_sizes --- src/impl/particles_impl_init_SD_with_distros.ipp | 3 +++ src/impl/particles_impl_src_dry_sizes.ipp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/impl/particles_impl_init_SD_with_distros.ipp b/src/impl/particles_impl_init_SD_with_distros.ipp index a6a8e94c4..0a20ba6c1 100644 --- a/src/impl/particles_impl_init_SD_with_distros.ipp +++ b/src/impl/particles_impl_init_SD_with_distros.ipp @@ -84,6 +84,9 @@ namespace libcloudphxx // initialising particle positions init_xyz(); + + if(opts_init.diag_incloud_time) + init_incloud_time(); } }; }; diff --git a/src/impl/particles_impl_src_dry_sizes.ipp b/src/impl/particles_impl_src_dry_sizes.ipp index 8957d7d5c..8b44eee62 100644 --- a/src/impl/particles_impl_src_dry_sizes.ipp +++ b/src/impl/particles_impl_src_dry_sizes.ipp @@ -79,6 +79,9 @@ namespace libcloudphxx // initialising particle positions init_xyz(); + + if(opts_init.diag_incloud_time) + init_incloud_time(); } } } From 4b46a5aca62676f4e0247bd2d8e0588718070740 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 9 Jun 2021 15:50:20 +0200 Subject: [PATCH 041/282] rlx dry distros: additional debug output --- src/impl/particles_impl_rlx_dry_distros.ipp | 49 ++++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index 6fd6487e9..dd93027d5 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -118,7 +118,7 @@ namespace libcloudphxx for (typename opts_init_t::rlx_dry_distros_t::const_iterator ddi = opts_init.rlx_dry_distros.begin(); ddi != opts_init.rlx_dry_distros.end(); ++ddi) { const auto &kappa(ddi->first); - //std::cerr << "rlx kappa: " << kappa << std::endl; + std::cerr << "rlx kappa: " << kappa << std::endl; assert(kappa >= 0); const auto &n_of_lnrd_stp(*(std::get<0>(ddi->second))); @@ -152,6 +152,8 @@ namespace libcloudphxx const auto n_part_pre_bins_loop = n_part; + real_t expected_STP_concentration_tot = 0; + // loop over the bins for(int bin_number=0; bin_number= 0); + expected_STP_concentration_tot += expected_STP_concentration; thrust::transform(zero, zero + opts_init.nz, expected_hor_sum.begin(), detail::hor_dv_eval(opts_init)); // fill with volume of the domain at this level thrust::transform(expected_hor_sum.begin(), expected_hor_sum.end(), expected_hor_sum.begin(), expected_STP_concentration * arg::_1); // multiply by the expected concentration @@ -229,36 +232,36 @@ namespace libcloudphxx // set to zero outside of the defined range of altitudes thrust::replace_if(expected_hor_sum.begin(), expected_hor_sum.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 >= z_max_index, real_t(0)); - //std::cerr << "bin number: " << bin_number ; - //std::cerr << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) ; - //std::cerr << " r_center: " << std::exp(bin_lnrd_center) ; - //std::cerr << " z_indices: (" << z_min_index << ", " << z_max_index << "), " ; - //std::cerr << " expected STD concentration: " << expected_STP_concentration ; - //std::cerr << std::endl; + std::cerr << "bin number: " << bin_number ; + std::cerr << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) ; + std::cerr << " r_center: " << std::exp(bin_lnrd_center) ; + std::cerr << " z_indices: (" << z_min_index << ", " << z_max_index << "), " ; + std::cerr << " expected STP concentration: " << expected_STP_concentration ; + std::cerr << std::endl; - //std::cerr << "hor_sum:" << std::endl; - //debug::print(hor_sum); + std::cerr << "hor_sum:" << std::endl; + debug::print(hor_sum); - //std::cerr << "expected_hor_sum:" << std::endl; - //debug::print(expected_hor_sum); + std::cerr << "expected_hor_sum:" << std::endl; + debug::print(expected_hor_sum); // calculate how many CCN are missing thrust::transform(expected_hor_sum.begin(), expected_hor_sum.end(), hor_sum.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); - //std::cerr << "hor_missing:" << std::endl; - //debug::print(hor_missing); + std::cerr << "hor_missing:" << std::endl; + debug::print(hor_missing); // set number of SDs to init; create only if concentration is lower than expected with a tolerance thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_sum.begin(), n_SD_to_create.begin(), detail::calc_n_sd_to_create(config.rlx_conc_tolerance, opts_init.rlx_sd_per_bin)); - //std::cerr << "n_SD_to_create:" << std::endl; - //debug::print(n_SD_to_create); + std::cerr << "n_SD_to_create:" << std::endl; + debug::print(n_SD_to_create); n_part_old = n_part; n_part_to_init = thrust::reduce(n_SD_to_create.begin(), n_SD_to_create.end()); n_part = n_part_old + n_part_to_init; - //std::cerr << "n_part_to_init: " << n_part_to_init << std::endl; + std::cerr << "n_part_to_init: " << n_part_to_init << std::endl; // resize arrays set in the bins loop: cell indices and rd3, resize should be cheap, because we allocate a large chunk of memory at the start ijk.resize(n_part); @@ -288,7 +291,7 @@ namespace libcloudphxx using std::min; #endif - //std::cerr << "dt: " << dt << " rlx_timescale: " << opts_init.rlx_timescale << " hor_avg fraction added: " << min(dt / opts_init.rlx_timescale, real_t(1)) << std::endl; + std::cerr << "dt: " << dt << " rlx_timescale: " << opts_init.rlx_timescale << " hor_avg fraction added: " << min(dt / opts_init.rlx_timescale, real_t(1)) << std::endl; thrust::for_each( thrust::make_zip_iterator(thrust::make_tuple( @@ -359,6 +362,8 @@ namespace libcloudphxx // NOTE: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } // end of the bins loop + std::cerr << "expected_STP_concentration_tot: " << expected_STP_concentration_tot << std::endl; + // init other SD characteristics that don't have to be initialized in the bins loop n_part_old = n_part_pre_bins_loop; n_part_to_init = n_part - n_part_old; From 39920a7dc5c8aa278f5f16cd1b1eb81ab1eaffde Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 11 Aug 2021 13:36:00 +0200 Subject: [PATCH 042/282] add a python binding for dynamic viscosity --- bindings/python/common.hpp | 6 ++++++ bindings/python/lib.cpp | 1 + 2 files changed, 7 insertions(+) diff --git a/bindings/python/common.hpp b/bindings/python/common.hpp index fb11862be..58c174c18 100644 --- a/bindings/python/common.hpp +++ b/bindings/python/common.hpp @@ -88,6 +88,12 @@ namespace libcloudphxx return cmn::theta_dry::p(rhod * si::kilograms / si::cubic_metres, r * si::kilograms / si::kilograms, T * si::kelvins) / si::pascals; } + template + real_t visc(const real_t &T) + { + return cmn::vterm::visc(T * si::kelvins) / si::pascals / si::seconds; + } + template real_t rw3_cr(const real_t &rd3, const real_t &kappa, const real_t &T) { diff --git a/bindings/python/lib.cpp b/bindings/python/lib.cpp index e285596c4..aad5039eb 100644 --- a/bindings/python/lib.cpp +++ b/bindings/python/lib.cpp @@ -135,6 +135,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) bp::def("l_v", &common::l_v); bp::def("T", &common::T); bp::def("p", &common::p); + bp::def("visc", &common::visc); bp::def("rw3_cr", &common::rw3_cr); bp::def("S_cr", &common::S_cr); bp::def("p_hydro", &common::p_hydro); From 29665be5f02b6cdc91f34f91600ee0e39df6520b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 15:04:22 +0200 Subject: [PATCH 043/282] Create cmake.yml --- .github/workflows/cmake.yml | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/cmake.yml diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml new file mode 100644 index 000000000..e282e3891 --- /dev/null +++ b/.github/workflows/cmake.yml @@ -0,0 +1,38 @@ +name: CMake + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) + BUILD_TYPE: Release + +jobs: + build: + # The CMake configure and build commands are platform agnostic and should work equally + # well on Windows or Mac. You can convert this to a matrix build if you need + # cross-platform coverage. + # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Configure CMake + # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. + # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + + - name: Build + # Build your program with the given configuration + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + + - name: Test + working-directory: ${{github.workspace}}/build + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: ctest -C ${{env.BUILD_TYPE}} + From c727e2736239e0530326f8e1c3c249dd63f4fd35 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 15:20:18 +0200 Subject: [PATCH 044/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index e282e3891..34bbfec99 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -20,6 +20,8 @@ jobs: steps: - uses: actions/checkout@v2 + + - uses: actions/cuda-toolkit - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. From 418f88fa5b1823b127f98545bfe09e48640c374e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 15:22:17 +0200 Subject: [PATCH 045/282] Update cmake.yml --- .github/workflows/cmake.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 34bbfec99..2882f011a 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -21,7 +21,8 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/cuda-toolkit + - name: cuda-toolkit + uses: Jimver/cuda-toolkit@v0.2.4 - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. From 8898ca9975312e4c752c4b4e45432a0201762884 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 15:27:30 +0200 Subject: [PATCH 046/282] Update cmake.yml --- .github/workflows/cmake.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 2882f011a..e362450f1 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -24,6 +24,9 @@ jobs: - name: cuda-toolkit uses: Jimver/cuda-toolkit@v0.2.4 + - name: Download and install Boost + uses: MarkusJx/install-boost@v2.0.0 + - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type From e3c9573f2f2550eb3bb2b5a6b41c7117baaab1f2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 15:41:27 +0200 Subject: [PATCH 047/282] Update cmake.yml --- .github/workflows/cmake.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index e362450f1..df8ccccfc 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -21,11 +21,21 @@ jobs: steps: - uses: actions/checkout@v2 - - name: cuda-toolkit + - name: Install cuda-toolkit uses: Jimver/cuda-toolkit@v0.2.4 - - name: Download and install Boost + - name: Install boost uses: MarkusJx/install-boost@v2.0.0 + id: install-boost + with: + # REQUIRED: Specify the required boost version + # A list of supported versions can be found here: + # https://github.com/actions/boost-versions/blob/main/versions-manifest.json + boost_version: 1.73.0 + # OPTIONAL: Specify a platform version + #platform_version: 18.04 + # OPTIONAL: Specify a custom install location + #boost_install_dir: /home/runner/some_directory - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. From 97be0ed3284896f66b52f17ac636b086299082ff Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 15:45:23 +0200 Subject: [PATCH 048/282] Update cmake.yml --- .github/workflows/cmake.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index df8ccccfc..b069528a3 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -36,6 +36,10 @@ jobs: #platform_version: 18.04 # OPTIONAL: Specify a custom install location #boost_install_dir: /home/runner/some_directory + + - name: Install hdf5 (Linux) + run: sudo apt-get install libhdf5-dev + #if: matrix.os == 'ubuntu-latest' - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. From 0c4d7519ce6d0c3b15980e1543459d3767d812e0 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 15:47:05 +0200 Subject: [PATCH 049/282] Update cmake.yml --- .github/workflows/cmake.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index b069528a3..423b74208 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -28,14 +28,14 @@ jobs: uses: MarkusJx/install-boost@v2.0.0 id: install-boost with: - # REQUIRED: Specify the required boost version - # A list of supported versions can be found here: - # https://github.com/actions/boost-versions/blob/main/versions-manifest.json - boost_version: 1.73.0 - # OPTIONAL: Specify a platform version - #platform_version: 18.04 - # OPTIONAL: Specify a custom install location - #boost_install_dir: /home/runner/some_directory + # REQUIRED: Specify the required boost version + # A list of supported versions can be found here: + # https://github.com/actions/boost-versions/blob/main/versions-manifest.json + boost_version: 1.73.0 + # OPTIONAL: Specify a platform version + #platform_version: 18.04 + # OPTIONAL: Specify a custom install location + #boost_install_dir: /home/runner/some_directory - name: Install hdf5 (Linux) run: sudo apt-get install libhdf5-dev From 77dcf9ccfc8a8fb1aedb6ea48df05a3dce67e6e7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 15:59:22 +0200 Subject: [PATCH 050/282] Update cmake.yml --- .github/workflows/cmake.yml | 40 ++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 423b74208..bb653410b 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -24,22 +24,34 @@ jobs: - name: Install cuda-toolkit uses: Jimver/cuda-toolkit@v0.2.4 - - name: Install boost - uses: MarkusJx/install-boost@v2.0.0 - id: install-boost + # - name: Install boost + # uses: MarkusJx/install-boost@v2.0.0 + # id: install-boost + # with: + # # REQUIRED: Specify the required boost version + # # A list of supported versions can be found here: + # # https://github.com/actions/boost-versions/blob/main/versions-manifest.json + # boost_version: 1.73.0 + # # OPTIONAL: Specify a platform version + # #platform_version: 18.04 + # # OPTIONAL: Specify a custom install location + # #boost_install_dir: /home/runner/some_directory + + - name: Cache Boost + uses: actions/cache@v2 with: - # REQUIRED: Specify the required boost version - # A list of supported versions can be found here: - # https://github.com/actions/boost-versions/blob/main/versions-manifest.json - boost_version: 1.73.0 - # OPTIONAL: Specify a platform version - #platform_version: 18.04 - # OPTIONAL: Specify a custom install location - #boost_install_dir: /home/runner/some_directory + path: '${{ runner.workspace }}/boost_*.tar.gz' + key: 'boost-1.72.0' + + - name: Build Boost + # This won't re-download the archive unnecessarily: + uses: egor-tensin/build-boost@v1 + with: + version: 1.72.0 - - name: Install hdf5 (Linux) - run: sudo apt-get install libhdf5-dev - #if: matrix.os == 'ubuntu-latest' + #- name: Install hdf5 (Linux) + # run: sudo apt-get install libhdf5-dev + # #if: matrix.os == 'ubuntu-latest' - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. From ea9cb6b88c921ac064d42eaa83b0bf39271eef5c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 16:02:48 +0200 Subject: [PATCH 051/282] Update cmake.yml --- .github/workflows/cmake.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index bb653410b..9ee3f68c3 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -48,6 +48,10 @@ jobs: uses: egor-tensin/build-boost@v1 with: version: 1.72.0 + #libraries: filesystem program_options system + libraries: + platform: x64 + configuration: Release #- name: Install hdf5 (Linux) # run: sudo apt-get install libhdf5-dev From 29700868cd196cd622e429de20f05de10ad45be2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 16:12:17 +0200 Subject: [PATCH 052/282] Update cmake.yml --- .github/workflows/cmake.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 9ee3f68c3..ab377289b 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -48,8 +48,7 @@ jobs: uses: egor-tensin/build-boost@v1 with: version: 1.72.0 - #libraries: filesystem program_options system - libraries: + libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built platform: x64 configuration: Release From c7895f2710219726c50b9393a3c4875855ca6607 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 16:20:24 +0200 Subject: [PATCH 053/282] Update cmake.yml --- .github/workflows/cmake.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index ab377289b..fcef25322 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -44,6 +44,7 @@ jobs: key: 'boost-1.72.0' - name: Build Boost + id: boost # This won't re-download the archive unnecessarily: uses: egor-tensin/build-boost@v1 with: @@ -59,7 +60,7 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBoost_INCLUDE_DIR=${{ steps.boost.outputs.includedir }} - name: Build # Build your program with the given configuration From dbe339083ab0b8aade8bc07ed711ad018cf7975c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 16:31:37 +0200 Subject: [PATCH 054/282] Update cmake.yml --- .github/workflows/cmake.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index fcef25322..24c610afc 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -21,8 +21,13 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Install cuda-toolkit - uses: Jimver/cuda-toolkit@v0.2.4 + # TODO: cache cuda-toolki + + #- name: Install cuda-toolkit + # id: cuda-toolkit + # uses: Jimver/cuda-toolkit@v0.2.4 + # with: + # linux-local-args: '["--toolkit"]' # - name: Install boost # uses: MarkusJx/install-boost@v2.0.0 @@ -60,7 +65,7 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBoost_INCLUDE_DIR=${{ steps.boost.outputs.includedir }} + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBoost_INCLUDE_DIR=${{ steps.boost.outputs.root }}/include - name: Build # Build your program with the given configuration From bb48c5b9fbb33eb6d0a340fcf36102ba6fdc2080 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 16:39:41 +0200 Subject: [PATCH 055/282] Update cmake.yml --- .github/workflows/cmake.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 24c610afc..8d7b54bdd 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -41,6 +41,9 @@ jobs: # #platform_version: 18.04 # # OPTIONAL: Specify a custom install location # #boost_install_dir: /home/runner/some_directory + + - name: Install Thrust + run: sudo apt-get install libthrust-dev - name: Cache Boost uses: actions/cache@v2 From 626f762dc657709c59405e8c95a7324f1f9f230a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 16:43:44 +0200 Subject: [PATCH 056/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 8d7b54bdd..ff8a6f200 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -68,7 +68,7 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBoost_INCLUDE_DIR=${{ steps.boost.outputs.root }}/include + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - name: Build # Build your program with the given configuration From 851ed46328d381b791459de31499e4bf68785c09 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 16:49:13 +0200 Subject: [PATCH 057/282] Update cmake.yml --- .github/workflows/cmake.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index ff8a6f200..884e40f92 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -43,7 +43,10 @@ jobs: # #boost_install_dir: /home/runner/some_directory - name: Install Thrust - run: sudo apt-get install libthrust-dev + #run: sudo apt-get install libthrust-dev + run: | + git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 + ln -s `pwd`/thrust/thrust /usr/local/include/thrust - name: Cache Boost uses: actions/cache@v2 From a515ef58025177f2f73362db5596a639441f784d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Oct 2021 16:54:58 +0200 Subject: [PATCH 058/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 884e40f92..19bdcc397 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -46,7 +46,7 @@ jobs: #run: sudo apt-get install libthrust-dev run: | git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 - ln -s `pwd`/thrust/thrust /usr/local/include/thrust + sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust - name: Cache Boost uses: actions/cache@v2 From c71efd453c115708eccfc8486fe9361413baca2a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 11:20:20 +0200 Subject: [PATCH 059/282] Update cmake.yml --- .github/workflows/cmake.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 19bdcc397..5097fe1d6 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -21,6 +21,17 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Install Singularity + uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: 3.6.4 + + - name: Download UWLCM Singularity container + run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 + + - name: Build UWLCM Singularity container + run: singularity build --remote sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 + # TODO: cache cuda-toolki #- name: Install cuda-toolkit From 0093b2c54afcf06397100aa3d99bc4fd61cb5043 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 11:32:26 +0200 Subject: [PATCH 060/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 5097fe1d6..2ff057eab 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -30,7 +30,7 @@ jobs: run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 - name: Build UWLCM Singularity container - run: singularity build --remote sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 + run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 # TODO: cache cuda-toolki From 1f09abc16eec0d5b570e8fd3c7f68fccfcb06d68 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:06:00 +0200 Subject: [PATCH 061/282] Update cmake.yml --- .github/workflows/cmake.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 2ff057eab..56836c940 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -25,12 +25,22 @@ jobs: uses: eWaterCycle/setup-singularity@v6 with: singularity-version: 3.6.4 + + - name: Cache Singularity container + uses: actions/cache@v2 + with: + path: '${{ runner.workspace }}/uwlcm_ubuntu*' + key: 'sng_ubuntu' - name: Download UWLCM Singularity container - run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 + run: singularity pull library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + # In truth it's cuda 11.4 + +# - name: Download UWLCM Singularity container +# run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 - - name: Build UWLCM Singularity container - run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 +# - name: Build UWLCM Singularity container +# run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 # TODO: cache cuda-toolki From fdb861ae3f48da4f072f318c06cb9e16d32957a1 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:07:42 +0200 Subject: [PATCH 062/282] Update cmake.yml --- .github/workflows/cmake.yml | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 56836c940..0c1814080 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -63,44 +63,44 @@ jobs: # # OPTIONAL: Specify a custom install location # #boost_install_dir: /home/runner/some_directory - - name: Install Thrust - #run: sudo apt-get install libthrust-dev - run: | - git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 - sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust +# - name: Install Thrust +# #run: sudo apt-get install libthrust-dev +# run: | +# git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 +# sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust - - name: Cache Boost - uses: actions/cache@v2 - with: - path: '${{ runner.workspace }}/boost_*.tar.gz' - key: 'boost-1.72.0' +# - name: Cache Boost +# uses: actions/cache@v2 +# with: +# path: '${{ runner.workspace }}/boost_*.tar.gz' +# key: 'boost-1.72.0' - - name: Build Boost - id: boost - # This won't re-download the archive unnecessarily: - uses: egor-tensin/build-boost@v1 - with: - version: 1.72.0 - libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built - platform: x64 - configuration: Release + # - name: Build Boost + # id: boost + # # This won't re-download the archive unnecessarily: + # uses: egor-tensin/build-boost@v1 + # with: + # version: 1.72.0 + # libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built + # platform: x64 + # configuration: Release #- name: Install hdf5 (Linux) # run: sudo apt-get install libhdf5-dev # #if: matrix.os == 'ubuntu-latest' - - name: Configure CMake +# - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - - name: Build + # - name: Build # Build your program with the given configuration - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + # run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build + # - name: Test + # working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: ctest -C ${{env.BUILD_TYPE}} + # run: ctest -C ${{env.BUILD_TYPE}} From 32b915c2a30cfe1bfd1ee3f268076a09ea427c0e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:15:16 +0200 Subject: [PATCH 063/282] Update cmake.yml --- .github/workflows/cmake.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 0c1814080..4eb064a4a 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -29,11 +29,13 @@ jobs: - name: Cache Singularity container uses: actions/cache@v2 with: - path: '${{ runner.workspace }}/uwlcm_ubuntu*' + path: '${{ runner.workspace }}/singularity_containers' key: 'sng_ubuntu' - name: Download UWLCM Singularity container - run: singularity pull library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + run: | + mkdir singularity_containers + singularity pull singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1.sif library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # In truth it's cuda 11.4 # - name: Download UWLCM Singularity container From d9ee12e3adb0226043679692f0038d53fae1fd8b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:20:41 +0200 Subject: [PATCH 064/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 4eb064a4a..64bfbe325 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -35,7 +35,7 @@ jobs: - name: Download UWLCM Singularity container run: | mkdir singularity_containers - singularity pull singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1.sif library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + singularity pull --dir '${{ runner.workspace }}/singularity_containters' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # In truth it's cuda 11.4 # - name: Download UWLCM Singularity container From 0a43baa5484ba7874a6c704bc8b7527e67e507dc Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:23:20 +0200 Subject: [PATCH 065/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 64bfbe325..52c01ed78 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -34,7 +34,7 @@ jobs: - name: Download UWLCM Singularity container run: | - mkdir singularity_containers + mkdir '${{ runner.workspace }}/singularity_containters' singularity pull --dir '${{ runner.workspace }}/singularity_containters' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # In truth it's cuda 11.4 From fa7dc3c09fc675dac6b1545294cdc27d5dc3f45b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:30:07 +0200 Subject: [PATCH 066/282] Update cmake.yml --- .github/workflows/cmake.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 52c01ed78..7e5bc6fd9 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -29,13 +29,14 @@ jobs: - name: Cache Singularity container uses: actions/cache@v2 with: - path: '${{ runner.workspace }}/singularity_containers' + #path: '${{ runner.workspace }}/singularity_containers' + path: '${{ runner.workspace }}/uwlcm_ubuntu_20_04_cuda_11_1*' key: 'sng_ubuntu' - name: Download UWLCM Singularity container run: | mkdir '${{ runner.workspace }}/singularity_containters' - singularity pull --dir '${{ runner.workspace }}/singularity_containters' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + singularity pull --dir '${{ runner.workspace }}' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # In truth it's cuda 11.4 # - name: Download UWLCM Singularity container From 209cfce3c8b0164b4d444402d17986996841ca35 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:48:34 +0200 Subject: [PATCH 067/282] Update cmake.yml --- .github/workflows/cmake.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 7e5bc6fd9..5d355ea56 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -35,8 +35,11 @@ jobs: - name: Download UWLCM Singularity container run: | - mkdir '${{ runner.workspace }}/singularity_containters' - singularity pull --dir '${{ runner.workspace }}' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + [ -f '${{ runner.workspace }}/singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1' ] && + {mkdir '${{ runner.workspace }}/singularity_containters' + singularity pull --disable-cache --dir '${{ runner.workspace }}' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0} + || echo "Cached Singularity container found, skipping Download" + # disable Singularity cache, we cache manually # In truth it's cuda 11.4 # - name: Download UWLCM Singularity container From 12eab92cf0a82e5754070a44aa705ae73aad2387 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:52:37 +0200 Subject: [PATCH 068/282] Update cmake.yml --- .github/workflows/cmake.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 5d355ea56..3b9587953 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -29,15 +29,15 @@ jobs: - name: Cache Singularity container uses: actions/cache@v2 with: - #path: '${{ runner.workspace }}/singularity_containers' - path: '${{ runner.workspace }}/uwlcm_ubuntu_20_04_cuda_11_1*' + path: '${{ runner.workspace }}/singularity_containers' + #path: '${{ runner.workspace }}/uwlcm_ubuntu_20_04_cuda_11_1*' key: 'sng_ubuntu' - name: Download UWLCM Singularity container run: | [ -f '${{ runner.workspace }}/singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1' ] && {mkdir '${{ runner.workspace }}/singularity_containters' - singularity pull --disable-cache --dir '${{ runner.workspace }}' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0} + singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0} || echo "Cached Singularity container found, skipping Download" # disable Singularity cache, we cache manually # In truth it's cuda 11.4 From adc31859dc532921d56a88993ffdbbcc2ac06a65 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 12:59:49 +0200 Subject: [PATCH 069/282] Update cmake.yml --- .github/workflows/cmake.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 3b9587953..99b7a2fb6 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -31,13 +31,13 @@ jobs: with: path: '${{ runner.workspace }}/singularity_containers' #path: '${{ runner.workspace }}/uwlcm_ubuntu_20_04_cuda_11_1*' - key: 'sng_ubuntu' + key: 'sng_ubuntu_test' - name: Download UWLCM Singularity container run: | [ -f '${{ runner.workspace }}/singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1' ] && - {mkdir '${{ runner.workspace }}/singularity_containters' - singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0} + {mkdir '${{ runner.workspace }}/singularity_containters'; + singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0;} || echo "Cached Singularity container found, skipping Download" # disable Singularity cache, we cache manually # In truth it's cuda 11.4 From 578e00715589d1c825a233c6c680feeb769d68a9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:06:22 +0200 Subject: [PATCH 070/282] Update cmake.yml --- .github/workflows/cmake.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 99b7a2fb6..d24532e5b 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -34,11 +34,10 @@ jobs: key: 'sng_ubuntu_test' - name: Download UWLCM Singularity container + if: ! test -f '${{ runner.workspace }}/singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1' run: | - [ -f '${{ runner.workspace }}/singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1' ] && - {mkdir '${{ runner.workspace }}/singularity_containters'; - singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0;} - || echo "Cached Singularity container found, skipping Download" + mkdir '${{ runner.workspace }}/singularity_containters' + singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # disable Singularity cache, we cache manually # In truth it's cuda 11.4 From a9004895bc2d6ee7fd645fc0a12eb6de52bbe8b7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:07:27 +0200 Subject: [PATCH 071/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index d24532e5b..7304945e0 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -34,7 +34,7 @@ jobs: key: 'sng_ubuntu_test' - name: Download UWLCM Singularity container - if: ! test -f '${{ runner.workspace }}/singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1' + if: test -f '${{ runner.workspace }}/singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1' run: | mkdir '${{ runner.workspace }}/singularity_containters' singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 From 930036e9e708ed6012cf848dae7cb4ae5841d7ae Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:13:15 +0200 Subject: [PATCH 072/282] Update cmake.yml --- .github/workflows/cmake.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 7304945e0..5c4af6c45 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -27,14 +27,15 @@ jobs: singularity-version: 3.6.4 - name: Cache Singularity container + id: cache_singularity uses: actions/cache@v2 with: path: '${{ runner.workspace }}/singularity_containers' #path: '${{ runner.workspace }}/uwlcm_ubuntu_20_04_cuda_11_1*' - key: 'sng_ubuntu_test' + key: 'sng_ubuntu' - name: Download UWLCM Singularity container - if: test -f '${{ runner.workspace }}/singularity_containters/uwlcm_ubuntu_20_04_cuda_11_1' + if: steps.cache_singularity.outputs.cache-hit != 'true' run: | mkdir '${{ runner.workspace }}/singularity_containters' singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 From f6ff635e2022b2ff6f019f6297993fcce6506afb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:14:21 +0200 Subject: [PATCH 073/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 5c4af6c45..8b177070c 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -32,7 +32,7 @@ jobs: with: path: '${{ runner.workspace }}/singularity_containers' #path: '${{ runner.workspace }}/uwlcm_ubuntu_20_04_cuda_11_1*' - key: 'sng_ubuntu' + key: 'sng_ubuntu_test' - name: Download UWLCM Singularity container if: steps.cache_singularity.outputs.cache-hit != 'true' From 4ef7e4e8018bc5242831e583aa873ef3547882b4 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:15:53 +0200 Subject: [PATCH 074/282] Update cmake.yml --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 8b177070c..c541706c5 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -37,7 +37,7 @@ jobs: - name: Download UWLCM Singularity container if: steps.cache_singularity.outputs.cache-hit != 'true' run: | - mkdir '${{ runner.workspace }}/singularity_containters' + mkdir '${{ runner.workspace }}/singularity_containers' singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # disable Singularity cache, we cache manually # In truth it's cuda 11.4 From fe312b7ed70d5668a293d9e473165b7359c62e9b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:19:50 +0200 Subject: [PATCH 075/282] Update cmake.yml --- .github/workflows/cmake.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index c541706c5..aca33e22e 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -31,8 +31,7 @@ jobs: uses: actions/cache@v2 with: path: '${{ runner.workspace }}/singularity_containers' - #path: '${{ runner.workspace }}/uwlcm_ubuntu_20_04_cuda_11_1*' - key: 'sng_ubuntu_test' + key: 'sng_ubuntu' - name: Download UWLCM Singularity container if: steps.cache_singularity.outputs.cache-hit != 'true' @@ -41,6 +40,11 @@ jobs: singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # disable Singularity cache, we cache manually # In truth it's cuda 11.4 + + - name: Run Singularity shell + run: | + ls singularity_containers + singularity shell singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1 # - name: Download UWLCM Singularity container # run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 From 9e70e3d06cfbda28977649b2c43243aaf336035f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:22:14 +0200 Subject: [PATCH 076/282] Update cmake.yml --- .github/workflows/cmake.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index aca33e22e..014ea5470 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -43,6 +43,8 @@ jobs: - name: Run Singularity shell run: | + pwd + ls ls singularity_containers singularity shell singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1 @@ -99,14 +101,14 @@ jobs: # run: sudo apt-get install libhdf5-dev # #if: matrix.os == 'ubuntu-latest' -# - name: Configure CMake + - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - # - name: Build + - name: Build # Build your program with the given configuration - # run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} # - name: Test # working-directory: ${{github.workspace}}/build From 39857743500d016edb738338114678fe0593c8d3 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:25:48 +0200 Subject: [PATCH 077/282] Update cmake.yml --- .github/workflows/cmake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 014ea5470..22b1eea01 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -45,8 +45,8 @@ jobs: run: | pwd ls - ls singularity_containers - singularity shell singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1 + ls '${{ runner.workspace }}/singularity_containers' + singularity shell '${{ runner.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1' # - name: Download UWLCM Singularity container # run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 From 2be201b197b576a7a4c76fa50c0fb5f6ee5cc3b1 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:31:07 +0200 Subject: [PATCH 078/282] Update cmake.yml --- .github/workflows/cmake.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 22b1eea01..1c42c1b59 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -26,18 +26,20 @@ jobs: with: singularity-version: 3.6.4 - - name: Cache Singularity container + - name: Cache Singularity id: cache_singularity uses: actions/cache@v2 with: - path: '${{ runner.workspace }}/singularity_containers' - key: 'sng_ubuntu' + #path: '${{ runner.workspace }}/singularity_containers' + path: '$HOME/.singularity/' + key: 'singularity' - name: Download UWLCM Singularity container if: steps.cache_singularity.outputs.cache-hit != 'true' - run: | - mkdir '${{ runner.workspace }}/singularity_containers' - singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + # run: | + # mkdir '${{ runner.workspace }}/singularity_containers' + # singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + run: singularity pull --disable-cache library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # disable Singularity cache, we cache manually # In truth it's cuda 11.4 @@ -45,8 +47,10 @@ jobs: run: | pwd ls - ls '${{ runner.workspace }}/singularity_containers' - singularity shell '${{ runner.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1' + ls '$HOME/.singularity' + singularity shell uwlcm_ubuntu_20_04_cuda_11_1 + # ls '${{ runner.workspace }}/singularity_containers' + # singularity shell '${{ runner.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1' # - name: Download UWLCM Singularity container # run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 From 3785c74e992a043a5a2b676597cede4f5237499b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:36:08 +0200 Subject: [PATCH 079/282] Update cmake.yml --- .github/workflows/cmake.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 1c42c1b59..416b71ac2 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -30,16 +30,16 @@ jobs: id: cache_singularity uses: actions/cache@v2 with: - #path: '${{ runner.workspace }}/singularity_containers' - path: '$HOME/.singularity/' - key: 'singularity' + path: '${{ github.workspace }}/singularity_containers' + #path: '$HOME/.singularity/' + key: 'sng_ubuntu' - name: Download UWLCM Singularity container if: steps.cache_singularity.outputs.cache-hit != 'true' - # run: | - # mkdir '${{ runner.workspace }}/singularity_containers' - # singularity pull --disable-cache --dir '${{ runner.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 - run: singularity pull --disable-cache library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + run: | + mkdir '${{ github.workspace }}/singularity_containers' + singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + # run: singularity pull --disable-cache library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # disable Singularity cache, we cache manually # In truth it's cuda 11.4 @@ -47,8 +47,9 @@ jobs: run: | pwd ls - ls '$HOME/.singularity' - singularity shell uwlcm_ubuntu_20_04_cuda_11_1 + ls '${{ github.workspace }}' + ls '${{ github.workspace }}/singularity_containers' + singularity shell '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' # ls '${{ runner.workspace }}/singularity_containers' # singularity shell '${{ runner.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1' From 41bbe214f70ea502543c2cf7c6ce5a6a555b26bb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:43:42 +0200 Subject: [PATCH 080/282] Update cmake.yml --- .github/workflows/cmake.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 416b71ac2..ec4bd1747 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -31,7 +31,6 @@ jobs: uses: actions/cache@v2 with: path: '${{ github.workspace }}/singularity_containers' - #path: '$HOME/.singularity/' key: 'sng_ubuntu' - name: Download UWLCM Singularity container @@ -39,19 +38,11 @@ jobs: run: | mkdir '${{ github.workspace }}/singularity_containers' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 - # run: singularity pull --disable-cache library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # disable Singularity cache, we cache manually # In truth it's cuda 11.4 - - name: Run Singularity shell - run: | - pwd - ls - ls '${{ github.workspace }}' - ls '${{ github.workspace }}/singularity_containers' - singularity shell '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' - # ls '${{ runner.workspace }}/singularity_containers' - # singularity shell '${{ runner.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1' + # - name: Run Singularity shell + # run: singularity shell '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' # - name: Download UWLCM Singularity container # run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 @@ -109,7 +100,8 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + run: singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - name: Build # Build your program with the given configuration From e726b275cda887a2ac698ac15e997ab2e2084c92 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:46:46 +0200 Subject: [PATCH 081/282] Update cmake.yml --- .github/workflows/cmake.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index ec4bd1747..c1c936a49 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -97,15 +97,15 @@ jobs: # run: sudo apt-get install libhdf5-dev # #if: matrix.os == 'ubuntu-latest' - - name: Configure CMake + # - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - - - name: Build + # run: singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} +# + # - name: Build # Build your program with the given configuration - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} +# run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} # - name: Test # working-directory: ${{github.workspace}}/build From e92d25e30dea479d0ceb204e4eee1ef06ebe2479 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 13:58:44 +0200 Subject: [PATCH 082/282] Update cmake.yml --- .github/workflows/cmake.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index c1c936a49..50520ea47 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -97,11 +97,11 @@ jobs: # run: sudo apt-get install libhdf5-dev # #if: matrix.os == 'ubuntu-latest' - # - name: Configure CMake + - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - # run: singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + run: singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} # # - name: Build # Build your program with the given configuration From 6db0352c6d6600aa4ff6c4263f80a58f4ecc6d9d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:00:21 +0200 Subject: [PATCH 083/282] Update cmake.yml --- .github/workflows/cmake.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 50520ea47..9ad5c3074 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -41,6 +41,22 @@ jobs: # disable Singularity cache, we cache manually # In truth it's cuda 11.4 + - name: Configure CMake + # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. + # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type + # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + run: singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} +# + # - name: Build + # Build your program with the given configuration +# run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + + # - name: Test + # working-directory: ${{github.workspace}}/build + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + # run: ctest -C ${{env.BUILD_TYPE}} + # - name: Run Singularity shell # run: singularity shell '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' @@ -96,20 +112,4 @@ jobs: #- name: Install hdf5 (Linux) # run: sudo apt-get install libhdf5-dev # #if: matrix.os == 'ubuntu-latest' - - - name: Configure CMake - # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} -# - # - name: Build - # Build your program with the given configuration -# run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - # - name: Test - # working-directory: ${{github.workspace}}/build - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - # run: ctest -C ${{env.BUILD_TYPE}} From 898d88da643923de36ba481192a9ffeef99af81e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:13:22 +0200 Subject: [PATCH 084/282] cmake: search for boost.numpy as numpy --- bindings/python/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 9ec6f7fed..ef399e8e7 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -48,8 +48,10 @@ if(${Boost_MINOR_VERSION} LESS 65) find_package(Boost COMPONENTS python3 REQUIRED) target_link_libraries(cloudphxx ${Boost_LIBRARIES}) else() - message(STATUS "boost numpy as numpy3") - find_package(Boost COMPONENTS numpy3 REQUIRED) # also finds python3 +# message(STATUS "boost numpy as numpy3") +# find_package(Boost COMPONENTS numpy3 REQUIRED) # also finds python3 + message(STATUS "boost numpy as numpy") + find_package(Boost COMPONENTS numpy REQUIRED) # also finds python3 target_link_libraries(cloudphxx ${Boost_LIBRARIES}) target_compile_options(cloudphxx PRIVATE -DBPNUMPY) # if(${Boost_MINOR_VERSION} LESS 67) From e9294ad1a8705158da346221daa40fdda7e3d77f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:18:29 +0200 Subject: [PATCH 085/282] GA: check nvidia driver --- .github/workflows/cmake.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 9ad5c3074..6e79a2bb6 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -38,6 +38,8 @@ jobs: run: | mkdir '${{ github.workspace }}/singularity_containers' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' nvidia-smi + nvidia-smi # disable Singularity cache, we cache manually # In truth it's cuda 11.4 From dcdcc836a4e64650dde1fc7b6e02a31c5a15e9be Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:23:26 +0200 Subject: [PATCH 086/282] GA --- .github/workflows/cmake.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 6e79a2bb6..1cc2256d0 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -30,24 +30,29 @@ jobs: id: cache_singularity uses: actions/cache@v2 with: - path: '${{ github.workspace }}/singularity_containers' + path: '${{ github.workspace }}/singularity_images' key: 'sng_ubuntu' - name: Download UWLCM Singularity container if: steps.cache_singularity.outputs.cache-hit != 'true' run: | - mkdir '${{ github.workspace }}/singularity_containers' - singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_containers' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 - singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' nvidia-smi - nvidia-smi + mkdir '${{ github.workspace }}/singularity_images' + singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 # disable Singularity cache, we cache manually # In truth it's cuda 11.4 + + - name: Set friendly Singularity image name + run: | + set SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' + echo $SI + singularity exec '$SI' nvidia-smi + nvidia-smi - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + run: singularity exec '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} # # - name: Build # Build your program with the given configuration @@ -60,7 +65,7 @@ jobs: # run: ctest -C ${{env.BUILD_TYPE}} # - name: Run Singularity shell - # run: singularity shell '${{ github.workspace }}/singularity_containers/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' + # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' # - name: Download UWLCM Singularity container # run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 From fa4cfd9b81bef5eaeaccf83636712e01c264e67c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:30:30 +0200 Subject: [PATCH 087/282] GA --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 1cc2256d0..bda21b2ec 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -43,7 +43,7 @@ jobs: - name: Set friendly Singularity image name run: | - set SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' + SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' echo $SI singularity exec '$SI' nvidia-smi nvidia-smi From bac26440f5abcfc8c45b488993cf42a9da351aaf Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:32:49 +0200 Subject: [PATCH 088/282] GA --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index bda21b2ec..546f13630 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -45,7 +45,7 @@ jobs: run: | SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' echo $SI - singularity exec '$SI' nvidia-smi + singularity exec $SI nvidia-smi nvidia-smi - name: Configure CMake From 8dd862ff3b9005d9931b13ce2e88a63db0782caa Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:39:55 +0200 Subject: [PATCH 089/282] GA --- .github/workflows/cmake.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 546f13630..3a5ca6a2d 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -20,6 +20,9 @@ jobs: steps: - uses: actions/checkout@v2 + + - name: Install Nvidia driver + run: sudo apt install nvidia-driver-470 - name: Install Singularity uses: eWaterCycle/setup-singularity@v6 From 283307d767a3152e4b2ea3f0af7d80b45e9cc826 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:51:00 +0200 Subject: [PATCH 090/282] GA --- .github/workflows/cmake.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 3a5ca6a2d..ce9c178b6 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -45,21 +45,17 @@ jobs: # In truth it's cuda 11.4 - name: Set friendly Singularity image name - run: | - SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' - echo $SI - singularity exec $SI nvidia-smi - nvidia-smi + run: SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} + run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} # - # - name: Build + - name: Build # Build your program with the given configuration -# run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} # - name: Test # working-directory: ${{github.workspace}}/build From e7dbfb5c7a488262ccd079991b429d234c5f6465 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 14:59:29 +0200 Subject: [PATCH 091/282] GA --- .github/workflows/cmake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index ce9c178b6..38d8e8c48 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -51,11 +51,11 @@ jobs: # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + run: singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' # - name: Build # Build your program with the given configuration - run: singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: singularity exec $SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' # - name: Test # working-directory: ${{github.workspace}}/build From 8599abff58471b67c67bb7e4cedb28e8eebb68bf Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:06:25 +0200 Subject: [PATCH 092/282] GA --- .github/workflows/cmake.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 38d8e8c48..a8e018321 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -11,6 +11,7 @@ env: BUILD_TYPE: Release jobs: + # 'build' is just a job name build: # The CMake configure and build commands are platform agnostic and should work equally # well on Windows or Mac. You can convert this to a matrix build if you need @@ -45,13 +46,15 @@ jobs: # In truth it's cuda 11.4 - name: Set friendly Singularity image name - run: SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' + run: export SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' + run: | + echo $SI + singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' # - name: Build # Build your program with the given configuration From 53089d68628d697fd265899009bd46de4fcbb150 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:09:36 +0200 Subject: [PATCH 093/282] GA --- .github/workflows/cmake.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index a8e018321..8191c8f19 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -34,19 +34,19 @@ jobs: id: cache_singularity uses: actions/cache@v2 with: - path: '${{ github.workspace }}/singularity_images' - key: 'sng_ubuntu' + path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + key: 'sng_ubuntu_20_04_cuda_11_4' - name: Download UWLCM Singularity container if: steps.cache_singularity.outputs.cache-hit != 'true' run: | mkdir '${{ github.workspace }}/singularity_images' - singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm_ubuntu_20_04_cuda_11_1:1.0 + singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 # disable Singularity cache, we cache manually # In truth it's cuda 11.4 - name: Set friendly Singularity image name - run: export SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' + run: export SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. From e2ebb0604a7cc7daa1c90e71d4ff4f459f9581ba Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:11:16 +0200 Subject: [PATCH 094/282] GA --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 8191c8f19..6c38ef34a 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Nvidia driver - run: sudo apt install nvidia-driver-470 + run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity uses: eWaterCycle/setup-singularity@v6 From 31dddacf130f9caf3f7fc601f5348513dfd8becf Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:26:02 +0200 Subject: [PATCH 095/282] GA --- .github/workflows/cmake.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 6c38ef34a..9be79156d 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -18,6 +18,8 @@ jobs: # cross-platform coverage. # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix runs-on: ubuntu-latest + env: + SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' steps: - uses: actions/checkout@v2 @@ -43,22 +45,20 @@ jobs: mkdir '${{ github.workspace }}/singularity_images' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 # disable Singularity cache, we cache manually - # In truth it's cuda 11.4 - - name: Set friendly Singularity image name - run: export SI='${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# - name: Set friendly Singularity image name - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} run: | - echo $SI - singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' + echo $env.SI + singularity exec $env.SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' # - name: Build # Build your program with the given configuration - run: singularity exec $SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' + run: singularity exec $env.SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' # - name: Test # working-directory: ${{github.workspace}}/build From b9c8480cc82b480b8a8ddc3eb1938dbf0dfbb76d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:34:55 +0200 Subject: [PATCH 096/282] GA --- .github/workflows/cmake.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 9be79156d..4885e9ff8 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -24,8 +24,8 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Install Nvidia driver - run: sudo apt install --no-install-recommends nvidia-driver-470 +# - name: Install Nvidia driver +# run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity uses: eWaterCycle/setup-singularity@v6 @@ -46,25 +46,33 @@ jobs: singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 # disable Singularity cache, we cache manually -# - name: Set friendly Singularity image name + - name: Set friendly Singularity image name + id: si + output: + name: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} run: | - echo $env.SI - singularity exec $env.SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' + echo ${{ si.name }} +# singularity exec ${{si.name}} 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' # - - name: Build +# - name: Build # Build your program with the given configuration - run: singularity exec $env.SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' +# run: singularity exec $env.SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' - # - name: Test - # working-directory: ${{github.workspace}}/build + test: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Test + working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - # run: ctest -C ${{env.BUILD_TYPE}} + run: ctest -C ${{env.BUILD_TYPE}} # - name: Run Singularity shell # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' From 59769657cb0df1cef276fdb8cef8330beb4ec263 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:38:33 +0200 Subject: [PATCH 097/282] GA --- .github/workflows/cmake.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 4885e9ff8..3741571ec 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -47,16 +47,16 @@ jobs: # disable Singularity cache, we cache manually - name: Set friendly Singularity image name - id: si - output: - name: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + uses: allenevans/set-env@v2.0.0 + with: + SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} run: | - echo ${{ si.name }} + echo $SI # singularity exec ${{si.name}} 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' # # - name: Build From 632efdb34fc2974eeeb35053bb5e3eccbee7ae54 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:44:20 +0200 Subject: [PATCH 098/282] GA --- .github/workflows/cmake.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 3741571ec..46f69ae2e 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -55,24 +55,30 @@ jobs: # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: | - echo $SI -# singularity exec ${{si.name}} 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' + run: singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' # -# - name: Build + - name: Build # Build your program with the given configuration -# run: singularity exec $env.SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' + run: singularity exec $SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' + + # job test test: needs: build runs-on: ubuntu-latest steps: + + - name: Set friendly Singularity image name + uses: allenevans/set-env@v2.0.0 + with: + SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + - name: Test working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: ctest -C ${{env.BUILD_TYPE}} + run: singularity exec $SI 'run: ctest -C ${{env.BUILD_TYPE}}' # - name: Run Singularity shell # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' From df8f52c944c9fe333fb744d55243a73e8ab8d4ea Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:45:29 +0200 Subject: [PATCH 099/282] GA --- .github/workflows/cmake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 46f69ae2e..a4f48293f 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -74,11 +74,11 @@ jobs: with: SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - - name: Test + - name: unit_test working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: singularity exec $SI 'run: ctest -C ${{env.BUILD_TYPE}}' + run: singularity exec $SI 'ctest -C ${{env.BUILD_TYPE}}' # - name: Run Singularity shell # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' From edea1fa5ce9fedd7e2d5b3b990b9d3da63c0f2c0 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:46:59 +0200 Subject: [PATCH 100/282] GA --- .github/workflows/cmake.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index a4f48293f..4a7203aab 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -11,7 +11,7 @@ env: BUILD_TYPE: Release jobs: - # 'build' is just a job name + # first job, build the library build: # The CMake configure and build commands are platform agnostic and should work equally # well on Windows or Mac. You can convert this to a matrix build if you need @@ -24,8 +24,8 @@ jobs: steps: - uses: actions/checkout@v2 -# - name: Install Nvidia driver -# run: sudo apt install --no-install-recommends nvidia-driver-470 + - name: Install Nvidia driver + run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity uses: eWaterCycle/setup-singularity@v6 @@ -57,7 +57,7 @@ jobs: # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} run: singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' # - - name: Build + - name: Build libcloudph++ # Build your program with the given configuration run: singularity exec $SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' @@ -74,7 +74,7 @@ jobs: with: SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - - name: unit_test + - name: Run unit tests working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail From 666ab146a06b435bed7549affbc1ec023a631a32 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:52:10 +0200 Subject: [PATCH 101/282] GA --- .../{cmake.yml => test_libclouphxx.yml} | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) rename .github/workflows/{cmake.yml => test_libclouphxx.yml} (99%) diff --git a/.github/workflows/cmake.yml b/.github/workflows/test_libclouphxx.yml similarity index 99% rename from .github/workflows/cmake.yml rename to .github/workflows/test_libclouphxx.yml index 4a7203aab..b75960873 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -1,4 +1,4 @@ -name: CMake +name: Test libcloudph++ on: push: @@ -80,6 +80,28 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: singularity exec $SI 'ctest -C ${{env.BUILD_TYPE}}' + + + + + + + + + + + + + + + + + + + + + + # - name: Run Singularity shell # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' From e1d483a91d894d3cf0b0961051434a227de79799 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 15:53:52 +0200 Subject: [PATCH 102/282] GA --- .github/workflows/test_libclouphxx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index b75960873..f2c0a0cf7 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -32,14 +32,14 @@ jobs: with: singularity-version: 3.6.4 - - name: Cache Singularity + - name: Cache UWLCM Singularity image id: cache_singularity uses: actions/cache@v2 with: path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' key: 'sng_ubuntu_20_04_cuda_11_4' - - name: Download UWLCM Singularity container + - name: Download UWLCM Singularity image if: steps.cache_singularity.outputs.cache-hit != 'true' run: | mkdir '${{ github.workspace }}/singularity_images' From 6cff877d6fa994741ae151f5c1507f853f21e773 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 16:02:06 +0200 Subject: [PATCH 103/282] GA --- .github/workflows/test_libclouphxx.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index f2c0a0cf7..fe5129c2f 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -55,7 +55,11 @@ jobs: # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' + run: | + pwd + singularity exec $SI 'cmake' + singularity exec $SI cmake + singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' # - name: Build libcloudph++ # Build your program with the given configuration From 889c3fb69aae3de83e18f31a74db0afe47144b76 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 16:27:42 +0200 Subject: [PATCH 104/282] GA --- .github/workflows/test_libclouphxx.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index fe5129c2f..55116eec9 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -55,15 +55,11 @@ jobs: # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: | - pwd - singularity exec $SI 'cmake' - singularity exec $SI cmake - singularity exec $SI 'cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}' + run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} # - name: Build libcloudph++ # Build your program with the given configuration - run: singularity exec $SI 'cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}' + run: singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} # job test @@ -82,7 +78,7 @@ jobs: working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: singularity exec $SI 'ctest -C ${{env.BUILD_TYPE}}' + run: singularity exec $SI ctest -C ${{env.BUILD_TYPE}} From 936a509d786b4dd967c981881ad86f5cd1c9f530 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 16:39:09 +0200 Subject: [PATCH 105/282] GA --- .github/workflows/test_libclouphxx.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 55116eec9..17d5a3ba9 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -28,9 +28,11 @@ jobs: run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity - uses: eWaterCycle/setup-singularity@v6 - with: - singularity-version: 3.6.4 +# when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) +# uses: eWaterCycle/setup-singularity@v6 +# with: +# singularity-version: 3.6.4 + run: sudo apt-get install -y singularity-container - name: Cache UWLCM Singularity image id: cache_singularity From e38788e1d254d241f4d1da8a3b9f5e5b7538a808 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 16:46:28 +0200 Subject: [PATCH 106/282] GA --- .github/workflows/test_libclouphxx.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 17d5a3ba9..944eee86b 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -28,11 +28,16 @@ jobs: run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity -# when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) + #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) # uses: eWaterCycle/setup-singularity@v6 # with: # singularity-version: 3.6.4 - run: sudo apt-get install -y singularity-container + #following https://sylabs.io/guides/3.0/user-guide/installation.html + run: | + wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list + sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 + sudo apt-get update + sudo apt-get install -y singularity-container - name: Cache UWLCM Singularity image id: cache_singularity From 53eede589d4979699b7b3d3d1f8cf0a4644b3b4d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 16:52:38 +0200 Subject: [PATCH 107/282] GA --- .github/workflows/test_libclouphxx.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 944eee86b..bdadd9f89 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -29,15 +29,15 @@ jobs: - name: Install Singularity #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) -# uses: eWaterCycle/setup-singularity@v6 -# with: -# singularity-version: 3.6.4 - #following https://sylabs.io/guides/3.0/user-guide/installation.html - run: | - wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list - sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 - sudo apt-get update - sudo apt-get install -y singularity-container + uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: 3.6.4 + #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +# run: | +# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +# sudo apt-get update +# sudo apt-get install -y singularity-container - name: Cache UWLCM Singularity image id: cache_singularity From f89015b79e29aa24c3b428c618293bc2b105cfa9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 16:53:44 +0200 Subject: [PATCH 108/282] GA --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index bdadd9f89..5cd9aea90 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -31,7 +31,7 @@ jobs: #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) uses: eWaterCycle/setup-singularity@v6 with: - singularity-version: 3.6.4 + singularity-version: 3.7.1 #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 # run: | # wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list From c60cd7c2cd51a27f87adc6e41cb08cef9de250dd Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 17:06:02 +0200 Subject: [PATCH 109/282] GA --- .github/workflows/test_libclouphxx.yml | 2 +- CMakeLists.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 5cd9aea90..b023d8b23 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -66,7 +66,7 @@ jobs: # - name: Build libcloudph++ # Build your program with the given configuration - run: singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: singularity exec $SI VERBOSE=1 cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} # job test diff --git a/CMakeLists.txt b/CMakeLists.txt index 03bd41a7a..6ac93ecd6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -214,10 +214,13 @@ if (CMAKE_CUDA_COMPILER) set (code "20") message(STATUS "CUDA capability check failed, assuming a default of ${code}") endif() + message(STATUS "CUDA capability check failed, assuming a default of code: ${code}") set(arch ${code}) + message(STATUS "CUDA capability check failed, assuming a default of arch: ${arch}") if (arch EQUAL "21") # see https://devtalk.nvidia.com/default/topic/606195/-solved-nvcc-fatal-unsupported-gpu-architecture-compute_21-/ set(arch "20") endif() + message(STATUS "CUDA capability check failed, assuming a default of arch: ${arch}") set(GPU_CODE_OPTIONS ${GPU_CODE_OPTIONS}sm_${arch},) endforeach(code) target_compile_options(cloudphxx_lgrngn PRIVATE $<$:${GPU_CODE_OPTIONS}>) From f78d51cb6be2887a77251e0b1e4e34eb8428e29b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 17:37:21 +0200 Subject: [PATCH 110/282] Update test_libclouphxx.yml --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index b023d8b23..eb32b7e4b 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -66,7 +66,7 @@ jobs: # - name: Build libcloudph++ # Build your program with the given configuration - run: singularity exec $SI VERBOSE=1 cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} # job test From 8e65e5e96a3ca45012d4230166e5a6feae12a8d2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 17:59:33 +0200 Subject: [PATCH 111/282] GA --- .github/workflows/test_libclouphxx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index b023d8b23..a750ed7f3 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -17,7 +17,7 @@ jobs: # well on Windows or Mac. You can convert this to a matrix build if you need # cross-platform coverage. # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' @@ -66,13 +66,13 @@ jobs: # - name: Build libcloudph++ # Build your program with the given configuration - run: singularity exec $SI VERBOSE=1 cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: singularity exec $SI VERBOSE=1 cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j4 # job test test: needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: From 0e6638a3d18e94b1fcca32243b4dfe14090accae Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 18:08:23 +0200 Subject: [PATCH 112/282] GA --- .github/workflows/test_libclouphxx.yml | 2 +- src/impl/particles_impl_bcnd.ipp | 4 ++-- src/impl/particles_impl_reserve_hskpng_npart.ipp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index a750ed7f3..d059da533 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -62,7 +62,7 @@ jobs: # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True # - name: Build libcloudph++ # Build your program with the given configuration diff --git a/src/impl/particles_impl_bcnd.ipp b/src/impl/particles_impl_bcnd.ipp index 8fcf255d5..6eebda9ee 100644 --- a/src/impl/particles_impl_bcnd.ipp +++ b/src/impl/particles_impl_bcnd.ipp @@ -124,8 +124,8 @@ namespace libcloudphxx arg::_1 >= opts_init.x1 ) - rgt_id.begin(); - const auto no_of_n_vctrs_copied(int(1)); - const auto no_of_real_vctrs_copied(distmem_real_vctrs.size()); + const int no_of_n_vctrs_copied(int(1)); + const int no_of_real_vctrs_copied(distmem_real_vctrs.size()); if(lft_count*no_of_n_vctrs_copied > in_n_bfr.size() || rgt_count*no_of_n_vctrs_copied > in_n_bfr.size()) { diff --git a/src/impl/particles_impl_reserve_hskpng_npart.ipp b/src/impl/particles_impl_reserve_hskpng_npart.ipp index ebf311072..063969e7f 100644 --- a/src/impl/particles_impl_reserve_hskpng_npart.ipp +++ b/src/impl/particles_impl_reserve_hskpng_npart.ipp @@ -78,8 +78,8 @@ namespace libcloudphxx // done using resize, because _bfr.end() is never used and we want to assert that buffer is large enough using the .size() function if(distmem()) { - const auto no_of_n_vctrs_copied(int(1)); - const auto no_of_real_vctrs_copied(distmem_real_vctrs.size()); + const int no_of_n_vctrs_copied(int(1)); + const int no_of_real_vctrs_copied(distmem_real_vctrs.size()); in_n_bfr.resize(no_of_n_vctrs_copied * opts_init.n_sd_max / opts_init.nx / config.bfr_fraction); // for n out_n_bfr.resize(no_of_n_vctrs_copied * opts_init.n_sd_max / opts_init.nx / config.bfr_fraction); From 915c1c9a4fb25a5a02546faedf9bd687ad48ce63 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 18:36:54 +0200 Subject: [PATCH 113/282] GA --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ac93ecd6..5a0f5df5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,9 @@ if (CMAKE_CUDA_COMPILER) # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#gpu-compilation list(SORT msg) list(GET msg 0 VIRT) + if (VIRT STREQUAL "?") + set (VIRT "20") + endif() target_compile_options(cloudphxx_lgrngn PRIVATE $<$: --gpu-architecture=compute_${VIRT}>) set(GPU_CODE_OPTIONS --gpu-code=) From 51c2e6f2e7703b6d7b3fd5ada36d69a71674d8cb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 4 Oct 2021 20:15:30 +0200 Subject: [PATCH 114/282] minimal gpu architecture: 35 --- CMakeLists.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a0f5df5f..7a9c99224 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -206,7 +206,7 @@ if (CMAKE_CUDA_COMPILER) list(SORT msg) list(GET msg 0 VIRT) if (VIRT STREQUAL "?") - set (VIRT "20") + set (VIRT "35") endif() target_compile_options(cloudphxx_lgrngn PRIVATE $<$: --gpu-architecture=compute_${VIRT}>) @@ -214,16 +214,13 @@ if (CMAKE_CUDA_COMPILER) foreach(code ${msg}) message(STATUS "CUDA capability: ${code}") if (code STREQUAL "?") - set (code "20") + set (code "35") message(STATUS "CUDA capability check failed, assuming a default of ${code}") endif() - message(STATUS "CUDA capability check failed, assuming a default of code: ${code}") set(arch ${code}) - message(STATUS "CUDA capability check failed, assuming a default of arch: ${arch}") if (arch EQUAL "21") # see https://devtalk.nvidia.com/default/topic/606195/-solved-nvcc-fatal-unsupported-gpu-architecture-compute_21-/ set(arch "20") endif() - message(STATUS "CUDA capability check failed, assuming a default of arch: ${arch}") set(GPU_CODE_OPTIONS ${GPU_CODE_OPTIONS}sm_${arch},) endforeach(code) target_compile_options(cloudphxx_lgrngn PRIVATE $<$:${GPU_CODE_OPTIONS}>) From ce94d361e60365b893003bf135167c5f899fcd23 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 13:15:58 +0200 Subject: [PATCH 115/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index aff7c980b..264919dfa 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -18,13 +18,18 @@ jobs: # cross-platform coverage. # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix runs-on: ubuntu-20.04 - env: - SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + + strategy: + matrix: + include: + - cuda: "True" + - cuda: "False" steps: - uses: actions/checkout@v2 - name: Install Nvidia driver + if: matrix.cuda == "true" run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity @@ -47,7 +52,7 @@ jobs: key: 'sng_ubuntu_20_04_cuda_11_4' - name: Download UWLCM Singularity image - if: steps.cache_singularity.outputs.cache-hit != 'true' + if: steps.cache_singularity.outputs.cache-hit != 'True' run: | mkdir '${{ github.workspace }}/singularity_images' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 @@ -61,8 +66,7 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBOOST_ROOT=${{ steps.boost.outputs.root }} - run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True + run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.cuda}} # - name: Build libcloudph++ # Build your program with the given configuration From 241dd80b529043d9fc54f579f905d74cddf40760 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 13:16:56 +0200 Subject: [PATCH 116/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 264919dfa..26613f884 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Nvidia driver - if: matrix.cuda == "true" + if: matrix.cuda == "True" run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From 1890a3fb88d8997fb03cf7a10397e888f765883a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 13:18:23 +0200 Subject: [PATCH 117/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 26613f884..72d6fd101 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Nvidia driver - if: matrix.cuda == "True" + if: matrix.cuda = 'True' run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity @@ -52,7 +52,7 @@ jobs: key: 'sng_ubuntu_20_04_cuda_11_4' - name: Download UWLCM Singularity image - if: steps.cache_singularity.outputs.cache-hit != 'True' + if: steps.cache_singularity.outputs.cache-hit != 'true' run: | mkdir '${{ github.workspace }}/singularity_images' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 From 8acc10eef17227310d90b6caccc49a47df5b032c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 13:19:56 +0200 Subject: [PATCH 118/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 72d6fd101..0748a2a48 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -22,14 +22,14 @@ jobs: strategy: matrix: include: - - cuda: "True" - - cuda: "False" + - cuda: True + - cuda: False steps: - uses: actions/checkout@v2 - name: Install Nvidia driver - if: matrix.cuda = 'True' + if: matrix.cuda == 'True' run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From 097c7b9d08285de8e008188a8fe199b9a4def393 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 13:23:21 +0200 Subject: [PATCH 119/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 0748a2a48..a5e68e121 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -21,15 +21,18 @@ jobs: strategy: matrix: + name: ["CUDA", "no CUDA"] include: - - cuda: True - - cuda: False + - name: "CUDA" + cuda: True + - name: "no CUDA" + cuda: False steps: - uses: actions/checkout@v2 - name: Install Nvidia driver - if: matrix.cuda == 'True' + if: matrix.cuda == True run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From 0bcfa7339ade21b9d5837e7db184a1078829db16 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 13:23:53 +0200 Subject: [PATCH 120/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index a5e68e121..bcbde813d 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Nvidia driver - if: matrix.cuda == True + if: matrix.cuda == "True" run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From f17e37fc8b23bf81169e90d3ec11e834b67062e2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 13:24:15 +0200 Subject: [PATCH 121/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index bcbde813d..999f4252f 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Nvidia driver - if: matrix.cuda == "True" + if: matrix.cuda == 'True' run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From eb2ddde96eb4c46726f985c27fe25c2d61c8468a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 13:31:37 +0200 Subject: [PATCH 122/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 999f4252f..a55eea0f6 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -24,15 +24,15 @@ jobs: name: ["CUDA", "no CUDA"] include: - name: "CUDA" - cuda: True + disable-cuda: false - name: "no CUDA" - cuda: False + disable-cuda: true steps: - uses: actions/checkout@v2 - name: Install Nvidia driver - if: matrix.cuda == 'True' + if: matrix.disable-cuda == 'false' run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity @@ -69,7 +69,7 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.cuda}} + run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable-cuda}} # - name: Build libcloudph++ # Build your program with the given configuration From 1b151edad0f2fc64bf25511c1eea17ea7350ba2a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 14:04:32 +0200 Subject: [PATCH 123/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index a55eea0f6..537bc470f 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Nvidia driver - if: matrix.disable-cuda == 'false' + if: !matrix.disable-cuda run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From 12241fc5160f60f0d64be6a1aa734fd52bdfb270 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 14:05:47 +0200 Subject: [PATCH 124/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 537bc470f..6a43a083a 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -31,8 +31,10 @@ jobs: steps: - uses: actions/checkout@v2 + - run: echo matrix.disable-cuda + - name: Install Nvidia driver - if: !matrix.disable-cuda + if: matrix.disable-cuda == 'false' run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From 7a0ed58674563716fed475d4bd1fecf0aca2d13b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 14:10:00 +0200 Subject: [PATCH 125/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 6a43a083a..a85e44e84 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -31,10 +31,10 @@ jobs: steps: - uses: actions/checkout@v2 - - run: echo matrix.disable-cuda + - run: echo ${{matrix.disable-cuda}} - name: Install Nvidia driver - if: matrix.disable-cuda == 'false' + if: ${{ matrix.disable-cuda == 'false'}} run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From 79af43debf2cd403ec3c8f9f2931ff8844d49ac6 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 14:11:25 +0200 Subject: [PATCH 126/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index a85e44e84..7f7d73652 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -33,8 +33,10 @@ jobs: - run: echo ${{matrix.disable-cuda}} + - run: echo ${{!matrix.disable-cuda}} + - name: Install Nvidia driver - if: ${{ matrix.disable-cuda == 'false'}} + if: ${{!matrix.disable-cuda}} run: sudo apt install --no-install-recommends nvidia-driver-470 - name: Install Singularity From 618b59c15a50a45294207f02fca36aba1c9eb6be Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 14:13:55 +0200 Subject: [PATCH 127/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7f7d73652..99578a5a2 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -6,9 +6,9 @@ on: pull_request: branches: [ master ] -env: +#env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release +# BUILD_TYPE: Release jobs: # first job, build the library @@ -22,6 +22,7 @@ jobs: strategy: matrix: name: ["CUDA", "no CUDA"] + build_type: ["Debug", "Release"] include: - name: "CUDA" disable-cuda: false @@ -31,10 +32,6 @@ jobs: steps: - uses: actions/checkout@v2 - - run: echo ${{matrix.disable-cuda}} - - - run: echo ${{!matrix.disable-cuda}} - - name: Install Nvidia driver if: ${{!matrix.disable-cuda}} run: sudo apt install --no-install-recommends nvidia-driver-470 @@ -73,11 +70,11 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable-cuda}} + run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable-cuda}} # - name: Build libcloudph++ # Build your program with the given configuration - run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j4 + run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j4 # job test test: @@ -95,7 +92,7 @@ jobs: working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: singularity exec $SI ctest -C ${{env.BUILD_TYPE}} + run: singularity exec $SI ctest -C ${{matrix.build_type}} From 962b093bb8fcbc8efb2850228cf55b330c40599a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 14:43:45 +0200 Subject: [PATCH 128/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 99578a5a2..ab654057b 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -23,9 +23,14 @@ jobs: matrix: name: ["CUDA", "no CUDA"] build_type: ["Debug", "Release"] + threads: 4 include: - name: "CUDA" disable-cuda: false + - name: "CUDA" + build_type: "Debug" + threads: 1 # not enough RAM to compile Debug CUDA + disable-cuda: false - name: "no CUDA" disable-cuda: true @@ -74,7 +79,7 @@ jobs: # - name: Build libcloudph++ # Build your program with the given configuration - run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j4 + run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} # job test test: From 16f1be2449100b3934723c3830650d9b49ff02c2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 14:44:12 +0200 Subject: [PATCH 129/282] GA: builds with and without cuda --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index ab654057b..252bd9e39 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -23,7 +23,7 @@ jobs: matrix: name: ["CUDA", "no CUDA"] build_type: ["Debug", "Release"] - threads: 4 + threads: [4] include: - name: "CUDA" disable-cuda: false From f06d9f754a4f557e2f462d7197abd1eda4ec8689 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 15:28:42 +0200 Subject: [PATCH 130/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 99 ++++++++++++++------------ 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 252bd9e39..fc32873cc 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -23,63 +23,68 @@ jobs: matrix: name: ["CUDA", "no CUDA"] build_type: ["Debug", "Release"] - threads: [4] include: - name: "CUDA" - disable-cuda: false - - name: "CUDA" - build_type: "Debug" - threads: 1 # not enough RAM to compile Debug CUDA - disable-cuda: false + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads + threads: 1 - name: "no CUDA" - disable-cuda: true + disable_cuda: true + threads: 4 steps: - uses: actions/checkout@v2 - - name: Install Nvidia driver - if: ${{!matrix.disable-cuda}} - run: sudo apt install --no-install-recommends nvidia-driver-470 - - - name: Install Singularity - #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) - uses: eWaterCycle/setup-singularity@v6 + - name: build libcloudph++ + uses: igfuw/libcloudphxx_build@v0.1-beta with: - singularity-version: 3.7.1 - #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + +# - name: Install Nvidia driver +# if: ${{!matrix.disable_cuda}} +# run: sudo apt install --no-install-recommends nvidia-driver-470 +# +# - name: Install Singularity +# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) +# uses: eWaterCycle/setup-singularity@v6 +# with: +# singularity-version: 3.7.1 +# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +## run: | +## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +## sudo apt-get update +## sudo apt-get install -y singularity-container +# +# - name: Cache UWLCM Singularity image +# id: cache_singularity +# uses: actions/cache@v2 +# with: +# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# key: 'sng_ubuntu_20_04_cuda_11_4' +# +# - name: Download UWLCM Singularity image +# if: steps.cache_singularity.outputs.cache-hit != 'true' # run: | -# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -# sudo apt-get update -# sudo apt-get install -y singularity-container - - - name: Cache UWLCM Singularity image - id: cache_singularity - uses: actions/cache@v2 - with: - path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - key: 'sng_ubuntu_20_04_cuda_11_4' - - - name: Download UWLCM Singularity image - if: steps.cache_singularity.outputs.cache-hit != 'true' - run: | - mkdir '${{ github.workspace }}/singularity_images' - singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 - # disable Singularity cache, we cache manually - - - name: Set friendly Singularity image name - uses: allenevans/set-env@v2.0.0 - with: - SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - - - name: Configure CMake - # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable-cuda}} +# mkdir '${{ github.workspace }}/singularity_images' +# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 +# # disable Singularity cache, we cache manually # - - name: Build libcloudph++ - # Build your program with the given configuration - run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} +# - name: Set friendly Singularity image name +# uses: allenevans/set-env@v2.0.0 +# with: +# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# +# - name: Configure CMake +# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. +# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type +# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} +## +# - name: Build libcloudph++ +# # Build your program with the given configuration +# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} # job test test: From 90855bddb408a0c3a37bb2d05316fd4d21cedd56 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 15:31:08 +0200 Subject: [PATCH 131/282] Revert "GA: use libcloudphxx build action" This reverts commit f06d9f754a4f557e2f462d7197abd1eda4ec8689. --- .github/workflows/test_libclouphxx.yml | 99 ++++++++++++-------------- 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index fc32873cc..252bd9e39 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -23,68 +23,63 @@ jobs: matrix: name: ["CUDA", "no CUDA"] build_type: ["Debug", "Release"] + threads: [4] include: - name: "CUDA" - disable_cuda: false - # not enough RAM to compile Debug CUDA on 4 threads - threads: 1 + disable-cuda: false + - name: "CUDA" + build_type: "Debug" + threads: 1 # not enough RAM to compile Debug CUDA + disable-cuda: false - name: "no CUDA" - disable_cuda: true - threads: 4 + disable-cuda: true steps: - uses: actions/checkout@v2 - - name: build libcloudph++ - uses: igfuw/libcloudphxx_build@v0.1-beta + - name: Install Nvidia driver + if: ${{!matrix.disable-cuda}} + run: sudo apt install --no-install-recommends nvidia-driver-470 + + - name: Install Singularity + #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) + uses: eWaterCycle/setup-singularity@v6 with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - -# - name: Install Nvidia driver -# if: ${{!matrix.disable_cuda}} -# run: sudo apt install --no-install-recommends nvidia-driver-470 -# -# - name: Install Singularity -# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) -# uses: eWaterCycle/setup-singularity@v6 -# with: -# singularity-version: 3.7.1 -# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 -## run: | -## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -## sudo apt-get update -## sudo apt-get install -y singularity-container -# -# - name: Cache UWLCM Singularity image -# id: cache_singularity -# uses: actions/cache@v2 -# with: -# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# key: 'sng_ubuntu_20_04_cuda_11_4' -# -# - name: Download UWLCM Singularity image -# if: steps.cache_singularity.outputs.cache-hit != 'true' + singularity-version: 3.7.1 + #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 # run: | -# mkdir '${{ github.workspace }}/singularity_images' -# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 -# # disable Singularity cache, we cache manually +# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +# sudo apt-get update +# sudo apt-get install -y singularity-container + + - name: Cache UWLCM Singularity image + id: cache_singularity + uses: actions/cache@v2 + with: + path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + key: 'sng_ubuntu_20_04_cuda_11_4' + + - name: Download UWLCM Singularity image + if: steps.cache_singularity.outputs.cache-hit != 'true' + run: | + mkdir '${{ github.workspace }}/singularity_images' + singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 + # disable Singularity cache, we cache manually + + - name: Set friendly Singularity image name + uses: allenevans/set-env@v2.0.0 + with: + SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + + - name: Configure CMake + # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. + # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type + run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable-cuda}} # -# - name: Set friendly Singularity image name -# uses: allenevans/set-env@v2.0.0 -# with: -# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# -# - name: Configure CMake -# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. -# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type -# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} -## -# - name: Build libcloudph++ -# # Build your program with the given configuration -# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} + - name: Build libcloudph++ + # Build your program with the given configuration + run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} # job test test: From 76d4a37c7dd764e782de18c097cb3e0d7dc5cdbe Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 15:34:31 +0200 Subject: [PATCH 132/282] Revert "Revert "GA: use libcloudphxx build action"" This reverts commit 90855bddb408a0c3a37bb2d05316fd4d21cedd56. --- .github/workflows/test_libclouphxx.yml | 99 ++++++++++++++------------ 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 252bd9e39..fc32873cc 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -23,63 +23,68 @@ jobs: matrix: name: ["CUDA", "no CUDA"] build_type: ["Debug", "Release"] - threads: [4] include: - name: "CUDA" - disable-cuda: false - - name: "CUDA" - build_type: "Debug" - threads: 1 # not enough RAM to compile Debug CUDA - disable-cuda: false + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads + threads: 1 - name: "no CUDA" - disable-cuda: true + disable_cuda: true + threads: 4 steps: - uses: actions/checkout@v2 - - name: Install Nvidia driver - if: ${{!matrix.disable-cuda}} - run: sudo apt install --no-install-recommends nvidia-driver-470 - - - name: Install Singularity - #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) - uses: eWaterCycle/setup-singularity@v6 + - name: build libcloudph++ + uses: igfuw/libcloudphxx_build@v0.1-beta with: - singularity-version: 3.7.1 - #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + +# - name: Install Nvidia driver +# if: ${{!matrix.disable_cuda}} +# run: sudo apt install --no-install-recommends nvidia-driver-470 +# +# - name: Install Singularity +# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) +# uses: eWaterCycle/setup-singularity@v6 +# with: +# singularity-version: 3.7.1 +# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +## run: | +## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +## sudo apt-get update +## sudo apt-get install -y singularity-container +# +# - name: Cache UWLCM Singularity image +# id: cache_singularity +# uses: actions/cache@v2 +# with: +# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# key: 'sng_ubuntu_20_04_cuda_11_4' +# +# - name: Download UWLCM Singularity image +# if: steps.cache_singularity.outputs.cache-hit != 'true' # run: | -# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -# sudo apt-get update -# sudo apt-get install -y singularity-container - - - name: Cache UWLCM Singularity image - id: cache_singularity - uses: actions/cache@v2 - with: - path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - key: 'sng_ubuntu_20_04_cuda_11_4' - - - name: Download UWLCM Singularity image - if: steps.cache_singularity.outputs.cache-hit != 'true' - run: | - mkdir '${{ github.workspace }}/singularity_images' - singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 - # disable Singularity cache, we cache manually - - - name: Set friendly Singularity image name - uses: allenevans/set-env@v2.0.0 - with: - SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - - - name: Configure CMake - # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable-cuda}} +# mkdir '${{ github.workspace }}/singularity_images' +# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 +# # disable Singularity cache, we cache manually # - - name: Build libcloudph++ - # Build your program with the given configuration - run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} +# - name: Set friendly Singularity image name +# uses: allenevans/set-env@v2.0.0 +# with: +# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# +# - name: Configure CMake +# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. +# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type +# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} +## +# - name: Build libcloudph++ +# # Build your program with the given configuration +# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} # job test test: From d6835222a3195b402d2a47c45c0ca420f33a7b82 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 15:43:43 +0200 Subject: [PATCH 133/282] GA: add actions for testing --- .github/actions/action.yml | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/actions/action.yml diff --git a/.github/actions/action.yml b/.github/actions/action.yml new file mode 100644 index 000000000..e78346a0a --- /dev/null +++ b/.github/actions/action.yml @@ -0,0 +1,66 @@ +name: 'build libcloudph++' +description: 'builds libcloudph++, libcloudph++ code needs to be check out before running this action' +inputs: + disable_cuda: + description: 'disable CUDA build?' + required: true + default: false + build_type: + description: 'CMake build type: Release, Debug, ...' + required: true + default: "Release" + threads: + description: "number of threads used by make" + required: true + default: 1 + +runs: + using: "composite" + steps: + - name: Install Nvidia driver + shell: bash + if: ${{!inputs.disable_cuda}} + run: sudo apt install --no-install-recommends nvidia-driver-470 + + - name: Install Singularity + #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) + uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: 3.7.1 + #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +# run: | +# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +# sudo apt-get update +# sudo apt-get install -y singularity-container + + - name: Cache UWLCM Singularity image + id: cache_singularity + uses: actions/cache@v2 + with: + path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + key: 'sng_ubuntu_20_04_cuda_11_4' + + - name: Download UWLCM Singularity image + shell: bash + if: steps.cache_singularity.outputs.cache-hit != 'true' + run: | + mkdir '${{ github.workspace }}/singularity_images' + singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 + # disable Singularity cache, we cache manually + + - name: Set friendly Singularity image name + uses: allenevans/set-env@v2.0.0 + with: + SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + + - name: Configure CMake + shell: bash + # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. + # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type + run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} +# + - name: Build libcloudph++ + shell: bash + # Build your program with the given configuration + run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{inputs.build_type}} -j${{inputs.threads}} From 7562f5a9b9c2c48d34d4aae82673b60d0ce659b8 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 15:46:00 +0200 Subject: [PATCH 134/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index fc32873cc..07b37a7e0 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -36,7 +36,8 @@ jobs: - uses: actions/checkout@v2 - name: build libcloudph++ - uses: igfuw/libcloudphxx_build@v0.1-beta + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build with: disable_cuda: ${{matrix.disable_cuda}} build_type: ${{matrix.build_type}} From 5f6f1ac4faec61bbcd759a4f65a3f05f3d114766 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 15:47:38 +0200 Subject: [PATCH 135/282] GA: add actions for testing --- .github/actions/{ => libcloudphxx_build}/action.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/actions/{ => libcloudphxx_build}/action.yml (100%) diff --git a/.github/actions/action.yml b/.github/actions/libcloudphxx_build/action.yml similarity index 100% rename from .github/actions/action.yml rename to .github/actions/libcloudphxx_build/action.yml From a08cfcc18c806dca3b262aba9a1359dd3944f81f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 15:49:18 +0200 Subject: [PATCH 136/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index e78346a0a..d78ff04c2 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -19,8 +19,8 @@ runs: steps: - name: Install Nvidia driver shell: bash - if: ${{!inputs.disable_cuda}} run: sudo apt install --no-install-recommends nvidia-driver-470 + if: ${{!inputs.disable_cuda}} - name: Install Singularity #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) From 2b5790adec9e14bf4a93c2b07a04619a1dc2e86a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:01:29 +0200 Subject: [PATCH 137/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index d78ff04c2..c339f5ae4 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -20,7 +20,7 @@ runs: - name: Install Nvidia driver shell: bash run: sudo apt install --no-install-recommends nvidia-driver-470 - if: ${{!inputs.disable_cuda}} + # if: ${{!inputs.disable_cuda}} - name: Install Singularity #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) @@ -43,7 +43,7 @@ runs: - name: Download UWLCM Singularity image shell: bash - if: steps.cache_singularity.outputs.cache-hit != 'true' +# if: steps.cache_singularity.outputs.cache-hit != 'true' run: | mkdir '${{ github.workspace }}/singularity_images' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 From f3843cdcfb6422844ef7bbe781c62fc43dedba7e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:06:05 +0200 Subject: [PATCH 138/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index c339f5ae4..a49b8eefb 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -19,7 +19,8 @@ runs: steps: - name: Install Nvidia driver shell: bash - run: sudo apt install --no-install-recommends nvidia-driver-470 + run: | + [[ "${{!inputs.disable_cuda}}" = true ]] && sudo apt install --no-install-recommends nvidia-driver-470 || true # if: ${{!inputs.disable_cuda}} - name: Install Singularity From 7b08f94af69d315f1054c3261bb0780541363a6b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:07:19 +0200 Subject: [PATCH 139/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index a49b8eefb..ab8cf07b8 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -46,8 +46,10 @@ runs: shell: bash # if: steps.cache_singularity.outputs.cache-hit != 'true' run: | + [[ "${{steps.cache_singularity.outputs.cache-hit}}" != 'true' ]] && mkdir '${{ github.workspace }}/singularity_images' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 + || true # disable Singularity cache, we cache manually - name: Set friendly Singularity image name From f3dc3556f83c59c4e1c678cdeb8d236c38a8295e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:10:29 +0200 Subject: [PATCH 140/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index ab8cf07b8..6742d2938 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -46,7 +46,7 @@ runs: shell: bash # if: steps.cache_singularity.outputs.cache-hit != 'true' run: | - [[ "${{steps.cache_singularity.outputs.cache-hit}}" != 'true' ]] && + [[ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ]] && mkdir '${{ github.workspace }}/singularity_images' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 || true From 87f28eb36ecfdf5b126d2b19cccfa76282c75aa0 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:15:36 +0200 Subject: [PATCH 141/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index 6742d2938..2520e6575 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -4,7 +4,7 @@ inputs: disable_cuda: description: 'disable CUDA build?' required: true - default: false + default: "false" build_type: description: 'CMake build type: Release, Debug, ...' required: true @@ -20,7 +20,7 @@ runs: - name: Install Nvidia driver shell: bash run: | - [[ "${{!inputs.disable_cuda}}" = true ]] && sudo apt install --no-install-recommends nvidia-driver-470 || true + [ "${{inputs.disable_cuda}}" = "false" ] && sudo apt install --no-install-recommends nvidia-driver-470 || true # if: ${{!inputs.disable_cuda}} - name: Install Singularity @@ -46,7 +46,7 @@ runs: shell: bash # if: steps.cache_singularity.outputs.cache-hit != 'true' run: | - [[ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ]] && + [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && mkdir '${{ github.workspace }}/singularity_images' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 || true From 76576f3b963ec959b9d127ab2a7facf9eae40e77 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:19:11 +0200 Subject: [PATCH 142/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index 2520e6575..e41150bcd 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -47,8 +47,9 @@ runs: # if: steps.cache_singularity.outputs.cache-hit != 'true' run: | [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && - mkdir '${{ github.workspace }}/singularity_images' + {mkdir '${{ github.workspace }}/singularity_images' singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 + } || true # disable Singularity cache, we cache manually From cc6fedd7813e9407d2b9a03ab9b30d8f6e877867 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:27:54 +0200 Subject: [PATCH 143/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index e41150bcd..e2fc219bf 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -45,12 +45,13 @@ runs: - name: Download UWLCM Singularity image shell: bash # if: steps.cache_singularity.outputs.cache-hit != 'true' - run: | - [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && - {mkdir '${{ github.workspace }}/singularity_images' - singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 - } - || true + run: [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ + { + mkdir '${{ github.workspace }}/singularity_images'; + singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; + } \ + || \ + true # disable Singularity cache, we cache manually - name: Set friendly Singularity image name From dbc28204c3e140a68e630c0f064c1cc4e8801ee7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:30:15 +0200 Subject: [PATCH 144/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index e2fc219bf..c47b4975e 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -45,12 +45,13 @@ runs: - name: Download UWLCM Singularity image shell: bash # if: steps.cache_singularity.outputs.cache-hit != 'true' - run: [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ + run: | + [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && { mkdir '${{ github.workspace }}/singularity_images'; singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; - } \ - || \ + } + || true # disable Singularity cache, we cache manually From ad795ae007d77d61170220a28c5a8f6e5fc912f9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:33:55 +0200 Subject: [PATCH 145/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index c47b4975e..b4adbbdff 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -48,8 +48,8 @@ runs: run: | [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && { - mkdir '${{ github.workspace }}/singularity_images'; - singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; + mkdir '${{ github.workspace }}/singularity_images'; + singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; } || true From 119ee3014e9ab5cc1f5c8b088dc3401ab443b73b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:35:56 +0200 Subject: [PATCH 146/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index b4adbbdff..15fbdf172 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -46,12 +46,12 @@ runs: shell: bash # if: steps.cache_singularity.outputs.cache-hit != 'true' run: | - [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && + [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ { mkdir '${{ github.workspace }}/singularity_images'; singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; - } - || + } \ + || \ true # disable Singularity cache, we cache manually From f990c2dab06987fecf0aa8892af117c7a26511ec Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:47:17 +0200 Subject: [PATCH 147/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 36 ++++++++++++++++++-------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 07b37a7e0..3b004e116 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,29 +7,43 @@ on: branches: [ master ] #env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) -# BUILD_TYPE: Release jobs: - # first job, build the library - build: - # The CMake configure and build commands are platform agnostic and should work equally - # well on Windows or Mac. You can convert this to a matrix build if you need - # cross-platform coverage. - # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix + build_CUDA: runs-on: ubuntu-20.04 strategy: matrix: - name: ["CUDA", "no CUDA"] + name: ["CUDA"] build_type: ["Debug", "Release"] include: - name: "CUDA" disable_cuda: false # not enough RAM to compile Debug CUDA on 4 threads threads: 1 - - name: "no CUDA" - disable_cuda: true + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + + build: + runs-on: ubuntu-20.04 + + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["Debug", "Release"] + include: + - name: "no_CUDA" + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads threads: 4 steps: From 22b4817cf1b3ca376549f2199f32a3df6c6d4038 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 16:50:51 +0200 Subject: [PATCH 148/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 3b004e116..6b71b7b45 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -106,12 +106,17 @@ jobs: needs: build runs-on: ubuntu-20.04 + strategy: + matrix: + build_type: ["Release"] + steps: - - name: Set friendly Singularity image name - uses: allenevans/set-env@v2.0.0 - with: - SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + # TODO: it is already done by the libcloudphxx_build action? + # - name: Set friendly Singularity image name + # uses: allenevans/set-env@v2.0.0 + # with: + # SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - name: Run unit tests working-directory: ${{github.workspace}}/build From bbba45bba0355e686bacd60d406f61749291a75b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 17:15:56 +0200 Subject: [PATCH 149/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 6b71b7b45..262b48504 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -42,7 +42,7 @@ jobs: build_type: ["Debug", "Release"] include: - name: "no_CUDA" - disable_cuda: false + disable_cuda: true # not enough RAM to compile Debug CUDA on 4 threads threads: 4 From 58f69eacb66b8cf8c7a466646636dfe0ef621f14 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 6 Oct 2021 17:31:42 +0200 Subject: [PATCH 150/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 58 ++++++++++++++------------ 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 262b48504..1bd60c0b6 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -111,12 +111,39 @@ jobs: build_type: ["Release"] steps: + - uses: actions/checkout@v2 - # TODO: it is already done by the libcloudphxx_build action? - # - name: Set friendly Singularity image name - # uses: allenevans/set-env@v2.0.0 - # with: - # SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# Singularity installation the same as in libcloudphxx_build action... + - name: Install Singularity + #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) + uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: 3.7.1 + #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +# run: | +# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +# sudo apt-get update +# sudo apt-get install -y singularity-container + + - name: Cache UWLCM Singularity image + id: cache_singularity + uses: actions/cache@v2 + with: + path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + key: 'sng_ubuntu_20_04_cuda_11_4' + + - name: Download UWLCM Singularity image + if: steps.cache_singularity.outputs.cache-hit != 'true' + run: | + mkdir '${{ github.workspace }}/singularity_images' + singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 + # disable Singularity cache, we cache manually + + - name: Set friendly Singularity image name + uses: allenevans/set-env@v2.0.0 + with: + SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - name: Run unit tests working-directory: ${{github.workspace}}/build @@ -124,27 +151,6 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: singularity exec $SI ctest -C ${{matrix.build_type}} - - - - - - - - - - - - - - - - - - - - - # - name: Run Singularity shell # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' From 7738acc1584a7bb8b1a3a7fcc687935cb7a55160 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 10:39:42 +0200 Subject: [PATCH 151/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 1bd60c0b6..d932eac86 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -149,7 +149,11 @@ jobs: working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: singularity exec $SI ctest -C ${{matrix.build_type}} + run: | + pwd + ls + singularity exec $SI make test + singularity exec $SI ctest -C ${{matrix.build_type}} # - name: Run Singularity shell From 60697742fc4e3dbb4b42da0f2bdb4b9c830431cc Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 11:17:45 +0200 Subject: [PATCH 152/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 57 +++++++++++++++----------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index d932eac86..09178e229 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -9,29 +9,29 @@ on: #env: jobs: - build_CUDA: - runs-on: ubuntu-20.04 - - strategy: - matrix: - name: ["CUDA"] - build_type: ["Debug", "Release"] - include: - - name: "CUDA" - disable_cuda: false - # not enough RAM to compile Debug CUDA on 4 threads - threads: 1 - - steps: - - uses: actions/checkout@v2 - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} +# build_CUDA: +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# name: ["CUDA"] +# build_type: ["Debug", "Release"] +# include: +# - name: "CUDA" +# disable_cuda: false +# # not enough RAM to compile Debug CUDA on 4 threads +# threads: 1 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} build: runs-on: ubuntu-20.04 @@ -57,6 +57,12 @@ jobs: build_type: ${{matrix.build_type}} threads: ${{matrix.threads}} + - name: Upload built libcloudph++ + uses: actions/upload-artifact@v2 + with: + name: libcloud_build + path: ${{github.workspace}}/build + # - name: Install Nvidia driver # if: ${{!matrix.disable_cuda}} # run: sudo apt install --no-install-recommends nvidia-driver-470 @@ -113,6 +119,11 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build + # Singularity installation the same as in libcloudphxx_build action... - name: Install Singularity #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) From 5d430f129ac7f9965ec9430319ee91da05f81a86 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 11:33:02 +0200 Subject: [PATCH 153/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 09178e229..0aa3f54b6 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -124,6 +124,12 @@ jobs: with: name: libcloud_build + - run: | + pwd + ls + echo ${{ github.workspace }} + ls ${{ github.workspace }} + # Singularity installation the same as in libcloudphxx_build action... - name: Install Singularity #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) From aa294f29ab59387586560d144b73d6294b1eb606 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 11:36:44 +0200 Subject: [PATCH 154/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 0aa3f54b6..e3fb411ca 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -125,10 +125,10 @@ jobs: name: libcloud_build - run: | - pwd - ls - echo ${{ github.workspace }} - ls ${{ github.workspace }} + pwd + ls + echo ${{ github.workspace }} + ls ${{ github.workspace }} # Singularity installation the same as in libcloudphxx_build action... - name: Install Singularity From 94c23d356f7026f2881723d43c596db3a98e1ba3 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 11:54:48 +0200 Subject: [PATCH 155/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index e3fb411ca..d3beef86d 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -119,10 +119,14 @@ jobs: steps: - uses: actions/checkout@v2 + - name: create the build directory + run: mkdir ${{ github.workspace }}/build + - name: Download libcloudph++ build uses: actions/download-artifact@v2 with: name: libcloud_build + path: ${{ github.workspace }}/build - run: | pwd From 1ea16eda3f41bfb5cc495099073f2397b51185ab Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 12:09:42 +0200 Subject: [PATCH 156/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index d3beef86d..a1296d8cc 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -128,12 +128,6 @@ jobs: name: libcloud_build path: ${{ github.workspace }}/build - - run: | - pwd - ls - echo ${{ github.workspace }} - ls ${{ github.workspace }} - # Singularity installation the same as in libcloudphxx_build action... - name: Install Singularity #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) @@ -171,10 +165,8 @@ jobs: # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | - pwd - ls - singularity exec $SI make test - singularity exec $SI ctest -C ${{matrix.build_type}} + singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + #singularity exec $SI make test # - name: Run Singularity shell From 5267ae24a3d1405d10da178cb38328101980daca Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 12:33:21 +0200 Subject: [PATCH 157/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index a1296d8cc..48af9d8a9 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -57,11 +57,15 @@ jobs: build_type: ${{matrix.build_type}} threads: ${{matrix.threads}} - - name: Upload built libcloudph++ + # tar build dir before upload as artifact to retain permission and case-sensitive names + - name: Compress libcloudph++ build + run: tar -cvf build.tar ${{github.workspace}}/build + + - name: Upload libcloudph++ build uses: actions/upload-artifact@v2 with: - name: libcloud_build - path: ${{github.workspace}}/build + name: libcloud_build_tar + path: build.tar # - name: Install Nvidia driver # if: ${{!matrix.disable_cuda}} @@ -119,14 +123,21 @@ jobs: steps: - uses: actions/checkout@v2 - - name: create the build directory - run: mkdir ${{ github.workspace }}/build +# - name: create the build directory +# run: mkdir ${{ github.workspace }}/build - name: Download libcloudph++ build uses: actions/download-artifact@v2 with: - name: libcloud_build - path: ${{ github.workspace }}/build + name: libcloud_build_tar +# path: ${{ github.workspace }}/build + + - name: Decompress libcloudph++ build + run: | + tar -xvf build.tar + pwd + ls + ls build # Singularity installation the same as in libcloudphxx_build action... - name: Install Singularity From 03dad5206eb3e09512b912a48d5abe97e2459179 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 13:13:06 +0200 Subject: [PATCH 158/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 48af9d8a9..5d12dce08 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -59,7 +59,7 @@ jobs: # tar build dir before upload as artifact to retain permission and case-sensitive names - name: Compress libcloudph++ build - run: tar -cvf build.tar ${{github.workspace}}/build + run: tar -cvf build.tar build - name: Upload libcloudph++ build uses: actions/upload-artifact@v2 From 286150190348f5836f68164cd649c178f8c9a49f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 13:35:20 +0200 Subject: [PATCH 159/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 5d12dce08..39a883dc4 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -64,7 +64,7 @@ jobs: - name: Upload libcloudph++ build uses: actions/upload-artifact@v2 with: - name: libcloud_build_tar + name: libcloud_build_${{matrix.build_type}}_tar path: build.tar # - name: Install Nvidia driver @@ -118,7 +118,7 @@ jobs: strategy: matrix: - build_type: ["Release"] + build_type: ["Release", "Debug"] steps: - uses: actions/checkout@v2 @@ -129,7 +129,7 @@ jobs: - name: Download libcloudph++ build uses: actions/download-artifact@v2 with: - name: libcloud_build_tar + name: libcloud_build_${{matrix.build_type}}_tar # path: ${{ github.workspace }}/build - name: Decompress libcloudph++ build From 73c31ab26c4deca09ffd9c74638a554b73f4a44f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 14:05:27 +0200 Subject: [PATCH 160/282] GA: add actions for testing --- .../load_UWLCM_singularity_image/action.yml | 47 ++++++++++++ .github/workflows/test_libclouphxx.yml | 76 ++++++++----------- 2 files changed, 79 insertions(+), 44 deletions(-) create mode 100644 .github/actions/load_UWLCM_singularity_image/action.yml diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml new file mode 100644 index 000000000..f71d913b3 --- /dev/null +++ b/.github/actions/load_UWLCM_singularity_image/action.yml @@ -0,0 +1,47 @@ +name: 'load UWLCM Singularity image' +description: 'set env var SI to point to the image location' +inputs: + path: + description: 'Where to put the image' + required: true + +runs: + using: "composite" + steps: + + - name: Install Singularity + #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) + uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: 3.7.1 + #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +# run: | +# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +# sudo apt-get update +# sudo apt-get install -y singularity-container + + - name: Cache UWLCM Singularity image + id: cache_singularity + uses: actions/cache@v2 + with: + path: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4.sif' + key: 'sng_ubuntu_20_04_cuda_11_4' + + - name: Download UWLCM Singularity image + shell: bash +# if: steps.cache_singularity.outputs.cache-hit != 'true' + run: | + [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ + { + mkdir '${{ inputs.path }}'; + singularity pull --disable-cache --dir '${{ inputs.path }}' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; + } \ + || \ + true + # disable Singularity cache, we cache manually + + - name: Set friendly Singularity image name + uses: allenevans/set-env@v2.0.0 + with: + SI: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4.sif' diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 39a883dc4..7e0c40525 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -111,8 +111,8 @@ jobs: # # Build your program with the given configuration # run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} - # job test - test: + # test jobs + unit_test: needs: build runs-on: ubuntu-20.04 @@ -123,61 +123,49 @@ jobs: steps: - uses: actions/checkout@v2 -# - name: create the build directory -# run: mkdir ${{ github.workspace }}/build - - name: Download libcloudph++ build uses: actions/download-artifact@v2 with: name: libcloud_build_${{matrix.build_type}}_tar -# path: ${{ github.workspace }}/build - name: Decompress libcloudph++ build - run: | - tar -xvf build.tar - pwd - ls - ls build + run: tar -xvf build.tar -# Singularity installation the same as in libcloudphxx_build action... - - name: Install Singularity - #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) - uses: eWaterCycle/setup-singularity@v6 + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image with: - singularity-version: 3.7.1 - #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 + path: ${{ github.workspace }}/singularity_images + +# Singularity installation the same as in libcloudphxx_build action... +# - name: Install Singularity +# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) +# uses: eWaterCycle/setup-singularity@v6 +# with: +# singularity-version: 3.7.1 +# +# - name: Cache UWLCM Singularity image +# id: cache_singularity +# uses: actions/cache@v2 +# with: +# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# key: 'sng_ubuntu_20_04_cuda_11_4' +# +# - name: Download UWLCM Singularity image +# if: steps.cache_singularity.outputs.cache-hit != 'true' # run: | -# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -# sudo apt-get update -# sudo apt-get install -y singularity-container - - - name: Cache UWLCM Singularity image - id: cache_singularity - uses: actions/cache@v2 - with: - path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - key: 'sng_ubuntu_20_04_cuda_11_4' - - - name: Download UWLCM Singularity image - if: steps.cache_singularity.outputs.cache-hit != 'true' - run: | - mkdir '${{ github.workspace }}/singularity_images' - singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 - # disable Singularity cache, we cache manually - - - name: Set friendly Singularity image name - uses: allenevans/set-env@v2.0.0 - with: - SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# mkdir '${{ github.workspace }}/singularity_images' +# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 +# +# - name: Set friendly Singularity image name +# uses: allenevans/set-env@v2.0.0 +# with: +# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - name: Run unit tests working-directory: ${{github.workspace}}/build - # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: | - singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - #singularity exec $SI make test + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) # - name: Run Singularity shell From f8822d5ce1e5d798e974e9f759e2964366bc2316 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 14:35:08 +0200 Subject: [PATCH 161/282] GA: add actions for testing --- .github/actions/libcloudphxx_build/action.yml | 84 ++++++++++--------- .github/workflows/test_libclouphxx.yml | 1 + 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index 15fbdf172..ba887ce00 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -1,5 +1,5 @@ name: 'build libcloudph++' -description: 'builds libcloudph++, libcloudph++ code needs to be check out before running this action' +description: 'builds libcloudph++, loads UWLCM Singularity image for dependencies' inputs: disable_cuda: description: 'disable CUDA build?' @@ -13,6 +13,9 @@ inputs: description: "number of threads used by make" required: true default: 1 + path: + description: "path to the dir with libcloudphxx code" + required: true runs: using: "composite" @@ -22,51 +25,56 @@ runs: run: | [ "${{inputs.disable_cuda}}" = "false" ] && sudo apt install --no-install-recommends nvidia-driver-470 || true # if: ${{!inputs.disable_cuda}} - - - name: Install Singularity - #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) - uses: eWaterCycle/setup-singularity@v6 - with: - singularity-version: 3.7.1 - #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 -# run: | -# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -# sudo apt-get update -# sudo apt-get install -y singularity-container - - - name: Cache UWLCM Singularity image - id: cache_singularity - uses: actions/cache@v2 - with: - path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - key: 'sng_ubuntu_20_04_cuda_11_4' - - - name: Download UWLCM Singularity image - shell: bash -# if: steps.cache_singularity.outputs.cache-hit != 'true' - run: | - [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ - { - mkdir '${{ github.workspace }}/singularity_images'; - singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; - } \ - || \ - true - # disable Singularity cache, we cache manually - - name: Set friendly Singularity image name - uses: allenevans/set-env@v2.0.0 + - name: load UWLCM Singularity image + uses: ./.github/actions/load_UWLCM_singularity_image with: - SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + path: ${{ inputs.path }}/singularity_images + +# - name: Install Singularity +# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) +# uses: eWaterCycle/setup-singularity@v6 +# with: +# singularity-version: 3.7.1 +# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +## run: | +## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +## sudo apt-get update +## sudo apt-get install -y singularity-container +# +# - name: Cache UWLCM Singularity image +# id: cache_singularity +# uses: actions/cache@v2 +# with: +# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# key: 'sng_ubuntu_20_04_cuda_11_4' +# +# - name: Download UWLCM Singularity image +# shell: bash +## if: steps.cache_singularity.outputs.cache-hit != 'true' +# run: | +# [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ +# { +# mkdir '${{ github.workspace }}/singularity_images'; +# singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; +# } \ +# || \ +# true +# # disable Singularity cache, we cache manually +# +# - name: Set friendly Singularity image name +# uses: allenevans/set-env@v2.0.0 +# with: +# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - name: Configure CMake shell: bash # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} + run: singularity exec $SI cmake -B ${{inputs.path}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} # - name: Build libcloudph++ shell: bash # Build your program with the given configuration - run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{inputs.build_type}} -j${{inputs.threads}} + run: VERBOSE=1 singularity exec $SI cmake --build ${{inputs.path}}/build --config ${{inputs.build_type}} -j${{inputs.threads}} diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7e0c40525..348497eee 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -56,6 +56,7 @@ jobs: disable_cuda: ${{matrix.disable_cuda}} build_type: ${{matrix.build_type}} threads: ${{matrix.threads}} + path: ${{ github.workspace }} # tar build dir before upload as artifact to retain permission and case-sensitive names - name: Compress libcloudph++ build From db1b1230cf4b8402064c10e89b30980f6bfb3924 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 16:15:01 +0200 Subject: [PATCH 162/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 134 +++++++++++++++---------- 1 file changed, 83 insertions(+), 51 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 348497eee..75f272d27 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -11,7 +11,6 @@ on: jobs: # build_CUDA: # runs-on: ubuntu-20.04 -# # strategy: # matrix: # name: ["CUDA"] @@ -35,7 +34,6 @@ jobs: build: runs-on: ubuntu-20.04 - strategy: matrix: name: ["no_CUDA"] @@ -43,7 +41,6 @@ jobs: include: - name: "no_CUDA" disable_cuda: true - # not enough RAM to compile Debug CUDA on 4 threads threads: 4 steps: @@ -68,58 +65,52 @@ jobs: name: libcloud_build_${{matrix.build_type}}_tar path: build.tar -# - name: Install Nvidia driver -# if: ${{!matrix.disable_cuda}} -# run: sudo apt install --no-install-recommends nvidia-driver-470 -# -# - name: Install Singularity -# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) -# uses: eWaterCycle/setup-singularity@v6 -# with: -# singularity-version: 3.7.1 -# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 -## run: | -## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -## sudo apt-get update -## sudo apt-get install -y singularity-container -# -# - name: Cache UWLCM Singularity image -# id: cache_singularity -# uses: actions/cache@v2 + # test jobs +# unit_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["Release", "Debug"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 # with: -# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# key: 'sng_ubuntu_20_04_cuda_11_4' -# -# - name: Download UWLCM Singularity image -# if: steps.cache_singularity.outputs.cache-hit != 'true' -# run: | -# mkdir '${{ github.workspace }}/singularity_images' -# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 -# # disable Singularity cache, we cache manually +# name: libcloud_build_${{matrix.build_type}}_tar # -# - name: Set friendly Singularity image name -# uses: allenevans/set-env@v2.0.0 +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image # with: -# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# -# - name: Configure CMake -# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. -# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type -# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} -## -# - name: Build libcloudph++ -# # Build your program with the given configuration -# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} +# path: ${{ github.workspace }}/singularity_images +# +# - name: Run unit tests +# working-directory: ${{github.workspace}}/build +# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail +# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - # test jobs - unit_test: + parcel_test: needs: build runs-on: ubuntu-20.04 strategy: matrix: build_type: ["Release", "Debug"] + include: + - build_type: "Release" + long_tests: true + debug_tests: false + - build_type: "Debug" + long_tests: false + debug_tests: true + steps: - uses: actions/checkout@v2 @@ -138,12 +129,48 @@ jobs: with: path: ${{ github.workspace }}/singularity_images -# Singularity installation the same as in libcloudphxx_build action... + - name: Install libcloudph++ + run: singularity exec $SI cmake --install build + + - name: Install python3-gnuplot + run: | + wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb + sudo dpkg -i python3-gnuplot_1.8-8_all.deb + + - name: checkout parcel repo + uses: actions/checkout@v2 + with: + repository: igfuw/parcel + + - run: mkdir parcel/plots/outputs + + - name: run parcel tests long + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: | + py.test -v unit_test + py.test -v long_test + + - name: run parcel tests debug + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.debug_tests}} + run: py.test -v unit_test_debug + +# - name: Install Nvidia driver +# if: ${{!matrix.disable_cuda}} +# run: sudo apt install --no-install-recommends nvidia-driver-470 +# # - name: Install Singularity # #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) # uses: eWaterCycle/setup-singularity@v6 # with: # singularity-version: 3.7.1 +# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +## run: | +## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +## sudo apt-get update +## sudo apt-get install -y singularity-container # # - name: Cache UWLCM Singularity image # id: cache_singularity @@ -157,16 +184,21 @@ jobs: # run: | # mkdir '${{ github.workspace }}/singularity_images' # singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 +# # disable Singularity cache, we cache manually # # - name: Set friendly Singularity image name # uses: allenevans/set-env@v2.0.0 # with: # SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - - - name: Run unit tests - working-directory: ${{github.workspace}}/build - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) +# +# - name: Configure CMake +# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. +# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type +# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} +## +# - name: Build libcloudph++ +# # Build your program with the given configuration +# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} # - name: Run Singularity shell From f0b1d51b43fb535a41cda758b05a7dd861ffc9d8 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 16:28:05 +0200 Subject: [PATCH 163/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 75f272d27..70265240f 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -130,7 +130,7 @@ jobs: path: ${{ github.workspace }}/singularity_images - name: Install libcloudph++ - run: singularity exec $SI cmake --install build + run: sudo singularity exec $SI cmake --install build - name: Install python3-gnuplot run: | From 33d1b3ad386e9a1996fe5e74cf73b25f2de91b0f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 16:45:31 +0200 Subject: [PATCH 164/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 73 +++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 70265240f..c830bb6e0 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -14,7 +14,7 @@ jobs: # strategy: # matrix: # name: ["CUDA"] -# build_type: ["Debug", "Release"] +# build_type: ["Debug", "RelWithDebInfo"] # include: # - name: "CUDA" # disable_cuda: false @@ -37,7 +37,7 @@ jobs: strategy: matrix: name: ["no_CUDA"] - build_type: ["Debug", "Release"] + build_type: ["Debug", "RelWithDebInfo"] include: - name: "no_CUDA" disable_cuda: true @@ -72,7 +72,7 @@ jobs: # # strategy: # matrix: -# build_type: ["Release", "Debug"] +# build_type: ["RelWithDebInfo", "Debug"] # # steps: # - uses: actions/checkout@v2 @@ -102,9 +102,9 @@ jobs: strategy: matrix: - build_type: ["Release", "Debug"] + build_type: ["RelWithDebInfo", "Debug"] include: - - build_type: "Release" + - build_type: "RelWithDebInfo" long_tests: true debug_tests: false - build_type: "Debug" @@ -130,7 +130,7 @@ jobs: path: ${{ github.workspace }}/singularity_images - name: Install libcloudph++ - run: sudo singularity exec $SI cmake --install build + run: sudo cmake --install build - name: Install python3-gnuplot run: | @@ -156,6 +156,65 @@ jobs: if: ${{matrix.debug_tests}} run: py.test -v unit_test_debug + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/libcloudphxx/models/kinematic_2D + run: singularity exec $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/libcloudphxx/models/kinematic_2D + run: VERBOSE=1 singularity exec $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/libcloudphxx/models/kinematic_2D/build + run: | + ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log + # - name: Install Nvidia driver # if: ${{!matrix.disable_cuda}} # run: sudo apt install --no-install-recommends nvidia-driver-470 @@ -251,7 +310,7 @@ jobs: # version: 1.72.0 # libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built # platform: x64 - # configuration: Release + # configuration: RelWithDebInfo #- name: Install hdf5 (Linux) # run: sudo apt-get install libhdf5-dev From 3d1b5a893495953eb1461beb4472365a4b9ff6eb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 17:02:15 +0200 Subject: [PATCH 165/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 55 ++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index c830bb6e0..27f1d9688 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -135,7 +135,7 @@ jobs: - name: Install python3-gnuplot run: | wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb - sudo dpkg -i python3-gnuplot_1.8-8_all.deb + sudo apt install python3-gnuplot_1.8-8_all.deb - name: checkout parcel repo uses: actions/checkout@v2 @@ -202,19 +202,66 @@ jobs: run: sudo cmake --install build - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/libcloudphxx/models/kinematic_2D + working-directory: ${{github.workspace}}models/kinematic_2D run: singularity exec $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - name: Build kinematic_2D - working-directory: ${{github.workspace}}/libcloudphxx/models/kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D run: VERBOSE=1 singularity exec $SI cmake --build build --config ${{matrix.build_type}} - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/libcloudphxx/models/kinematic_2D/build + working-directory: ${{github.workspace}}/models/kinematic_2D/build run: | ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) cat Testing/Temporary/LastTest.log + build_and_test_KiD-A: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["RelWithDebInfo"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - uses: actions/checkout@v2 + + - name: merge with the KiD-A branch +# working-directory: ${{github.workspace}}/libcloudphxx + run: | + git ls-remote --heads origin + git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch --all + git branch -r + git branch -v -a + git stash + git merge --verbose --progress origin/kida-1d + git stash apply + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout KiD-libcloud repo + uses: actions/checkout@v2 + with: + repository: igfuw/kid-libcloud + + - name: run KiD LWP test + run: \ # TODO + + # - name: Install Nvidia driver # if: ${{!matrix.disable_cuda}} # run: sudo apt install --no-install-recommends nvidia-driver-470 From f85560417ad37766cd064fd1fb5c4caa4a9902af Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 7 Oct 2021 17:14:48 +0200 Subject: [PATCH 166/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 35 +++++++++++++------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 27f1d9688..3b0ed5ae4 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -135,7 +135,7 @@ jobs: - name: Install python3-gnuplot run: | wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb - sudo apt install python3-gnuplot_1.8-8_all.deb + sudo apt install ./python3-gnuplot_1.8-8_all.deb - name: checkout parcel repo uses: actions/checkout@v2 @@ -165,40 +165,40 @@ jobs: build_type: ["RelWithDebInfo"] steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - name: load UWLCM Singularity image #uses: igfuw/libcloudphxx_build@v0.1-beta uses: ./.github/actions/load_UWLCM_singularity_image with: path: ${{ github.workspace }}/singularity_images - - name: Install libcloudph++ - run: sudo cmake --install build - - name: checkout libmpdata++ repo uses: actions/checkout@v2 with: repository: igfuw/libmpdataxx - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + working-directory: ${{github.workspace}}/libmpdata++ run: singularity exec $SI cmake -B build - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + working-directory: ${{github.workspace}}/libmpdata++ run: VERBOSE=1 singularity exec $SI cmake --build build - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + working-directory: ${{github.workspace}}/libmpdata++ + run: sudo cmake --install build + + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ run: sudo cmake --install build - name: Configure kinematic_2D CMake @@ -232,6 +232,7 @@ jobs: - name: merge with the KiD-A branch # working-directory: ${{github.workspace}}/libcloudphxx run: | + git checkout -b tmp_branch # to get rid of the detached HEAD git ls-remote --heads origin git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" git fetch --all From 78b80807977e1093b9da26e682f46d9441d9fcfb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 09:17:38 +0200 Subject: [PATCH 167/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 3b0ed5ae4..eec0dc730 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -141,6 +141,7 @@ jobs: uses: actions/checkout@v2 with: repository: igfuw/parcel + path: parcel - run: mkdir parcel/plots/outputs @@ -175,6 +176,7 @@ jobs: uses: actions/checkout@v2 with: repository: igfuw/libmpdataxx + path: libmpdataxx - name: Configure libmpdata++ CMake working-directory: ${{github.workspace}}/libmpdata++ @@ -232,14 +234,14 @@ jobs: - name: merge with the KiD-A branch # working-directory: ${{github.workspace}}/libcloudphxx run: | - git checkout -b tmp_branch # to get rid of the detached HEAD +# git checkout -b tmp_branch # to get rid of the detached HEAD git ls-remote --heads origin git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" git fetch --all git branch -r git branch -v -a git stash - git merge --verbose --progress origin/kida-1d + git merge --verbose --progress --allow-unrelated-histories origin/kida-1d git stash apply - name: build libcloudph++ @@ -258,6 +260,7 @@ jobs: uses: actions/checkout@v2 with: repository: igfuw/kid-libcloud + path: kid-libcloud - name: run KiD LWP test run: \ # TODO From 5dee25c989ae9f51d10f6701aee1f2b6bcd2e9bf Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 09:24:49 +0200 Subject: [PATCH 168/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index eec0dc730..c378c6e5c 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -233,8 +233,9 @@ jobs: - name: merge with the KiD-A branch # working-directory: ${{github.workspace}}/libcloudphxx - run: | + # git checkout -b tmp_branch # to get rid of the detached HEAD + run: | git ls-remote --heads origin git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" git fetch --all From 12bfb62122ca65f50da0c716feef4dea3faa6f6e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 09:51:00 +0200 Subject: [PATCH 169/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 34 ++++++++++++++------------ 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index c378c6e5c..7559a0f86 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -155,7 +155,7 @@ jobs: - name: run parcel tests debug working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} - run: py.test -v unit_test_debug + run: singularity exec $SI py.test -v unit_test_debug kinematic_2D_test: needs: build @@ -166,6 +166,19 @@ jobs: build_type: ["RelWithDebInfo"] steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + - name: load UWLCM Singularity image #uses: igfuw/libcloudphxx_build@v0.1-beta uses: ./.github/actions/load_UWLCM_singularity_image @@ -190,19 +203,6 @@ jobs: working-directory: ${{github.workspace}}/libmpdata++ run: sudo cmake --install build - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}models/kinematic_2D run: singularity exec $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} @@ -214,8 +214,8 @@ jobs: - name: Run kinematic_2D tests working-directory: ${{github.workspace}}/models/kinematic_2D/build run: | - ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - cat Testing/Temporary/LastTest.log + singularity exec $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + singularity exec $SI cat Testing/Temporary/LastTest.log build_and_test_KiD-A: runs-on: ubuntu-20.04 @@ -236,6 +236,8 @@ jobs: # git checkout -b tmp_branch # to get rid of the detached HEAD run: | + git config user.email "pdziekan@fuw.edu.pl" + git config user.name "Piotr Dziekan" git ls-remote --heads origin git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" git fetch --all From a9e06738e5cf6dffbfe152cf070bbe3b5b133447 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 10:18:01 +0200 Subject: [PATCH 170/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 34 +++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7559a0f86..e3aaaa145 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -149,8 +149,8 @@ jobs: working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} run: | - py.test -v unit_test - py.test -v long_test + singularity exec $SI py.test -v unit_test + singularity exec $SI py.test -v long_test - name: run parcel tests debug working-directory: ${{github.workspace}}/parcel @@ -191,6 +191,8 @@ jobs: repository: igfuw/libmpdataxx path: libmpdataxx + - run: echo ${{ github.workspace }} + - name: Configure libmpdata++ CMake working-directory: ${{github.workspace}}/libmpdata++ run: singularity exec $SI cmake -B build @@ -232,20 +234,24 @@ jobs: - uses: actions/checkout@v2 - name: merge with the KiD-A branch + uses: devmasx/merge-branch@v1.4.0 + with: + type: now + target_branch: kida-1d + github_token: ${{ github.token }} # working-directory: ${{github.workspace}}/libcloudphxx -# git checkout -b tmp_branch # to get rid of the detached HEAD - run: | - git config user.email "pdziekan@fuw.edu.pl" - git config user.name "Piotr Dziekan" - git ls-remote --heads origin - git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --all - git branch -r - git branch -v -a - git stash - git merge --verbose --progress --allow-unrelated-histories origin/kida-1d - git stash apply +# run: | +# git config user.email "pdziekan@fuw.edu.pl" +# git config user.name "Piotr Dziekan" +# git ls-remote --heads origin +# git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" +# git fetch --all +# git branch -r +# git branch -v -a +# git stash +# git merge --verbose --progress --allow-unrelated-histories origin/kida-1d +# git stash apply - name: build libcloudph++ #uses: igfuw/libcloudphxx_build@v0.1-beta From 66e91c4c82fe5d1957d5eb00e729540368f32542 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 10:37:57 +0200 Subject: [PATCH 171/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index e3aaaa145..541f3cb87 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -149,13 +149,13 @@ jobs: working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} run: | - singularity exec $SI py.test -v unit_test - singularity exec $SI py.test -v long_test + singularity exec $SI py.test-3 -v unit_test + singularity exec $SI py.test-3 -v long_test - name: run parcel tests debug working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} - run: singularity exec $SI py.test -v unit_test_debug + run: singularity exec $SI py.test-3 -v unit_test_debug kinematic_2D_test: needs: build @@ -194,19 +194,19 @@ jobs: - run: echo ${{ github.workspace }} - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ run: singularity exec $SI cmake -B build - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ run: VERBOSE=1 singularity exec $SI cmake --build build - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ run: sudo cmake --install build - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}models/kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D run: singularity exec $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - name: Build kinematic_2D @@ -234,7 +234,7 @@ jobs: - uses: actions/checkout@v2 - name: merge with the KiD-A branch - uses: devmasx/merge-branch@v1.4.0 + uses: devmasx/merge-branch@1.4.0 with: type: now target_branch: kida-1d From 55540980478364016920175361e523d69d85dd36 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 10:54:19 +0200 Subject: [PATCH 172/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 541f3cb87..2ac16f50b 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -145,6 +145,8 @@ jobs: - run: mkdir parcel/plots/outputs + - run: singularity exec $SI ls /usr/local/lib/python3/dist-packages/ + - name: run parcel tests long working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} @@ -207,7 +209,7 @@ jobs: - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} + run: singularity exec $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=/usr/local/share/libcloudph++/ - name: Build kinematic_2D working-directory: ${{github.workspace}}/models/kinematic_2D From 1180434a944468a8313a3356517e5aa1db6fd0c5 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 10:54:41 +0200 Subject: [PATCH 173/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 95 +++++++++++--------------- 1 file changed, 41 insertions(+), 54 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 2ac16f50b..0165784f7 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -221,60 +221,47 @@ jobs: singularity exec $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) singularity exec $SI cat Testing/Temporary/LastTest.log - build_and_test_KiD-A: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["RelWithDebInfo"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - uses: actions/checkout@v2 - - - name: merge with the KiD-A branch - uses: devmasx/merge-branch@1.4.0 - with: - type: now - target_branch: kida-1d - github_token: ${{ github.token }} -# working-directory: ${{github.workspace}}/libcloudphxx - -# run: | -# git config user.email "pdziekan@fuw.edu.pl" -# git config user.name "Piotr Dziekan" -# git ls-remote --heads origin -# git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" -# git fetch --all -# git branch -r -# git branch -v -a -# git stash -# git merge --verbose --progress --allow-unrelated-histories origin/kida-1d -# git stash apply - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout KiD-libcloud repo - uses: actions/checkout@v2 - with: - repository: igfuw/kid-libcloud - path: kid-libcloud - - - name: run KiD LWP test - run: \ # TODO +# build_and_test_KiD-A: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["RelWithDebInfo"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: merge with the KiD-A branch +# uses: devmasx/merge-branch@1.4.0 +# with: +# type: now +# target_branch: kida-1d +# github_token: ${{ github.token }} +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout KiD-libcloud repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/kid-libcloud +# path: kid-libcloud +# +# - name: run KiD LWP test +# run: \ # TODO # - name: Install Nvidia driver From fc6a2644b199f8ee973f6057530fa0df145d29a6 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 11:49:26 +0200 Subject: [PATCH 174/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 0165784f7..09e79df3b 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -145,19 +145,19 @@ jobs: - run: mkdir parcel/plots/outputs - - run: singularity exec $SI ls /usr/local/lib/python3/dist-packages/ +# - run: singularity exec $SI ls /usr/local/lib/python3/dist-packages/ - name: run parcel tests long working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} run: | - singularity exec $SI py.test-3 -v unit_test - singularity exec $SI py.test-3 -v long_test + singularity exec $SI -B /usr/local py.test-3 -v unit_test + singularity exec $SI -B /usr/local py.test-3 -v long_test - name: run parcel tests debug working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} - run: singularity exec $SI py.test-3 -v unit_test_debug + run: singularity exec $SI -B /usr/local py.test-3 -v unit_test_debug kinematic_2D_test: needs: build @@ -209,17 +209,17 @@ jobs: - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=/usr/local/share/libcloudph++/ + run: singularity exec $SI -B /usr/local cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - name: Build kinematic_2D working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec $SI cmake --build build --config ${{matrix.build_type}} + run: VERBOSE=1 singularity exec $SI -B /usr/local cmake --build build --config ${{matrix.build_type}} - name: Run kinematic_2D tests working-directory: ${{github.workspace}}/models/kinematic_2D/build run: | - singularity exec $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - singularity exec $SI cat Testing/Temporary/LastTest.log + singularity exec $SI -B /usr/local ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + singularity exec $SI -B /usr/local cat Testing/Temporary/LastTest.log # build_and_test_KiD-A: # runs-on: ubuntu-20.04 From 604437b840fdc1c5dd60fa3d6f3c97a54aa9cc2f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 12:01:02 +0200 Subject: [PATCH 175/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 09e79df3b..9ef01d333 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -151,13 +151,13 @@ jobs: working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} run: | - singularity exec $SI -B /usr/local py.test-3 -v unit_test - singularity exec $SI -B /usr/local py.test-3 -v long_test + singularity exec -B /usr/local $SI py.test-3 -v unit_test + singularity exec -B /usr/local $SI py.test-3 -v long_test - name: run parcel tests debug working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} - run: singularity exec $SI -B /usr/local py.test-3 -v unit_test_debug + run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug kinematic_2D_test: needs: build @@ -209,17 +209,17 @@ jobs: - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec $SI -B /usr/local cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} + run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - name: Build kinematic_2D working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec $SI -B /usr/local cmake --build build --config ${{matrix.build_type}} + run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} - name: Run kinematic_2D tests working-directory: ${{github.workspace}}/models/kinematic_2D/build run: | - singularity exec $SI -B /usr/local ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - singularity exec $SI -B /usr/local cat Testing/Temporary/LastTest.log + singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log # build_and_test_KiD-A: # runs-on: ubuntu-20.04 From ee152305bd23f562959946542c5aa6d856f3b5cf Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 8 Oct 2021 12:30:52 +0200 Subject: [PATCH 176/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 124 ++++++++++++------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 9ef01d333..6008156bb 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -145,7 +145,7 @@ jobs: - run: mkdir parcel/plots/outputs -# - run: singularity exec $SI ls /usr/local/lib/python3/dist-packages/ + - run: singularity exec -B /usr/local $SI ls /usr/local/lib/python3/dist-packages/ - name: run parcel tests long working-directory: ${{github.workspace}}/parcel @@ -159,67 +159,67 @@ jobs: if: ${{matrix.debug_tests}} run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - run: echo ${{ github.workspace }} - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log +# kinematic_2D_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: checkout libmpdata++ repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/libmpdataxx +# path: libmpdataxx +# +# - run: echo ${{ github.workspace }} +# +# - name: Configure libmpdata++ CMake +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: singularity exec $SI cmake -B build +# +# - name: Build libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: VERBOSE=1 singularity exec $SI cmake --build build +# +# - name: Install libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: sudo cmake --install build +# +# - name: Configure kinematic_2D CMake +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} +# +# - name: Build kinematic_2D +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} +# +# - name: Run kinematic_2D tests +# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# run: | +# singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) +# singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log # build_and_test_KiD-A: # runs-on: ubuntu-20.04 From 6172cf18641bda98f4a4fe7a77f59ea9769cf524 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 11:04:40 +0200 Subject: [PATCH 177/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 6008156bb..cbd55be8e 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -102,7 +102,8 @@ jobs: strategy: matrix: - build_type: ["RelWithDebInfo", "Debug"] + # build_type: ["RelWithDebInfo", "Debug"] + build_type: ["Debug"] include: - build_type: "RelWithDebInfo" long_tests: true @@ -157,7 +158,8 @@ jobs: - name: run parcel tests debug working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} - run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug + #run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug + run: singularity exec -B /usr/local $SI python3 -m pytest -v unit_test_debug # kinematic_2D_test: # needs: build From a294ab757316f4972c7484ed9e0a750e9e28de41 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 11:31:54 +0200 Subject: [PATCH 178/282] GA: add actions for testing --- .github/workflows/test_libclouphxx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index cbd55be8e..52513ed9c 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -105,9 +105,9 @@ jobs: # build_type: ["RelWithDebInfo", "Debug"] build_type: ["Debug"] include: - - build_type: "RelWithDebInfo" - long_tests: true - debug_tests: false + # - build_type: "RelWithDebInfo" + # long_tests: true + # debug_tests: false - build_type: "Debug" long_tests: false debug_tests: true From cc6cf86b24ddf0d6ff77c5d7d714898912cd9758 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 14:02:33 +0200 Subject: [PATCH 179/282] GA: add actions for testing --- bindings/python/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index ef399e8e7..2c44fffdd 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -95,7 +95,7 @@ set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH_USE_LINK_PATH TRUE) execute_process( COMMAND "${PYTHON_EXECUTABLE}" -c "if True: from distutils import sysconfig as sc - print(sc.get_python_lib(prefix='', plat_specific=True))" + print(sc.get_python_lib(prefix='', plat_specific=False))" OUTPUT_VARIABLE PYTHON_SITE OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -104,3 +104,5 @@ install ( TARGETS cloudphxx DESTINATION ${PYTHON_SITE} COMPONENT library ) + +#execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install ${cloudphxx}) From ad6c68882572328a04e185e58007543438a3d00c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 14:04:48 +0200 Subject: [PATCH 180/282] GA: add actions for testing --- bindings/python/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 2c44fffdd..9f3068e3d 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -95,9 +95,10 @@ set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH_USE_LINK_PATH TRUE) execute_process( COMMAND "${PYTHON_EXECUTABLE}" -c "if True: from distutils import sysconfig as sc - print(sc.get_python_lib(prefix='', plat_specific=False))" + print((sysconfig.get_paths()['purelib'])" OUTPUT_VARIABLE PYTHON_SITE OUTPUT_STRIP_TRAILING_WHITESPACE) +# print(sc.get_python_lib(prefix='', plat_specific=False))" install ( TARGETS cloudphxx LIBRARY From 1b417a1a6c6d33206b4d311c55f5d55687cdaa3c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 14:45:58 +0200 Subject: [PATCH 181/282] GA: add actions for testing --- bindings/python/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 9f3068e3d..c638c253e 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -95,7 +95,7 @@ set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH_USE_LINK_PATH TRUE) execute_process( COMMAND "${PYTHON_EXECUTABLE}" -c "if True: from distutils import sysconfig as sc - print((sysconfig.get_paths()['purelib'])" + print((sc.get_paths()['purelib'])" OUTPUT_VARIABLE PYTHON_SITE OUTPUT_STRIP_TRAILING_WHITESPACE) # print(sc.get_python_lib(prefix='', plat_specific=False))" From cb06bfc2579e7d66944ba1d5b68d3540adf83181 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 15:48:14 +0200 Subject: [PATCH 182/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 3 ++- bindings/python/CMakeLists.txt | 24 ++++-------------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 52513ed9c..07012c6dc 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -159,7 +159,8 @@ jobs: working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} #run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug - run: singularity exec -B /usr/local $SI python3 -m pytest -v unit_test_debug + run: singularity exec -B/usr/lib/python3/dist-packages:/usr/local/lib/python3.8/dist-packages $SI python3 -m pytest -v unit_test_debug + #libcloudphxx python bindings are installed in /usr/lib/python3/dist-packages, we need them mounted in a sys.path directory within the image, but without shadowing other packages. /usr/local/lib/python3.8/dist-packages is (hopefully) empty within the image # kinematic_2D_test: # needs: build diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index c638c253e..225f2a0c0 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -2,14 +2,8 @@ add_library(cloudphxx SHARED lib.cpp) add_dependencies(cloudphxx git_revision.h) set_target_properties(cloudphxx PROPERTIES SUFFIX ".so") # e.g. Mac defaults to .dylib which is not looked for by Python -# informing the Python bindings where to find Python -find_package(PythonLibs) -if (NOT PYTHON_LIBRARIES) - message(FATAL_ERROR " - Python libraries not found. - Please install them (e.g. sudo apt-get install python-dev). - ") -endif() +find_package(Python REQUIRED) + target_include_directories(cloudphxx PRIVATE ${PYTHON_INCLUDE_DIRS}) target_link_libraries(cloudphxx ${PYTHON_LIBRARIES}) @@ -91,19 +85,9 @@ target_link_libraries(cloudphxx cloudphxx_lgrngn) set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH_USE_LINK_PATH TRUE) -# where to install python modules (see http://stackoverflow.com/questions/1242904/finding-python-site-packages-directory-with-cmake) -execute_process( - COMMAND "${PYTHON_EXECUTABLE}" -c "if True: - from distutils import sysconfig as sc - print((sc.get_paths()['purelib'])" - OUTPUT_VARIABLE PYTHON_SITE - OUTPUT_STRIP_TRAILING_WHITESPACE) -# print(sc.get_python_lib(prefix='', plat_specific=False))" - +# Python_SITELIB is an absolute path, we manually prepend install prefix install ( TARGETS cloudphxx LIBRARY - DESTINATION ${PYTHON_SITE} + DESTINATION "${CMAKE_INSTALL_PREFIX}${Python_SITELIB}" COMPONENT library ) - -#execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install ${cloudphxx}) From 4385aa456f92b6d44308f135d69441d3ab95b801 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 16:04:37 +0200 Subject: [PATCH 183/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 2 +- bindings/python/CMakeLists.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 07012c6dc..4779a594a 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -160,7 +160,7 @@ jobs: if: ${{matrix.debug_tests}} #run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug run: singularity exec -B/usr/lib/python3/dist-packages:/usr/local/lib/python3.8/dist-packages $SI python3 -m pytest -v unit_test_debug - #libcloudphxx python bindings are installed in /usr/lib/python3/dist-packages, we need them mounted in a sys.path directory within the image, but without shadowing other packages. /usr/local/lib/python3.8/dist-packages is (hopefully) empty within the image + #libcloudphxx python bindings are installed in /usr/lib/python3/dist-packages, we need them mounted in a sys.path directory within the image, but without shadowing other packages. /usr/local/lib/python3.8/dist-packages is (hopefully) empty within the image. NOTE: hardcode paths may change in future... # kinematic_2D_test: # needs: build diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 225f2a0c0..b45940e8e 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -2,10 +2,10 @@ add_library(cloudphxx SHARED lib.cpp) add_dependencies(cloudphxx git_revision.h) set_target_properties(cloudphxx PROPERTIES SUFFIX ".so") # e.g. Mac defaults to .dylib which is not looked for by Python -find_package(Python REQUIRED) +find_package(Python3 REQUIRED COMPONENTS Interpreter Development NumPy) -target_include_directories(cloudphxx PRIVATE ${PYTHON_INCLUDE_DIRS}) -target_link_libraries(cloudphxx ${PYTHON_LIBRARIES}) +target_include_directories(cloudphxx PRIVATE ${Python3_INCLUDE_DIRS}) +target_link_libraries(cloudphxx ${Python3_LIBRARIES}) # informing the Python bindings where to find Boost.Python #if(${Boost_MINOR_VERSION} LESS 67) From 6b4f53e2bca94e6465bf9908fe421ea87dc4cf92 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 16:18:55 +0200 Subject: [PATCH 184/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 4779a594a..4d35f6eda 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -146,8 +146,6 @@ jobs: - run: mkdir parcel/plots/outputs - - run: singularity exec -B /usr/local $SI ls /usr/local/lib/python3/dist-packages/ - - name: run parcel tests long working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} From 63a21d5e0a6b6142704cc452f9c83fca88c24a96 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 11 Oct 2021 16:55:50 +0200 Subject: [PATCH 185/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 4 ++-- bindings/python/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 4d35f6eda..703a18896 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -157,8 +157,8 @@ jobs: working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} #run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug - run: singularity exec -B/usr/lib/python3/dist-packages:/usr/local/lib/python3.8/dist-packages $SI python3 -m pytest -v unit_test_debug - #libcloudphxx python bindings are installed in /usr/lib/python3/dist-packages, we need them mounted in a sys.path directory within the image, but without shadowing other packages. /usr/local/lib/python3.8/dist-packages is (hopefully) empty within the image. NOTE: hardcode paths may change in future... + run: singularity exec -B/usr/local/usr/lib/python3/dist-packages:/usr/local/lib/python3.8/dist-packages $SI python3 -m pytest -v unit_test_debug + #libcloudphxx python bindings are installed in /usr/local/usr/lib/python3/dist-packages, we need them mounted in a sys.path directory within the image, but without shadowing other packages. /usr/local/lib/python3.8/dist-packages is (hopefully) empty within the image. NOTE: hardcode paths may change in future... # kinematic_2D_test: # needs: build diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index b45940e8e..327c4ca06 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -88,6 +88,6 @@ set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH_USE_LINK_PATH TRUE) # Python_SITELIB is an absolute path, we manually prepend install prefix install ( TARGETS cloudphxx LIBRARY - DESTINATION "${CMAKE_INSTALL_PREFIX}${Python_SITELIB}" + DESTINATION "${CMAKE_INSTALL_PREFIX}${Python3_SITELIB}" COMPONENT library ) From 630368a849ae8b161d45c1a421e861e1848c3b9c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 10:13:18 +0200 Subject: [PATCH 186/282] GA: use libcloudphxx build action --- .github/actions/libcloudphxx_build/action.yml | 6 +++++- .github/workflows/test_libclouphxx.yml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index ba887ce00..1a14d15d8 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -16,6 +16,10 @@ inputs: path: description: "path to the dir with libcloudphxx code" required: true + install_prefix: + description: "CMake install prefix" + required: true + default: "/usr/local" runs: using: "composite" @@ -72,7 +76,7 @@ runs: shell: bash # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{inputs.path}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} + run: singularity exec $SI cmake -B ${{inputs.path}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} -DCMAKE_INSTALL_PREFIX==${{inputs.install_prefix}} # - name: Build libcloudph++ shell: bash diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 703a18896..7619cb076 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -54,6 +54,7 @@ jobs: build_type: ${{matrix.build_type}} threads: ${{matrix.threads}} path: ${{ github.workspace }} + install_prefix: ${{ github.workspace }}/installed # tar build dir before upload as artifact to retain permission and case-sensitive names - name: Compress libcloudph++ build @@ -157,7 +158,10 @@ jobs: working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} #run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug - run: singularity exec -B/usr/local/usr/lib/python3/dist-packages:/usr/local/lib/python3.8/dist-packages $SI python3 -m pytest -v unit_test_debug +# run: singularity exec -B/usr/local/usr/lib/python3/dist-packages:/usr/local/lib/python3.8/dist-packages $SI python3 -m pytest -v unit_test_debug + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -v unit_test_debug + + #libcloudphxx python bindings are installed in /usr/local/usr/lib/python3/dist-packages, we need them mounted in a sys.path directory within the image, but without shadowing other packages. /usr/local/lib/python3.8/dist-packages is (hopefully) empty within the image. NOTE: hardcode paths may change in future... # kinematic_2D_test: From 2c300061c52b814911fbe754757dce70d661b775 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 10:25:22 +0200 Subject: [PATCH 187/282] GA: use libcloudphxx build action --- .github/actions/libcloudphxx_build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index 1a14d15d8..e1be7e0b6 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -76,7 +76,7 @@ runs: shell: bash # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{inputs.path}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} -DCMAKE_INSTALL_PREFIX==${{inputs.install_prefix}} + run: singularity exec $SI cmake -B ${{inputs.path}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} -DCMAKE_INSTALL_PREFIX=${{inputs.install_prefix}} # - name: Build libcloudph++ shell: bash From f8c587022ca021066ca8877e8ad3c9f713ce9494 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 10:45:43 +0200 Subject: [PATCH 188/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7619cb076..519ee92a7 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -147,6 +147,11 @@ jobs: - run: mkdir parcel/plots/outputs + - run: ls /usr/lib/python3/dist-packages/ + - run: ls /usr/lib/python3/site-packages/ + - run: ls /usr/local/lib/python3/dist-packages/ + - run: ls /usr/local/lib/python3/site-packages/ + - name: run parcel tests long working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} @@ -159,7 +164,7 @@ jobs: if: ${{matrix.debug_tests}} #run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug # run: singularity exec -B/usr/local/usr/lib/python3/dist-packages:/usr/local/lib/python3.8/dist-packages $SI python3 -m pytest -v unit_test_debug - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -v unit_test_debug + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug #libcloudphxx python bindings are installed in /usr/local/usr/lib/python3/dist-packages, we need them mounted in a sys.path directory within the image, but without shadowing other packages. /usr/local/lib/python3.8/dist-packages is (hopefully) empty within the image. NOTE: hardcode paths may change in future... From 860d3ba1d34b7e27f6396de0211aaf54feef8166 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 10:58:30 +0200 Subject: [PATCH 189/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 519ee92a7..2c2a71b8e 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -147,11 +147,6 @@ jobs: - run: mkdir parcel/plots/outputs - - run: ls /usr/lib/python3/dist-packages/ - - run: ls /usr/lib/python3/site-packages/ - - run: ls /usr/local/lib/python3/dist-packages/ - - run: ls /usr/local/lib/python3/site-packages/ - - name: run parcel tests long working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} From d89c0fd77b02357fe905914799dc4e6c01e04195 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 11:25:39 +0200 Subject: [PATCH 190/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 323 ++++++++++++------------- 1 file changed, 159 insertions(+), 164 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 2c2a71b8e..4692d2bcc 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -9,28 +9,28 @@ on: #env: jobs: -# build_CUDA: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "CUDA" -# disable_cuda: false -# # not enough RAM to compile Debug CUDA on 4 threads -# threads: 1 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} + build_CUDA: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["CUDA"] + build_type: ["Debug", "RelWithDebInfo"] + include: + - name: "CUDA" + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads + threads: 1 + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} build: runs-on: ubuntu-20.04 @@ -67,35 +67,35 @@ jobs: path: build.tar # test jobs -# unit_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Run unit tests -# working-directory: ${{github.workspace}}/build -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + unit_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Run unit tests + working-directory: ${{github.workspace}}/build + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) parcel_test: needs: build @@ -103,12 +103,11 @@ jobs: strategy: matrix: - # build_type: ["RelWithDebInfo", "Debug"] - build_type: ["Debug"] + build_type: ["RelWithDebInfo", "Debug"] include: - # - build_type: "RelWithDebInfo" - # long_tests: true - # debug_tests: false + - build_type: "RelWithDebInfo" + long_tests: true + debug_tests: false - build_type: "Debug" long_tests: false debug_tests: true @@ -151,122 +150,118 @@ jobs: working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} run: | - singularity exec -B /usr/local $SI py.test-3 -v unit_test - singularity exec -B /usr/local $SI py.test-3 -v long_test + PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test + PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test - name: run parcel tests debug working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} - #run: singularity exec -B /usr/local $SI py.test-3 -v unit_test_debug -# run: singularity exec -B/usr/local/usr/lib/python3/dist-packages:/usr/local/lib/python3.8/dist-packages $SI python3 -m pytest -v unit_test_debug run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug - #libcloudphxx python bindings are installed in /usr/local/usr/lib/python3/dist-packages, we need them mounted in a sys.path directory within the image, but without shadowing other packages. /usr/local/lib/python3.8/dist-packages is (hopefully) empty within the image. NOTE: hardcode paths may change in future... + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 -# kinematic_2D_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: checkout libmpdata++ repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/libmpdataxx -# path: libmpdataxx -# -# - run: echo ${{ github.workspace }} -# -# - name: Configure libmpdata++ CMake -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: singularity exec $SI cmake -B build -# -# - name: Build libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: VERBOSE=1 singularity exec $SI cmake --build build -# -# - name: Install libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: sudo cmake --install build -# -# - name: Configure kinematic_2D CMake -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -# -# - name: Build kinematic_2D -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} -# -# - name: Run kinematic_2D tests -# working-directory: ${{github.workspace}}/models/kinematic_2D/build -# run: | -# singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) -# singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log - -# build_and_test_KiD-A: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["RelWithDebInfo"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: merge with the KiD-A branch -# uses: devmasx/merge-branch@1.4.0 -# with: -# type: now -# target_branch: kida-1d -# github_token: ${{ github.token }} -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout KiD-libcloud repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/kid-libcloud -# path: kid-libcloud -# -# - name: run KiD LWP test -# run: \ # TODO + strategy: + matrix: + build_type: ["RelWithDebInfo"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - run: echo ${{ github.workspace }} + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log + + build_and_test_KiD-A: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["RelWithDebInfo"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - uses: actions/checkout@v2 + + - name: merge with the KiD-A branch + uses: devmasx/merge-branch@1.4.0 + with: + type: now + target_branch: kida-1d + github_token: ${{ github.token }} + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout KiD-libcloud repo + uses: actions/checkout@v2 + with: + repository: igfuw/kid-libcloud + path: kid-libcloud + + - name: run KiD LWP test + run: \ # TODO # - name: Install Nvidia driver From 68baa89787907eb30561ea75a421c7f247de385e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 11:38:45 +0200 Subject: [PATCH 191/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 651 +++++++++++++------------ 1 file changed, 326 insertions(+), 325 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 4692d2bcc..5bab13782 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -24,224 +24,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - - build: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - uses: actions/checkout@v2 - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - install_prefix: ${{ github.workspace }}/installed - - # tar build dir before upload as artifact to retain permission and case-sensitive names - - name: Compress libcloudph++ build - run: tar -cvf build.tar build - - - name: Upload libcloudph++ build - uses: actions/upload-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - path: build.tar - - # test jobs - unit_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Run unit tests - working-directory: ${{github.workspace}}/build - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - - parcel_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - include: - - build_type: "RelWithDebInfo" - long_tests: true - debug_tests: false - - build_type: "Debug" - long_tests: false - debug_tests: true - - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: Install python3-gnuplot - run: | - wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb - sudo apt install ./python3-gnuplot_1.8-8_all.deb - - - name: checkout parcel repo - uses: actions/checkout@v2 - with: - repository: igfuw/parcel - path: parcel - - - run: mkdir parcel/plots/outputs - - - name: run parcel tests long - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: | - PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test - - - name: run parcel tests debug - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.debug_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug - - - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - run: echo ${{ github.workspace }} - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log - - build_and_test_KiD-A: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["RelWithDebInfo"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - uses: actions/checkout@v2 - - - name: merge with the KiD-A branch - uses: devmasx/merge-branch@1.4.0 - with: - type: now - target_branch: kida-1d - github_token: ${{ github.token }} - - name: build libcloudph++ #uses: igfuw/libcloudphxx_build@v0.1-beta uses: ./.github/actions/libcloudphxx_build @@ -251,117 +33,336 @@ jobs: threads: ${{matrix.threads}} path: ${{ github.workspace }} - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout KiD-libcloud repo - uses: actions/checkout@v2 - with: - repository: igfuw/kid-libcloud - path: kid-libcloud - - - name: run KiD LWP test - run: \ # TODO - - -# - name: Install Nvidia driver -# if: ${{!matrix.disable_cuda}} -# run: sudo apt install --no-install-recommends nvidia-driver-470 -# -# - name: Install Singularity -# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) -# uses: eWaterCycle/setup-singularity@v6 +# build: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["Debug", "RelWithDebInfo"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build # with: -# singularity-version: 3.7.1 -# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 -## run: | -## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -## sudo apt-get update -## sudo apt-get install -y singularity-container -# -# - name: Cache UWLCM Singularity image -# id: cache_singularity -# uses: actions/cache@v2 +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# install_prefix: ${{ github.workspace }}/installed +# +# # tar build dir before upload as artifact to retain permission and case-sensitive names +# - name: Compress libcloudph++ build +# run: tar -cvf build.tar build +# +# - name: Upload libcloudph++ build +# uses: actions/upload-artifact@v2 # with: -# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# key: 'sng_ubuntu_20_04_cuda_11_4' -# -# - name: Download UWLCM Singularity image -# if: steps.cache_singularity.outputs.cache-hit != 'true' +# name: libcloud_build_${{matrix.build_type}}_tar +# path: build.tar +# +# # test jobs +# unit_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Run unit tests +# working-directory: ${{github.workspace}}/build +# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail +# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) +# +# parcel_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# include: +# - build_type: "RelWithDebInfo" +# long_tests: true +# debug_tests: false +# - build_type: "Debug" +# long_tests: false +# debug_tests: true +# +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: Install python3-gnuplot # run: | -# mkdir '${{ github.workspace }}/singularity_images' -# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 -# # disable Singularity cache, we cache manually +# wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb +# sudo apt install ./python3-gnuplot_1.8-8_all.deb # -# - name: Set friendly Singularity image name -# uses: allenevans/set-env@v2.0.0 +# - name: checkout parcel repo +# uses: actions/checkout@v2 # with: -# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# -# - name: Configure CMake -# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. -# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type -# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} -## -# - name: Build libcloudph++ -# # Build your program with the given configuration -# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} - - - # - name: Run Singularity shell - # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' - -# - name: Download UWLCM Singularity container -# run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 - -# - name: Build UWLCM Singularity container -# run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 - - # TODO: cache cuda-toolki - - #- name: Install cuda-toolkit - # id: cuda-toolkit - # uses: Jimver/cuda-toolkit@v0.2.4 - # with: - # linux-local-args: '["--toolkit"]' - - # - name: Install boost - # uses: MarkusJx/install-boost@v2.0.0 - # id: install-boost - # with: - # # REQUIRED: Specify the required boost version - # # A list of supported versions can be found here: - # # https://github.com/actions/boost-versions/blob/main/versions-manifest.json - # boost_version: 1.73.0 - # # OPTIONAL: Specify a platform version - # #platform_version: 18.04 - # # OPTIONAL: Specify a custom install location - # #boost_install_dir: /home/runner/some_directory - -# - name: Install Thrust -# #run: sudo apt-get install libthrust-dev +# repository: igfuw/parcel +# path: parcel +# +# - run: mkdir parcel/plots/outputs +# +# - name: run parcel tests long +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} # run: | -# git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 -# sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust - -# - name: Cache Boost -# uses: actions/cache@v2 +# PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test +# PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test +# +# - name: run parcel tests debug +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.debug_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug +# +# +# kinematic_2D_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 # with: -# path: '${{ runner.workspace }}/boost_*.tar.gz' -# key: 'boost-1.72.0' - - # - name: Build Boost - # id: boost - # # This won't re-download the archive unnecessarily: - # uses: egor-tensin/build-boost@v1 - # with: - # version: 1.72.0 - # libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built - # platform: x64 - # configuration: RelWithDebInfo - - #- name: Install hdf5 (Linux) - # run: sudo apt-get install libhdf5-dev - # #if: matrix.os == 'ubuntu-latest' - +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: checkout libmpdata++ repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/libmpdataxx +# path: libmpdataxx +# +# - run: echo ${{ github.workspace }} +# +# - name: Configure libmpdata++ CMake +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: singularity exec $SI cmake -B build +# +# - name: Build libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: VERBOSE=1 singularity exec $SI cmake --build build +# +# - name: Install libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: sudo cmake --install build +# +# - name: Configure kinematic_2D CMake +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} +# +# - name: Build kinematic_2D +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} +# +# - name: Run kinematic_2D tests +# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# run: | +# singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) +# singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log +# +# build_and_test_KiD-A: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["RelWithDebInfo"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: merge with the KiD-A branch +# uses: devmasx/merge-branch@1.4.0 +# with: +# type: now +# target_branch: kida-1d +# github_token: ${{ github.token }} +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout KiD-libcloud repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/kid-libcloud +# path: kid-libcloud +# +# - name: run KiD LWP test +# run: \ # TODO +# +# +## - name: Install Nvidia driver +## if: ${{!matrix.disable_cuda}} +## run: sudo apt install --no-install-recommends nvidia-driver-470 +## +## - name: Install Singularity +## #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) +## uses: eWaterCycle/setup-singularity@v6 +## with: +## singularity-version: 3.7.1 +## #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +### run: | +### wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +### sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +### sudo apt-get update +### sudo apt-get install -y singularity-container +## +## - name: Cache UWLCM Singularity image +## id: cache_singularity +## uses: actions/cache@v2 +## with: +## path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +## key: 'sng_ubuntu_20_04_cuda_11_4' +## +## - name: Download UWLCM Singularity image +## if: steps.cache_singularity.outputs.cache-hit != 'true' +## run: | +## mkdir '${{ github.workspace }}/singularity_images' +## singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 +## # disable Singularity cache, we cache manually +## +## - name: Set friendly Singularity image name +## uses: allenevans/set-env@v2.0.0 +## with: +## SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +## +## - name: Configure CMake +## # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. +## # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type +## run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} +### +## - name: Build libcloudph++ +## # Build your program with the given configuration +## run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} +# +# +# # - name: Run Singularity shell +# # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' +# +## - name: Download UWLCM Singularity container +## run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 +# +## - name: Build UWLCM Singularity container +## run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 +# +# # TODO: cache cuda-toolki +# +# #- name: Install cuda-toolkit +# # id: cuda-toolkit +# # uses: Jimver/cuda-toolkit@v0.2.4 +# # with: +# # linux-local-args: '["--toolkit"]' +# +# # - name: Install boost +# # uses: MarkusJx/install-boost@v2.0.0 +# # id: install-boost +# # with: +# # # REQUIRED: Specify the required boost version +# # # A list of supported versions can be found here: +# # # https://github.com/actions/boost-versions/blob/main/versions-manifest.json +# # boost_version: 1.73.0 +# # # OPTIONAL: Specify a platform version +# # #platform_version: 18.04 +# # # OPTIONAL: Specify a custom install location +# # #boost_install_dir: /home/runner/some_directory +# +## - name: Install Thrust +## #run: sudo apt-get install libthrust-dev +## run: | +## git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 +## sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust +# +## - name: Cache Boost +## uses: actions/cache@v2 +## with: +## path: '${{ runner.workspace }}/boost_*.tar.gz' +## key: 'boost-1.72.0' +# +# # - name: Build Boost +# # id: boost +# # # This won't re-download the archive unnecessarily: +# # uses: egor-tensin/build-boost@v1 +# # with: +# # version: 1.72.0 +# # libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built +# # platform: x64 +# # configuration: RelWithDebInfo +# +# #- name: Install hdf5 (Linux) +# # run: sudo apt-get install libhdf5-dev +# # #if: matrix.os == 'ubuntu-latest' +# From 169d63c63f82355cb23b15aa8dbfaebea25d2cb3 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 12:02:03 +0200 Subject: [PATCH 192/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 667 +++++++++++++------------ 1 file changed, 335 insertions(+), 332 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 5bab13782..16dd72cba 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -9,17 +9,40 @@ on: #env: jobs: - build_CUDA: +# build_CUDA: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["CUDA"] +# build_type: ["Debug", "RelWithDebInfo"] +# include: +# - name: "CUDA" +# disable_cuda: false +# # not enough RAM to compile Debug CUDA on 4 threads +# threads: 1 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} + + build: runs-on: ubuntu-20.04 strategy: matrix: - name: ["CUDA"] + name: ["no_CUDA"] build_type: ["Debug", "RelWithDebInfo"] include: - - name: "CUDA" - disable_cuda: false - # not enough RAM to compile Debug CUDA on 4 threads - threads: 1 + - name: "no_CUDA" + disable_cuda: true + threads: 4 steps: - uses: actions/checkout@v2 @@ -32,337 +55,317 @@ jobs: build_type: ${{matrix.build_type}} threads: ${{matrix.threads}} path: ${{ github.workspace }} + install_prefix: ${{ github.workspace }}/installed -# build: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# install_prefix: ${{ github.workspace }}/installed -# -# # tar build dir before upload as artifact to retain permission and case-sensitive names -# - name: Compress libcloudph++ build -# run: tar -cvf build.tar build -# -# - name: Upload libcloudph++ build -# uses: actions/upload-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# path: build.tar -# -# # test jobs -# unit_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Run unit tests -# working-directory: ${{github.workspace}}/build -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) -# -# parcel_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# include: -# - build_type: "RelWithDebInfo" -# long_tests: true -# debug_tests: false -# - build_type: "Debug" -# long_tests: false -# debug_tests: true -# -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image + # tar build dir before upload as artifact to retain permission and case-sensitive names + - name: Compress libcloudph++ build + run: tar -cvf build.tar build + + - name: Upload libcloudph++ build + uses: actions/upload-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + path: build.tar + + # test jobs + unit_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Run unit tests + working-directory: ${{github.workspace}}/build + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + + parcel_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + include: + - build_type: "RelWithDebInfo" + long_tests: true + debug_tests: false + - build_type: "Debug" + long_tests: false + debug_tests: true + + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: Install python3-gnuplot + run: | + wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb + sudo apt install ./python3-gnuplot_1.8-8_all.deb + + - name: checkout parcel repo + uses: actions/checkout@v2 + with: + repository: igfuw/parcel + path: parcel + + - run: mkdir parcel/plots/outputs + + - name: run parcel unit_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test + + - name: run parcel long_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test + + - name: run parcel unit_test_debug + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.debug_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug + + + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - run: echo ${{ github.workspace }} + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log + + build_and_test_KiD-A: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["RelWithDebInfo"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - uses: actions/checkout@v2 + + - name: merge with the KiD-A branch + uses: devmasx/merge-branch@1.4.0 + with: + type: now + target_branch: kida-1d + github_token: ${{ github.token }} + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout KiD-libcloud repo + uses: actions/checkout@v2 + with: + repository: igfuw/kid-libcloud + path: kid-libcloud + + - name: run KiD LWP test + run: \ # TODO + + +# - name: Install Nvidia driver +# if: ${{!matrix.disable_cuda}} +# run: sudo apt install --no-install-recommends nvidia-driver-470 +# +# - name: Install Singularity +# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) +# uses: eWaterCycle/setup-singularity@v6 # with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: Install python3-gnuplot -# run: | -# wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb -# sudo apt install ./python3-gnuplot_1.8-8_all.deb -# -# - name: checkout parcel repo -# uses: actions/checkout@v2 +# singularity-version: 3.7.1 +# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +## run: | +## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +## sudo apt-get update +## sudo apt-get install -y singularity-container +# +# - name: Cache UWLCM Singularity image +# id: cache_singularity +# uses: actions/cache@v2 # with: -# repository: igfuw/parcel -# path: parcel -# -# - run: mkdir parcel/plots/outputs -# -# - name: run parcel tests long -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} +# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# key: 'sng_ubuntu_20_04_cuda_11_4' +# +# - name: Download UWLCM Singularity image +# if: steps.cache_singularity.outputs.cache-hit != 'true' # run: | -# PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test -# PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test -# -# - name: run parcel tests debug -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.debug_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug -# -# -# kinematic_2D_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo"] +# mkdir '${{ github.workspace }}/singularity_images' +# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 +# # disable Singularity cache, we cache manually # -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image +# - name: Set friendly Singularity image name +# uses: allenevans/set-env@v2.0.0 # with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: checkout libmpdata++ repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/libmpdataxx -# path: libmpdataxx -# -# - run: echo ${{ github.workspace }} -# -# - name: Configure libmpdata++ CMake -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: singularity exec $SI cmake -B build -# -# - name: Build libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: VERBOSE=1 singularity exec $SI cmake --build build -# -# - name: Install libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: sudo cmake --install build -# -# - name: Configure kinematic_2D CMake -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -# -# - name: Build kinematic_2D -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} -# -# - name: Run kinematic_2D tests -# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# +# - name: Configure CMake +# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. +# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type +# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} +## +# - name: Build libcloudph++ +# # Build your program with the given configuration +# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} + + + # - name: Run Singularity shell + # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' + +# - name: Download UWLCM Singularity container +# run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 + +# - name: Build UWLCM Singularity container +# run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 + + # TODO: cache cuda-toolki + + #- name: Install cuda-toolkit + # id: cuda-toolkit + # uses: Jimver/cuda-toolkit@v0.2.4 + # with: + # linux-local-args: '["--toolkit"]' + + # - name: Install boost + # uses: MarkusJx/install-boost@v2.0.0 + # id: install-boost + # with: + # # REQUIRED: Specify the required boost version + # # A list of supported versions can be found here: + # # https://github.com/actions/boost-versions/blob/main/versions-manifest.json + # boost_version: 1.73.0 + # # OPTIONAL: Specify a platform version + # #platform_version: 18.04 + # # OPTIONAL: Specify a custom install location + # #boost_install_dir: /home/runner/some_directory + +# - name: Install Thrust +# #run: sudo apt-get install libthrust-dev # run: | -# singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) -# singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log -# -# build_and_test_KiD-A: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["RelWithDebInfo"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: merge with the KiD-A branch -# uses: devmasx/merge-branch@1.4.0 +# git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 +# sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust + +# - name: Cache Boost +# uses: actions/cache@v2 # with: -# type: now -# target_branch: kida-1d -# github_token: ${{ github.token }} -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout KiD-libcloud repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/kid-libcloud -# path: kid-libcloud -# -# - name: run KiD LWP test -# run: \ # TODO -# -# -## - name: Install Nvidia driver -## if: ${{!matrix.disable_cuda}} -## run: sudo apt install --no-install-recommends nvidia-driver-470 -## -## - name: Install Singularity -## #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) -## uses: eWaterCycle/setup-singularity@v6 -## with: -## singularity-version: 3.7.1 -## #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 -### run: | -### wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -### sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -### sudo apt-get update -### sudo apt-get install -y singularity-container -## -## - name: Cache UWLCM Singularity image -## id: cache_singularity -## uses: actions/cache@v2 -## with: -## path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -## key: 'sng_ubuntu_20_04_cuda_11_4' -## -## - name: Download UWLCM Singularity image -## if: steps.cache_singularity.outputs.cache-hit != 'true' -## run: | -## mkdir '${{ github.workspace }}/singularity_images' -## singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 -## # disable Singularity cache, we cache manually -## -## - name: Set friendly Singularity image name -## uses: allenevans/set-env@v2.0.0 -## with: -## SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -## -## - name: Configure CMake -## # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. -## # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type -## run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} -### -## - name: Build libcloudph++ -## # Build your program with the given configuration -## run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} -# -# -# # - name: Run Singularity shell -# # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' -# -## - name: Download UWLCM Singularity container -## run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 -# -## - name: Build UWLCM Singularity container -## run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 -# -# # TODO: cache cuda-toolki -# -# #- name: Install cuda-toolkit -# # id: cuda-toolkit -# # uses: Jimver/cuda-toolkit@v0.2.4 -# # with: -# # linux-local-args: '["--toolkit"]' -# -# # - name: Install boost -# # uses: MarkusJx/install-boost@v2.0.0 -# # id: install-boost -# # with: -# # # REQUIRED: Specify the required boost version -# # # A list of supported versions can be found here: -# # # https://github.com/actions/boost-versions/blob/main/versions-manifest.json -# # boost_version: 1.73.0 -# # # OPTIONAL: Specify a platform version -# # #platform_version: 18.04 -# # # OPTIONAL: Specify a custom install location -# # #boost_install_dir: /home/runner/some_directory -# -## - name: Install Thrust -## #run: sudo apt-get install libthrust-dev -## run: | -## git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 -## sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust -# -## - name: Cache Boost -## uses: actions/cache@v2 -## with: -## path: '${{ runner.workspace }}/boost_*.tar.gz' -## key: 'boost-1.72.0' -# -# # - name: Build Boost -# # id: boost -# # # This won't re-download the archive unnecessarily: -# # uses: egor-tensin/build-boost@v1 -# # with: -# # version: 1.72.0 -# # libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built -# # platform: x64 -# # configuration: RelWithDebInfo -# -# #- name: Install hdf5 (Linux) -# # run: sudo apt-get install libhdf5-dev -# # #if: matrix.os == 'ubuntu-latest' -# +# path: '${{ runner.workspace }}/boost_*.tar.gz' +# key: 'boost-1.72.0' + + # - name: Build Boost + # id: boost + # # This won't re-download the archive unnecessarily: + # uses: egor-tensin/build-boost@v1 + # with: + # version: 1.72.0 + # libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built + # platform: x64 + # configuration: RelWithDebInfo + + #- name: Install hdf5 (Linux) + # run: sudo apt-get install libhdf5-dev + # #if: matrix.os == 'ubuntu-latest' + From acf3ce8eaebeaa5a567e9eb906a181c151581bdb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 12:25:15 +0200 Subject: [PATCH 193/282] Init action --- .github/backup/test_libclouphxx.yml | 101 ++++++++++++++ .github/workflows/test_libclouphxx.yml | 179 +++++-------------------- 2 files changed, 138 insertions(+), 142 deletions(-) create mode 100644 .github/backup/test_libclouphxx.yml diff --git a/.github/backup/test_libclouphxx.yml b/.github/backup/test_libclouphxx.yml new file mode 100644 index 000000000..f533ef025 --- /dev/null +++ b/.github/backup/test_libclouphxx.yml @@ -0,0 +1,101 @@ +# - name: Install Nvidia driver +# if: ${{!matrix.disable_cuda}} +# run: sudo apt install --no-install-recommends nvidia-driver-470 +# +# - name: Install Singularity +# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) +# uses: eWaterCycle/setup-singularity@v6 +# with: +# singularity-version: 3.7.1 +# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 +## run: | +## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list +## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 +## sudo apt-get update +## sudo apt-get install -y singularity-container +# +# - name: Cache UWLCM Singularity image +# id: cache_singularity +# uses: actions/cache@v2 +# with: +# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# key: 'sng_ubuntu_20_04_cuda_11_4' +# +# - name: Download UWLCM Singularity image +# if: steps.cache_singularity.outputs.cache-hit != 'true' +# run: | +# mkdir '${{ github.workspace }}/singularity_images' +# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 +# # disable Singularity cache, we cache manually +# +# - name: Set friendly Singularity image name +# uses: allenevans/set-env@v2.0.0 +# with: +# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' +# +# - name: Configure CMake +# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. +# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type +# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} +## +# - name: Build libcloudph++ +# # Build your program with the given configuration +# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} + + + # - name: Run Singularity shell + # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' + +# - name: Download UWLCM Singularity container +# run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 + +# - name: Build UWLCM Singularity container +# run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 + + # TODO: cache cuda-toolki + + #- name: Install cuda-toolkit + # id: cuda-toolkit + # uses: Jimver/cuda-toolkit@v0.2.4 + # with: + # linux-local-args: '["--toolkit"]' + + # - name: Install boost + # uses: MarkusJx/install-boost@v2.0.0 + # id: install-boost + # with: + # # REQUIRED: Specify the required boost version + # # A list of supported versions can be found here: + # # https://github.com/actions/boost-versions/blob/main/versions-manifest.json + # boost_version: 1.73.0 + # # OPTIONAL: Specify a platform version + # #platform_version: 18.04 + # # OPTIONAL: Specify a custom install location + # #boost_install_dir: /home/runner/some_directory + +# - name: Install Thrust +# #run: sudo apt-get install libthrust-dev +# run: | +# git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 +# sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust + +# - name: Cache Boost +# uses: actions/cache@v2 +# with: +# path: '${{ runner.workspace }}/boost_*.tar.gz' +# key: 'boost-1.72.0' + + # - name: Build Boost + # id: boost + # # This won't re-download the archive unnecessarily: + # uses: egor-tensin/build-boost@v1 + # with: + # version: 1.72.0 + # libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built + # platform: x64 + # configuration: RelWithDebInfo + + #- name: Install hdf5 (Linux) + # run: sudo apt-get install libhdf5-dev + # #if: matrix.os == 'ubuntu-latest' + diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 16dd72cba..bd5912299 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -6,8 +6,6 @@ on: pull_request: branches: [ master ] -#env: - jobs: # build_CUDA: # runs-on: ubuntu-20.04 @@ -225,147 +223,44 @@ jobs: singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log - build_and_test_KiD-A: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["RelWithDebInfo"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - uses: actions/checkout@v2 - - - name: merge with the KiD-A branch - uses: devmasx/merge-branch@1.4.0 - with: - type: now - target_branch: kida-1d - github_token: ${{ github.token }} - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout KiD-libcloud repo - uses: actions/checkout@v2 - with: - repository: igfuw/kid-libcloud - path: kid-libcloud - - - name: run KiD LWP test - run: \ # TODO - - -# - name: Install Nvidia driver -# if: ${{!matrix.disable_cuda}} -# run: sudo apt install --no-install-recommends nvidia-driver-470 -# -# - name: Install Singularity -# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) -# uses: eWaterCycle/setup-singularity@v6 -# with: -# singularity-version: 3.7.1 -# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 -## run: | -## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -## sudo apt-get update -## sudo apt-get install -y singularity-container -# -# - name: Cache UWLCM Singularity image -# id: cache_singularity -# uses: actions/cache@v2 +# build_and_test_KiD-A: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["RelWithDebInfo"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: merge with the KiD-A branch +# uses: devmasx/merge-branch@1.4.0 # with: -# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# key: 'sng_ubuntu_20_04_cuda_11_4' -# -# - name: Download UWLCM Singularity image -# if: steps.cache_singularity.outputs.cache-hit != 'true' -# run: | -# mkdir '${{ github.workspace }}/singularity_images' -# singularity pull --disable_cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4 -# # disable Singularity cache, we cache manually +# type: now +# target_branch: kida-1d +# github_token: ${{ github.token }} # -# - name: Set friendly Singularity image name -# uses: allenevans/set-env@v2.0.0 +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build # with: -# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# -# - name: Configure CMake -# # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. -# # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type -# run: singularity exec $SI cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{matrix.disable_cuda}} -## -# - name: Build libcloudph++ -# # Build your program with the given configuration -# run: VERBOSE=1 singularity exec $SI cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j${{matrix.threads}} - - - # - name: Run Singularity shell - # run: singularity shell '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_1_1.0.sif' - -# - name: Download UWLCM Singularity container -# run: wget https://github.com/igfuw/UWLCM/blob/0fc77ec68053936e36eea4b49f11b3dd2cb1a827/singularity/sng_ubuntu_20_04_cuda_11_1 - -# - name: Build UWLCM Singularity container -# run: singularity build sng_ubuntu_20_04_cuda_11_1.sif sng_ubuntu_20_04_cuda_11_1 - - # TODO: cache cuda-toolki - - #- name: Install cuda-toolkit - # id: cuda-toolkit - # uses: Jimver/cuda-toolkit@v0.2.4 - # with: - # linux-local-args: '["--toolkit"]' - - # - name: Install boost - # uses: MarkusJx/install-boost@v2.0.0 - # id: install-boost - # with: - # # REQUIRED: Specify the required boost version - # # A list of supported versions can be found here: - # # https://github.com/actions/boost-versions/blob/main/versions-manifest.json - # boost_version: 1.73.0 - # # OPTIONAL: Specify a platform version - # #platform_version: 18.04 - # # OPTIONAL: Specify a custom install location - # #boost_install_dir: /home/runner/some_directory - -# - name: Install Thrust -# #run: sudo apt-get install libthrust-dev -# run: | -# git clone --depth=1 git://github.com/thrust/thrust.git --branch 1.9.10-1 -# sudo ln -s `pwd`/thrust/thrust /usr/local/include/thrust - -# - name: Cache Boost -# uses: actions/cache@v2 +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout KiD-libcloud repo +# uses: actions/checkout@v2 # with: -# path: '${{ runner.workspace }}/boost_*.tar.gz' -# key: 'boost-1.72.0' - - # - name: Build Boost - # id: boost - # # This won't re-download the archive unnecessarily: - # uses: egor-tensin/build-boost@v1 - # with: - # version: 1.72.0 - # libraries: date_time # in fact we don't need any compiled libs, only header libs, but when nothing is passed all libs are built - # platform: x64 - # configuration: RelWithDebInfo - - #- name: Install hdf5 (Linux) - # run: sudo apt-get install libhdf5-dev - # #if: matrix.os == 'ubuntu-latest' - +# repository: igfuw/kid-libcloud +# path: kid-libcloud +# +# - name: run KiD LWP test +# run: \ # TODO From bb199c4aef0cf7ff5ea30250a82c0ef3e3d03c19 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 12:28:03 +0200 Subject: [PATCH 194/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 83 +++++++++++++------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index bd5912299..4e7651c37 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -223,44 +223,45 @@ jobs: singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log -# build_and_test_KiD-A: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["RelWithDebInfo"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: merge with the KiD-A branch -# uses: devmasx/merge-branch@1.4.0 -# with: -# type: now -# target_branch: kida-1d -# github_token: ${{ github.token }} -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout KiD-libcloud repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/kid-libcloud -# path: kid-libcloud -# -# - name: run KiD LWP test -# run: \ # TODO + build_and_test_KiD-A: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["RelWithDebInfo"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - uses: actions/checkout@v2 + + - name: merge with the KiD-A branch + uses: devmasx/merge-branch@1.4.0 + with: + type: now + target_branch: kida-1d + github_token: ${{ github.token }} + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout KiD-libcloud repo + uses: actions/checkout@v2 + with: + repository: igfuw/kid-libcloud + path: kid-libcloud + + - name: run KiD LWP test + working-directory: ${{github.workspace}}/kid-libcloud + run: . ./.travis_scripts/lwp_test.sh From 1c3cb7cea56a37efd8b4cc49cab9bd9ffa451535 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 13:04:56 +0200 Subject: [PATCH 195/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 4e7651c37..7de7dbb2c 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -148,7 +148,8 @@ jobs: - name: run parcel unit_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI py.test-3 --verbosity=100 unit_test + #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - name: run parcel long_test working-directory: ${{github.workspace}}/parcel @@ -211,17 +212,17 @@ jobs: - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B /usr/local $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ + run: singularity exec -B /usr/local,${{ github.workspace }}/installed/share/libcloudph++ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ - name: Build kinematic_2D working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B /usr/local $SI cmake --build build --config ${{matrix.build_type}} + run: VERBOSE=1 singularity exec -B /usr/local,${{ github.workspace }}/installed/share/libcloudph++ $SI cmake --build build --config ${{matrix.build_type}} - name: Run kinematic_2D tests working-directory: ${{github.workspace}}/models/kinematic_2D/build run: | - singularity exec -B /usr/local $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - singularity exec -B /usr/local $SI cat Testing/Temporary/LastTest.log + singularity exec -B /usr/local,${{ github.workspace }}/installed/share/libcloudph++ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log build_and_test_KiD-A: runs-on: ubuntu-20.04 @@ -264,4 +265,6 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud - run: . ./.travis_scripts/lwp_test.sh + run: singularity exec $SI . ./.travis_scripts/lwp_test.sh +# sudo apt install nco + From c0eb968f917be02c8fba902008f3d7c01465306e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 13:24:28 +0200 Subject: [PATCH 196/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7de7dbb2c..332a2f154 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -148,7 +148,7 @@ jobs: - name: run parcel unit_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI py.test-3 --verbosity=100 unit_test + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI py.test-3 -s --verbosity=100 unit_test #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - name: run parcel long_test @@ -265,6 +265,6 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud - run: singularity exec $SI . ./.travis_scripts/lwp_test.sh + run: singularity exec $SI bash ./.travis_scripts/lwp_test.sh # sudo apt install nco From 7845209c94165dd6c7e06f4fec818a49fd6e7350 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 14:50:26 +0200 Subject: [PATCH 197/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 332a2f154..517fb129a 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -265,6 +265,6 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud - run: singularity exec $SI bash ./.travis_scripts/lwp_test.sh + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh # sudo apt install nco From af86f20a0de7d76657681909407052c3ef805077 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 15:20:17 +0200 Subject: [PATCH 198/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 517fb129a..9b5885139 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -196,8 +196,6 @@ jobs: repository: igfuw/libmpdataxx path: libmpdataxx - - run: echo ${{ github.workspace }} - - name: Configure libmpdata++ CMake working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ run: singularity exec $SI cmake -B build @@ -212,16 +210,16 @@ jobs: - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B /usr/local,${{ github.workspace }}/installed/share/libcloudph++ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ + run: singularity exec -B ${{ github.workspace }}/installed/share/libcloudph++ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ - name: Build kinematic_2D working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B /usr/local,${{ github.workspace }}/installed/share/libcloudph++ $SI cmake --build build --config ${{matrix.build_type}} + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/share/libcloudph++ $SI cmake --build build --config ${{matrix.build_type}} - name: Run kinematic_2D tests working-directory: ${{github.workspace}}/models/kinematic_2D/build run: | - singularity exec -B /usr/local,${{ github.workspace }}/installed/share/libcloudph++ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + singularity exec -B ${{ github.workspace }}/installed/share/libcloudph++ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) cat Testing/Temporary/LastTest.log build_and_test_KiD-A: @@ -253,6 +251,7 @@ jobs: build_type: ${{matrix.build_type}} threads: ${{matrix.threads}} path: ${{ github.workspace }} + install_prefix: ${{ github.workspace }}/installed - name: Install libcloudph++ run: sudo cmake --install build From a3e07159ea65384a17ef17cfe34e852688cbf1a3 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 16:25:21 +0200 Subject: [PATCH 199/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 9b5885139..59c960a51 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -198,7 +198,7 @@ jobs: - name: Configure libmpdata++ CMake working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build + run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - name: Build libmpdata++ working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ @@ -210,16 +210,17 @@ jobs: - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B ${{ github.workspace }}/installed/share/libcloudph++ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ + run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ + - name: Build kinematic_2D working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/share/libcloudph++ $SI cmake --build build --config ${{matrix.build_type}} + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} - name: Run kinematic_2D tests working-directory: ${{github.workspace}}/models/kinematic_2D/build run: | - singularity exec -B ${{ github.workspace }}/installed/share/libcloudph++ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) cat Testing/Temporary/LastTest.log build_and_test_KiD-A: @@ -264,6 +265,6 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI "sudo apt install nco; bash ./.travis_scripts/lwp_test.sh" # sudo apt install nco From d94d2f59a9ee59c9ac619d2b0de69878dfa1e404 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 12 Oct 2021 17:01:25 +0200 Subject: [PATCH 200/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 59c960a51..3a68f4dda 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -265,6 +265,6 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI "sudo apt install nco; bash ./.travis_scripts/lwp_test.sh" + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash -c "sudo apt install nco; bash ./.travis_scripts/lwp_test.sh" # sudo apt install nco From 8b78282220b5e0b68120f2d3dfc9f40693185c6e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 13 Oct 2021 11:14:59 +0200 Subject: [PATCH 201/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 388 ++++++++++++------------- 1 file changed, 194 insertions(+), 194 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 3a68f4dda..1928cdaa2 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -31,199 +31,199 @@ jobs: # threads: ${{matrix.threads}} # path: ${{ github.workspace }} - build: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - uses: actions/checkout@v2 - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - install_prefix: ${{ github.workspace }}/installed - - # tar build dir before upload as artifact to retain permission and case-sensitive names - - name: Compress libcloudph++ build - run: tar -cvf build.tar build - - - name: Upload libcloudph++ build - uses: actions/upload-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - path: build.tar - - # test jobs - unit_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Run unit tests - working-directory: ${{github.workspace}}/build - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - - parcel_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - include: - - build_type: "RelWithDebInfo" - long_tests: true - debug_tests: false - - build_type: "Debug" - long_tests: false - debug_tests: true - - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: Install python3-gnuplot - run: | - wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb - sudo apt install ./python3-gnuplot_1.8-8_all.deb - - - name: checkout parcel repo - uses: actions/checkout@v2 - with: - repository: igfuw/parcel - path: parcel - - - run: mkdir parcel/plots/outputs - - - name: run parcel unit_test - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI py.test-3 -s --verbosity=100 unit_test - #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - - - name: run parcel long_test - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test - - - name: run parcel unit_test_debug - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.debug_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug - - - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ - - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - cat Testing/Temporary/LastTest.log - - build_and_test_KiD-A: +# build: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["Debug", "RelWithDebInfo"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# install_prefix: ${{ github.workspace }}/installed +# +# # tar build dir before upload as artifact to retain permission and case-sensitive names +# - name: Compress libcloudph++ build +# run: tar -cvf build.tar build +# +# - name: Upload libcloudph++ build +# uses: actions/upload-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# path: build.tar +# +# # test jobs +# unit_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Run unit tests +# working-directory: ${{github.workspace}}/build +# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail +# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) +# +# parcel_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# include: +# - build_type: "RelWithDebInfo" +# long_tests: true +# debug_tests: false +# - build_type: "Debug" +# long_tests: false +# debug_tests: true +# +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: Install python3-gnuplot +# run: | +# wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb +# sudo apt install ./python3-gnuplot_1.8-8_all.deb +# +# - name: checkout parcel repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/parcel +# path: parcel +# +# - run: mkdir parcel/plots/outputs +# +# - name: run parcel unit_test +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI py.test-3 -s --verbosity=100 unit_test +# #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test +# +# - name: run parcel long_test +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test +# +# - name: run parcel unit_test_debug +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.debug_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug +# +# +# kinematic_2D_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: checkout libmpdata++ repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/libmpdataxx +# path: libmpdataxx +# +# - name: Configure libmpdata++ CMake +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed +# +# - name: Build libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: VERBOSE=1 singularity exec $SI cmake --build build +# +# - name: Install libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: sudo cmake --install build +# +# - name: Configure kinematic_2D CMake +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ +# +# +# - name: Build kinematic_2D +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} +# +# - name: Run kinematic_2D tests +# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# run: | +# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) +# cat Testing/Temporary/LastTest.log +# +# build_and_test_KiD-A: runs-on: ubuntu-20.04 strategy: matrix: @@ -265,6 +265,6 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash -c "sudo apt install nco; bash ./.travis_scripts/lwp_test.sh" + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages sudo -H singularity exec -B${{ github.workspace }}/installed $SI bash -c "sudo apt install nco; bash ./.travis_scripts/lwp_test.sh" # sudo apt install nco From 09aaf1475b76361cb237d68d310b9f62229cc46d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 13 Oct 2021 11:26:55 +0200 Subject: [PATCH 202/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 1928cdaa2..c1329b40b 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -223,7 +223,7 @@ jobs: # singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) # cat Testing/Temporary/LastTest.log # -# build_and_test_KiD-A: + build_and_test_KiD-A: runs-on: ubuntu-20.04 strategy: matrix: From 1382313c594a566f54da047cc83d2906484448f4 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 13 Oct 2021 11:46:01 +0200 Subject: [PATCH 203/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index c1329b40b..fe3c748a9 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -263,8 +263,10 @@ jobs: repository: igfuw/kid-libcloud path: kid-libcloud + - name: Install nco + run: sudo apt install nco + - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages sudo -H singularity exec -B${{ github.workspace }}/installed $SI bash -c "sudo apt install nco; bash ./.travis_scripts/lwp_test.sh" -# sudo apt install nco + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh From 481e4c1421932c8dc08560140cf8085e28ca5cd2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 13 Oct 2021 16:41:23 +0200 Subject: [PATCH 204/282] GA: use libcloudphxx build action --- .../load_UWLCM_singularity_image/action.yml | 8 +- .github/workflows/test_libclouphxx.yml | 430 +++++++++--------- 2 files changed, 215 insertions(+), 223 deletions(-) diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml index f71d913b3..11adfa0b4 100644 --- a/.github/actions/load_UWLCM_singularity_image/action.yml +++ b/.github/actions/load_UWLCM_singularity_image/action.yml @@ -25,8 +25,8 @@ runs: id: cache_singularity uses: actions/cache@v2 with: - path: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4.sif' - key: 'sng_ubuntu_20_04_cuda_11_4' + path: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v2.sif' + key: 'sng_ubuntu_20_04_cuda_11_4_v2' - name: Download UWLCM Singularity image shell: bash @@ -35,7 +35,7 @@ runs: [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ { mkdir '${{ inputs.path }}'; - singularity pull --disable-cache --dir '${{ inputs.path }}' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; + singularity pull --disable-cache --dir '${{ inputs.path }}' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4_v2; } \ || \ true @@ -44,4 +44,4 @@ runs: - name: Set friendly Singularity image name uses: allenevans/set-env@v2.0.0 with: - SI: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4.sif' + SI: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v2.sif' diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index fe3c748a9..9ad360066 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,222 +7,217 @@ on: branches: [ master ] jobs: -# build_CUDA: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "CUDA" -# disable_cuda: false -# # not enough RAM to compile Debug CUDA on 4 threads -# threads: 1 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} - -# build: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# install_prefix: ${{ github.workspace }}/installed -# -# # tar build dir before upload as artifact to retain permission and case-sensitive names -# - name: Compress libcloudph++ build -# run: tar -cvf build.tar build -# -# - name: Upload libcloudph++ build -# uses: actions/upload-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# path: build.tar -# -# # test jobs -# unit_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Run unit tests -# working-directory: ${{github.workspace}}/build -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) -# -# parcel_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# include: -# - build_type: "RelWithDebInfo" -# long_tests: true -# debug_tests: false -# - build_type: "Debug" -# long_tests: false -# debug_tests: true -# -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: Install python3-gnuplot -# run: | -# wget http://ftp.pl.debian.org/debian/pool/main/p/python-gnuplot/python3-gnuplot_1.8-8_all.deb -# sudo apt install ./python3-gnuplot_1.8-8_all.deb -# -# - name: checkout parcel repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/parcel -# path: parcel -# -# - run: mkdir parcel/plots/outputs -# -# - name: run parcel unit_test -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI py.test-3 -s --verbosity=100 unit_test -# #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test -# -# - name: run parcel long_test -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test -# -# - name: run parcel unit_test_debug -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.debug_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug -# -# -# kinematic_2D_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: checkout libmpdata++ repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/libmpdataxx -# path: libmpdataxx -# -# - name: Configure libmpdata++ CMake -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed -# -# - name: Build libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: VERBOSE=1 singularity exec $SI cmake --build build -# -# - name: Install libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: sudo cmake --install build -# -# - name: Configure kinematic_2D CMake -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ -# -# -# - name: Build kinematic_2D -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} -# -# - name: Run kinematic_2D tests -# working-directory: ${{github.workspace}}/models/kinematic_2D/build -# run: | -# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) -# cat Testing/Temporary/LastTest.log -# + build_CUDA: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["CUDA"] + build_type: ["Debug", "RelWithDebInfo"] + include: + - name: "CUDA" + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads + threads: 1 + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + + build: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["Debug", "RelWithDebInfo"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + install_prefix: ${{ github.workspace }}/installed + + # tar build dir before upload as artifact to retain permission and case-sensitive names + - name: Compress libcloudph++ build + run: tar -cvf build.tar build + + - name: Upload libcloudph++ build + uses: actions/upload-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + path: build.tar + + # test jobs + unit_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Run unit tests + working-directory: ${{github.workspace}}/build + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + + parcel_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + include: + - build_type: "RelWithDebInfo" + long_tests: true + debug_tests: false + - build_type: "Debug" + long_tests: false + debug_tests: true + + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout parcel repo + uses: actions/checkout@v2 + with: + repository: igfuw/parcel + path: parcel + + - run: mkdir parcel/plots/outputs + + - name: run parcel unit_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI py.test-3 -s --verbosity=100 unit_test + #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test + + - name: run parcel long_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test + + - name: run parcel unit_test_debug + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.debug_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug + + + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ + + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log + build_and_test_KiD-A: runs-on: ubuntu-20.04 strategy: @@ -263,9 +258,6 @@ jobs: repository: igfuw/kid-libcloud path: kid-libcloud - - name: Install nco - run: sudo apt install nco - - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh From 800b49a8b9388e4da5d02c3b39b16ca7f93fe052 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 13 Oct 2021 16:57:30 +0200 Subject: [PATCH 205/282] GA: use libcloudphxx build action --- .github/workflows/test_libclouphxx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 9ad360066..035829afc 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -143,18 +143,18 @@ jobs: - name: run parcel unit_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI py.test-3 -s --verbosity=100 unit_test + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - name: run parcel long_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v long_test + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test - name: run parcel unit_test_debug working-directory: ${{github.workspace}}/parcel if: ${{matrix.debug_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test_debug + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug kinematic_2D_test: From 84e7871654697841c9ad37a5a7d771099be94f28 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 19 Oct 2021 11:52:16 +0200 Subject: [PATCH 206/282] odeint in chem: never_resizer -> initially_resizer + test this in CMake --- CMakeLists.txt | 4 +++- src/impl/particles_impl.ipp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a9c99224..7cb4ea8da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -317,6 +317,7 @@ file(WRITE "${tmpdir}/test.cpp" " const float /* t */ ) { + assert(psi.size() == dot_psi.size()); } }; @@ -328,7 +329,8 @@ file(WRITE "${tmpdir}/test.cpp" " thrust::cpp::vector, // deriv_type float, // time_type boost::numeric::odeint::thrust_algebra, - boost::numeric::odeint::thrust_operations + boost::numeric::odeint::thrust_operations, + boost::numeric::odeint::initially_resizer > chem_stepper; thrust::cpp::vector v(2); diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index d03ccca57..690a6813b 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -200,7 +200,7 @@ namespace libcloudphxx real_t, // time_type boost::numeric::odeint::thrust_algebra, boost::numeric::odeint::thrust_operations, - boost::numeric::odeint::never_resizer + boost::numeric::odeint::initially_resizer > chem_stepper; // temporary data From 18396ab6875933c6ab91b45735c7b23dcb5aa89d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 19 Oct 2021 13:28:03 +0200 Subject: [PATCH 207/282] odeint in chem: always_resizer --- CMakeLists.txt | 2 +- src/impl/particles_impl.ipp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7cb4ea8da..6f5548ab8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -330,7 +330,7 @@ file(WRITE "${tmpdir}/test.cpp" " float, // time_type boost::numeric::odeint::thrust_algebra, boost::numeric::odeint::thrust_operations, - boost::numeric::odeint::initially_resizer + boost::numeric::odeint::always_resizer > chem_stepper; thrust::cpp::vector v(2); diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 690a6813b..2d6611a7f 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -200,7 +200,7 @@ namespace libcloudphxx real_t, // time_type boost::numeric::odeint::thrust_algebra, boost::numeric::odeint::thrust_operations, - boost::numeric::odeint::initially_resizer + boost::numeric::odeint::always_resizer > chem_stepper; // temporary data From 039bec3d6f1405eaca1baefe2255f8c944c0b9a4 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 19 Oct 2021 15:27:47 +0200 Subject: [PATCH 208/282] in build, link python bindings to built libcloud --- bindings/python/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 327c4ca06..b61a21a60 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -81,7 +81,12 @@ endif() target_link_libraries(cloudphxx cloudphxx_lgrngn) -#to retain rpath to libcloudphxx_lgrngn.so linked by libcloudphxx.so after installation +# build rpath to libcloudphxx_lgrngn.so in the build directory +set_property(TARGET cloudphxx PROPERTY BUILD_RPATH_USE_ORIGIN TRUE) +set_property(TARGET cloudphxx PROPERTY BUILD_RPATH "${ORIGIN}/") +target_link_options(cloudphxx PRIVATE "-Wl,--disable-new-dtags") # otherwise BUILD_RPATH sets RUNPATH instead of RPATH, and LD_LIBRARY_PATH has precedence over RUNPATH + +#install rpath to libcloudphxx_lgrngn.so linked by libcloudphxx.so after installation set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH_USE_LINK_PATH TRUE) From 0b20a9f464fad355d45fe6a7cf22b2103aff4341 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 20 Oct 2021 11:34:55 +0200 Subject: [PATCH 209/282] sanity check: ccn relaxation is not compatible with chemistry --- src/impl/particles_impl_init_sanity_check.ipp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/impl/particles_impl_init_sanity_check.ipp b/src/impl/particles_impl_init_sanity_check.ipp index 30216f68a..5f2316a29 100644 --- a/src/impl/particles_impl_init_sanity_check.ipp +++ b/src/impl/particles_impl_init_sanity_check.ipp @@ -138,7 +138,9 @@ namespace libcloudphxx if(opts_init.rlx_switch && opts_init.rlx_sd_per_bin <= 0) throw std::runtime_error("rlx_sd_per_bin <= 0"); if(opts_init.rlx_switch && opts_init.rlx_timescale <= 0) - throw std::runtime_error("rlx_timescale <= 0"); + throw std::runtime_error("rlx_timescale <= 0"); + if(opts_init.rlx_switch && opts_init.chem_switch) + throw std::runtime_error("CCN relaxation does not work with chemistry"); } }; }; From 7bac911bc377ee349ab1a78c69f8b1f32d9babc3 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 20 Oct 2021 13:57:27 +0200 Subject: [PATCH 210/282] GA: use singu image v3 --- .../actions/load_UWLCM_singularity_image/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml index 11adfa0b4..01ba99306 100644 --- a/.github/actions/load_UWLCM_singularity_image/action.yml +++ b/.github/actions/load_UWLCM_singularity_image/action.yml @@ -23,10 +23,10 @@ runs: - name: Cache UWLCM Singularity image id: cache_singularity - uses: actions/cache@v2 + uses: actions/cache@v3 with: - path: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v2.sif' - key: 'sng_ubuntu_20_04_cuda_11_4_v2' + path: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v3.sif' + key: 'sng_ubuntu_20_04_cuda_11_4_v3' - name: Download UWLCM Singularity image shell: bash @@ -35,7 +35,7 @@ runs: [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ { mkdir '${{ inputs.path }}'; - singularity pull --disable-cache --dir '${{ inputs.path }}' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4_v2; + singularity pull --disable-cache --dir '${{ inputs.path }}' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4_v3; } \ || \ true @@ -44,4 +44,4 @@ runs: - name: Set friendly Singularity image name uses: allenevans/set-env@v2.0.0 with: - SI: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v2.sif' + SI: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v3.sif' From c2777f7024b8456ebaac13b4ee11138f463f60b7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 20 Oct 2021 14:03:47 +0200 Subject: [PATCH 211/282] chem odeint fix --- CMakeLists.txt | 7 +-- bindings/python/CMakeLists.txt | 2 +- src/impl/particles_impl.ipp | 5 +-- src/impl/particles_impl_init_chem.ipp | 44 ++++++++++--------- src/impl/particles_impl_init_sanity_check.ipp | 4 ++ 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f5548ab8..03bd4634c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -305,9 +305,7 @@ file(WRITE "${tmpdir}/test.cpp" " #include #include - #include - #include - #include + #include struct rhs { @@ -329,8 +327,7 @@ file(WRITE "${tmpdir}/test.cpp" " thrust::cpp::vector, // deriv_type float, // time_type boost::numeric::odeint::thrust_algebra, - boost::numeric::odeint::thrust_operations, - boost::numeric::odeint::always_resizer + boost::numeric::odeint::thrust_operations > chem_stepper; thrust::cpp::vector v(2); diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index b61a21a60..039533ae1 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -84,7 +84,7 @@ target_link_libraries(cloudphxx cloudphxx_lgrngn) # build rpath to libcloudphxx_lgrngn.so in the build directory set_property(TARGET cloudphxx PROPERTY BUILD_RPATH_USE_ORIGIN TRUE) set_property(TARGET cloudphxx PROPERTY BUILD_RPATH "${ORIGIN}/") -target_link_options(cloudphxx PRIVATE "-Wl,--disable-new-dtags") # otherwise BUILD_RPATH sets RUNPATH instead of RPATH, and LD_LIBRARY_PATH has precedence over RUNPATH +target_link_options(cloudphxx PRIVATE "-Wl,--disable-new-dtags") # otherwise BUILD_RPATH sets RUNPATH instead of RPATH, and LD_LIBRARY_PATH has precedence over RUNPATH. TODO: when installing, it is probably better to set RUNPATH... #install rpath to libcloudphxx_lgrngn.so linked by libcloudphxx.so after installation set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 2d6611a7f..4eab706a9 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -11,8 +11,7 @@ #include #include -#include -#include +#include #include #include @@ -200,7 +199,7 @@ namespace libcloudphxx real_t, // time_type boost::numeric::odeint::thrust_algebra, boost::numeric::odeint::thrust_operations, - boost::numeric::odeint::always_resizer + boost::numeric::odeint::never_resizer > chem_stepper; // temporary data diff --git a/src/impl/particles_impl_init_chem.ipp b/src/impl/particles_impl_init_chem.ipp index b5cabf27d..ec2b8e1ee 100644 --- a/src/impl/particles_impl_init_chem.ipp +++ b/src/impl/particles_impl_init_chem.ipp @@ -41,8 +41,8 @@ namespace libcloudphxx chem_rho(chem_rho) {} - BOOST_GPU_ENABLED - real_t operator()(const real_t &rd3) const + BOOST_GPU_ENABLED + real_t operator()(const real_t &rd3) const { using namespace common::molar_mass; @@ -55,7 +55,7 @@ namespace libcloudphxx #endif * chem_rho * rd3 * (M_NH3_H2O() / (M_NH4() + M_HSO4())); - } + } }; /* template @@ -65,8 +65,8 @@ namespace libcloudphxx chem_init_SO4(const real_t &chem_rho) : chem_rho(chem_rho) {} - BOOST_GPU_ENABLED - real_t operator()(const real_t &rd3) const + BOOST_GPU_ENABLED + real_t operator()(const real_t &rd3) const { using namespace common::molar_mass; @@ -79,7 +79,7 @@ namespace libcloudphxx #endif * chem_rho * rd3 * (M_SO4() / (M_NH4() + M_SO4())); - } + } }; */ template @@ -90,8 +90,8 @@ namespace libcloudphxx chem_init_S6(const real_t &chem_rho) : chem_rho(chem_rho) {} - BOOST_GPU_ENABLED - real_t operator()(const real_t &rd3) const + BOOST_GPU_ENABLED + real_t operator()(const real_t &rd3) const { using namespace common::molar_mass; @@ -104,7 +104,7 @@ namespace libcloudphxx #endif * chem_rho * rd3 * (M_H2SO4() / (M_NH4() + M_HSO4())); - } + } }; template @@ -114,8 +114,8 @@ namespace libcloudphxx chem_init_H(const real_t &chem_rho) : chem_rho(chem_rho) {} - BOOST_GPU_ENABLED - real_t operator()(const real_t &rd3) const + BOOST_GPU_ENABLED + real_t operator()(const real_t &rd3) const { using namespace common::molar_mass; @@ -128,10 +128,12 @@ namespace libcloudphxx #endif * chem_rho * rd3 * (M_H() / (M_NH4() + M_HSO4())); - } + } }; }; + // NOTE: init_chem does not work for SD created during the simulation, hence chemistry is not compatible with distributed memory (multi_CUDA or MPI), src nor ccn relaxation + // however, it works with removing SD in remove_n0 (?) template void particles_t::impl::init_chem() { @@ -153,8 +155,8 @@ namespace libcloudphxx i < chem_rhs_beg ? chem_ante_rhs : i < chem_rhs_fin - ? chem_rhs - : chem_post_rhs + ? chem_rhs + : chem_post_rhs ); const int offset = i < chem_rhs_beg @@ -170,6 +172,7 @@ namespace libcloudphxx assert(chem_end[chem_all-1] == chem_post_rhs.end()); } + // NOTE: valid only at init, not afterwards when SD are created e.g. from src, ccn_relax or distmem copies template void particles_t::impl::init_chem_aq() { @@ -183,34 +186,33 @@ namespace libcloudphxx using namespace common::molar_mass; switch (i) { - case NH3: { - thrust::transform( + thrust::transform( rd3.begin(), rd3.end(), // input chem_bgn[i], // output detail::chem_init_NH4(opts_init.chem_rho) - ); + ); } break; case H: { - thrust::transform( + thrust::transform( rd3.begin(), rd3.end(), // input chem_bgn[i], // output detail::chem_init_H(opts_init.chem_rho) - ); + ); } break; case S_VI: { - thrust::transform( + thrust::transform( rd3.begin(), rd3.end(), // input chem_bgn[i], // output detail::chem_init_S6(opts_init.chem_rho) - ); + ); } break; diff --git a/src/impl/particles_impl_init_sanity_check.ipp b/src/impl/particles_impl_init_sanity_check.ipp index 40a68b371..e094a8940 100644 --- a/src/impl/particles_impl_init_sanity_check.ipp +++ b/src/impl/particles_impl_init_sanity_check.ipp @@ -127,6 +127,10 @@ namespace libcloudphxx throw std::runtime_error("at least one of opts_init.turb_adve_switch, opts_init.turb_cond_switch is true, but SGS mixing length profile size != nz"); if(opts_init.SGS_mix_len.size() > 0 && *std::min(opts_init.SGS_mix_len.begin(), opts_init.SGS_mix_len.end()) <= 0) throw std::runtime_error("SGS_mix_len <= 0"); + #if defined(USE_MPI) + if(opts_init.chem_switch) + throw std::runtime_error("chemistry is not compatible with MPI"); + #endif } }; }; From 83f9431a894239ceba6065e0524c6521b693b374 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 20 Oct 2021 14:04:54 +0200 Subject: [PATCH 212/282] GA: fix --- .github/actions/load_UWLCM_singularity_image/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml index 01ba99306..e4471680f 100644 --- a/.github/actions/load_UWLCM_singularity_image/action.yml +++ b/.github/actions/load_UWLCM_singularity_image/action.yml @@ -23,7 +23,7 @@ runs: - name: Cache UWLCM Singularity image id: cache_singularity - uses: actions/cache@v3 + uses: actions/cache@v2 with: path: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v3.sif' key: 'sng_ubuntu_20_04_cuda_11_4_v3' From c59918c7a0be5733ac22db67ab38ebccb97377fe Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 13:55:36 +0200 Subject: [PATCH 213/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 424 +++++++++++++------------ 1 file changed, 214 insertions(+), 210 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 035829afc..043187f0b 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,216 +7,216 @@ on: branches: [ master ] jobs: - build_CUDA: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - include: - - name: "CUDA" - disable_cuda: false - # not enough RAM to compile Debug CUDA on 4 threads - threads: 1 - - steps: - - uses: actions/checkout@v2 - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - - build: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - uses: actions/checkout@v2 - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - install_prefix: ${{ github.workspace }}/installed - - # tar build dir before upload as artifact to retain permission and case-sensitive names - - name: Compress libcloudph++ build - run: tar -cvf build.tar build - - - name: Upload libcloudph++ build - uses: actions/upload-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - path: build.tar - - # test jobs - unit_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Run unit tests - working-directory: ${{github.workspace}}/build - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - - parcel_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - include: - - build_type: "RelWithDebInfo" - long_tests: true - debug_tests: false - - build_type: "Debug" - long_tests: false - debug_tests: true - - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout parcel repo - uses: actions/checkout@v2 - with: - repository: igfuw/parcel - path: parcel - - - run: mkdir parcel/plots/outputs - - - name: run parcel unit_test - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test - #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - - - name: run parcel long_test - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test - - - name: run parcel unit_test_debug - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.debug_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug - - - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ - - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - cat Testing/Temporary/LastTest.log +# build_CUDA: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["CUDA"] +# build_type: ["Debug", "RelWithDebInfo"] +# include: +# - name: "CUDA" +# disable_cuda: false +# # not enough RAM to compile Debug CUDA on 4 threads +# threads: 1 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# +# build: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["Debug", "RelWithDebInfo"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# install_prefix: ${{ github.workspace }}/installed +# +# # tar build dir before upload as artifact to retain permission and case-sensitive names +# - name: Compress libcloudph++ build +# run: tar -cvf build.tar build +# +# - name: Upload libcloudph++ build +# uses: actions/upload-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# path: build.tar +# +# # test jobs +# unit_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Run unit tests +# working-directory: ${{github.workspace}}/build +# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail +# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) +# +# parcel_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# include: +# - build_type: "RelWithDebInfo" +# long_tests: true +# debug_tests: false +# - build_type: "Debug" +# long_tests: false +# debug_tests: true +# +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout parcel repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/parcel +# path: parcel +# +# - run: mkdir parcel/plots/outputs +# +# - name: run parcel unit_test +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test +# #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test +# +# - name: run parcel long_test +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test +# +# - name: run parcel unit_test_debug +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.debug_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug +# +# +# kinematic_2D_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: checkout libmpdata++ repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/libmpdataxx +# path: libmpdataxx +# +# - name: Configure libmpdata++ CMake +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed +# +# - name: Build libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: VERBOSE=1 singularity exec $SI cmake --build build +# +# - name: Install libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: sudo cmake --install build +# +# - name: Configure kinematic_2D CMake +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ +# +# +# - name: Build kinematic_2D +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} +# +# - name: Run kinematic_2D tests +# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# run: | +# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) +# cat Testing/Temporary/LastTest.log build_and_test_KiD-A: runs-on: ubuntu-20.04 @@ -239,6 +239,10 @@ jobs: target_branch: kida-1d github_token: ${{ github.token }} + - run: pwd + - run: git status + - run: git log + - name: build libcloudph++ #uses: igfuw/libcloudphxx_build@v0.1-beta uses: ./.github/actions/libcloudphxx_build From 19ae5eb2ae0df8b489b9507c286fc5cb45b60d5f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 14:49:03 +0200 Subject: [PATCH 214/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 043187f0b..3a0f49c53 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -262,6 +262,10 @@ jobs: repository: igfuw/kid-libcloud path: kid-libcloud + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh From 19daaf847ae66680612a71a4ce2338dad79f3220 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 15:34:39 +0200 Subject: [PATCH 215/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 3a0f49c53..f24c72a2d 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -233,15 +233,16 @@ jobs: - uses: actions/checkout@v2 - name: merge with the KiD-A branch - uses: devmasx/merge-branch@1.4.0 - with: - type: now - target_branch: kida-1d - github_token: ${{ github.token }} + run: | + git fetch origin + git merge origin/kida-1d +# uses: devmasx/merge-branch@1.4.0 +# with: +# type: now +# target_branch: kida-1d +# github_token: ${{ github.token }} - - run: pwd - - run: git status - - run: git log + - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp - name: build libcloudph++ #uses: igfuw/libcloudphxx_build@v0.1-beta @@ -262,9 +263,9 @@ jobs: repository: igfuw/kid-libcloud path: kid-libcloud - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 +# Debugging ssh session +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud From d1849916f056b7cfa498dce538525ba3e8eabb0a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 15:36:19 +0200 Subject: [PATCH 216/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index f24c72a2d..3a1eb0400 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -235,7 +235,7 @@ jobs: - name: merge with the KiD-A branch run: | git fetch origin - git merge origin/kida-1d + git merge --allow-unrelated-histories origin/kida-1d # uses: devmasx/merge-branch@1.4.0 # with: # type: now From e5590ac0887503d138df992886333c712a491fb4 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 15:39:54 +0200 Subject: [PATCH 217/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 3a1eb0400..c72c55887 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -235,7 +235,7 @@ jobs: - name: merge with the KiD-A branch run: | git fetch origin - git merge --allow-unrelated-histories origin/kida-1d + git merge --verbose --progress --allow-unrelated-histories origin/kida-1d # uses: devmasx/merge-branch@1.4.0 # with: # type: now From efa11bee1e5d52bcd3efcd883d6061147ab45f19 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 15:44:34 +0200 Subject: [PATCH 218/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index c72c55887..ff602fbcf 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -234,6 +234,8 @@ jobs: - name: merge with the KiD-A branch run: | + git config --global user.email "pdziekan@fuw.edu.pl" + git config --global user.name "Piotr Dziekan" git fetch origin git merge --verbose --progress --allow-unrelated-histories origin/kida-1d # uses: devmasx/merge-branch@1.4.0 From 86d7bbd813be7f521c33e3fd6c6104b70eaaa753 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 15:48:17 +0200 Subject: [PATCH 219/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index ff602fbcf..35c63e317 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -236,8 +236,14 @@ jobs: run: | git config --global user.email "pdziekan@fuw.edu.pl" git config --global user.name "Piotr Dziekan" - git fetch origin - git merge --verbose --progress --allow-unrelated-histories origin/kida-1d + git ls-remote --heads origin + git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch --all + git branch -r + git branch -v -a + git stash + git merge --verbose --progress origin/kida-1d + git stash apply # uses: devmasx/merge-branch@1.4.0 # with: # type: now From 533386db24992de62f3bb02a4b3738c86c8849de Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 15:50:18 +0200 Subject: [PATCH 220/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 35c63e317..623482510 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -242,7 +242,7 @@ jobs: git branch -r git branch -v -a git stash - git merge --verbose --progress origin/kida-1d + git merge --verbose --progress --allow-unrelated-histories origin/kida-1d git stash apply # uses: devmasx/merge-branch@1.4.0 # with: From 6a21622696587b5ed70228352b010786f7f67b98 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Oct 2021 15:55:12 +0200 Subject: [PATCH 221/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 623482510..f258b15ec 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -232,6 +232,10 @@ jobs: steps: - uses: actions/checkout@v2 +# Debugging ssh session + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: merge with the KiD-A branch run: | git config --global user.email "pdziekan@fuw.edu.pl" From af751c1c6fa01ab44a398c5dedceb6445f419dd9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 22 Oct 2021 13:03:54 +0200 Subject: [PATCH 222/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index f258b15ec..6af42568a 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -233,21 +233,25 @@ jobs: - uses: actions/checkout@v2 # Debugging ssh session - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 - name: merge with the KiD-A branch run: | - git config --global user.email "pdziekan@fuw.edu.pl" - git config --global user.name "Piotr Dziekan" - git ls-remote --heads origin - git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" git fetch --all - git branch -r - git branch -v -a - git stash - git merge --verbose --progress --allow-unrelated-histories origin/kida-1d - git stash apply + git merge origin/kida-1d + +# git config --global user.email "pdziekan@fuw.edu.pl" +# git config --global user.name "Piotr Dziekan" +# git ls-remote --heads origin +# git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" +# git fetch --all +# git branch -r +# git branch -v -a +# git stash +# git merge --verbose --progress --allow-unrelated-histories origin/kida-1d +# git stash apply + # uses: devmasx/merge-branch@1.4.0 # with: # type: now From 43d15fc41b5deda153938fb9869ea60bff589c68 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 22 Oct 2021 13:08:02 +0200 Subject: [PATCH 223/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 6af42568a..07e18f4dd 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -232,9 +232,9 @@ jobs: steps: - uses: actions/checkout@v2 -# Debugging ssh session -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 + # Debugging ssh session + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 - name: merge with the KiD-A branch run: | From 448310e6c6e8b881e438c76fddad2239c797d5fc Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 25 Oct 2021 11:23:22 +0200 Subject: [PATCH 224/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 07e18f4dd..bc38d6258 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -230,7 +230,10 @@ jobs: threads: 4 steps: - - uses: actions/checkout@v2 + - name: Checkout libcloudphxx PR + uses: actions/checkout@v2 + with: + fetch-depth: 0 # Debugging ssh session - name: Setup tmate session From 272fcda6cbf603913775a109464420d13a257e64 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 25 Oct 2021 11:36:18 +0200 Subject: [PATCH 225/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index bc38d6258..166795afd 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -236,12 +236,14 @@ jobs: fetch-depth: 0 # Debugging ssh session - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 - name: merge with the KiD-A branch run: | - git fetch --all + git config --global user.email "pdziekan@fuw.edu.pl" + git config --global user.name "Piotr Dziekan" + git fetch origin git merge origin/kida-1d # git config --global user.email "pdziekan@fuw.edu.pl" From 83a32713d2345a9614f4d35865c44c88066b2d47 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 25 Oct 2021 12:14:03 +0200 Subject: [PATCH 226/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 442 ++++++++++++------------- 1 file changed, 210 insertions(+), 232 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 166795afd..31efff189 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,216 +7,216 @@ on: branches: [ master ] jobs: -# build_CUDA: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "CUDA" -# disable_cuda: false -# # not enough RAM to compile Debug CUDA on 4 threads -# threads: 1 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# -# build: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# install_prefix: ${{ github.workspace }}/installed -# -# # tar build dir before upload as artifact to retain permission and case-sensitive names -# - name: Compress libcloudph++ build -# run: tar -cvf build.tar build -# -# - name: Upload libcloudph++ build -# uses: actions/upload-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# path: build.tar -# -# # test jobs -# unit_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Run unit tests -# working-directory: ${{github.workspace}}/build -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) -# -# parcel_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# include: -# - build_type: "RelWithDebInfo" -# long_tests: true -# debug_tests: false -# - build_type: "Debug" -# long_tests: false -# debug_tests: true -# -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout parcel repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/parcel -# path: parcel -# -# - run: mkdir parcel/plots/outputs -# -# - name: run parcel unit_test -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test -# #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test -# -# - name: run parcel long_test -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test -# -# - name: run parcel unit_test_debug -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.debug_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug -# -# -# kinematic_2D_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: checkout libmpdata++ repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/libmpdataxx -# path: libmpdataxx -# -# - name: Configure libmpdata++ CMake -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed -# -# - name: Build libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: VERBOSE=1 singularity exec $SI cmake --build build -# -# - name: Install libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: sudo cmake --install build -# -# - name: Configure kinematic_2D CMake -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ -# -# -# - name: Build kinematic_2D -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} -# -# - name: Run kinematic_2D tests -# working-directory: ${{github.workspace}}/models/kinematic_2D/build -# run: | -# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) -# cat Testing/Temporary/LastTest.log + build_CUDA: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["CUDA"] + build_type: ["Debug", "RelWithDebInfo"] + include: + - name: "CUDA" + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads + threads: 1 + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + + build: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["Debug", "RelWithDebInfo"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + install_prefix: ${{ github.workspace }}/installed + + # tar build dir before upload as artifact to retain permission and case-sensitive names + - name: Compress libcloudph++ build + run: tar -cvf build.tar build + + - name: Upload libcloudph++ build + uses: actions/upload-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + path: build.tar + + # test jobs + unit_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Run unit tests + working-directory: ${{github.workspace}}/build + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + + parcel_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + include: + - build_type: "RelWithDebInfo" + long_tests: true + debug_tests: false + - build_type: "Debug" + long_tests: false + debug_tests: true + + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout parcel repo + uses: actions/checkout@v2 + with: + repository: igfuw/parcel + path: parcel + + - run: mkdir parcel/plots/outputs + + - name: run parcel unit_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test + #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test + + - name: run parcel long_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test + + - name: run parcel unit_test_debug + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.debug_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug + + + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ + + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log build_and_test_KiD-A: runs-on: ubuntu-20.04 @@ -246,23 +246,6 @@ jobs: git fetch origin git merge origin/kida-1d -# git config --global user.email "pdziekan@fuw.edu.pl" -# git config --global user.name "Piotr Dziekan" -# git ls-remote --heads origin -# git config --replace-all remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" -# git fetch --all -# git branch -r -# git branch -v -a -# git stash -# git merge --verbose --progress --allow-unrelated-histories origin/kida-1d -# git stash apply - -# uses: devmasx/merge-branch@1.4.0 -# with: -# type: now -# target_branch: kida-1d -# github_token: ${{ github.token }} - - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp - name: build libcloudph++ @@ -284,11 +267,6 @@ jobs: repository: igfuw/kid-libcloud path: kid-libcloud -# Debugging ssh session -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 - - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh - From 3887690504c96468e98a46c0b0b28bb62f4eb56a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 09:02:48 +0200 Subject: [PATCH 227/282] GA: fix --- .github/actions/libcloudphxx_build/action.yml | 5 + .../load_UWLCM_singularity_image/action.yml | 10 +- .github/workflows/test_libclouphxx.yml | 462 +++++++++--------- CMakeLists.txt | 110 ++--- 4 files changed, 301 insertions(+), 286 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index e1be7e0b6..a7797caa6 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -20,6 +20,10 @@ inputs: description: "CMake install prefix" required: true default: "/usr/local" + tag: + description: 'Singularity image tag' + required: true + default: 'ubuntu_20_04_cuda_11_4_v3' runs: using: "composite" @@ -34,6 +38,7 @@ runs: uses: ./.github/actions/load_UWLCM_singularity_image with: path: ${{ inputs.path }}/singularity_images + tag: ${{ inputs.tag }} # - name: Install Singularity # #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml index e4471680f..54bf61942 100644 --- a/.github/actions/load_UWLCM_singularity_image/action.yml +++ b/.github/actions/load_UWLCM_singularity_image/action.yml @@ -4,6 +4,10 @@ inputs: path: description: 'Where to put the image' required: true + tag: + description: 'Image tag' + required: true + default: 'ubuntu_20_04_cuda_11_4_v3' runs: using: "composite" @@ -25,8 +29,8 @@ runs: id: cache_singularity uses: actions/cache@v2 with: - path: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v3.sif' - key: 'sng_ubuntu_20_04_cuda_11_4_v3' + path: '${{ inputs.path }}/uwlcm_${{ inputs.tag }}.sif' + key: '${{ inputs.tag }}' - name: Download UWLCM Singularity image shell: bash @@ -35,7 +39,7 @@ runs: [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ { mkdir '${{ inputs.path }}'; - singularity pull --disable-cache --dir '${{ inputs.path }}' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4_v3; + singularity pull --disable-cache --dir '${{ inputs.path }}' library://pdziekan/default/uwlcm:${{ inputs.tag }}; } \ || \ true diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 31efff189..f56d497fb 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,29 +7,29 @@ on: branches: [ master ] jobs: - build_CUDA: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - include: - - name: "CUDA" - disable_cuda: false - # not enough RAM to compile Debug CUDA on 4 threads - threads: 1 - - steps: - - uses: actions/checkout@v2 - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} +# build_CUDA: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["CUDA"] +# build_type: ["Debug", "RelWithDebInfo"] +# include: +# - name: "CUDA" +# disable_cuda: false +# # not enough RAM to compile Debug CUDA on 4 threads +# threads: 1 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} build: runs-on: ubuntu-20.04 @@ -37,10 +37,15 @@ jobs: matrix: name: ["no_CUDA"] build_type: ["Debug", "RelWithDebInfo"] + mpi: ["none", "mvapich2"] include: - name: "no_CUDA" disable_cuda: true threads: 4 + - mpi: "none" + tag: "ubuntu_20_04_cuda_11_4_v3" + - mpi: "mvapich2" + tag: "ubuntu_20_04_cuda_11_4_mvapich2" steps: - uses: actions/checkout@v2 @@ -54,6 +59,7 @@ jobs: threads: ${{matrix.threads}} path: ${{ github.workspace }} install_prefix: ${{ github.workspace }}/installed + tag: ${{ matrix.tag }} # tar build dir before upload as artifact to retain permission and case-sensitive names - name: Compress libcloudph++ build @@ -62,211 +68,211 @@ jobs: - name: Upload libcloudph++ build uses: actions/upload-artifact@v2 with: - name: libcloud_build_${{matrix.build_type}}_tar + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar path: build.tar # test jobs - unit_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Run unit tests - working-directory: ${{github.workspace}}/build - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - - parcel_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - include: - - build_type: "RelWithDebInfo" - long_tests: true - debug_tests: false - - build_type: "Debug" - long_tests: false - debug_tests: true - - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout parcel repo - uses: actions/checkout@v2 - with: - repository: igfuw/parcel - path: parcel - - - run: mkdir parcel/plots/outputs - - - name: run parcel unit_test - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test - #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - - - name: run parcel long_test - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test - - - name: run parcel unit_test_debug - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.debug_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug - - - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ - - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - cat Testing/Temporary/LastTest.log - - build_and_test_KiD-A: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["RelWithDebInfo"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - name: Checkout libcloudphxx PR - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - # Debugging ssh session -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 - - - name: merge with the KiD-A branch - run: | - git config --global user.email "pdziekan@fuw.edu.pl" - git config --global user.name "Piotr Dziekan" - git fetch origin - git merge origin/kida-1d - - - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - install_prefix: ${{ github.workspace }}/installed - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout KiD-libcloud repo - uses: actions/checkout@v2 - with: - repository: igfuw/kid-libcloud - path: kid-libcloud - - - name: run KiD LWP test - working-directory: ${{github.workspace}}/kid-libcloud - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh +# unit_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Run unit tests +# working-directory: ${{github.workspace}}/build +# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail +# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) +# +# parcel_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# include: +# - build_type: "RelWithDebInfo" +# long_tests: true +# debug_tests: false +# - build_type: "Debug" +# long_tests: false +# debug_tests: true +# +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout parcel repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/parcel +# path: parcel +# +# - run: mkdir parcel/plots/outputs +# +# - name: run parcel unit_test +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test +# #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test +# +# - name: run parcel long_test +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test +# +# - name: run parcel unit_test_debug +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.debug_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug +# +# +# kinematic_2D_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: checkout libmpdata++ repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/libmpdataxx +# path: libmpdataxx +# +# - name: Configure libmpdata++ CMake +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed +# +# - name: Build libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: VERBOSE=1 singularity exec $SI cmake --build build +# +# - name: Install libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: sudo cmake --install build +# +# - name: Configure kinematic_2D CMake +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ +# +# +# - name: Build kinematic_2D +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} +# +# - name: Run kinematic_2D tests +# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# run: | +# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) +# cat Testing/Temporary/LastTest.log +# +# build_and_test_KiD-A: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["RelWithDebInfo"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - name: Checkout libcloudphxx PR +# uses: actions/checkout@v2 +# with: +# fetch-depth: 0 +# +# # Debugging ssh session +## - name: Setup tmate session +## uses: mxschmitt/action-tmate@v3 +# +# - name: merge with the KiD-A branch +# run: | +# git config --global user.email "pdziekan@fuw.edu.pl" +# git config --global user.name "Piotr Dziekan" +# git fetch origin +# git merge origin/kida-1d +# +# - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# install_prefix: ${{ github.workspace }}/installed +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout KiD-libcloud repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/kid-libcloud +# path: kid-libcloud +# +# - name: run KiD LWP test +# working-directory: ${{github.workspace}}/kid-libcloud +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 03bd4634c..5b381a88f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -293,59 +293,59 @@ endif() ############################################################################################ # BOOST ODEINT VERSION TEST -message(STATUS "Testing if Boost ODEINT version >= 1.58") -set(pfx "boost odeint check") -execute_process(COMMAND "mktemp" "-d" "/tmp/tmp.XXX" RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir) -if (NOT status EQUAL 0) - message(FATAL_ERROR "${pfx}: mkdtemp failed") -endif() -file(WRITE "${tmpdir}/test.cpp" " - #define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_CPP - - #include - - #include - #include - - struct rhs - { - void operator()( - const thrust::cpp::vector &psi, - thrust::cpp::vector &dot_psi, - const float /* t */ - ) - { - assert(psi.size() == dot_psi.size()); - } - }; - - int main() - { - boost::numeric::odeint::euler< - thrust::cpp::vector, // state_type - float, // value_type - thrust::cpp::vector, // deriv_type - float, // time_type - boost::numeric::odeint::thrust_algebra, - boost::numeric::odeint::thrust_operations - > chem_stepper; - - thrust::cpp::vector v(2); - chem_stepper.do_step(rhs(), v, 0, 1); - } -") -execute_process(COMMAND "${CMAKE_CXX_COMPILER}" "-std=c++14" "test.cpp" "-I${Boost_INCLUDE_DIR}" "-I${THRUST_INCLUDE_DIR}" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status ERROR_VARIABLE msg) -if (NOT status EQUAL 0) - message(FATAL_ERROR "${pfx}: c++ compiler failed\n ${msg}") -endif() -execute_process(COMMAND "./a.out" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status OUTPUT_VARIABLE msg) -if (NOT status EQUAL 0) - message(FATAL_ERROR "${pfx}: test program failed, install Boost odeint version >= 1.58") -endif() -unset(pfx) -unset(tmpdir) -unset(msg) -unset(status) +#message(STATUS "Testing if Boost ODEINT version >= 1.58") +#set(pfx "boost odeint check") +#execute_process(COMMAND "mktemp" "-d" "/tmp/tmp.XXX" RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir) +#if (NOT status EQUAL 0) +# message(FATAL_ERROR "${pfx}: mkdtemp failed") +#endif() +#file(WRITE "${tmpdir}/test.cpp" " +# #define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_CPP +# +# #include +# +# #include +# #include +# +# struct rhs +# { +# void operator()( +# const thrust::cpp::vector &psi, +# thrust::cpp::vector &dot_psi, +# const float /* t */ +# ) +# { +# assert(psi.size() == dot_psi.size()); +# } +# }; +# +# int main() +# { +# boost::numeric::odeint::euler< +# thrust::cpp::vector, // state_type +# float, // value_type +# thrust::cpp::vector, // deriv_type +# float, // time_type +# boost::numeric::odeint::thrust_algebra, +# boost::numeric::odeint::thrust_operations +# > chem_stepper; +# +# thrust::cpp::vector v(2); +# chem_stepper.do_step(rhs(), v, 0, 1); +# } +#") +#execute_process(COMMAND "${CMAKE_CXX_COMPILER}" "-std=c++14" "test.cpp" "-I${Boost_INCLUDE_DIR}" "-I${THRUST_INCLUDE_DIR}" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status ERROR_VARIABLE msg) +#if (NOT status EQUAL 0) +# message(FATAL_ERROR "${pfx}: c++ compiler failed\n ${msg}") +#endif() +#execute_process(COMMAND "./a.out" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status OUTPUT_VARIABLE msg) +#if (NOT status EQUAL 0) +# message(FATAL_ERROR "${pfx}: test program failed, install Boost odeint version >= 1.58") +#endif() +#unset(pfx) +#unset(tmpdir) +#unset(msg) +#unset(status) # generate a header file with git revision id if (EXISTS "${CMAKE_SOURCE_DIR}/.git") @@ -357,9 +357,9 @@ endif() enable_testing() -add_subdirectory(tests) +#add_subdirectory(tests) add_subdirectory(include) -add_subdirectory(bindings) +#add_subdirectory(bindings) ############################################################################################ # installation From aa41c93fc074285d40654461579ea38f848c0be7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 09:14:25 +0200 Subject: [PATCH 228/282] GA: fix --- .github/actions/load_UWLCM_singularity_image/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml index 54bf61942..6a89364eb 100644 --- a/.github/actions/load_UWLCM_singularity_image/action.yml +++ b/.github/actions/load_UWLCM_singularity_image/action.yml @@ -48,4 +48,4 @@ runs: - name: Set friendly Singularity image name uses: allenevans/set-env@v2.0.0 with: - SI: '${{ inputs.path }}/uwlcm_ubuntu_20_04_cuda_11_4_v3.sif' + SI: '${{ inputs.path }}/uwlcm_${{ inputs.tag }}.sif' From fcecf19af700263d69108eb784bab1deef69f955 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 09:36:17 +0200 Subject: [PATCH 229/282] GA: fix --- .github/actions/libcloudphxx_build/action.yml | 6 +++++- .github/workflows/test_libclouphxx.yml | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index a7797caa6..63d1faf93 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -24,6 +24,10 @@ inputs: description: 'Singularity image tag' required: true default: 'ubuntu_20_04_cuda_11_4_v3' + cxx: + description: 'C++ compiler' + required: true + default: 'g++' runs: using: "composite" @@ -81,7 +85,7 @@ runs: shell: bash # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{inputs.path}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} -DCMAKE_INSTALL_PREFIX=${{inputs.install_prefix}} + run: singularity exec $SI cmake -B ${{inputs.path}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} -DCMAKE_INSTALL_PREFIX=${{inputs.install_prefix}} -DCMAKE_CXX_COMPILER=${{inputs.cxx}} # - name: Build libcloudph++ shell: bash diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index f56d497fb..e29b1e661 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -44,8 +44,10 @@ jobs: threads: 4 - mpi: "none" tag: "ubuntu_20_04_cuda_11_4_v3" + cxx: "g++" - mpi: "mvapich2" tag: "ubuntu_20_04_cuda_11_4_mvapich2" + cxx: "mpic++" steps: - uses: actions/checkout@v2 @@ -60,6 +62,7 @@ jobs: path: ${{ github.workspace }} install_prefix: ${{ github.workspace }}/installed tag: ${{ matrix.tag }} + cxx: ${{ matrix.cxx }} # tar build dir before upload as artifact to retain permission and case-sensitive names - name: Compress libcloudph++ build From 784273320ebe044ea27e8d584f2284802262cba5 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 11:17:53 +0200 Subject: [PATCH 230/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 412 +++++++++++++------------ 1 file changed, 208 insertions(+), 204 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index e29b1e661..36574a28f 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -75,207 +75,211 @@ jobs: path: build.tar # test jobs -# unit_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Run unit tests -# working-directory: ${{github.workspace}}/build -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) -# -# parcel_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# include: -# - build_type: "RelWithDebInfo" -# long_tests: true -# debug_tests: false -# - build_type: "Debug" -# long_tests: false -# debug_tests: true -# -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout parcel repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/parcel -# path: parcel -# -# - run: mkdir parcel/plots/outputs -# -# - name: run parcel unit_test -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test -# #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test -# -# - name: run parcel long_test -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test -# -# - name: run parcel unit_test_debug -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.debug_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug -# -# -# kinematic_2D_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: checkout libmpdata++ repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/libmpdataxx -# path: libmpdataxx -# -# - name: Configure libmpdata++ CMake -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed -# -# - name: Build libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: VERBOSE=1 singularity exec $SI cmake --build build -# -# - name: Install libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: sudo cmake --install build -# -# - name: Configure kinematic_2D CMake -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ -# -# -# - name: Build kinematic_2D -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} -# -# - name: Run kinematic_2D tests -# working-directory: ${{github.workspace}}/models/kinematic_2D/build -# run: | -# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) -# cat Testing/Temporary/LastTest.log -# -# build_and_test_KiD-A: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["RelWithDebInfo"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - name: Checkout libcloudphxx PR -# uses: actions/checkout@v2 -# with: -# fetch-depth: 0 -# -# # Debugging ssh session -## - name: Setup tmate session -## uses: mxschmitt/action-tmate@v3 -# -# - name: merge with the KiD-A branch -# run: | -# git config --global user.email "pdziekan@fuw.edu.pl" -# git config --global user.name "Piotr Dziekan" -# git fetch origin -# git merge origin/kida-1d -# -# - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# install_prefix: ${{ github.workspace }}/installed -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout KiD-libcloud repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/kid-libcloud -# path: kid-libcloud -# -# - name: run KiD LWP test -# working-directory: ${{github.workspace}}/kid-libcloud -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh + unit_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + mpi: ["none", "mvapich2"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Run unit tests + working-directory: ${{github.workspace}}/build + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + + parcel_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + mpi: ["none"] + include: + - build_type: "RelWithDebInfo" + long_tests: true + debug_tests: false + - build_type: "Debug" + long_tests: false + debug_tests: true + + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout parcel repo + uses: actions/checkout@v2 + with: + repository: igfuw/parcel + path: parcel + + - run: mkdir parcel/plots/outputs + + - name: run parcel unit_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test + #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test + + - name: run parcel long_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test + + - name: run parcel unit_test_debug + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.debug_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug + + + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo"] + mpi: ["none"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ + + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log + + build_and_test_KiD-A: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["RelWithDebInfo"] + mpi: ["none"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - name: Checkout libcloudphxx PR + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # Debugging ssh session +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 + + - name: merge with the KiD-A branch + run: | + git config --global user.email "pdziekan@fuw.edu.pl" + git config --global user.name "Piotr Dziekan" + git fetch origin + git merge origin/kida-1d + + - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + install_prefix: ${{ github.workspace }}/installed + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout KiD-libcloud repo + uses: actions/checkout@v2 + with: + repository: igfuw/kid-libcloud + path: kid-libcloud + + - name: run KiD LWP test + working-directory: ${{github.workspace}}/kid-libcloud + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh From 3f868e9a574befdaf34966a4a93a55cae0934867 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 12:10:07 +0200 Subject: [PATCH 231/282] GA: fix --- .github/workflows/test_libclouphxx.yml | 46 +++++------ CMakeLists.txt | 110 ++++++++++++------------- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 36574a28f..8737d2375 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,29 +7,29 @@ on: branches: [ master ] jobs: -# build_CUDA: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "CUDA" -# disable_cuda: false -# # not enough RAM to compile Debug CUDA on 4 threads -# threads: 1 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} + build_CUDA: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["CUDA"] + build_type: ["Debug", "RelWithDebInfo"] + include: + - name: "CUDA" + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads + threads: 1 + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} build: runs-on: ubuntu-20.04 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b381a88f..03bd4634c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -293,59 +293,59 @@ endif() ############################################################################################ # BOOST ODEINT VERSION TEST -#message(STATUS "Testing if Boost ODEINT version >= 1.58") -#set(pfx "boost odeint check") -#execute_process(COMMAND "mktemp" "-d" "/tmp/tmp.XXX" RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir) -#if (NOT status EQUAL 0) -# message(FATAL_ERROR "${pfx}: mkdtemp failed") -#endif() -#file(WRITE "${tmpdir}/test.cpp" " -# #define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_CPP -# -# #include -# -# #include -# #include -# -# struct rhs -# { -# void operator()( -# const thrust::cpp::vector &psi, -# thrust::cpp::vector &dot_psi, -# const float /* t */ -# ) -# { -# assert(psi.size() == dot_psi.size()); -# } -# }; -# -# int main() -# { -# boost::numeric::odeint::euler< -# thrust::cpp::vector, // state_type -# float, // value_type -# thrust::cpp::vector, // deriv_type -# float, // time_type -# boost::numeric::odeint::thrust_algebra, -# boost::numeric::odeint::thrust_operations -# > chem_stepper; -# -# thrust::cpp::vector v(2); -# chem_stepper.do_step(rhs(), v, 0, 1); -# } -#") -#execute_process(COMMAND "${CMAKE_CXX_COMPILER}" "-std=c++14" "test.cpp" "-I${Boost_INCLUDE_DIR}" "-I${THRUST_INCLUDE_DIR}" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status ERROR_VARIABLE msg) -#if (NOT status EQUAL 0) -# message(FATAL_ERROR "${pfx}: c++ compiler failed\n ${msg}") -#endif() -#execute_process(COMMAND "./a.out" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status OUTPUT_VARIABLE msg) -#if (NOT status EQUAL 0) -# message(FATAL_ERROR "${pfx}: test program failed, install Boost odeint version >= 1.58") -#endif() -#unset(pfx) -#unset(tmpdir) -#unset(msg) -#unset(status) +message(STATUS "Testing if Boost ODEINT version >= 1.58") +set(pfx "boost odeint check") +execute_process(COMMAND "mktemp" "-d" "/tmp/tmp.XXX" RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir) +if (NOT status EQUAL 0) + message(FATAL_ERROR "${pfx}: mkdtemp failed") +endif() +file(WRITE "${tmpdir}/test.cpp" " + #define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_CPP + + #include + + #include + #include + + struct rhs + { + void operator()( + const thrust::cpp::vector &psi, + thrust::cpp::vector &dot_psi, + const float /* t */ + ) + { + assert(psi.size() == dot_psi.size()); + } + }; + + int main() + { + boost::numeric::odeint::euler< + thrust::cpp::vector, // state_type + float, // value_type + thrust::cpp::vector, // deriv_type + float, // time_type + boost::numeric::odeint::thrust_algebra, + boost::numeric::odeint::thrust_operations + > chem_stepper; + + thrust::cpp::vector v(2); + chem_stepper.do_step(rhs(), v, 0, 1); + } +") +execute_process(COMMAND "${CMAKE_CXX_COMPILER}" "-std=c++14" "test.cpp" "-I${Boost_INCLUDE_DIR}" "-I${THRUST_INCLUDE_DIR}" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status ERROR_VARIABLE msg) +if (NOT status EQUAL 0) + message(FATAL_ERROR "${pfx}: c++ compiler failed\n ${msg}") +endif() +execute_process(COMMAND "./a.out" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status OUTPUT_VARIABLE msg) +if (NOT status EQUAL 0) + message(FATAL_ERROR "${pfx}: test program failed, install Boost odeint version >= 1.58") +endif() +unset(pfx) +unset(tmpdir) +unset(msg) +unset(status) # generate a header file with git revision id if (EXISTS "${CMAKE_SOURCE_DIR}/.git") @@ -357,9 +357,9 @@ endif() enable_testing() -#add_subdirectory(tests) +add_subdirectory(tests) add_subdirectory(include) -#add_subdirectory(bindings) +add_subdirectory(bindings) ############################################################################################ # installation From 0970a4a78aa4f0d7ab3aa2485f667010f4a5973f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 12:22:01 +0200 Subject: [PATCH 232/282] GA: fix --- .github/actions/libcloudphxx_build/action.yml | 6 + .github/workflows/test_libclouphxx.yml | 464 +++++++++--------- 2 files changed, 238 insertions(+), 232 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index 63d1faf93..259fc5fa8 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -80,6 +80,12 @@ runs: # uses: allenevans/set-env@v2.0.0 # with: # SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' + + + # Debugging with a ssh session + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Configure CMake shell: bash diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 8737d2375..5d0fd34dd 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,29 +7,29 @@ on: branches: [ master ] jobs: - build_CUDA: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - include: - - name: "CUDA" - disable_cuda: false - # not enough RAM to compile Debug CUDA on 4 threads - threads: 1 - - steps: - - uses: actions/checkout@v2 - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} +# build_CUDA: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["CUDA"] +# build_type: ["Debug", "RelWithDebInfo"] +# include: +# - name: "CUDA" +# disable_cuda: false +# # not enough RAM to compile Debug CUDA on 4 threads +# threads: 1 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} build: runs-on: ubuntu-20.04 @@ -52,6 +52,10 @@ jobs: steps: - uses: actions/checkout@v2 + # Debugging with a ssh session +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 + - name: build libcloudph++ #uses: igfuw/libcloudphxx_build@v0.1-beta uses: ./.github/actions/libcloudphxx_build @@ -74,212 +78,208 @@ jobs: name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar path: build.tar - # test jobs - unit_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - mpi: ["none", "mvapich2"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Run unit tests - working-directory: ${{github.workspace}}/build - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - - parcel_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - mpi: ["none"] - include: - - build_type: "RelWithDebInfo" - long_tests: true - debug_tests: false - - build_type: "Debug" - long_tests: false - debug_tests: true - - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout parcel repo - uses: actions/checkout@v2 - with: - repository: igfuw/parcel - path: parcel - - - run: mkdir parcel/plots/outputs - - - name: run parcel unit_test - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test - #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - - - name: run parcel long_test - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.long_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test - - - name: run parcel unit_test_debug - working-directory: ${{github.workspace}}/parcel - if: ${{matrix.debug_tests}} - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug - - - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - mpi: ["none"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ - - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - cat Testing/Temporary/LastTest.log - - build_and_test_KiD-A: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["RelWithDebInfo"] - mpi: ["none"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - name: Checkout libcloudphxx PR - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - # Debugging ssh session -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 - - - name: merge with the KiD-A branch - run: | - git config --global user.email "pdziekan@fuw.edu.pl" - git config --global user.name "Piotr Dziekan" - git fetch origin - git merge origin/kida-1d - - - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - install_prefix: ${{ github.workspace }}/installed - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout KiD-libcloud repo - uses: actions/checkout@v2 - with: - repository: igfuw/kid-libcloud - path: kid-libcloud - - - name: run KiD LWP test - working-directory: ${{github.workspace}}/kid-libcloud - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh +# # test jobs +# unit_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# mpi: ["none", "mvapich2"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Run unit tests +# working-directory: ${{github.workspace}}/build +# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail +# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) +# +# parcel_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# mpi: ["none"] +# include: +# - build_type: "RelWithDebInfo" +# long_tests: true +# debug_tests: false +# - build_type: "Debug" +# long_tests: false +# debug_tests: true +# +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout parcel repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/parcel +# path: parcel +# +# - run: mkdir parcel/plots/outputs +# +# - name: run parcel unit_test +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test +# #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test +# +# - name: run parcel long_test +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.long_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test +# +# - name: run parcel unit_test_debug +# working-directory: ${{github.workspace}}/parcel +# if: ${{matrix.debug_tests}} +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug +# +# +# kinematic_2D_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo"] +# mpi: ["none"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: checkout libmpdata++ repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/libmpdataxx +# path: libmpdataxx +# +# - name: Configure libmpdata++ CMake +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed +# +# - name: Build libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: VERBOSE=1 singularity exec $SI cmake --build build +# +# - name: Install libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: sudo cmake --install build +# +# - name: Configure kinematic_2D CMake +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ +# +# +# - name: Build kinematic_2D +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} +# +# - name: Run kinematic_2D tests +# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# run: | +# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) +# cat Testing/Temporary/LastTest.log +# +# build_and_test_KiD-A: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["RelWithDebInfo"] +# mpi: ["none"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - name: Checkout libcloudphxx PR +# uses: actions/checkout@v2 +# with: +# fetch-depth: 0 +# +# - name: merge with the KiD-A branch +# run: | +# git config --global user.email "pdziekan@fuw.edu.pl" +# git config --global user.name "Piotr Dziekan" +# git fetch origin +# git merge origin/kida-1d +# +# - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# install_prefix: ${{ github.workspace }}/installed +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout KiD-libcloud repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/kid-libcloud +# path: kid-libcloud +# +# - name: run KiD LWP test +# working-directory: ${{github.workspace}}/kid-libcloud +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh From 763175ec0f930d82eb0cd9a9b950039f714ce486 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 12:36:30 +0200 Subject: [PATCH 233/282] GA: fix --- .github/actions/libcloudphxx_build/action.yml | 2 +- .github/actions/load_UWLCM_singularity_image/action.yml | 4 ++-- .github/workflows/test_libclouphxx.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index 259fc5fa8..954ae16c6 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -23,7 +23,7 @@ inputs: tag: description: 'Singularity image tag' required: true - default: 'ubuntu_20_04_cuda_11_4_v3' + default: 'ubuntu_20_04_cuda_11_4' cxx: description: 'C++ compiler' required: true diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml index 6a89364eb..31f3fdf3f 100644 --- a/.github/actions/load_UWLCM_singularity_image/action.yml +++ b/.github/actions/load_UWLCM_singularity_image/action.yml @@ -7,7 +7,7 @@ inputs: tag: description: 'Image tag' required: true - default: 'ubuntu_20_04_cuda_11_4_v3' + default: 'ubuntu_20_04_cuda_11_4' runs: using: "composite" @@ -15,7 +15,7 @@ runs: - name: Install Singularity #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) - uses: eWaterCycle/setup-singularity@v6 + uses: eWaterCycle/setup-singularity@v7 with: singularity-version: 3.7.1 #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 5d0fd34dd..d1c5ecc9d 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -43,7 +43,7 @@ jobs: disable_cuda: true threads: 4 - mpi: "none" - tag: "ubuntu_20_04_cuda_11_4_v3" + tag: "ubuntu_20_04_cuda_11_4" cxx: "g++" - mpi: "mvapich2" tag: "ubuntu_20_04_cuda_11_4_mvapich2" From 9516cfdfdf8a5f30900d61b0b366fe3988d23743 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 14:53:18 +0200 Subject: [PATCH 234/282] GA --- .github/actions/libcloudphxx_build/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml index 954ae16c6..8a6ff5902 100644 --- a/.github/actions/libcloudphxx_build/action.yml +++ b/.github/actions/libcloudphxx_build/action.yml @@ -83,8 +83,8 @@ runs: # Debugging with a ssh session - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 - name: Configure CMake From 75d212deee6485337686a65b419272f77f24a7ed Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Oct 2021 14:58:29 +0200 Subject: [PATCH 235/282] GA: fix --- .github/actions/load_UWLCM_singularity_image/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml index 31f3fdf3f..f3876e8e5 100644 --- a/.github/actions/load_UWLCM_singularity_image/action.yml +++ b/.github/actions/load_UWLCM_singularity_image/action.yml @@ -17,7 +17,7 @@ runs: #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) uses: eWaterCycle/setup-singularity@v7 with: - singularity-version: 3.7.1 + singularity-version: 3.8.3 #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 # run: | # wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list From 8f6ab13abbf7ae8515e19a3b7f5d44e64fcf9eb7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 27 Oct 2021 12:49:23 +0200 Subject: [PATCH 236/282] toms: include stdint --- include/libcloudph++/common/detail/toms748.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/libcloudph++/common/detail/toms748.hpp b/include/libcloudph++/common/detail/toms748.hpp index a571445dd..8e475c62d 100644 --- a/include/libcloudph++/common/detail/toms748.hpp +++ b/include/libcloudph++/common/detail/toms748.hpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace libcloudphxx { namespace common { From d6755edcc167102c162edd9ab04f3e98456fff73 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 27 Oct 2021 13:14:11 +0200 Subject: [PATCH 237/282] GA --- .github/workflows/test_libclouphxx.yml | 462 +++++++++++++------------ 1 file changed, 234 insertions(+), 228 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index d1c5ecc9d..7f510accb 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,29 +7,29 @@ on: branches: [ master ] jobs: -# build_CUDA: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "CUDA" -# disable_cuda: false -# # not enough RAM to compile Debug CUDA on 4 threads -# threads: 1 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} + build_CUDA: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["CUDA"] + build_type: ["Debug", "RelWithDebInfo"] + include: + - name: "CUDA" + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads + threads: 1 + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} build: runs-on: ubuntu-20.04 @@ -78,208 +78,214 @@ jobs: name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar path: build.tar -# # test jobs -# unit_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# mpi: ["none", "mvapich2"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Run unit tests -# working-directory: ${{github.workspace}}/build -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) -# -# parcel_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# mpi: ["none"] -# include: -# - build_type: "RelWithDebInfo" -# long_tests: true -# debug_tests: false -# - build_type: "Debug" -# long_tests: false -# debug_tests: true -# -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout parcel repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/parcel -# path: parcel -# -# - run: mkdir parcel/plots/outputs -# -# - name: run parcel unit_test -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test -# #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test -# -# - name: run parcel long_test -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.long_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test -# -# - name: run parcel unit_test_debug -# working-directory: ${{github.workspace}}/parcel -# if: ${{matrix.debug_tests}} -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug -# -# -# kinematic_2D_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo"] -# mpi: ["none"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: checkout libmpdata++ repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/libmpdataxx -# path: libmpdataxx -# -# - name: Configure libmpdata++ CMake -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed -# -# - name: Build libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: VERBOSE=1 singularity exec $SI cmake --build build -# -# - name: Install libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: sudo cmake --install build -# -# - name: Configure kinematic_2D CMake -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ -# -# -# - name: Build kinematic_2D -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} -# -# - name: Run kinematic_2D tests -# working-directory: ${{github.workspace}}/models/kinematic_2D/build -# run: | -# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) -# cat Testing/Temporary/LastTest.log -# -# build_and_test_KiD-A: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["RelWithDebInfo"] -# mpi: ["none"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - name: Checkout libcloudphxx PR -# uses: actions/checkout@v2 -# with: -# fetch-depth: 0 -# -# - name: merge with the KiD-A branch -# run: | -# git config --global user.email "pdziekan@fuw.edu.pl" -# git config --global user.name "Piotr Dziekan" -# git fetch origin -# git merge origin/kida-1d -# -# - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# install_prefix: ${{ github.workspace }}/installed -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout KiD-libcloud repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/kid-libcloud -# path: kid-libcloud -# -# - name: run KiD LWP test -# working-directory: ${{github.workspace}}/kid-libcloud -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh + # test jobs + unit_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + mpi: ["none", "mvapich2"] + include: + - mpi: "none" + tag: "ubuntu_20_04_cuda_11_4" + - mpi: "mvapich2" + tag: "ubuntu_20_04_cuda_11_4_mvapich2" + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + tag: ${{ matrix.tag }} + + - name: Run unit tests + working-directory: ${{github.workspace}}/build + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + + parcel_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + mpi: ["none"] + include: + - build_type: "RelWithDebInfo" + long_tests: true + debug_tests: false + - build_type: "Debug" + long_tests: false + debug_tests: true + + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout parcel repo + uses: actions/checkout@v2 + with: + repository: igfuw/parcel + path: parcel + + - run: mkdir parcel/plots/outputs + + - name: run parcel unit_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test + #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test + + - name: run parcel long_test + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.long_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test + + - name: run parcel unit_test_debug + working-directory: ${{github.workspace}}/parcel + if: ${{matrix.debug_tests}} + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug + + + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo"] + mpi: ["none"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ + + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log + + build_and_test_KiD-A: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["RelWithDebInfo"] + mpi: ["none"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - name: Checkout libcloudphxx PR + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: merge with the KiD-A branch + run: | + git config --global user.email "pdziekan@fuw.edu.pl" + git config --global user.name "Piotr Dziekan" + git fetch origin + git merge origin/kida-1d + + - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + install_prefix: ${{ github.workspace }}/installed + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout KiD-libcloud repo + uses: actions/checkout@v2 + with: + repository: igfuw/kid-libcloud + path: kid-libcloud + + - name: run KiD LWP test + working-directory: ${{github.workspace}}/kid-libcloud + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh From e05083f5aff40d6933505e1a2bef54cd4d341c3d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 27 Oct 2021 17:13:34 +0200 Subject: [PATCH 238/282] dont run chem_coal and SD_removal tests with MPI --- tests/python/unit/CMakeLists.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/python/unit/CMakeLists.txt b/tests/python/unit/CMakeLists.txt index c3d337546..fa1f0a6fc 100644 --- a/tests/python/unit/CMakeLists.txt +++ b/tests/python/unit/CMakeLists.txt @@ -1,5 +1,5 @@ # non-pytest tests -foreach(test api_blk_1m api_blk_2m api_lgrngn api_common segfault_20150216 col_kernels terminal_velocities SD_removal uniform_init source chem_coal sstp_cond multiple_kappas adve_scheme lgrngn_subsidence sat_adj_blk_1m diag_incloud_time) +foreach(test api_blk_1m api_blk_2m api_lgrngn api_common segfault_20150216 col_kernels terminal_velocities uniform_init source sstp_cond multiple_kappas adve_scheme lgrngn_subsidence sat_adj_blk_1m diag_incloud_time) #TODO: indicate that tests depend on the lib add_test( NAME ${test} @@ -8,6 +8,17 @@ foreach(test api_blk_1m api_blk_2m api_lgrngn api_common segfault_20150216 col_k ) endforeach() +# tests with chemistry, chemistry does not work with MPI +if(NOT USE_MPI) + foreach(SD_removal chem_coal) + #TODO: indicate that tests depend on the lib + add_test( + NAME ${test} + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/bindings/python" + COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_SOURCE_DIR}/tests/python/unit/${test}.py" + ) +endif() + ## pytest tests run with "python -m pytest" foreach(test lgrngn_adve) #TODO: indicate that tests depend on the lib From 25ee2c1354064b8085a1a93dbc61aed3d3b8cb95 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 27 Oct 2021 17:15:03 +0200 Subject: [PATCH 239/282] GA debug --- .github/workflows/test_libclouphxx.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7f510accb..b4721cb09 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -161,6 +161,10 @@ jobs: - run: mkdir parcel/plots/outputs + # Debugging with a ssh session + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: run parcel unit_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} From 482835db87755cc7f2156ec3a7db600896aaf342 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 27 Oct 2021 17:27:16 +0200 Subject: [PATCH 240/282] GA debug --- tests/python/unit/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/unit/CMakeLists.txt b/tests/python/unit/CMakeLists.txt index fa1f0a6fc..e7d9dee39 100644 --- a/tests/python/unit/CMakeLists.txt +++ b/tests/python/unit/CMakeLists.txt @@ -10,13 +10,14 @@ endforeach() # tests with chemistry, chemistry does not work with MPI if(NOT USE_MPI) - foreach(SD_removal chem_coal) + foreach(test SD_removal chem_coal) #TODO: indicate that tests depend on the lib add_test( NAME ${test} WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/bindings/python" COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_SOURCE_DIR}/tests/python/unit/${test}.py" ) + endforeach() endif() ## pytest tests run with "python -m pytest" From f243f4bece91e52b6e57527469b36115bcb35174 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 12:29:46 +0200 Subject: [PATCH 241/282] GA debug --- .github/workflows/test_libclouphxx.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index b4721cb09..7f510accb 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -161,10 +161,6 @@ jobs: - run: mkdir parcel/plots/outputs - # Debugging with a ssh session - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: run parcel unit_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} From f165ad499f55405f71bbdac2ad0a76448cf1b70c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 12:45:17 +0200 Subject: [PATCH 242/282] GA debug --- .github/workflows/test_libclouphxx.yml | 349 +++++++++++++------------ 1 file changed, 177 insertions(+), 172 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7f510accb..e54520836 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,36 +7,37 @@ on: branches: [ master ] jobs: - build_CUDA: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - include: - - name: "CUDA" - disable_cuda: false - # not enough RAM to compile Debug CUDA on 4 threads - threads: 1 - - steps: - - uses: actions/checkout@v2 - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - +# build_CUDA: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["CUDA"] +# build_type: ["Debug", "RelWithDebInfo"] +# include: +# - name: "CUDA" +# disable_cuda: false +# # not enough RAM to compile Debug CUDA on 4 threads +# threads: 1 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# build: runs-on: ubuntu-20.04 strategy: matrix: name: ["no_CUDA"] - build_type: ["Debug", "RelWithDebInfo"] + #build_type: ["Debug", "RelWithDebInfo"] + build_type: ["RelWithDebInfo"] mpi: ["none", "mvapich2"] include: - name: "no_CUDA" @@ -79,42 +80,42 @@ jobs: path: build.tar # test jobs - unit_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - mpi: ["none", "mvapich2"] - include: - - mpi: "none" - tag: "ubuntu_20_04_cuda_11_4" - - mpi: "mvapich2" - tag: "ubuntu_20_04_cuda_11_4_mvapich2" - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - tag: ${{ matrix.tag }} - - - name: Run unit tests - working-directory: ${{github.workspace}}/build - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) +# unit_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# mpi: ["none", "mvapich2"] +# include: +# - mpi: "none" +# tag: "ubuntu_20_04_cuda_11_4" +# - mpi: "mvapich2" +# tag: "ubuntu_20_04_cuda_11_4_mvapich2" +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# tag: ${{ matrix.tag }} +# +# - name: Run unit tests +# working-directory: ${{github.workspace}}/build +# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail +# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) parcel_test: needs: build @@ -161,6 +162,10 @@ jobs: - run: mkdir parcel/plots/outputs + # Debugging with a ssh session + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: run parcel unit_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} @@ -178,114 +183,114 @@ jobs: run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - mpi: ["none"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ - - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - cat Testing/Temporary/LastTest.log - - build_and_test_KiD-A: - runs-on: ubuntu-20.04 - strategy: - matrix: - name: ["no_CUDA"] - build_type: ["RelWithDebInfo"] - mpi: ["none"] - include: - - name: "no_CUDA" - disable_cuda: true - threads: 4 - - steps: - - name: Checkout libcloudphxx PR - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: merge with the KiD-A branch - run: | - git config --global user.email "pdziekan@fuw.edu.pl" - git config --global user.name "Piotr Dziekan" - git fetch origin - git merge origin/kida-1d - - - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp - - - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build - with: - disable_cuda: ${{matrix.disable_cuda}} - build_type: ${{matrix.build_type}} - threads: ${{matrix.threads}} - path: ${{ github.workspace }} - install_prefix: ${{ github.workspace }}/installed - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: checkout KiD-libcloud repo - uses: actions/checkout@v2 - with: - repository: igfuw/kid-libcloud - path: kid-libcloud - - - name: run KiD LWP test - working-directory: ${{github.workspace}}/kid-libcloud - run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh +# kinematic_2D_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo"] +# mpi: ["none"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: checkout libmpdata++ repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/libmpdataxx +# path: libmpdataxx +# +# - name: Configure libmpdata++ CMake +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed +# +# - name: Build libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: VERBOSE=1 singularity exec $SI cmake --build build +# +# - name: Install libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: sudo cmake --install build +# +# - name: Configure kinematic_2D CMake +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ +# +# +# - name: Build kinematic_2D +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} +# +# - name: Run kinematic_2D tests +# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# run: | +# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) +# cat Testing/Temporary/LastTest.log +# +# build_and_test_KiD-A: +# runs-on: ubuntu-20.04 +# strategy: +# matrix: +# name: ["no_CUDA"] +# build_type: ["RelWithDebInfo"] +# mpi: ["none"] +# include: +# - name: "no_CUDA" +# disable_cuda: true +# threads: 4 +# +# steps: +# - name: Checkout libcloudphxx PR +# uses: actions/checkout@v2 +# with: +# fetch-depth: 0 +# +# - name: merge with the KiD-A branch +# run: | +# git config --global user.email "pdziekan@fuw.edu.pl" +# git config --global user.name "Piotr Dziekan" +# git fetch origin +# git merge origin/kida-1d +# +# - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp +# +# - name: build libcloudph++ +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/libcloudphxx_build +# with: +# disable_cuda: ${{matrix.disable_cuda}} +# build_type: ${{matrix.build_type}} +# threads: ${{matrix.threads}} +# path: ${{ github.workspace }} +# install_prefix: ${{ github.workspace }}/installed +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: checkout KiD-libcloud repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/kid-libcloud +# path: kid-libcloud +# +# - name: run KiD LWP test +# working-directory: ${{github.workspace}}/kid-libcloud +# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh From 00ae509c46092d3f2cd286fda5fa3ffb159e29d2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 12:45:37 +0200 Subject: [PATCH 243/282] GA debug --- .github/workflows/test_libclouphxx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index e54520836..26c38f131 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -123,7 +123,8 @@ jobs: strategy: matrix: - build_type: ["RelWithDebInfo", "Debug"] + #build_type: ["RelWithDebInfo", "Debug"] + build_type: ["RelWithDebInfo"] mpi: ["none"] include: - build_type: "RelWithDebInfo" From 93963d56a2eb0ae0b95ed19fec08329aad15663f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 13:40:06 +0200 Subject: [PATCH 244/282] GA debug --- .github/workflows/test_libclouphxx.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 26c38f131..145a4baa0 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -38,7 +38,8 @@ jobs: name: ["no_CUDA"] #build_type: ["Debug", "RelWithDebInfo"] build_type: ["RelWithDebInfo"] - mpi: ["none", "mvapich2"] + #mpi: ["none", "mvapich2"] + mpi: ["none"] include: - name: "no_CUDA" disable_cuda: true @@ -130,9 +131,9 @@ jobs: - build_type: "RelWithDebInfo" long_tests: true debug_tests: false - - build_type: "Debug" - long_tests: false - debug_tests: true +# - build_type: "Debug" +# long_tests: false +# debug_tests: true steps: From 87ce84c900b115b0f4999d43c2a93e8f9d9362c0 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 13:57:32 +0200 Subject: [PATCH 245/282] GA debug --- .github/workflows/test_libclouphxx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 145a4baa0..1d423b0e3 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -47,9 +47,9 @@ jobs: - mpi: "none" tag: "ubuntu_20_04_cuda_11_4" cxx: "g++" - - mpi: "mvapich2" - tag: "ubuntu_20_04_cuda_11_4_mvapich2" - cxx: "mpic++" +# - mpi: "mvapich2" +# tag: "ubuntu_20_04_cuda_11_4_mvapich2" +# cxx: "mpic++" steps: - uses: actions/checkout@v2 From afc7278437a9ce2fb6974c9c2f30db5d76f7bff4 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 14:50:43 +0200 Subject: [PATCH 246/282] GA debug --- .github/workflows/test_libclouphxx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 1d423b0e3..0202e4060 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -165,8 +165,8 @@ jobs: - run: mkdir parcel/plots/outputs # Debugging with a ssh session - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 - name: run parcel unit_test working-directory: ${{github.workspace}}/parcel From 4ba7bfb485eb7297dcfd522cdc4afc311e38ebd7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 15:53:41 +0200 Subject: [PATCH 247/282] GA debug --- .github/workflows/test_libclouphxx.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 0202e4060..4068ef1c0 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -36,10 +36,10 @@ jobs: strategy: matrix: name: ["no_CUDA"] - #build_type: ["Debug", "RelWithDebInfo"] - build_type: ["RelWithDebInfo"] - #mpi: ["none", "mvapich2"] - mpi: ["none"] + build_type: ["Debug", "RelWithDebInfo"] + #build_type: ["RelWithDebInfo"] + mpi: ["none", "mvapich2"] + #mpi: ["none"] include: - name: "no_CUDA" disable_cuda: true @@ -47,9 +47,9 @@ jobs: - mpi: "none" tag: "ubuntu_20_04_cuda_11_4" cxx: "g++" -# - mpi: "mvapich2" -# tag: "ubuntu_20_04_cuda_11_4_mvapich2" -# cxx: "mpic++" + - mpi: "mvapich2" + tag: "ubuntu_20_04_cuda_11_4_mvapich2" + cxx: "mpic++" steps: - uses: actions/checkout@v2 From c81b72846a126a8ed7812009ade5b63187db1692 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 16:17:06 +0200 Subject: [PATCH 248/282] GA debug --- .github/workflows/test_libclouphxx.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 4068ef1c0..983b8ac95 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -124,16 +124,16 @@ jobs: strategy: matrix: - #build_type: ["RelWithDebInfo", "Debug"] - build_type: ["RelWithDebInfo"] + build_type: ["RelWithDebInfo", "Debug"] + # build_type: ["RelWithDebInfo"] mpi: ["none"] include: - build_type: "RelWithDebInfo" long_tests: true debug_tests: false -# - build_type: "Debug" -# long_tests: false -# debug_tests: true + - build_type: "Debug" + long_tests: false + debug_tests: true steps: From 50a2068dfd32e9b6bacb36634e4319551c2e7f61 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Oct 2021 16:37:52 +0200 Subject: [PATCH 249/282] GA debug --- .github/workflows/test_libclouphxx.yml | 342 ++++++++++++------------- 1 file changed, 171 insertions(+), 171 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 983b8ac95..8d1f3730e 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -7,30 +7,30 @@ on: branches: [ master ] jobs: -# build_CUDA: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["CUDA"] -# build_type: ["Debug", "RelWithDebInfo"] -# include: -# - name: "CUDA" -# disable_cuda: false -# # not enough RAM to compile Debug CUDA on 4 threads -# threads: 1 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# + build_CUDA: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["CUDA"] + build_type: ["Debug", "RelWithDebInfo"] + include: + - name: "CUDA" + disable_cuda: false + # not enough RAM to compile Debug CUDA on 4 threads + threads: 1 + + steps: + - uses: actions/checkout@v2 + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + build: runs-on: ubuntu-20.04 strategy: @@ -81,42 +81,42 @@ jobs: path: build.tar # test jobs -# unit_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# mpi: ["none", "mvapich2"] -# include: -# - mpi: "none" -# tag: "ubuntu_20_04_cuda_11_4" -# - mpi: "mvapich2" -# tag: "ubuntu_20_04_cuda_11_4_mvapich2" -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# tag: ${{ matrix.tag }} -# -# - name: Run unit tests -# working-directory: ${{github.workspace}}/build -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + unit_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + mpi: ["none", "mvapich2"] + include: + - mpi: "none" + tag: "ubuntu_20_04_cuda_11_4" + - mpi: "mvapich2" + tag: "ubuntu_20_04_cuda_11_4_mvapich2" + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + tag: ${{ matrix.tag }} + + - name: Run unit tests + working-directory: ${{github.workspace}}/build + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) parcel_test: needs: build @@ -185,114 +185,114 @@ jobs: run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug -# kinematic_2D_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo"] -# mpi: ["none"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: checkout libmpdata++ repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/libmpdataxx -# path: libmpdataxx -# -# - name: Configure libmpdata++ CMake -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed -# -# - name: Build libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: VERBOSE=1 singularity exec $SI cmake --build build -# -# - name: Install libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: sudo cmake --install build -# -# - name: Configure kinematic_2D CMake -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ -# -# -# - name: Build kinematic_2D -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} -# -# - name: Run kinematic_2D tests -# working-directory: ${{github.workspace}}/models/kinematic_2D/build -# run: | -# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) -# cat Testing/Temporary/LastTest.log -# -# build_and_test_KiD-A: -# runs-on: ubuntu-20.04 -# strategy: -# matrix: -# name: ["no_CUDA"] -# build_type: ["RelWithDebInfo"] -# mpi: ["none"] -# include: -# - name: "no_CUDA" -# disable_cuda: true -# threads: 4 -# -# steps: -# - name: Checkout libcloudphxx PR -# uses: actions/checkout@v2 -# with: -# fetch-depth: 0 -# -# - name: merge with the KiD-A branch -# run: | -# git config --global user.email "pdziekan@fuw.edu.pl" -# git config --global user.name "Piotr Dziekan" -# git fetch origin -# git merge origin/kida-1d -# -# - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp -# -# - name: build libcloudph++ -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/libcloudphxx_build -# with: -# disable_cuda: ${{matrix.disable_cuda}} -# build_type: ${{matrix.build_type}} -# threads: ${{matrix.threads}} -# path: ${{ github.workspace }} -# install_prefix: ${{ github.workspace }}/installed -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: checkout KiD-libcloud repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/kid-libcloud -# path: kid-libcloud -# -# - name: run KiD LWP test -# working-directory: ${{github.workspace}}/kid-libcloud -# run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo"] + mpi: ["none"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ + + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log + + build_and_test_KiD-A: + runs-on: ubuntu-20.04 + strategy: + matrix: + name: ["no_CUDA"] + build_type: ["RelWithDebInfo"] + mpi: ["none"] + include: + - name: "no_CUDA" + disable_cuda: true + threads: 4 + + steps: + - name: Checkout libcloudphxx PR + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: merge with the KiD-A branch + run: | + git config --global user.email "pdziekan@fuw.edu.pl" + git config --global user.name "Piotr Dziekan" + git fetch origin + git merge origin/kida-1d + + - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp + + - name: build libcloudph++ + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/libcloudphxx_build + with: + disable_cuda: ${{matrix.disable_cuda}} + build_type: ${{matrix.build_type}} + threads: ${{matrix.threads}} + path: ${{ github.workspace }} + install_prefix: ${{ github.workspace }}/installed + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: checkout KiD-libcloud repo + uses: actions/checkout@v2 + with: + repository: igfuw/kid-libcloud + path: kid-libcloud + + - name: run KiD LWP test + working-directory: ${{github.workspace}}/kid-libcloud + run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh From 3dc8c7b71470ff91abc70f2d48e64ef588551431 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Oct 2021 10:10:09 +0200 Subject: [PATCH 250/282] GA debug --- .github/workflows/test_libclouphxx.yml | 130 +++++++++++++------------ 1 file changed, 66 insertions(+), 64 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 8d1f3730e..9024bb867 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -118,10 +118,75 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - parcel_test: + + kinematic_2D_test: needs: build runs-on: ubuntu-20.04 + strategy: + matrix: + build_type: ["RelWithDebInfo"] + mpi: ["none"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ + + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log + + + parcel_test: +# needs: build + needs: kinematic_2D_test + runs-on: ubuntu-20.04 + strategy: matrix: build_type: ["RelWithDebInfo", "Debug"] @@ -184,69 +249,6 @@ jobs: if: ${{matrix.debug_tests}} run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug - - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - mpi: ["none"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ - - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - cat Testing/Temporary/LastTest.log - build_and_test_KiD-A: runs-on: ubuntu-20.04 strategy: From 2ff0112392da870f85b0ad3e9cd065e1752a2b2a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Oct 2021 11:02:32 +0200 Subject: [PATCH 251/282] GA debug --- .github/workflows/test_libclouphxx.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 9024bb867..3029b4cc4 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -183,6 +183,7 @@ jobs: parcel_test: +# done after kinematic_2D_test, because otherwise we occasionally get "illegal instruction" errors, probably because libcloudphxx is installed simultaneously on the same runner at the same location by two different jobs (?) # needs: build needs: kinematic_2D_test runs-on: ubuntu-20.04 From 9851a4a7bceb6b84af41fb42f54a93645b141b4d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Oct 2021 14:22:56 +0200 Subject: [PATCH 252/282] GA debug --- .github/workflows/test_libclouphxx.yml | 214 +++++++++++++------------ 1 file changed, 108 insertions(+), 106 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 3029b4cc4..c2b55ca8f 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -81,111 +81,111 @@ jobs: path: build.tar # test jobs - unit_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo", "Debug"] - mpi: ["none", "mvapich2"] - include: - - mpi: "none" - tag: "ubuntu_20_04_cuda_11_4" - - mpi: "mvapich2" - tag: "ubuntu_20_04_cuda_11_4_mvapich2" - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - tag: ${{ matrix.tag }} - - - name: Run unit tests - working-directory: ${{github.workspace}}/build - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) - - - kinematic_2D_test: - needs: build - runs-on: ubuntu-20.04 - - strategy: - matrix: - build_type: ["RelWithDebInfo"] - mpi: ["none"] - - steps: - - uses: actions/checkout@v2 - - - name: Download libcloudph++ build - uses: actions/download-artifact@v2 - with: - name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar - - - name: Decompress libcloudph++ build - run: tar -xvf build.tar - - - name: Install libcloudph++ - run: sudo cmake --install build - - - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ github.workspace }}/singularity_images - - - name: checkout libmpdata++ repo - uses: actions/checkout@v2 - with: - repository: igfuw/libmpdataxx - path: libmpdataxx - - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build - - - name: Configure kinematic_2D CMake - working-directory: ${{github.workspace}}/models/kinematic_2D - run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ - - - - name: Build kinematic_2D - working-directory: ${{github.workspace}}/models/kinematic_2D - run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} - - - name: Run kinematic_2D tests - working-directory: ${{github.workspace}}/models/kinematic_2D/build - run: | - singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) - cat Testing/Temporary/LastTest.log +# unit_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo", "Debug"] +# mpi: ["none", "mvapich2"] +# include: +# - mpi: "none" +# tag: "ubuntu_20_04_cuda_11_4" +# - mpi: "mvapich2" +# tag: "ubuntu_20_04_cuda_11_4_mvapich2" +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# tag: ${{ matrix.tag }} +# +# - name: Run unit tests +# working-directory: ${{github.workspace}}/build +# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail +# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) +# +# +# kinematic_2D_test: +# needs: build +# runs-on: ubuntu-20.04 +# +# strategy: +# matrix: +# build_type: ["RelWithDebInfo"] +# mpi: ["none"] +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Download libcloudph++ build +# uses: actions/download-artifact@v2 +# with: +# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar +# +# - name: Decompress libcloudph++ build +# run: tar -xvf build.tar +# +# - name: Install libcloudph++ +# run: sudo cmake --install build +# +# - name: load UWLCM Singularity image +# #uses: igfuw/libcloudphxx_build@v0.1-beta +# uses: ./.github/actions/load_UWLCM_singularity_image +# with: +# path: ${{ github.workspace }}/singularity_images +# +# - name: checkout libmpdata++ repo +# uses: actions/checkout@v2 +# with: +# repository: igfuw/libmpdataxx +# path: libmpdataxx +# +# - name: Configure libmpdata++ CMake +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed +# +# - name: Build libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: VERBOSE=1 singularity exec $SI cmake --build build +# +# - name: Install libmpdata++ +# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ +# run: sudo cmake --install build +# +# - name: Configure kinematic_2D CMake +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ +# +# +# - name: Build kinematic_2D +# working-directory: ${{github.workspace}}/models/kinematic_2D +# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} +# +# - name: Run kinematic_2D tests +# working-directory: ${{github.workspace}}/models/kinematic_2D/build +# run: | +# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) +# cat Testing/Temporary/LastTest.log parcel_test: -# done after kinematic_2D_test, because otherwise we occasionally get "illegal instruction" errors, probably because libcloudphxx is installed simultaneously on the same runner at the same location by two different jobs (?) -# needs: build - needs: kinematic_2D_test + needs: build +# do after kinematic_2D_test, because otherwise we occasionally get "illegal instruction" errors, probably because libcloudphxx is installed simultaneously on the same runner at the same location by two different jobs (?) +# needs: kinematic_2D_test runs-on: ubuntu-20.04 strategy: @@ -230,16 +230,18 @@ jobs: - run: mkdir parcel/plots/outputs - # Debugging with a ssh session -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 - - name: run parcel unit_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test + + # Debugging with a ssh session if previous step failed + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + - name: run parcel long_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} From c0867174a301ad5c7f0b4d82ed0e081af35a56fc Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Oct 2021 15:26:25 +0200 Subject: [PATCH 253/282] GA debug --- .github/workflows/test_libclouphxx.yml | 214 +++++++++++++------------ 1 file changed, 109 insertions(+), 105 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index c2b55ca8f..7ffab5268 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -81,105 +81,110 @@ jobs: path: build.tar # test jobs -# unit_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo", "Debug"] -# mpi: ["none", "mvapich2"] -# include: -# - mpi: "none" -# tag: "ubuntu_20_04_cuda_11_4" -# - mpi: "mvapich2" -# tag: "ubuntu_20_04_cuda_11_4_mvapich2" -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# tag: ${{ matrix.tag }} -# -# - name: Run unit tests -# working-directory: ${{github.workspace}}/build -# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) -# -# -# kinematic_2D_test: -# needs: build -# runs-on: ubuntu-20.04 -# -# strategy: -# matrix: -# build_type: ["RelWithDebInfo"] -# mpi: ["none"] -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Download libcloudph++ build -# uses: actions/download-artifact@v2 -# with: -# name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar -# -# - name: Decompress libcloudph++ build -# run: tar -xvf build.tar -# -# - name: Install libcloudph++ -# run: sudo cmake --install build -# -# - name: load UWLCM Singularity image -# #uses: igfuw/libcloudphxx_build@v0.1-beta -# uses: ./.github/actions/load_UWLCM_singularity_image -# with: -# path: ${{ github.workspace }}/singularity_images -# -# - name: checkout libmpdata++ repo -# uses: actions/checkout@v2 -# with: -# repository: igfuw/libmpdataxx -# path: libmpdataxx -# -# - name: Configure libmpdata++ CMake -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed -# -# - name: Build libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: VERBOSE=1 singularity exec $SI cmake --build build -# -# - name: Install libmpdata++ -# working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ -# run: sudo cmake --install build -# -# - name: Configure kinematic_2D CMake -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ -# -# -# - name: Build kinematic_2D -# working-directory: ${{github.workspace}}/models/kinematic_2D -# run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} -# -# - name: Run kinematic_2D tests -# working-directory: ${{github.workspace}}/models/kinematic_2D/build -# run: | -# singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) -# cat Testing/Temporary/LastTest.log + unit_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo", "Debug"] + mpi: ["none", "mvapich2"] + include: + - mpi: "none" + tag: "ubuntu_20_04_cuda_11_4" + - mpi: "mvapich2" + tag: "ubuntu_20_04_cuda_11_4_mvapich2" + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + tag: ${{ matrix.tag }} + + - name: Run unit tests + working-directory: ${{github.workspace}}/build + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code) + + + kinematic_2D_test: + needs: build + runs-on: ubuntu-20.04 + + strategy: + matrix: + build_type: ["RelWithDebInfo"] + mpi: ["none"] + + steps: + - uses: actions/checkout@v2 + + - name: Download libcloudph++ build + uses: actions/download-artifact@v2 + with: + name: libcloud_build_${{matrix.build_type}}_mpi_${{matrix.mpi}}_tar + + - name: Decompress libcloudph++ build + run: tar -xvf build.tar + + - name: Install libcloudph++ + run: sudo cmake --install build + + - name: load UWLCM Singularity image + #uses: igfuw/libcloudphxx_build@v0.1-beta + uses: ./.github/actions/load_UWLCM_singularity_image + with: + path: ${{ github.workspace }}/singularity_images + + - name: checkout libmpdata++ repo + uses: actions/checkout@v2 + with: + repository: igfuw/libmpdataxx + path: libmpdataxx + + - name: Configure libmpdata++ CMake + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed + + - name: Build libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: VERBOSE=1 singularity exec $SI cmake --build build + + - name: Install libmpdata++ + working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ + run: sudo cmake --install build + + - name: Configure kinematic_2D CMake + working-directory: ${{github.workspace}}/models/kinematic_2D + run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ + + + - name: Build kinematic_2D + working-directory: ${{github.workspace}}/models/kinematic_2D + run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}} + + - name: Run kinematic_2D tests + working-directory: ${{github.workspace}}/models/kinematic_2D/build + run: | + singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian) + cat Testing/Temporary/LastTest.log + + # Debugging with a ssh session on failure + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 parcel_test: @@ -236,12 +241,6 @@ jobs: run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test #run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test - - # Debugging with a ssh session if previous step failed - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - - name: run parcel long_test working-directory: ${{github.workspace}}/parcel if: ${{matrix.long_tests}} @@ -252,6 +251,11 @@ jobs: if: ${{matrix.debug_tests}} run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug + # Debugging with a ssh session on failure + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + build_and_test_KiD-A: runs-on: ubuntu-20.04 strategy: From 4632c3d65e520da95efc3f5405bcee75b1f6b176 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Oct 2021 16:34:31 +0200 Subject: [PATCH 254/282] GA debug --- .github/workflows/test_libclouphxx.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7ffab5268..e2c082145 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -305,3 +305,7 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh + + # Debugging with a ssh session + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 From e4f5d0f72145fa3c603939a77b9ce61a9d7627a7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Oct 2021 17:02:55 +0200 Subject: [PATCH 255/282] GA debug --- .github/workflows/test_libclouphxx.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index e2c082145..7ffab5268 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -305,7 +305,3 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh - - # Debugging with a ssh session - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 From 29c03b4d5b159256011c65b1ba36aef4f7e7d93f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Oct 2021 17:08:45 +0200 Subject: [PATCH 256/282] include whole boost odeint --- include/libcloudph++/blk_1m/extincl.hpp | 9 +++++---- src/impl/particles_impl.ipp | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/libcloudph++/blk_1m/extincl.hpp b/include/libcloudph++/blk_1m/extincl.hpp index 76f3dba49..11e8338d7 100644 --- a/include/libcloudph++/blk_1m/extincl.hpp +++ b/include/libcloudph++/blk_1m/extincl.hpp @@ -2,10 +2,11 @@ #include -#include -#include -#include -#include +#include +//#include +//#include +//#include +//#include #include "../common/const_cp.hpp" #include "../common/theta_dry.hpp" diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 4eab706a9..a45a69944 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -10,9 +10,10 @@ #include #include -#include +#include +//#include +//#include #include -#include #include #include From 4898fe2c288dafeb94c1210425824783a0bc23b6 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 5 Nov 2021 11:15:14 +0100 Subject: [PATCH 257/282] libcloud build and singu load actions from github marketplace --- .github/actions/libcloudphxx_build/action.yml | 99 ------------------- .../load_UWLCM_singularity_image/action.yml | 51 ---------- .github/workflows/test_libclouphxx.yml | 18 ++-- 3 files changed, 6 insertions(+), 162 deletions(-) delete mode 100644 .github/actions/libcloudphxx_build/action.yml delete mode 100644 .github/actions/load_UWLCM_singularity_image/action.yml diff --git a/.github/actions/libcloudphxx_build/action.yml b/.github/actions/libcloudphxx_build/action.yml deleted file mode 100644 index 8a6ff5902..000000000 --- a/.github/actions/libcloudphxx_build/action.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: 'build libcloudph++' -description: 'builds libcloudph++, loads UWLCM Singularity image for dependencies' -inputs: - disable_cuda: - description: 'disable CUDA build?' - required: true - default: "false" - build_type: - description: 'CMake build type: Release, Debug, ...' - required: true - default: "Release" - threads: - description: "number of threads used by make" - required: true - default: 1 - path: - description: "path to the dir with libcloudphxx code" - required: true - install_prefix: - description: "CMake install prefix" - required: true - default: "/usr/local" - tag: - description: 'Singularity image tag' - required: true - default: 'ubuntu_20_04_cuda_11_4' - cxx: - description: 'C++ compiler' - required: true - default: 'g++' - -runs: - using: "composite" - steps: - - name: Install Nvidia driver - shell: bash - run: | - [ "${{inputs.disable_cuda}}" = "false" ] && sudo apt install --no-install-recommends nvidia-driver-470 || true - # if: ${{!inputs.disable_cuda}} - - - name: load UWLCM Singularity image - uses: ./.github/actions/load_UWLCM_singularity_image - with: - path: ${{ inputs.path }}/singularity_images - tag: ${{ inputs.tag }} - -# - name: Install Singularity -# #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) -# uses: eWaterCycle/setup-singularity@v6 -# with: -# singularity-version: 3.7.1 -# #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 -## run: | -## wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -## sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -## sudo apt-get update -## sudo apt-get install -y singularity-container -# -# - name: Cache UWLCM Singularity image -# id: cache_singularity -# uses: actions/cache@v2 -# with: -# path: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' -# key: 'sng_ubuntu_20_04_cuda_11_4' -# -# - name: Download UWLCM Singularity image -# shell: bash -## if: steps.cache_singularity.outputs.cache-hit != 'true' -# run: | -# [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ -# { -# mkdir '${{ github.workspace }}/singularity_images'; -# singularity pull --disable-cache --dir '${{ github.workspace }}/singularity_images' library://pdziekan/default/uwlcm:ubuntu_20_04_cuda_11_4; -# } \ -# || \ -# true -# # disable Singularity cache, we cache manually -# -# - name: Set friendly Singularity image name -# uses: allenevans/set-env@v2.0.0 -# with: -# SI: '${{ github.workspace }}/singularity_images/uwlcm_ubuntu_20_04_cuda_11_4.sif' - - - # Debugging with a ssh session -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 - - - - name: Configure CMake - shell: bash - # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: singularity exec $SI cmake -B ${{inputs.path}}/build -DCMAKE_BUILD_TYPE=${{inputs.build_type}} -DLIBCLOUDPHXX_FORCE_MULTI_CUDA=True -DLIBCLOUDPHXX_DISABLE_CUDA=${{inputs.disable_cuda}} -DCMAKE_INSTALL_PREFIX=${{inputs.install_prefix}} -DCMAKE_CXX_COMPILER=${{inputs.cxx}} -# - - name: Build libcloudph++ - shell: bash - # Build your program with the given configuration - run: VERBOSE=1 singularity exec $SI cmake --build ${{inputs.path}}/build --config ${{inputs.build_type}} -j${{inputs.threads}} diff --git a/.github/actions/load_UWLCM_singularity_image/action.yml b/.github/actions/load_UWLCM_singularity_image/action.yml deleted file mode 100644 index f3876e8e5..000000000 --- a/.github/actions/load_UWLCM_singularity_image/action.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: 'load UWLCM Singularity image' -description: 'set env var SI to point to the image location' -inputs: - path: - description: 'Where to put the image' - required: true - tag: - description: 'Image tag' - required: true - default: 'ubuntu_20_04_cuda_11_4' - -runs: - using: "composite" - steps: - - - name: Install Singularity - #when installed from this action, .SIF is always converted to sandbox (could be related to: https://githubmemory.com/repo/hpcng/singularity/issues/6065) - uses: eWaterCycle/setup-singularity@v7 - with: - singularity-version: 3.8.3 - #apt installation following https://sylabs.io/guides/3.0/user-guide/installation.html, but this is a too old version and uninstalls python-is-python3 -# run: | -# wget -O- http://neuro.debian.net/lists/focal.de-fzj.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -# sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 -# sudo apt-get update -# sudo apt-get install -y singularity-container - - - name: Cache UWLCM Singularity image - id: cache_singularity - uses: actions/cache@v2 - with: - path: '${{ inputs.path }}/uwlcm_${{ inputs.tag }}.sif' - key: '${{ inputs.tag }}' - - - name: Download UWLCM Singularity image - shell: bash -# if: steps.cache_singularity.outputs.cache-hit != 'true' - run: | - [ "${{steps.cache_singularity.outputs.cache-hit}}" != "true" ] && \ - { - mkdir '${{ inputs.path }}'; - singularity pull --disable-cache --dir '${{ inputs.path }}' library://pdziekan/default/uwlcm:${{ inputs.tag }}; - } \ - || \ - true - # disable Singularity cache, we cache manually - - - name: Set friendly Singularity image name - uses: allenevans/set-env@v2.0.0 - with: - SI: '${{ inputs.path }}/uwlcm_${{ inputs.tag }}.sif' diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 7ffab5268..a0e026d7d 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -23,8 +23,7 @@ jobs: - uses: actions/checkout@v2 - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build + uses: igfuw/libcloudphxx_build@v0.2-beta with: disable_cuda: ${{matrix.disable_cuda}} build_type: ${{matrix.build_type}} @@ -59,8 +58,7 @@ jobs: # uses: mxschmitt/action-tmate@v3 - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build + uses: igfuw/libcloudphxx_build@v0.2-beta with: disable_cuda: ${{matrix.disable_cuda}} build_type: ${{matrix.build_type}} @@ -107,8 +105,7 @@ jobs: run: tar -xvf build.tar - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image + uses: igfuw/load_UWLCM_singularity_image@v0.1 with: path: ${{ github.workspace }}/singularity_images tag: ${{ matrix.tag }} @@ -143,8 +140,7 @@ jobs: run: sudo cmake --install build - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image + uses: igfuw/load_UWLCM_singularity_image@v0.1 with: path: ${{ github.workspace }}/singularity_images @@ -219,8 +215,7 @@ jobs: run: tar -xvf build.tar - name: load UWLCM Singularity image - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/load_UWLCM_singularity_image + uses: igfuw/load_UWLCM_singularity_image@v0.1 with: path: ${{ github.workspace }}/singularity_images @@ -284,8 +279,7 @@ jobs: - run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp - name: build libcloudph++ - #uses: igfuw/libcloudphxx_build@v0.1-beta - uses: ./.github/actions/libcloudphxx_build + uses: igfuw/libcloudphxx_build@v0.2-beta with: disable_cuda: ${{matrix.disable_cuda}} build_type: ${{matrix.build_type}} From a4116d89cd27b5c0d11b27aad1ab2bb7cdb7170f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 17 Nov 2021 14:03:10 +0100 Subject: [PATCH 258/282] GA: libmpdata++ install from marketplace --- .github/workflows/test_libclouphxx.yml | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index a0e026d7d..308667e3a 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -139,28 +139,20 @@ jobs: - name: Install libcloudph++ run: sudo cmake --install build - - name: load UWLCM Singularity image - uses: igfuw/load_UWLCM_singularity_image@v0.1 - with: - path: ${{ github.workspace }}/singularity_images - - name: checkout libmpdata++ repo uses: actions/checkout@v2 with: repository: igfuw/libmpdataxx path: libmpdataxx - - name: Configure libmpdata++ CMake - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: singularity exec $SI cmake -B build -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed - - - name: Build libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: VERBOSE=1 singularity exec $SI cmake --build build - - name: Install libmpdata++ - working-directory: ${{github.workspace}}/libmpdataxx/libmpdata++ - run: sudo cmake --install build + uses: igfuw/libmpdataxx_install@v0.1 + with: + build_type: ${{matrix.build_type}} + threads: 4 + path: ${{ github.workspace }}/libmpdataxx/libmpdata++ + install_prefix: ${{ github.workspace }}/installed + tag: ${{ matrix.tag }} - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}/models/kinematic_2D From 33506f7a7cb9e86fb91c496d2f91731f851123c0 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 17 Nov 2021 14:52:54 +0100 Subject: [PATCH 259/282] GA: libmpdata++ install, remove tag --- .github/workflows/test_libclouphxx.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 308667e3a..e766243be 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -152,7 +152,6 @@ jobs: threads: 4 path: ${{ github.workspace }}/libmpdataxx/libmpdata++ install_prefix: ${{ github.workspace }}/installed - tag: ${{ matrix.tag }} - name: Configure kinematic_2D CMake working-directory: ${{github.workspace}}/models/kinematic_2D From 7eeeb40332ff82d3f9d0fc19ad04aac3d38fa4bc Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 17 Nov 2021 15:18:51 +0100 Subject: [PATCH 260/282] GA: libmpdata_install v0.2 --- .github/workflows/test_libclouphxx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index e766243be..24ccc2567 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -146,7 +146,7 @@ jobs: path: libmpdataxx - name: Install libmpdata++ - uses: igfuw/libmpdataxx_install@v0.1 + uses: igfuw/libmpdataxx_install@v0.2 with: build_type: ${{matrix.build_type}} threads: 4 From 214a7a44f50d0ce6ff2cb4c2b19866167611d7d9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 18 Nov 2021 11:31:36 +0100 Subject: [PATCH 261/282] GA: libmpdata_install v0.3 + no ssh debug --- .github/workflows/test_libclouphxx.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 24ccc2567..2d95d0be7 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -146,7 +146,7 @@ jobs: path: libmpdataxx - name: Install libmpdata++ - uses: igfuw/libmpdataxx_install@v0.2 + uses: igfuw/libmpdataxx_install@v0.3 with: build_type: ${{matrix.build_type}} threads: 4 @@ -169,9 +169,9 @@ jobs: cat Testing/Temporary/LastTest.log # Debugging with a ssh session on failure - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 +# - name: Setup tmate session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 parcel_test: @@ -237,11 +237,6 @@ jobs: if: ${{matrix.debug_tests}} run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug - # Debugging with a ssh session on failure - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - build_and_test_KiD-A: runs-on: ubuntu-20.04 strategy: From eea756cefc554d63281b3656d8dee7d63519bf8c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 18 Nov 2021 12:17:58 +0100 Subject: [PATCH 262/282] GA: ssh debug --- .github/workflows/test_libclouphxx.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 2d95d0be7..271c1ae97 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -53,10 +53,6 @@ jobs: steps: - uses: actions/checkout@v2 - # Debugging with a ssh session -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 - - name: build libcloudph++ uses: igfuw/libcloudphxx_build@v0.2-beta with: @@ -136,6 +132,10 @@ jobs: - name: Decompress libcloudph++ build run: tar -xvf build.tar + # Debugging with a ssh session + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Install libcloudph++ run: sudo cmake --install build From 1512a4345997f616155ebe7b4ec886101e8bf873 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 18 Nov 2021 12:37:28 +0100 Subject: [PATCH 263/282] GA: new install libmpdata++ action version --- .github/workflows/test_libclouphxx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 271c1ae97..5ff8c0a4c 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -133,8 +133,8 @@ jobs: run: tar -xvf build.tar # Debugging with a ssh session - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 - name: Install libcloudph++ run: sudo cmake --install build @@ -146,7 +146,7 @@ jobs: path: libmpdataxx - name: Install libmpdata++ - uses: igfuw/libmpdataxx_install@v0.3 + uses: igfuw/libmpdataxx_install@v0.4 with: build_type: ${{matrix.build_type}} threads: 4 From f8f6ea28bbb063323d915aed7c786f997ff8038d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 13 Dec 2021 20:10:42 +0100 Subject: [PATCH 264/282] GA: test uwlcm --- .github/workflows/test_libclouphxx.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 5ff8c0a4c..ccb3ed43b 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -285,3 +285,10 @@ jobs: - name: run KiD LWP test working-directory: ${{github.workspace}}/kid-libcloud run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh + + call_test_uwlcm_hlpr: + uses: igfuw/UWLCM/.github/workflows/test_uwlcm_hlpr.yml@master + with: + UWLCM_sha: "master" + libcloudphxx_sha: ${{ github.sha }} # merge PR SHA + libmpdataxx_sha: "master" From f1e3d6dcd43b8662aa03839197837aa6e1705b1d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 14 Dec 2021 16:20:30 +0100 Subject: [PATCH 265/282] add the relwithdebinfoportable config --- CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03bd4634c..4b47d5d98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ ENDIF(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE) set(CMAKE_CXX_FLAGS_DEBUG "") set(CMAKE_CXX_FLAGS_RELEASE "") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFOPORTABLE "") # without march=native ############################################################################################ # see if CUDA is available @@ -61,6 +62,7 @@ else() set(CMAKE_CUDA_FLAGS_DEBUG "") set(CMAKE_CUDA_FLAGS_RELEASE "") set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "") + set(CMAKE_CUDA_FLAGS_RELWITHDEBINFOPORTABLE "") endif() ############################################################################################# @@ -114,9 +116,10 @@ target_include_directories(cloudphxx_lgrngn # enabling c++14 target_compile_features(cloudphxx_lgrngn PUBLIC cxx_std_14) # config-specific flags -target_compile_options(cloudphxx_lgrngn PRIVATE $<$,$>: -g -Og -DTHRUST_DEBUG>) -target_compile_options(cloudphxx_lgrngn PRIVATE $<$,$>: -Ofast -march=native>) -target_compile_options(cloudphxx_lgrngn PRIVATE $<$,$>: -DNDEBUG -Ofast -march=native -Winline>) +target_compile_options(cloudphxx_lgrngn PRIVATE $<$,$>: -g -Og -DTHRUST_DEBUG>) +target_compile_options(cloudphxx_lgrngn PRIVATE $<$,$>: -Ofast -march=native -Winline -DNDEBUG >) +target_compile_options(cloudphxx_lgrngn PRIVATE $<$,$>: -Ofast -march=native>) +target_compile_options(cloudphxx_lgrngn PRIVATE $<$,$>: -Ofast>) # add dependencies if (CMAKE_CUDA_COMPILER) @@ -363,7 +366,7 @@ add_subdirectory(bindings) ############################################################################################ # installation -set_target_properties(cloudphxx_lgrngn PROPERTIES DEBUG_POSTFIX _dbg RELWITHDEBINFO_POSTFIX _relwithdbg) +set_target_properties(cloudphxx_lgrngn PROPERTIES DEBUG_POSTFIX _dbg RELWITHDEBINFO_POSTFIX _relwithdbg RELWITHDEBINFOPORTABLE_POSTFIX _relwithdbg) # install files and export targets install(TARGETS cloudphxx_lgrngn EXPORT cloudphxx_lgrngn-targets From 08cb43cc81e88f69042ae29e6649d0b4a100bb45 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 14 Dec 2021 17:00:15 +0100 Subject: [PATCH 266/282] GA: Portable version of the Relwithdebinfo config --- .github/workflows/test_libclouphxx.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index ccb3ed43b..debd04cb7 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: name: ["CUDA"] - build_type: ["Debug", "RelWithDebInfo"] + build_type: ["Debug", "RelWithDebInfoPortable"] include: - name: "CUDA" disable_cuda: false @@ -35,8 +35,7 @@ jobs: strategy: matrix: name: ["no_CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - #build_type: ["RelWithDebInfo"] + build_type: ["Debug", "RelWithDebInfoPortable"] mpi: ["none", "mvapich2"] #mpi: ["none"] include: @@ -81,7 +80,7 @@ jobs: strategy: matrix: - build_type: ["RelWithDebInfo", "Debug"] + build_type: ["Debug", "RelWithDebInfoPortable"] mpi: ["none", "mvapich2"] include: - mpi: "none" @@ -118,7 +117,7 @@ jobs: strategy: matrix: - build_type: ["RelWithDebInfo"] + build_type: ["RelWithDebInfoPortable"] mpi: ["none"] steps: @@ -182,11 +181,10 @@ jobs: strategy: matrix: - build_type: ["RelWithDebInfo", "Debug"] - # build_type: ["RelWithDebInfo"] + build_type: ["Debug", "RelWithDebInfoPortable"] mpi: ["none"] include: - - build_type: "RelWithDebInfo" + - build_type: "RelWithDebInfoPortable" long_tests: true debug_tests: false - build_type: "Debug" From 5f285e261af8d26b8752f58a9e4ec0f42198849b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 21 Dec 2021 16:13:32 +0100 Subject: [PATCH 267/282] remove debug output in SD relax --- bindings/python/lgrngn.hpp | 4 -- src/impl/particles_impl_rlx_dry_distros.ipp | 78 --------------------- 2 files changed, 82 deletions(-) diff --git a/bindings/python/lgrngn.hpp b/bindings/python/lgrngn.hpp index 08a93883e..58c1fe84e 100644 --- a/bindings/python/lgrngn.hpp +++ b/bindings/python/lgrngn.hpp @@ -332,18 +332,14 @@ namespace libcloudphxx // loop over kappas for (int i = 0; i < len(kappa_func.keys()); ++i) { - std::cerr << i << std::endl; const real_t kappa = bp::extract(kappa_func.keys()[i]); - std::cerr << kappa << std::endl; const bp::list nlnrd_kparange_zrange_list = bp::extract(kappa_func.values()[i]); assert(len(nlnrd_kparange_zrange_list) == 3); auto nlnrd = std::make_shared>(nlnrd_kparange_zrange_list[0]); const bp::list kparange_list = bp::extract(nlnrd_kparange_zrange_list[1]); std::pair kparange{bp::extract(kparange_list[0]), bp::extract(kparange_list[1])}; - std::cerr << kparange.first << " " << kparange.second << std::endl; const bp::list zrange_list = bp::extract(nlnrd_kparange_zrange_list[2]); std::pair zrange{bp::extract(zrange_list[0]), bp::extract(zrange_list[1])}; - std::cerr << zrange.first << " " << zrange.second << std::endl; arg->rlx_dry_distros[kappa] = std::make_tuple(nlnrd, kparange, zrange); } } diff --git a/src/impl/particles_impl_rlx_dry_distros.ipp b/src/impl/particles_impl_rlx_dry_distros.ipp index dd93027d5..528b9eca7 100644 --- a/src/impl/particles_impl_rlx_dry_distros.ipp +++ b/src/impl/particles_impl_rlx_dry_distros.ipp @@ -44,10 +44,6 @@ namespace libcloudphxx BOOST_GPU_ENABLED int operator()(const real_t &a1, const real_t &a2) { -// printf("missing %g expected %g tolerance %g result %d\n", a1, a2, tolerance, a2 > real_t(0) ? -// a1 / a2 > tolerance ? 1 : 0 -// : 0); -// //std::cerr << "a1: " << a1 << " a2: " << a2 << " tol: " << tolerance << " result: " << bool(a1 / a2 > tolerance) << std::endl; return a2 > real_t(0) ? a1 / a2 > tolerance ? n_sd_per_bin : 0 : 0; @@ -118,7 +114,6 @@ namespace libcloudphxx for (typename opts_init_t::rlx_dry_distros_t::const_iterator ddi = opts_init.rlx_dry_distros.begin(); ddi != opts_init.rlx_dry_distros.end(); ++ddi) { const auto &kappa(ddi->first); - std::cerr << "rlx kappa: " << kappa << std::endl; assert(kappa >= 0); const auto &n_of_lnrd_stp(*(std::get<0>(ddi->second))); @@ -168,38 +163,15 @@ namespace libcloudphxx // calculate 0-th non-specific moment of rd3 (number of droplets in a cell) of droplets in this rd3 and kappa range moms_calc(rd3.begin(), n_part_pre_relax, 0, false); -// // divide by volume -// thrust::transform( -// count_mom.begin(), count_mom.begin() + count_n, // input - first arg -// thrust::make_permutation_iterator( // input - second arg -// dv.begin(), -// count_ijk.begin() -// ), -// count_mom.begin(), // output (in place) -// thrust::divides() -// ); - // horizontal sum of this moment thrust::fill(hor_sum.begin(), hor_sum.end(), 0); thrust_device::vector &count_k(tmp_device_size_cell); // NOTE: tmp_device_size_cell is also used in some other inits, careful not to overwrite it! thrust::transform(count_ijk.begin(), count_ijk.begin() + count_n, count_k.begin(), arg::_1 % opts_init.nz); thrust::sort_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin()); - //std::cerr << "count_n: " << count_n << std::endl; - //std::cerr << "count_k:" << std::endl; - //debug::print(count_k.begin(), count_k.end()); - //std::cerr << "count_mom:" << std::endl; - //debug::print(count_mom.begin(), count_mom.end()); - auto new_end = thrust::reduce_by_key(count_k.begin(), count_k.begin() + count_n, count_mom.begin(), hor_sum_k.begin(), hor_sum_count.begin()); - std::cerr << "hor_sum_k:" << std::endl; - debug::print(hor_sum_k.begin(), hor_sum_k.end()); - std::cerr << "hor_sum_count:" << std::endl; - debug::print(hor_sum_count.begin(), hor_sum_count.end()); - int number_of_levels_with_droplets = new_end.first - hor_sum_k.begin(); // number of levels with any SD, not with SD in this size and kappa range - std::cerr << "number_of_levels_with_droplets: " << number_of_levels_with_droplets << std::endl; assert(number_of_levels_with_droplets <= opts_init.nz); thrust::copy(hor_sum_count.begin(), hor_sum_count.begin() + number_of_levels_with_droplets, thrust::make_permutation_iterator(hor_sum.begin(), hor_sum_k.begin())); @@ -232,37 +204,17 @@ namespace libcloudphxx // set to zero outside of the defined range of altitudes thrust::replace_if(expected_hor_sum.begin(), expected_hor_sum.begin()+opts_init.nz, zero, arg::_1 < z_min_index || arg::_1 >= z_max_index, real_t(0)); - std::cerr << "bin number: " << bin_number ; - std::cerr << " rd_range: (" << std::pow(rd3_min, 1./3.) << ", " << std::pow(rd3_max, 1./3.) ; - std::cerr << " r_center: " << std::exp(bin_lnrd_center) ; - std::cerr << " z_indices: (" << z_min_index << ", " << z_max_index << "), " ; - std::cerr << " expected STP concentration: " << expected_STP_concentration ; - std::cerr << std::endl; - - std::cerr << "hor_sum:" << std::endl; - debug::print(hor_sum); - - std::cerr << "expected_hor_sum:" << std::endl; - debug::print(expected_hor_sum); // calculate how many CCN are missing thrust::transform(expected_hor_sum.begin(), expected_hor_sum.end(), hor_sum.begin(), hor_missing.begin(), arg::_1 - arg::_2); thrust::replace_if(hor_missing.begin(), hor_missing.end(), arg::_1 < 0, 0); - std::cerr << "hor_missing:" << std::endl; - debug::print(hor_missing); - // set number of SDs to init; create only if concentration is lower than expected with a tolerance thrust::transform(hor_missing.begin(), hor_missing.end(), expected_hor_sum.begin(), n_SD_to_create.begin(), detail::calc_n_sd_to_create(config.rlx_conc_tolerance, opts_init.rlx_sd_per_bin)); - - std::cerr << "n_SD_to_create:" << std::endl; - debug::print(n_SD_to_create); n_part_old = n_part; n_part_to_init = thrust::reduce(n_SD_to_create.begin(), n_SD_to_create.end()); n_part = n_part_old + n_part_to_init; - std::cerr << "n_part_to_init: " << n_part_to_init << std::endl; - // resize arrays set in the bins loop: cell indices and rd3, resize should be cheap, because we allocate a large chunk of memory at the start ijk.resize(n_part); i.resize(n_part); @@ -291,8 +243,6 @@ namespace libcloudphxx using std::min; #endif - std::cerr << "dt: " << dt << " rlx_timescale: " << opts_init.rlx_timescale << " hor_avg fraction added: " << min(dt / opts_init.rlx_timescale, real_t(1)) << std::endl; - thrust::for_each( thrust::make_zip_iterator(thrust::make_tuple( n_SD_to_create.begin(), ptr.begin(), @@ -305,9 +255,6 @@ namespace libcloudphxx detail::arbitrary_sequence(&(n[n_part_old])) ); - //std::cerr << "n:" << std::endl; - //debug::print(n.begin()+n_part_old, n.end()); - // detecting possible overflows of n type { thrust_size_t ix = thrust::max_element(n.begin() + n_part_old, n.end()) - (n.begin() + n_part_old); @@ -318,31 +265,12 @@ namespace libcloudphxx // tossing random numbers [0,1) TODO: do it once for all bins rand_u01(n_part_to_init * (n_dims)); // random numbers for: i, rd, j (j only in 3D) - //std::cerr << "u01:" << std::endl; - //debug::print(u01.begin(), u01.begin()+n_part_to_init); - thrust::transform(u01.begin(), u01.begin() + n_part_to_init, i.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.nx)); if(n_dims==3) thrust::transform(u01.begin() + 2*n_part_to_init, u01.begin() + 3*n_part_to_init, j.begin() + n_part_old, detail::multiply_by_constant_and_cast(opts_init.ny)); // raveling i, j & k into ijk; only of the new SD ravel_ijk(n_part_old); - // set count_num to the number of SD to init per cell -// thrust::fill(count_num.begin(), count_num.end(), 0); -// thrust::scatter(thrust::make_constant_iterator(opts_init.rlx_sd_per_bin), thrust::make_constant_iterator(opts_init.rlx_sd_per_bin) + n_part_to_init, ijk.begin() + n_part_old, count_num.begin()); - - //std::cerr << "i:" << std::endl; - //debug::print(i.begin()+n_part_old, i.end()); - - //std::cerr << "k:" << std::endl; - //debug::print(k.begin()+n_part_old, k.end()); - - //std::cerr << "ijk:" << std::endl; - //debug::print(ijk.begin()+n_part_old, ijk.end()); -// -// //std::cerr << "count_num:" << std::endl; -// //debug::print(count_num); - // init dry radius // set rd3 randomized within the bin, uniformly distributed on the log(rd) axis const real_t lnrd_min = std::log(std::pow(rd3_min, 1./3.)); @@ -356,18 +284,12 @@ namespace libcloudphxx detail::exp3x() ); - - //std::cerr << "rd3:" << std::endl; - //debug::print(rd3.begin()+n_part_old, rd3.end()); // NOTE: watch out not to mess up sorting while adding SDs to the bins, because moms_X functions require sorted data... } // end of the bins loop - std::cerr << "expected_STP_concentration_tot: " << expected_STP_concentration_tot << std::endl; - // init other SD characteristics that don't have to be initialized in the bins loop n_part_old = n_part_pre_bins_loop; n_part_to_init = n_part - n_part_old; - //std::cerr << "n_part: " << n_part << " n_part_old: " << n_part_old << " n_part_to_init: " << n_part_to_init << std::endl; hskpng_resize_npart(); init_SD_with_distros_finalize(kappa, false); // no need to unravel ijk there, becaues i j k are already initialized From 6dc61967c92d867fea5bddbb24278dfc36f5ff36 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 21 Dec 2021 16:16:12 +0100 Subject: [PATCH 268/282] comment about ante/post_adding_SD() --- src/impl/particles_impl.ipp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index c7754964d..f15bd2c6b 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -593,6 +593,8 @@ namespace libcloudphxx void post_copy(const opts_t&); + // two functions for calculating changes in rv and th due to condensation on SDs initialized during simulation, e.g. via source or relaxation + // NOTE: curently not used, because of small sizes of these droplets void ante_adding_SD(); void post_adding_SD(); From d93234f669ef32330eb5355e3de7f2e96a554337 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 27 Dec 2021 14:37:55 +0100 Subject: [PATCH 269/282] add a target for headers only --- CMakeLists.txt | 33 ++++++++++++++++++++++++++------- libcloudph++-config.cmake.in | 4 ++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b47d5d98..00d9fa58a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,8 +97,8 @@ endif() set(CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) # mpic++ will not work ############################################################################################ -# create the library target -add_library(cloudphxx_lgrngn SHARED ${files}) +# create the lagrangian microphysics library target +add_library(cloudphxx_lgrngn SHARED ${files}) ############################################################################################ @@ -364,22 +364,42 @@ add_subdirectory(tests) add_subdirectory(include) add_subdirectory(bindings) +############################################################################################ + +# create a header-only library target (enough for blk_1m blk_2m and common) +add_library(cloudphxx_headers INTERFACE) + +target_include_directories(cloudphxx_headers + INTERFACE + $ +) + +target_compile_features(cloudphxx_headers INTERFACE cxx_std_14) + +target_link_libraries(cloudphxx_headers + INTERFACE + Boost::boost +) + ############################################################################################ # installation set_target_properties(cloudphxx_lgrngn PROPERTIES DEBUG_POSTFIX _dbg RELWITHDEBINFO_POSTFIX _relwithdbg RELWITHDEBINFOPORTABLE_POSTFIX _relwithdbg) # install files and export targets -install(TARGETS cloudphxx_lgrngn EXPORT cloudphxx_lgrngn-targets +install(TARGETS cloudphxx_lgrngn cloudphxx_headers EXPORT cloudphxx-targets LIBRARY DESTINATION lib INCLUDES DESTINATION include) -install(EXPORT cloudphxx_lgrngn-targets NAMESPACE clphxx:: DESTINATION share/libcloudph++) +install(EXPORT cloudphxx-targets NAMESPACE clphxx:: DESTINATION share/libcloudph++) + +#install(TARGETS cloudphxx_headers EXPORT cloudphxx_headers-targets) +# INCLUDES DESTINATION include) # these files were already installed by lgrngn + +#install(EXPORT cloudphxx_headers-targets NAMESPACE clphxx:: DESTINATION share/libcloudph++) # generate and install a config file include(CMakePackageConfigHelpers) -#export(TARGETS cloudphxx_lgrngn FILE MyLibraryConfig.cmake) - configure_package_config_file( libcloudph++-config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/libcloudph++-config.cmake" @@ -392,4 +412,3 @@ install( DESTINATION share/libcloudph++ ) -############################################################################################ diff --git a/libcloudph++-config.cmake.in b/libcloudph++-config.cmake.in index 1bf38e806..3011c7529 100644 --- a/libcloudph++-config.cmake.in +++ b/libcloudph++-config.cmake.in @@ -5,6 +5,6 @@ include(CMakeFindDependencyMacro) find_dependency(Boost REQUIRED) -if(NOT TARGET clphxx::cloudphxx_lgrngn) - include("${cloudphxx_CMAKE_DIR}/cloudphxx_lgrngn-targets.cmake") +if(NOT TARGET clphxx::cloudphxx_lgrngn OR NOT TARGET clphxx::cloudphxx_headers) + include("${cloudphxx_CMAKE_DIR}/cloudphxx-targets.cmake") endif() From 2612dcefb58c2cd40e80c3f6a1ef058b272f90ff Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 27 Dec 2021 20:37:31 +0100 Subject: [PATCH 270/282] GA: relwithdebinfo -> relwithdebinfoportable --- .github/workflows/test_libclouphxx.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_libclouphxx.yml b/.github/workflows/test_libclouphxx.yml index 5ff8c0a4c..6f76ea191 100644 --- a/.github/workflows/test_libclouphxx.yml +++ b/.github/workflows/test_libclouphxx.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: name: ["CUDA"] - build_type: ["Debug", "RelWithDebInfo"] + build_type: ["Debug", "RelWithDebInfoPortable"] include: - name: "CUDA" disable_cuda: false @@ -35,8 +35,8 @@ jobs: strategy: matrix: name: ["no_CUDA"] - build_type: ["Debug", "RelWithDebInfo"] - #build_type: ["RelWithDebInfo"] + build_type: ["Debug", "RelWithDebInfoPortable"] + #build_type: ["RelWithDebInfoPortable"] mpi: ["none", "mvapich2"] #mpi: ["none"] include: @@ -81,7 +81,7 @@ jobs: strategy: matrix: - build_type: ["RelWithDebInfo", "Debug"] + build_type: ["RelWithDebInfoPortable", "Debug"] mpi: ["none", "mvapich2"] include: - mpi: "none" @@ -118,7 +118,7 @@ jobs: strategy: matrix: - build_type: ["RelWithDebInfo"] + build_type: ["RelWithDebInfoPortable"] mpi: ["none"] steps: @@ -182,11 +182,11 @@ jobs: strategy: matrix: - build_type: ["RelWithDebInfo", "Debug"] - # build_type: ["RelWithDebInfo"] + build_type: ["RelWithDebInfoPortable", "Debug"] + # build_type: ["RelWithDebInfoPortable"] mpi: ["none"] include: - - build_type: "RelWithDebInfo" + - build_type: "RelWithDebInfoPortable" long_tests: true debug_tests: false - build_type: "Debug" @@ -242,7 +242,7 @@ jobs: strategy: matrix: name: ["no_CUDA"] - build_type: ["RelWithDebInfo"] + build_type: ["RelWithDebInfoPortable"] mpi: ["none"] include: - name: "no_CUDA" From e420ffa2b6f5ca8645900e55dffa9739c99b1014 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 Jan 2022 11:22:44 +0100 Subject: [PATCH 271/282] new src options definition: off, simple, matching --- include/libcloudph++/lgrngn/ccn_source.hpp | 14 ++++++++++ include/libcloudph++/lgrngn/opts_init.hpp | 6 +++-- src/impl/particles_impl_init_sanity_check.ipp | 26 +++++++++---------- src/particles_step.ipp | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 include/libcloudph++/lgrngn/ccn_source.hpp diff --git a/include/libcloudph++/lgrngn/ccn_source.hpp b/include/libcloudph++/lgrngn/ccn_source.hpp new file mode 100644 index 000000000..67b3c90d3 --- /dev/null +++ b/include/libcloudph++/lgrngn/ccn_source.hpp @@ -0,0 +1,14 @@ +#pragma once + +namespace libcloudphxx +{ + namespace lgrngn + { +//+ enum class src_t { off, simple, matching }; +// + // off - no source + // simple - src_dry_distros: new SD are added; src_dry_sizes: new SD are added + // matching - src_dry_distros: find similar SD and increase their multiplicity. Add new SD if match not found; src_dry_sizes: new SD are added + }; +}; diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index 1348e0cc3..d3b584789 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -91,7 +91,6 @@ namespace libcloudphxx coal_switch, // if false no coalescence throughout the whole simulation sedi_switch, // if false no sedimentation throughout the whole simulation subs_switch, // if false no subsidence throughout the whole simulation - src_switch, // if false no source throughout the whole simulation rlx_switch, // if false no relaxation throughout the whole simulation turb_adve_switch, // if true, turbulent motion of SDs is modeled turb_cond_switch, // if true, turbulent condensation of SDs is modeled @@ -136,6 +135,9 @@ namespace libcloudphxx // --- aerosol source stuff --- + // type of CCN source + src_t src_type; + // source distro per unit time dry_distros_t src_dry_distros; @@ -198,7 +200,7 @@ namespace libcloudphxx sedi_switch(true), // sedimentation turned on by default subs_switch(false), // subsidence turned off by default coal_switch(true), // coalescence turned on by default - src_switch(false), // source turned off by default + src_type(src_t::off), // source turned off by default rlx_switch(false), exact_sstp_cond(false), turb_cond_switch(false), diff --git a/src/impl/particles_impl_init_sanity_check.ipp b/src/impl/particles_impl_init_sanity_check.ipp index df81f76b7..5d309dafa 100644 --- a/src/impl/particles_impl_init_sanity_check.ipp +++ b/src/impl/particles_impl_init_sanity_check.ipp @@ -54,36 +54,34 @@ namespace libcloudphxx if (!opts_init.chem_switch && ambient_chem.size() != 0) throw std::runtime_error("chemistry was switched off and ambient_chem is not empty"); - // TODO: in source match chemical composition - if (opts_init.chem_switch && opts_init.src_switch) + if (opts_init.chem_switch && opts_init.src_type!=src_t::off) throw std::runtime_error("chemistry and aerosol source are not compatible"); - if (opts_init.src_switch && opts_init.src_dry_distros.empty() && opts_init.src_dry_sizes.empty()) - throw std::runtime_error("src_switch==True, but src_dry_distros and src_dry_sizes are empty"); + if (opts_init.src_type!=src_t::off && opts_init.src_dry_distros.empty() && opts_init.src_dry_sizes.empty()) + throw std::runtime_error("CCN source enabled, but src_dry_distros and src_dry_sizes are empty"); - if (opts_init.src_switch && opts_init.src_dry_distros.size() > 1) + if (opts_init.src_type!=src_t::off && opts_init.src_dry_distros.size() > 1) throw std::runtime_error("src_dry_distros can only have a single kappa value."); - if (opts_init.src_switch && n_dims<2) - throw std::runtime_error("src_switch==True and n_dims<2. Source only works in 2D and 3D."); + if (opts_init.src_type==src_t::matching && opts_init.dry_distros.size() > 1) + throw std::runtime_error("For 'matching' CCN source, the initial aerosol distribution can only have one kappa value (na kappa matching done)."); - if (opts_init.src_switch && !opts_init.src_dry_distros.empty() && - opts_init.src_dry_distros.begin()->first != opts_init.dry_distros.begin()->first) throw std::runtime_error("Kappa of the source has to be the same as that of the initial profile"); + if (opts_init.src_type!=src_t::off && n_dims<2) + throw std::runtime_error("CCN source works in 2D and 3D only."); + + if (opts_init.src_type==src_t::matching && !opts_init.src_dry_distros.empty() && + opts_init.src_dry_distros.begin()->first != opts_init.dry_distros.begin()->first) throw std::runtime_error("For 'matching' CCN source, kappa of the source has to be the same as that of the initial profile (no kappa matching done)"); if(opts_init.dry_distros.size() > 1 && opts_init.chem_switch) throw std::runtime_error("chemistry and multiple kappa distributions are not compatible"); - // TODO: in source match kappas - if(opts_init.dry_distros.size() > 1 && opts_init.src_switch) - throw std::runtime_error("aerosol source and multiple kappa distributions are not compatible"); - if(opts_init.dry_distros.size() == 0 && opts_init.dry_sizes.size() == 0) throw std::runtime_error("Both dry_distros and dry_sizes are undefined"); if(opts_init.sd_conc_large_tail && opts_init.sd_conc == 0) throw std::runtime_error("Sd_conc_large_tail make sense only with sd_conc init (i.e. sd_conc>0)"); - if(opts_init.sd_const_multi > 0 && opts_init.src_switch) + if(opts_init.sd_const_multi > 0 && opts_init.src_type!=src_t::off) throw std::runtime_error("aerosol source and constant multiplicity option are not compatible"); // NOTE: why not? if (n_dims > 0) diff --git a/src/particles_step.ipp b/src/particles_step.ipp index 8d46af8fa..0ffee2b0e 100644 --- a/src/particles_step.ipp +++ b/src/particles_step.ipp @@ -399,7 +399,7 @@ namespace libcloudphxx if (opts.src && !(pimpl->opts_init.src_x0 == 0 && pimpl->opts_init.src_x1 == 0)) // src_x0=0 and src_x1=0 is a way of disabling source in some domains in distmem simulations { // sanity check - if (pimpl->opts_init.src_switch == false) throw std::runtime_error("aerosol source was switched off in opts_init"); + if (pimpl->opts_init.src_type == src_t::off) throw std::runtime_error("aerosol source was switched off in opts_init"); // introduce new particles with the given time interval if(pimpl->src_stp_ctr % pimpl->opts_init.supstp_src == 0) From 86eecf275d646ebdfbc2767b6fb1983dbbb847fe Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 Jan 2022 12:15:46 +0100 Subject: [PATCH 272/282] actual code for new src options --- include/libcloudph++/lgrngn/opts_init.hpp | 1 + src/impl/particles_impl.ipp | 2 + src/impl/particles_impl_src_dry_distros.ipp | 381 +---------------- ...articles_impl_src_dry_distros_matching.ipp | 396 ++++++++++++++++++ .../particles_impl_src_dry_distros_simple.ipp | 69 +++ src/particles.tpp | 2 + 6 files changed, 475 insertions(+), 376 deletions(-) create mode 100644 src/impl/particles_impl_src_dry_distros_matching.ipp create mode 100644 src/impl/particles_impl_src_dry_distros_simple.ipp diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index d3b584789..9c7f2b121 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -12,6 +12,7 @@ #include "terminal_velocity.hpp" #include "advection_scheme.hpp" #include "RH_formula.hpp" +#include "ccn_source.hpp" #include "../common/chem.hpp" namespace libcloudphxx diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index f15bd2c6b..8ff6178f3 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -578,6 +578,8 @@ namespace libcloudphxx void bcnd(); void src(const real_t &dt); + void src_dry_distros_simple(const real_t &dt); + void src_dry_distros_matching(const real_t &dt); void src_dry_distros(const real_t &dt); void src_dry_sizes(const real_t &dt); diff --git a/src/impl/particles_impl_src_dry_distros.ipp b/src/impl/particles_impl_src_dry_distros.ipp index c22f59e67..ffd4ea560 100644 --- a/src/impl/particles_impl_src_dry_distros.ipp +++ b/src/impl/particles_impl_src_dry_distros.ipp @@ -12,385 +12,14 @@ namespace libcloudphxx { namespace lgrngn { - namespace detail - { - template - struct two_keys_sort - { - BOOST_GPU_ENABLED - bool operator()(const thrust::tuple &a, const thrust::tuple &b) - { - if(a.head < b.head) return true; - if(a.head == b.head) return a.tail < b.tail; - return false; - } - }; - - template - struct get_bin_no - { - real_t log_rd_min, log_rd_max, log_diff; - get_bin_no(const real_t &log_rd_min, const real_t &log_rd_max): log_rd_min(log_rd_min), log_rd_max(log_rd_max), log_diff(log_rd_max - log_rd_min) {} - - BOOST_GPU_ENABLED - res_t operator()(real_t rd3, n_t count_num) - { - real_t log_rd = log(rd3) / 3.; - if(log_rd < log_rd_min || log_rd >= log_rd_max || count_num == 0) return out_of_bins; - else return ( (log_rd - log_rd_min) / log_diff * count_num); //count num is the number of bins in the cell - } - }; - }; - // create new aerosol particles based on a size distribution - // if any SDs with dry radius similar to the one to be added are present, - // we increase their multiplicity instead of adding new SDs template void particles_t::impl::src_dry_distros(const real_t &dt) - { - // set number of SDs to init; use count_num as storage - init_count_num_src(opts_init.src_sd_conc); - - // -------- TODO: match not only sizes of old particles, but also kappas and chemical composition... -------- - - // --- sort already existing SDs; primary key ijk, secondary rd --- - // TODO: do all of this only on SDs in cells below src_z1? - - // filling-in sorted_id with a sequence - thrust::sequence(sorted_id.begin(), sorted_id.end()); - - // tmp vector with sorted rd3 - thrust_device::vector &sorted_rd3(tmp_device_real_part); - - // use sorted_rd3 as tmp copy of rd3 - thrust::copy( - rd3.begin(), rd3.end(), // from - sorted_rd3.begin() // to - ); - - // copy ijk to sorted ijk - thrust::copy( - ijk.begin(), ijk.end(), // from - sorted_ijk.begin() // to - ); - - // sorting by ijk and rd3 - thrust::sort_by_key( - thrust::make_zip_iterator(thrust::make_tuple( - sorted_ijk.begin(), - sorted_rd3.begin() - )), - thrust::make_zip_iterator(thrust::make_tuple( - sorted_ijk.begin(), - sorted_rd3.begin() - )) + n_part, // keys - sorted_id.begin(), // values - detail::two_keys_sort() - ); - - // analyze distribution to get rd_min and max needed for bin sizes - // TODO: this could be done once at the beginning of the simulation - dist_analysis_sd_conc( - *(opts_init.src_dry_distros.begin()->second), - opts_init.src_sd_conc, - dt - ); - - // --- see how many already existing SDs match size bins --- - { - namespace arg = thrust::placeholders; - - // set no of particles to init - n_part_old = n_part; - n_part_to_init = thrust::reduce(count_num.begin(), count_num.end()); - n_part = n_part_old + n_part_to_init; - hskpng_resize_npart(); - - thrust_size_t n_part_bfr_src = n_part_old, - n_part_tot_in_src = n_part_to_init; - - // tmp vector with bin number of existing SDs - thrust_device::vector bin_no(n_part); - - const thrust_size_t out_of_bins = 4444444444; // would cause an error for src_sd_conc > out_of_bins - // calc bin no - thrust::transform( - sorted_rd3.begin(), - sorted_rd3.end(), - thrust::make_permutation_iterator(count_num.begin(), sorted_ijk.begin()), - bin_no.begin(), - detail::get_bin_no(log_rd_min, log_rd_max) - ); - - // init ijk and rd3 of new particles - init_ijk(); - init_dry_sd_conc(); - - // translate these new rd3 into bin no; bin_no just got resized - thrust::transform( - rd3.begin() + n_part_old, - rd3.end(), - thrust::make_permutation_iterator(count_num.begin(), ijk.begin() + n_part_old), - bin_no.begin() + n_part_old, - detail::get_bin_no(log_rd_min, log_rd_max) - ); - - // -- init new SDs that didnt have a match -- - { - thrust_device::vector tmp_bin_no(n_part_old); - thrust::copy(bin_no.begin(), bin_no.begin() + n_part_old, tmp_bin_no.begin()); - - thrust_size_t n_out_of_bins = thrust::count(tmp_bin_no.begin(), tmp_bin_no.end(), out_of_bins); - - // remove reference to those outside of bins from tmp_bin_no and sorted_ijk - thrust::remove_if( - sorted_ijk.begin(), - sorted_ijk.begin() + n_part_old, - tmp_bin_no.begin(), - arg::_1 == out_of_bins - ); - - thrust::remove( - tmp_bin_no.begin(), - tmp_bin_no.begin() + n_part_old, - out_of_bins - ); // if these two removes are done in a single step with a tuple, it fails on CUDA; TODO: report this? - - thrust_size_t count_bins; - { - // remove duplicates from tmp_bin_no - thrust::pair< - thrust_device::vector::iterator, - typename thrust_device::vector::iterator - > np = thrust::unique_by_key(tmp_bin_no.begin(), tmp_bin_no.begin() + n_part_old - n_out_of_bins, sorted_ijk.begin()); - count_bins = np.first - tmp_bin_no.begin(); // total no of bins with a match - } - - // --- remove rd3 and ijk of newly added SDs that have counterparts --- - thrust_device::vector have_match(n_part_to_init); - // find those with a match - thrust::binary_search( - thrust::make_zip_iterator(thrust::make_tuple( - sorted_ijk.begin(), - tmp_bin_no.begin() - )), - thrust::make_zip_iterator(thrust::make_tuple( - sorted_ijk.begin(), - tmp_bin_no.begin() - )) + count_bins, - thrust::make_zip_iterator(thrust::make_tuple( - ijk.begin() + n_part_old, - bin_no.begin() + n_part_old - )), - thrust::make_zip_iterator(thrust::make_tuple( - ijk.begin() + n_part_old, - bin_no.begin() + n_part_old - )) + n_part_to_init, - have_match.begin(), - detail::two_keys_sort() - ); - // remove those with a match - thrust::remove_if( - thrust::make_zip_iterator(thrust::make_tuple( - rd3.begin() + n_part_old, - ijk.begin() + n_part_old - )), - thrust::make_zip_iterator(thrust::make_tuple( - rd3.begin() + n_part_old, - ijk.begin() + n_part_old - )) + n_part_to_init, - have_match.begin(), - thrust::identity() - ); - - n_part_to_init -= count_bins; - n_part -= count_bins; - hskpng_resize_npart(); - - // init other peoperties of SDs that didnt have a match - init_kappa( - opts_init.src_dry_distros.begin()->first - ); - - if(opts_init.diag_incloud_time) - init_incloud_time(); - - init_n_sd_conc( - *(opts_init.src_dry_distros.begin()->second) - ); // TODO: document that n_of_lnrd_stp is expected! - - // init rw - init_wet(); - - // ijk -> i, j, k - unravel_ijk(n_part_old); - - // init x, y, z, i, j, k - init_xyz(); - - // TODO: init chem - - { - // count number of matched bins per cell - thrust::pair< - thrust_device::vector::iterator, - typename thrust_device::vector::iterator - > it_pair = thrust::reduce_by_key( - sorted_ijk.begin(), sorted_ijk.begin() + count_bins, - thrust::make_constant_iterator(1), - sorted_ijk.begin(), - tmp_bin_no.begin() - ); - count_bins = it_pair.first - sorted_ijk.begin(); // now it counts no of cells that have any bins matched - } - - // set count_num to the number of SDs matched per cell - // they still need to be initialized - thrust::copy( - tmp_bin_no.begin(), - tmp_bin_no.begin() + count_bins, - thrust::make_permutation_iterator(count_num.begin(), sorted_ijk.begin()) - ); - // sorted_ijk no longer valid - } - - // tmp vector to hold number of particles in a given size bin in a given cell - thrust_device::vector bin_cell_count(n_part_tot_in_src + n_cell + 1); // needs space for out_of_bins - // tmp vector for number of particles in bins up to this one - thrust_device::vector bin_cell_count_ptr(n_part_tot_in_src + n_cell + 1); - - thrust_size_t count_bins; - { - thrust_device::vector &out(bin_cell_count_ptr); // use it temporarily - // calc no of SDs in bins/cells - thrust::pair< - thrust_device::vector::iterator, - typename thrust_device::vector::iterator - > np = thrust::reduce_by_key( - bin_no.begin(), - bin_no.begin() + n_part_bfr_src, - thrust::make_constant_iterator(1), - out.begin(),// output bin no - in place didn't work well, why? - bin_cell_count.begin()// output number of SDs - ); - count_bins = np.second - bin_cell_count.begin(); // number of bins with SDs inside, includes the out_of_bins - thrust::copy(out.begin(), out.begin() + count_bins, bin_no.begin()); - } - - // number of SDs (incl. out_of_bins) in bins up to (i-1) - thrust::exclusive_scan( - bin_cell_count.begin(), - bin_cell_count.begin() + count_bins, - bin_cell_count_ptr.begin() - ); - - // remove out of bins from bin_cell_count, bins_no and bin cell_count_ptr - count_bins = - thrust::remove_if( - thrust::make_zip_iterator(thrust::make_tuple( - bin_no.begin(), - bin_cell_count.begin(), - bin_cell_count_ptr.begin() - )), - thrust::make_zip_iterator(thrust::make_tuple( - bin_no.begin(), - bin_cell_count.begin(), - bin_cell_count_ptr.begin() - )) + count_bins, - bin_no.begin(), - arg::_1 == out_of_bins - ) - - thrust::make_zip_iterator(thrust::make_tuple( - bin_no.begin(), - bin_cell_count.begin(), - bin_cell_count_ptr.begin() - )); // count_bins now does not count out_of_bins - - // randomly select which old SD will be increased - // overwrites sorted_rd3 - rand_u01(count_bins); - - // TODO: merge the following transforms into one - - // randomly choose one SD per bin - thrust::transform( - u01.begin(), - u01.begin() + count_bins, - bin_cell_count.begin(), - bin_cell_count.begin(), - thrust::multiplies() - ); - // translate no in bin to the total id - thrust::transform( - bin_cell_count_ptr.begin(), - bin_cell_count_ptr.begin() + count_bins, - bin_cell_count.begin(), - bin_cell_count.begin(), - thrust::plus() - ); - - // --- increase multiplicity of existing SDs --- - - n_part_old = n_part; // number of those before src + no of those w/o match - n_part_to_init = count_bins; // number of matched SDs - n_part = n_part_old + n_part_to_init; - hskpng_resize_npart(); - - // copy rd3 and ijk of the selected SDs to the end of the respective vectors - thrust::copy( - thrust::make_permutation_iterator( - rd3.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id - ) - ), - thrust::make_permutation_iterator( - rd3.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() - ) - ) + count_bins, - rd3.begin() + n_part_old // output - ); - thrust::copy( - thrust::make_permutation_iterator( - ijk.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id - ) - ), - thrust::make_permutation_iterator( - ijk.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() - ) - ) + count_bins, - ijk.begin() + n_part_old // output - ); - - // init n of the copied SDs, but using the src distribution - init_n_sd_conc( - *(opts_init.src_dry_distros.begin()->second) - ); // TODO: document that n_of_lnrd_stp is expected! - - // add the just-initialized multiplicities to the old ones - thrust::transform( - n.begin() + n_part_old, - n.end(), - thrust::make_permutation_iterator( - n.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id - ) - ), - thrust::make_permutation_iterator( - n.begin(), thrust::make_permutation_iterator( - sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id - ) - ), //in-place - thrust::plus() - ); - // TODO: check for overflows of na after addition - - // --- properly reduce size of the vectors back to no before src + no w/o match --- - n_part = n_part_old; - hskpng_resize_npart(); - } + { + if(opts_init.src_type == src_t::matching) + src_dry_distros_matching(dt); + if(opts_init.src_type == src_t::simple) + src_dry_distros_simple(dt); } }; }; diff --git a/src/impl/particles_impl_src_dry_distros_matching.ipp b/src/impl/particles_impl_src_dry_distros_matching.ipp new file mode 100644 index 000000000..61faa20bb --- /dev/null +++ b/src/impl/particles_impl_src_dry_distros_matching.ipp @@ -0,0 +1,396 @@ +// vim:filetype=cpp +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ +// #include +#include +#include + +namespace libcloudphxx +{ + namespace lgrngn + { + namespace detail + { + template + struct two_keys_sort + { + BOOST_GPU_ENABLED + bool operator()(const thrust::tuple &a, const thrust::tuple &b) + { + if(a.head < b.head) return true; + if(a.head == b.head) return a.tail < b.tail; + return false; + } + }; + + template + struct get_bin_no + { + real_t log_rd_min, log_rd_max, log_diff; + get_bin_no(const real_t &log_rd_min, const real_t &log_rd_max): log_rd_min(log_rd_min), log_rd_max(log_rd_max), log_diff(log_rd_max - log_rd_min) {} + + BOOST_GPU_ENABLED + res_t operator()(real_t rd3, n_t count_num) + { + real_t log_rd = log(rd3) / 3.; + if(log_rd < log_rd_min || log_rd >= log_rd_max || count_num == 0) return out_of_bins; + else return ( (log_rd - log_rd_min) / log_diff * count_num); //count num is the number of bins in the cell + } + }; + }; + + // create new aerosol particles based on a size distribution + // if any SDs with dry radius similar to the one to be added are present, + // we increase their multiplicity instead of adding new SDs + template + void particles_t::impl::src_dry_distros_matching(const real_t &dt) + { + // set number of SDs to init; use count_num as storage + init_count_num_src(opts_init.src_sd_conc); + + // -------- TODO: match not only sizes of old particles, but also kappas and chemical composition... -------- + + // --- sort already existing SDs; primary key ijk, secondary rd --- + // TODO: do all of this only on SDs in cells below src_z1? + + // filling-in sorted_id with a sequence + thrust::sequence(sorted_id.begin(), sorted_id.end()); + + // tmp vector with sorted rd3 + thrust_device::vector &sorted_rd3(tmp_device_real_part); + + // use sorted_rd3 as tmp copy of rd3 + thrust::copy( + rd3.begin(), rd3.end(), // from + sorted_rd3.begin() // to + ); + + // copy ijk to sorted ijk + thrust::copy( + ijk.begin(), ijk.end(), // from + sorted_ijk.begin() // to + ); + + // sorting by ijk and rd3 + thrust::sort_by_key( + thrust::make_zip_iterator(thrust::make_tuple( + sorted_ijk.begin(), + sorted_rd3.begin() + )), + thrust::make_zip_iterator(thrust::make_tuple( + sorted_ijk.begin(), + sorted_rd3.begin() + )) + n_part, // keys + sorted_id.begin(), // values + detail::two_keys_sort() + ); + + // analyze distribution to get rd_min and max needed for bin sizes + // TODO: this could be done once at the beginning of the simulation + dist_analysis_sd_conc( + *(opts_init.src_dry_distros.begin()->second), + opts_init.src_sd_conc, + dt + ); + + // --- see how many already existing SDs match size bins --- + { + namespace arg = thrust::placeholders; + + // set no of particles to init + n_part_old = n_part; + n_part_to_init = thrust::reduce(count_num.begin(), count_num.end()); + n_part = n_part_old + n_part_to_init; + hskpng_resize_npart(); + + thrust_size_t n_part_bfr_src = n_part_old, + n_part_tot_in_src = n_part_to_init; + + // tmp vector with bin number of existing SDs + thrust_device::vector bin_no(n_part); + + const thrust_size_t out_of_bins = 4444444444; // would cause an error for src_sd_conc > out_of_bins + // calc bin no + thrust::transform( + sorted_rd3.begin(), + sorted_rd3.end(), + thrust::make_permutation_iterator(count_num.begin(), sorted_ijk.begin()), + bin_no.begin(), + detail::get_bin_no(log_rd_min, log_rd_max) + ); + + // init ijk and rd3 of new particles + init_ijk(); + init_dry_sd_conc(); + + // translate these new rd3 into bin no; bin_no just got resized + thrust::transform( + rd3.begin() + n_part_old, + rd3.end(), + thrust::make_permutation_iterator(count_num.begin(), ijk.begin() + n_part_old), + bin_no.begin() + n_part_old, + detail::get_bin_no(log_rd_min, log_rd_max) + ); + + // -- init new SDs that didnt have a match -- + { + thrust_device::vector tmp_bin_no(n_part_old); + thrust::copy(bin_no.begin(), bin_no.begin() + n_part_old, tmp_bin_no.begin()); + + thrust_size_t n_out_of_bins = thrust::count(tmp_bin_no.begin(), tmp_bin_no.end(), out_of_bins); + + // remove reference to those outside of bins from tmp_bin_no and sorted_ijk + thrust::remove_if( + sorted_ijk.begin(), + sorted_ijk.begin() + n_part_old, + tmp_bin_no.begin(), + arg::_1 == out_of_bins + ); + + thrust::remove( + tmp_bin_no.begin(), + tmp_bin_no.begin() + n_part_old, + out_of_bins + ); // if these two removes are done in a single step with a tuple, it fails on CUDA; TODO: report this? + + thrust_size_t count_bins; + { + // remove duplicates from tmp_bin_no + thrust::pair< + thrust_device::vector::iterator, + typename thrust_device::vector::iterator + > np = thrust::unique_by_key(tmp_bin_no.begin(), tmp_bin_no.begin() + n_part_old - n_out_of_bins, sorted_ijk.begin()); + count_bins = np.first - tmp_bin_no.begin(); // total no of bins with a match + } + + // --- remove rd3 and ijk of newly added SDs that have counterparts --- + thrust_device::vector have_match(n_part_to_init); + // find those with a match + thrust::binary_search( + thrust::make_zip_iterator(thrust::make_tuple( + sorted_ijk.begin(), + tmp_bin_no.begin() + )), + thrust::make_zip_iterator(thrust::make_tuple( + sorted_ijk.begin(), + tmp_bin_no.begin() + )) + count_bins, + thrust::make_zip_iterator(thrust::make_tuple( + ijk.begin() + n_part_old, + bin_no.begin() + n_part_old + )), + thrust::make_zip_iterator(thrust::make_tuple( + ijk.begin() + n_part_old, + bin_no.begin() + n_part_old + )) + n_part_to_init, + have_match.begin(), + detail::two_keys_sort() + ); + // remove those with a match + thrust::remove_if( + thrust::make_zip_iterator(thrust::make_tuple( + rd3.begin() + n_part_old, + ijk.begin() + n_part_old + )), + thrust::make_zip_iterator(thrust::make_tuple( + rd3.begin() + n_part_old, + ijk.begin() + n_part_old + )) + n_part_to_init, + have_match.begin(), + thrust::identity() + ); + + n_part_to_init -= count_bins; + n_part -= count_bins; + hskpng_resize_npart(); + + // init other peoperties of SDs that didnt have a match + init_kappa( + opts_init.src_dry_distros.begin()->first + ); + + if(opts_init.diag_incloud_time) + init_incloud_time(); + + init_n_sd_conc( + *(opts_init.src_dry_distros.begin()->second) + ); // TODO: document that n_of_lnrd_stp is expected! + + // init rw + init_wet(); + + // ijk -> i, j, k + unravel_ijk(n_part_old); + + // init x, y, z, i, j, k + init_xyz(); + + // TODO: init chem + + { + // count number of matched bins per cell + thrust::pair< + thrust_device::vector::iterator, + typename thrust_device::vector::iterator + > it_pair = thrust::reduce_by_key( + sorted_ijk.begin(), sorted_ijk.begin() + count_bins, + thrust::make_constant_iterator(1), + sorted_ijk.begin(), + tmp_bin_no.begin() + ); + count_bins = it_pair.first - sorted_ijk.begin(); // now it counts no of cells that have any bins matched + } + + // set count_num to the number of SDs matched per cell + // they still need to be initialized + thrust::copy( + tmp_bin_no.begin(), + tmp_bin_no.begin() + count_bins, + thrust::make_permutation_iterator(count_num.begin(), sorted_ijk.begin()) + ); + // sorted_ijk no longer valid + } + + // tmp vector to hold number of particles in a given size bin in a given cell + thrust_device::vector bin_cell_count(n_part_tot_in_src + n_cell + 1); // needs space for out_of_bins + // tmp vector for number of particles in bins up to this one + thrust_device::vector bin_cell_count_ptr(n_part_tot_in_src + n_cell + 1); + + thrust_size_t count_bins; + { + thrust_device::vector &out(bin_cell_count_ptr); // use it temporarily + // calc no of SDs in bins/cells + thrust::pair< + thrust_device::vector::iterator, + typename thrust_device::vector::iterator + > np = thrust::reduce_by_key( + bin_no.begin(), + bin_no.begin() + n_part_bfr_src, + thrust::make_constant_iterator(1), + out.begin(),// output bin no - in place didn't work well, why? + bin_cell_count.begin()// output number of SDs + ); + count_bins = np.second - bin_cell_count.begin(); // number of bins with SDs inside, includes the out_of_bins + thrust::copy(out.begin(), out.begin() + count_bins, bin_no.begin()); + } + + // number of SDs (incl. out_of_bins) in bins up to (i-1) + thrust::exclusive_scan( + bin_cell_count.begin(), + bin_cell_count.begin() + count_bins, + bin_cell_count_ptr.begin() + ); + + // remove out of bins from bin_cell_count, bins_no and bin cell_count_ptr + count_bins = + thrust::remove_if( + thrust::make_zip_iterator(thrust::make_tuple( + bin_no.begin(), + bin_cell_count.begin(), + bin_cell_count_ptr.begin() + )), + thrust::make_zip_iterator(thrust::make_tuple( + bin_no.begin(), + bin_cell_count.begin(), + bin_cell_count_ptr.begin() + )) + count_bins, + bin_no.begin(), + arg::_1 == out_of_bins + ) - + thrust::make_zip_iterator(thrust::make_tuple( + bin_no.begin(), + bin_cell_count.begin(), + bin_cell_count_ptr.begin() + )); // count_bins now does not count out_of_bins + + // randomly select which old SD will be increased + // overwrites sorted_rd3 + rand_u01(count_bins); + + // TODO: merge the following transforms into one + + // randomly choose one SD per bin + thrust::transform( + u01.begin(), + u01.begin() + count_bins, + bin_cell_count.begin(), + bin_cell_count.begin(), + thrust::multiplies() + ); + // translate no in bin to the total id + thrust::transform( + bin_cell_count_ptr.begin(), + bin_cell_count_ptr.begin() + count_bins, + bin_cell_count.begin(), + bin_cell_count.begin(), + thrust::plus() + ); + + // --- increase multiplicity of existing SDs --- + + n_part_old = n_part; // number of those before src + no of those w/o match + n_part_to_init = count_bins; // number of matched SDs + n_part = n_part_old + n_part_to_init; + hskpng_resize_npart(); + + // copy rd3 and ijk of the selected SDs to the end of the respective vectors + thrust::copy( + thrust::make_permutation_iterator( + rd3.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id + ) + ), + thrust::make_permutation_iterator( + rd3.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() + ) + ) + count_bins, + rd3.begin() + n_part_old // output + ); + thrust::copy( + thrust::make_permutation_iterator( + ijk.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id + ) + ), + thrust::make_permutation_iterator( + ijk.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() + ) + ) + count_bins, + ijk.begin() + n_part_old // output + ); + + // init n of the copied SDs, but using the src distribution + init_n_sd_conc( + *(opts_init.src_dry_distros.begin()->second) + ); // TODO: document that n_of_lnrd_stp is expected! + + // add the just-initialized multiplicities to the old ones + thrust::transform( + n.begin() + n_part_old, + n.end(), + thrust::make_permutation_iterator( + n.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id + ) + ), + thrust::make_permutation_iterator( + n.begin(), thrust::make_permutation_iterator( + sorted_id.begin(), bin_cell_count.begin() // translates no of sorted SD into id + ) + ), //in-place + thrust::plus() + ); + // TODO: check for overflows of na after addition + + // --- properly reduce size of the vectors back to no before src + no w/o match --- + n_part = n_part_old; + hskpng_resize_npart(); + } + } + }; +}; diff --git a/src/impl/particles_impl_src_dry_distros_simple.ipp b/src/impl/particles_impl_src_dry_distros_simple.ipp new file mode 100644 index 000000000..8b7d733b8 --- /dev/null +++ b/src/impl/particles_impl_src_dry_distros_simple.ipp @@ -0,0 +1,69 @@ +// vim:filetype=cpp +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ +// #include +#include +#include + +namespace libcloudphxx +{ + namespace lgrngn + { + // create new aerosol particles based on a size distribution + template + void particles_t::impl::src_dry_distros_simple(const real_t &dt) + { + // set number of SDs to init; use count_num as storage + init_count_num_src(opts_init.src_sd_conc); + + // analyze distribution to get rd_min and max needed for bin sizes + // TODO: this could be done once at the beginning of the simulation + dist_analysis_sd_conc( + *(opts_init.src_dry_distros.begin()->second), + opts_init.src_sd_conc, + dt + ); + + namespace arg = thrust::placeholders; + + // set no of particles to init + n_part_old = n_part; + n_part_to_init = thrust::reduce(count_num.begin(), count_num.end()); + n_part = n_part_old + n_part_to_init; + hskpng_resize_npart(); + + thrust_size_t n_part_bfr_src = n_part_old, + n_part_tot_in_src = n_part_to_init; + + // init ijk and rd3 of new particles + init_ijk(); + init_dry_sd_conc(); + + // init other peoperties of SDs that didnt have a match + init_kappa( + opts_init.src_dry_distros.begin()->first + ); + + if(opts_init.diag_incloud_time) + init_incloud_time(); + + init_n_sd_conc( + *(opts_init.src_dry_distros.begin()->second) + ); // TODO: document that n_of_lnrd_stp is expected! + + // init rw + init_wet(); + + // ijk -> i, j, k + unravel_ijk(n_part_old); + + // init x, y, z, i, j, k + init_xyz(); + + // TODO: init chem + } + }; +}; diff --git a/src/particles.tpp b/src/particles.tpp index c6ea78a11..477f7a295 100644 --- a/src/particles.tpp +++ b/src/particles.tpp @@ -117,6 +117,8 @@ #include "impl/particles_impl_ante_adding_SD.ipp" #include "impl/particles_impl_post_adding_SD.ipp" #include "impl/particles_impl_src.ipp" +#include "impl/particles_impl_src_dry_distros_simple.ipp" +#include "impl/particles_impl_src_dry_distros_matching.ipp" #include "impl/particles_impl_src_dry_distros.ipp" #include "impl/particles_impl_src_dry_sizes.ipp" #include "impl/particles_impl_rlx.ipp" From a36f6b4dd1e8d0a53cb70129242df3870a6e5118 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 Jan 2022 12:29:17 +0100 Subject: [PATCH 273/282] python bindings for src types --- bindings/python/lib.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bindings/python/lib.cpp b/bindings/python/lib.cpp index 3191c2993..5f68efd4d 100644 --- a/bindings/python/lib.cpp +++ b/bindings/python/lib.cpp @@ -225,6 +225,10 @@ BOOST_PYTHON_MODULE(libcloudphxx) .value("implicit", lgr::as_t::implicit) .value("euler", lgr::as_t::euler) .value("pred_corr", lgr::as_t::pred_corr); + bp::enum_("src_type") + .value("off", lgr::src_t::off) + .value("simple", lgr::src_t::simple) + .value("matching", lgr::src_t::matching); bp::enum_("chem_species_t") .value("H", cmn::chem::H) @@ -287,7 +291,6 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("coal_switch", &lgr::opts_init_t::coal_switch) .def_readwrite("sedi_switch", &lgr::opts_init_t::sedi_switch) .def_readwrite("subs_switch", &lgr::opts_init_t::subs_switch) - .def_readwrite("src_switch", &lgr::opts_init_t::src_switch) .def_readwrite("rlx_switch", &lgr::opts_init_t::rlx_switch) .def_readwrite("turb_adve_switch", &lgr::opts_init_t::turb_adve_switch) .def_readwrite("turb_cond_switch", &lgr::opts_init_t::turb_cond_switch) From 86fb2101894574e0710d15340a2c6ff5738afe0e Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 Jan 2022 12:48:37 +0100 Subject: [PATCH 274/282] source test: test dry_distros simple too --- bindings/python/lib.cpp | 3 ++- tests/python/unit/source.py | 38 +++++++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/bindings/python/lib.cpp b/bindings/python/lib.cpp index 5f68efd4d..b505c8026 100644 --- a/bindings/python/lib.cpp +++ b/bindings/python/lib.cpp @@ -225,7 +225,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .value("implicit", lgr::as_t::implicit) .value("euler", lgr::as_t::euler) .value("pred_corr", lgr::as_t::pred_corr); - bp::enum_("src_type") + bp::enum_("src_t") .value("off", lgr::src_t::off) .value("simple", lgr::src_t::simple) .value("matching", lgr::src_t::matching); @@ -313,6 +313,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .def_readwrite("rlx_timescale", &lgr::opts_init_t::rlx_timescale) .def_readwrite("n_sd_max", &lgr::opts_init_t::n_sd_max) .def_readwrite("terminal_velocity", &lgr::opts_init_t::terminal_velocity) + .def_readwrite("src_type", &lgr::opts_init_t::src_type) .def_readwrite("RH_formula", &lgr::opts_init_t::RH_formula) .def_readwrite("chem_rho", &lgr::opts_init_t::chem_rho) .def_readwrite("RH_max", &lgr::opts_init_t::RH_max) diff --git a/tests/python/unit/source.py b/tests/python/unit/source.py index 8e6b79911..2b12e5ff5 100644 --- a/tests/python/unit/source.py +++ b/tests/python/unit/source.py @@ -51,7 +51,6 @@ def test(opts_init): opts_init.adve_switch = 0; opts_init.cond_switch = 0; opts_init.sedi_switch = 0; - opts_init.src_switch = 1; opts = lgrngn.opts_t() @@ -92,16 +91,43 @@ def test(opts_init): return sd_conc, wet_mom0, wet_mom1 -# test source with dry_distros kappa = .61 + +# ----------- test source with dry_distros simple ------------------ +print(' --- dry_distros simple src ---') opts_init = lgrngn.opts_init_t() opts_init.dry_distros = {kappa:lognormal} opts_init.src_dry_distros = {kappa:lognormal_src} opts_init.sd_conc = 1024 opts_init.src_sd_conc = 512 opts_init.n_sd_max = int((opts_init.sd_conc * 2 + opts_init.src_sd_conc * 2) * 2) # assuming nx=nz=2 +opts_init.src_type = lgrngn.src_t.simple; + +sd_conc, wet_mom0, wet_mom1 = test(opts_init) -print(' --- dry_distros src ---') +print('diag_sd_conc', sd_conc) +if not(sd_conc[0] == 2048 and sd_conc[2] == 2048): + raise Exception("wrong amount of SDs were added") +if not(sd_conc[1] == 1024 and sd_conc[3] == 1024): + raise Exception("SDs were added in wrong cells") + +print(('wet mom0', wet_mom0)) +if (abs( 2 - (wet_mom0[0] + wet_mom0[2]) / (wet_mom0[1] + wet_mom0[3]) ) > 0.015): + raise Exception("incorrect multiplicity after source") + +print(('wet mom1', wet_mom1)) +if (abs( (7.84 / 2.12) - (wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) ) > 0.015): + raise Exception("incorrect radius after source") + +# --------------- test source with dry_distros matching ------------------ +print(' --- dry_distros matching src ---') +opts_init = lgrngn.opts_init_t() +opts_init.dry_distros = {kappa:lognormal} +opts_init.src_dry_distros = {kappa:lognormal_src} +opts_init.sd_conc = 1024 +opts_init.src_sd_conc = 512 +opts_init.n_sd_max = int((opts_init.sd_conc * 2 + opts_init.src_sd_conc * 2) * 2) # assuming nx=nz=2 +opts_init.src_type = lgrngn.src_t.matching; sd_conc, wet_mom0, wet_mom1 = test(opts_init) @@ -119,14 +145,14 @@ def test(opts_init): if (abs( (7.84 / 2.12) - (wet_mom1[0] + wet_mom1[2]) / (wet_mom1[1] + wet_mom1[3]) ) > 0.015): raise Exception("incorrect radius after source") -# test source with dry_sizes -kappa = .61 +# --------- test source with dry_sizes ------------ +print(' --- dry_sizes src ---') opts_init = lgrngn.opts_init_t() opts_init.dry_sizes = {kappa : {1.e-6 : [30., 20], 15.e-6 : [10., 10]}} opts_init.src_dry_sizes = {kappa : {1.e-6 : [0.3, 10], 15.e-6 : [0.1, 5]}} opts_init.n_sd_max=240 +opts_init.src_type = lgrngn.src_t.simple; # dry sizes works the same for simple and matching (no matching done) -print(' --- dry_sizes src ---') sd_conc, wet_mom0, wet_mom1 = test(opts_init) From 1090180e512048b3eaf686517db461e07cef62dd Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 Jan 2022 13:04:23 +0100 Subject: [PATCH 275/282] src swtich -> src type update in tests --- tests/python/unit/api_lgrngn.py | 2 +- tests/python/unit/relax.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/python/unit/api_lgrngn.py b/tests/python/unit/api_lgrngn.py index fc0130742..ce5c912a8 100644 --- a/tests/python/unit/api_lgrngn.py +++ b/tests/python/unit/api_lgrngn.py @@ -69,7 +69,7 @@ def lognormal(lnr): print("coal_switch = ", opts_init.coal_switch) print("sedi_switch = ", opts_init.sedi_switch) print("subs_switch = ", opts_init.subs_switch) -print("src_switch = ", opts_init.src_switch) +print("src_type = ", opts_init.src_type) print("exact_sstp_cond = ", opts_init.exact_sstp_cond) print("sstp_cond =", opts_init.sstp_cond) diff --git a/tests/python/unit/relax.py b/tests/python/unit/relax.py index 6a1228f5a..ee0289794 100644 --- a/tests/python/unit/relax.py +++ b/tests/python/unit/relax.py @@ -60,7 +60,6 @@ def test(opts_init): opts_init.adve_switch = 0; opts_init.cond_switch = 0; opts_init.sedi_switch = 0; - opts_init.src_switch = 0; opts_init.rlx_switch = 1; opts = lgrngn.opts_t() From 87c759a304b54faa88de32fb0ab629cbb42fa894 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 Jan 2022 14:06:28 +0100 Subject: [PATCH 276/282] enums -> enum classes --- bindings/python/lib.cpp | 8 ++++---- include/libcloudph++/lgrngn/RH_formula.hpp | 13 +++++-------- include/libcloudph++/lgrngn/advection_scheme.hpp | 5 +---- include/libcloudph++/lgrngn/backend.hpp | 1 - include/libcloudph++/lgrngn/kernel.hpp | 5 +---- include/libcloudph++/lgrngn/opts_init.hpp | 8 ++++---- include/libcloudph++/lgrngn/terminal_velocity.hpp | 5 +---- src/impl/particles_impl.ipp | 2 +- src/impl/particles_impl_cond_sstp.ipp | 2 +- src/impl/particles_impl_hskpng_Tpr.ipp | 4 ++-- src/impl/particles_impl_hskpng_vterm.ipp | 8 ++++---- 11 files changed, 24 insertions(+), 37 deletions(-) diff --git a/bindings/python/lib.cpp b/bindings/python/lib.cpp index b505c8026..8d86cd566 100644 --- a/bindings/python/lib.cpp +++ b/bindings/python/lib.cpp @@ -198,7 +198,7 @@ BOOST_PYTHON_MODULE(libcloudphxx) .value("OpenMP", lgr::OpenMP) .value("CUDA", lgr::CUDA) .value("multi_CUDA", lgr::multi_CUDA); - bp::enum_("kernel_t") + bp::enum_("kernel_t") .value("geometric", lgr::kernel_t::geometric) .value("golovin", lgr::kernel_t::golovin) .value("hall", lgr::kernel_t::hall) @@ -210,18 +210,18 @@ BOOST_PYTHON_MODULE(libcloudphxx) .value("vohl_davis_no_waals", lgr::kernel_t::vohl_davis_no_waals) .value("hall_pinsky_stratocumulus", lgr::kernel_t::hall_pinsky_stratocumulus) .value("hall_pinsky_cumulonimbus", lgr::kernel_t::hall_pinsky_cumulonimbus); - bp::enum_("vt_t") + bp::enum_("vt_t") .value("beard76", lgr::vt_t::beard76) .value("beard77", lgr::vt_t::beard77) .value("beard77fast", lgr::vt_t::beard77fast) .value("khvorostyanov_spherical", lgr::vt_t::khvorostyanov_spherical) .value("khvorostyanov_nonspherical", lgr::vt_t::khvorostyanov_nonspherical); - bp::enum_("RH_formula_t") + bp::enum_("RH_formula_t") .value("pv_cc", lgr::RH_formula_t::pv_cc) .value("rv_cc", lgr::RH_formula_t::rv_cc) .value("pv_tet", lgr::RH_formula_t::pv_tet) .value("rv_tet", lgr::RH_formula_t::rv_tet); - bp::enum_("as_t") + bp::enum_("as_t") .value("implicit", lgr::as_t::implicit) .value("euler", lgr::as_t::euler) .value("pred_corr", lgr::as_t::pred_corr); diff --git a/include/libcloudph++/lgrngn/RH_formula.hpp b/include/libcloudph++/lgrngn/RH_formula.hpp index 79305d05e..ed6e85ade 100644 --- a/include/libcloudph++/lgrngn/RH_formula.hpp +++ b/include/libcloudph++/lgrngn/RH_formula.hpp @@ -4,15 +4,12 @@ namespace libcloudphxx { namespace lgrngn { - namespace RH_formula_t //separate namespace to avoid member name conflicts with kernel enumerator, TODO: in c++11 change it to an enum class - { //- enum RH_formula_t { pv_cc, // RH = pv / pvs with pvs from the Clausius-Clapeyron equation - rv_cc, // RH = rv / rvs with rvs from the Clausius-Clapeyron equation - pv_tet, // RH = pv / pvs with pvs from the Tetens equation - rv_tet // RH = rv / rvs with rvs from the Tetens equation - }; + enum class RH_formula_t { pv_cc, // RH = pv / pvs with pvs from the Clausius-Clapeyron equation + rv_cc, // RH = rv / rvs with rvs from the Clausius-Clapeyron equation + pv_tet, // RH = pv / pvs with pvs from the Tetens equation + rv_tet // RH = rv / rvs with rvs from the Tetens equation + }; // - }; }; }; diff --git a/include/libcloudph++/lgrngn/advection_scheme.hpp b/include/libcloudph++/lgrngn/advection_scheme.hpp index 21af39313..4f707dd5f 100644 --- a/include/libcloudph++/lgrngn/advection_scheme.hpp +++ b/include/libcloudph++/lgrngn/advection_scheme.hpp @@ -4,11 +4,8 @@ namespace libcloudphxx { namespace lgrngn { - namespace as_t //separate namespace to avoid member name conflicts with kernel enumerator, TODO: in c++11 change it to an enum class - { //- enum as_t { undefined, implicit, euler, pred_corr }; + enum class as_t { undefined, implicit, euler, pred_corr }; // - }; }; }; diff --git a/include/libcloudph++/lgrngn/backend.hpp b/include/libcloudph++/lgrngn/backend.hpp index 756099b4e..792dd0167 100644 --- a/include/libcloudph++/lgrngn/backend.hpp +++ b/include/libcloudph++/lgrngn/backend.hpp @@ -4,7 +4,6 @@ namespace libcloudphxx { namespace lgrngn { - // to make inclusion of Thrust not neccesarry here // enum backend_t { serial, OpenMP, CUDA, multi_CUDA }; // diff --git a/include/libcloudph++/lgrngn/kernel.hpp b/include/libcloudph++/lgrngn/kernel.hpp index 65aa01dd5..768db84be 100644 --- a/include/libcloudph++/lgrngn/kernel.hpp +++ b/include/libcloudph++/lgrngn/kernel.hpp @@ -4,11 +4,8 @@ namespace libcloudphxx { namespace lgrngn { - namespace kernel_t //separate namespace to avoid member name conflicts with terminal_velocity enumerator, TODO: in c++11 change it to an enum class - { //- enum kernel_t { undefined, geometric, golovin, hall, hall_davis_no_waals, Long, onishi_hall, onishi_hall_davis_no_waals, hall_pinsky_1000mb_grav, hall_pinsky_cumulonimbus, hall_pinsky_stratocumulus, vohl_davis_no_waals}; + enum class kernel_t { undefined, geometric, golovin, hall, hall_davis_no_waals, Long, onishi_hall, onishi_hall_davis_no_waals, hall_pinsky_1000mb_grav, hall_pinsky_cumulonimbus, hall_pinsky_stratocumulus, vohl_davis_no_waals}; // - }; }; }; diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index 9c7f2b121..c012d8823 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -73,16 +73,16 @@ namespace libcloudphxx unsigned long long n_sd_max; // coalescence Kernel type - kernel_t::kernel_t kernel; + kernel_t kernel; // terminal velocity formula - vt_t::vt_t terminal_velocity; + vt_t terminal_velocity; // super-droplet advection scheme - as_t::as_t adve_scheme; + as_t adve_scheme; // RH formula - RH_formula_t::RH_formula_t RH_formula; + RH_formula_t RH_formula; // // coalescence kernel parameters diff --git a/include/libcloudph++/lgrngn/terminal_velocity.hpp b/include/libcloudph++/lgrngn/terminal_velocity.hpp index 8ecc59b88..6bfaedc4a 100644 --- a/include/libcloudph++/lgrngn/terminal_velocity.hpp +++ b/include/libcloudph++/lgrngn/terminal_velocity.hpp @@ -4,11 +4,8 @@ namespace libcloudphxx { namespace lgrngn { - namespace vt_t //separate namespace to avoid member name conflicts with kernel enumerator, TODO: in c++11 change it to an enum class - { //- enum vt_t { undefined, beard76, beard77, beard77fast, khvorostyanov_spherical, khvorostyanov_nonspherical }; + enum class vt_t { undefined, beard76, beard77, beard77fast, khvorostyanov_spherical, khvorostyanov_nonspherical }; // - }; }; }; diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 8ff6178f3..b07d3332c 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -51,7 +51,7 @@ namespace libcloudphxx n_part_to_init; // number of SDs to be initialized by source detail::rng rng; detail::config config; - as_t::as_t adve_scheme; // actual advection scheme used, might be different from opts_init.adve_scheme if courant>halo + as_t adve_scheme; // actual advection scheme used, might be different from opts_init.adve_scheme if courant>halo // pointer to collision kernel kernel_base *p_kernel; diff --git a/src/impl/particles_impl_cond_sstp.ipp b/src/impl/particles_impl_cond_sstp.ipp index a322458ca..01c145724 100644 --- a/src/impl/particles_impl_cond_sstp.ipp +++ b/src/impl/particles_impl_cond_sstp.ipp @@ -16,7 +16,7 @@ namespace libcloudphxx { RH resolved_RH; - RH_sgs(RH_formula_t::RH_formula_t RH_formula): + RH_sgs(RH_formula_t RH_formula): resolved_RH(RH_formula) {} diff --git a/src/impl/particles_impl_hskpng_Tpr.ipp b/src/impl/particles_impl_hskpng_Tpr.ipp index f5a161ef7..38433bcad 100644 --- a/src/impl/particles_impl_hskpng_Tpr.ipp +++ b/src/impl/particles_impl_hskpng_Tpr.ipp @@ -133,9 +133,9 @@ namespace libcloudphxx */ // an alternative implementation with formula choice at functor call - const RH_formula_t::RH_formula_t RH_formula; + const RH_formula_t RH_formula; // the type of formula to be used for RH - RH(RH_formula_t::RH_formula_t RH_formula): + RH(RH_formula_t RH_formula): RH_formula(RH_formula) {} diff --git a/src/impl/particles_impl_hskpng_vterm.ipp b/src/impl/particles_impl_hskpng_vterm.ipp index 7103d1d6e..765d7768c 100644 --- a/src/impl/particles_impl_hskpng_vterm.ipp +++ b/src/impl/particles_impl_hskpng_vterm.ipp @@ -37,10 +37,10 @@ namespace libcloudphxx template struct common__vterm__vt { - vt_t::vt_t vt_eq; //type of terminal velocity formula to use + vt_t vt_eq; //type of terminal velocity formula to use //ctor - common__vterm__vt(const vt_t::vt_t &vt_eq): vt_eq(vt_eq) {} + common__vterm__vt(const vt_t &vt_eq): vt_eq(vt_eq) {} BOOST_GPU_ENABLED real_t operator()( @@ -98,10 +98,10 @@ namespace libcloudphxx template struct common__vterm__vt__cached { - vt_t::vt_t vt_eq; //type of terminal velocity formula to use + vt_t vt_eq; //type of terminal velocity formula to use //ctor - common__vterm__vt__cached(const vt_t::vt_t &vt_eq): vt_eq(vt_eq) {} + common__vterm__vt__cached(const vt_t &vt_eq): vt_eq(vt_eq) {} BOOST_GPU_ENABLED real_t operator()( From 02d853c628dccdfbbb92bbe9a8c052b48f533cbb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 13 Jan 2022 14:59:08 +0100 Subject: [PATCH 277/282] dicts with enum names --- include/libcloudph++/lgrngn/RH_formula.hpp | 6 ++++++ include/libcloudph++/lgrngn/advection_scheme.hpp | 6 ++++++ include/libcloudph++/lgrngn/backend.hpp | 6 ++++++ include/libcloudph++/lgrngn/ccn_source.hpp | 6 ++++++ include/libcloudph++/lgrngn/kernel.hpp | 15 +++++++++++++++ include/libcloudph++/lgrngn/terminal_velocity.hpp | 8 ++++++++ 6 files changed, 47 insertions(+) diff --git a/include/libcloudph++/lgrngn/RH_formula.hpp b/include/libcloudph++/lgrngn/RH_formula.hpp index ed6e85ade..2d41835c7 100644 --- a/include/libcloudph++/lgrngn/RH_formula.hpp +++ b/include/libcloudph++/lgrngn/RH_formula.hpp @@ -11,5 +11,11 @@ namespace libcloudphxx rv_tet // RH = rv / rvs with rvs from the Tetens equation }; // + const std::unordered_map RH_formula_name = { + {RH_formula_t::pv_cc, "pv_cc"}, + {RH_formula_t::rv_cc, "rv_cc"}, + {RH_formula_t::pv_tet, "pv_tet"}, + {RH_formula_t::rv_tet, "rv_tet"} + }; }; }; diff --git a/include/libcloudph++/lgrngn/advection_scheme.hpp b/include/libcloudph++/lgrngn/advection_scheme.hpp index 4f707dd5f..b3f124711 100644 --- a/include/libcloudph++/lgrngn/advection_scheme.hpp +++ b/include/libcloudph++/lgrngn/advection_scheme.hpp @@ -7,5 +7,11 @@ namespace libcloudphxx // enum class as_t { undefined, implicit, euler, pred_corr }; // + const std::unordered_map as_name = { + {as_t::undefined, "undefined"}, + {as_t::implicit, "implicit"}, + {as_t::euler, "euler"}, + {as_t::pred_corr, "pred_corr"} + }; }; }; diff --git a/include/libcloudph++/lgrngn/backend.hpp b/include/libcloudph++/lgrngn/backend.hpp index 792dd0167..103c87227 100644 --- a/include/libcloudph++/lgrngn/backend.hpp +++ b/include/libcloudph++/lgrngn/backend.hpp @@ -7,5 +7,11 @@ namespace libcloudphxx // enum backend_t { serial, OpenMP, CUDA, multi_CUDA }; // + const std::unordered_map backend_name = { + {serial, "serial"}, + {OpenMP, "OpenMP"}, + {CUDA, "CUDA"}, + {multi_CUDA, "multi_CUDA"} + }; }; }; diff --git a/include/libcloudph++/lgrngn/ccn_source.hpp b/include/libcloudph++/lgrngn/ccn_source.hpp index 67b3c90d3..02f4a1c9a 100644 --- a/include/libcloudph++/lgrngn/ccn_source.hpp +++ b/include/libcloudph++/lgrngn/ccn_source.hpp @@ -10,5 +10,11 @@ namespace libcloudphxx // off - no source // simple - src_dry_distros: new SD are added; src_dry_sizes: new SD are added // matching - src_dry_distros: find similar SD and increase their multiplicity. Add new SD if match not found; src_dry_sizes: new SD are added + + const std::unordered_map src_name = { + {src_t::off, "off"}, + {src_t::simple, "simple"}, + {src_t::matching, "matching"} + }; }; }; diff --git a/include/libcloudph++/lgrngn/kernel.hpp b/include/libcloudph++/lgrngn/kernel.hpp index 768db84be..59d8899ea 100644 --- a/include/libcloudph++/lgrngn/kernel.hpp +++ b/include/libcloudph++/lgrngn/kernel.hpp @@ -7,5 +7,20 @@ namespace libcloudphxx // enum class kernel_t { undefined, geometric, golovin, hall, hall_davis_no_waals, Long, onishi_hall, onishi_hall_davis_no_waals, hall_pinsky_1000mb_grav, hall_pinsky_cumulonimbus, hall_pinsky_stratocumulus, vohl_davis_no_waals}; // + + const std::unordered_map kernel_name = { + {kernel_t::undefined, "undefined"}, + {kernel_t::geometric, "geometric"}, + {kernel_t::golovin, "golovin"}, + {kernel_t::hall, "hall"}, + {kernel_t::hall_davis_no_waals, "hall_davis_no_waals"}, + {kernel_t::Long, "Long"}, + {kernel_t::onishi_hall, "onishi_hall"}, + {kernel_t::onishi_hall_davis_no_waals, "onishi_hall_davis_no_waals"}, + {kernel_t::hall_pinsky_1000mb_grav, "hall_pinsky_1000mb_grav"}, + {kernel_t::hall_pinsky_cumulonimbus, "hall_pinsky_cumulonimbus"}, + {kernel_t::hall_pinsky_stratocumulus, "hall_pinsky_stratocumulus"}, + {kernel_t::vohl_davis_no_waals, "vohl_davis_no_waals"} + }; }; }; diff --git a/include/libcloudph++/lgrngn/terminal_velocity.hpp b/include/libcloudph++/lgrngn/terminal_velocity.hpp index 6bfaedc4a..3a21417d9 100644 --- a/include/libcloudph++/lgrngn/terminal_velocity.hpp +++ b/include/libcloudph++/lgrngn/terminal_velocity.hpp @@ -7,5 +7,13 @@ namespace libcloudphxx // enum class vt_t { undefined, beard76, beard77, beard77fast, khvorostyanov_spherical, khvorostyanov_nonspherical }; // + const std::unordered_map vt_name = { + {vt_t::undefined, "undefined"}, + {vt_t::beard76, "beard76"}, + {vt_t::beard77, "beard77"}, + {vt_t::beard77fast, "beard77fast"}, + {vt_t::khvorostyanov_spherical, "khvorostyanov_spherical"}, + {vt_t::khvorostyanov_nonspherical, "khvorostyanov_nonspherical"} + }; }; }; From 8acff8734905cfbc65d7422d3a1e48ee2ea5780f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 17 Jan 2022 12:25:34 +0100 Subject: [PATCH 278/282] backend: undefined --- include/libcloudph++/lgrngn/backend.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/libcloudph++/lgrngn/backend.hpp b/include/libcloudph++/lgrngn/backend.hpp index 103c87227..94b438f4f 100644 --- a/include/libcloudph++/lgrngn/backend.hpp +++ b/include/libcloudph++/lgrngn/backend.hpp @@ -5,9 +5,10 @@ namespace libcloudphxx namespace lgrngn { //- enum backend_t { serial, OpenMP, CUDA, multi_CUDA }; + enum backend_t { undefined, serial, OpenMP, CUDA, multi_CUDA }; // const std::unordered_map backend_name = { + {undefined, "undefined"}, {serial, "serial"}, {OpenMP, "OpenMP"}, {CUDA, "CUDA"}, From 45c4e1fd87bd4f07f130c36f1b8f84f150323138 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 19 Oct 2022 10:27:24 +0200 Subject: [PATCH 279/282] more debug output for count_n > n_cell --- src/impl/particles_impl_hskpng_count.ipp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/impl/particles_impl_hskpng_count.ipp b/src/impl/particles_impl_hskpng_count.ipp index 2619094c1..f2e4398b6 100644 --- a/src/impl/particles_impl_hskpng_count.ipp +++ b/src/impl/particles_impl_hskpng_count.ipp @@ -17,6 +17,7 @@ namespace libcloudphxx { hskpng_sort(); + // computing count_* - number of particles per grid cell thrust::pair< thrust_device::vector::iterator, @@ -28,6 +29,21 @@ namespace libcloudphxx count_num.begin() // output - values ); count_n = it_pair.first - count_ijk.begin(); +#if !defined(NDEBUG) + if(count_n > n_cell) + { + std::cerr << "count_n: " << count_n << std::endl; + std::cerr << "n_cell: " << n_cell << std::endl; + /* + std::cerr << "ijk:" << std::endl; + debug::print(ijk); + */ + std::cerr << "sorted_ijk:" << std::endl; + debug::print(sorted_ijk); + std::cerr << "count_ijk:" << std::endl; + debug::print(count_ijk); + } +#endif assert(count_n <= n_cell); } }; From 05d0a0b88301607528ca6ab484e7d24a7ae8edaf Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 28 Oct 2022 14:44:50 +0200 Subject: [PATCH 280/282] bump bcond tolerance --- src/detail/config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detail/config.hpp b/src/detail/config.hpp index 5c396b9df..44c01d5bd 100644 --- a/src/detail/config.hpp +++ b/src/detail/config.hpp @@ -28,7 +28,7 @@ namespace libcloudphxx // range of beard77fast bins: const real_t vt0_ln_r_min, vt0_ln_r_max; - const real_t bcond_tolerance = 2e-4; // [m]; error tolerance for position near bcond after distmem copy + const real_t bcond_tolerance = 5e-4; // [m]; error tolerance for position near bcond after distmem copy const real_t rlx_conc_tolerance = 0.1; // tolerance of the relaxation scheme; new SD will be created if missing_conc/expected_conc > tolerance From 8e98d21e11b78cf1c7154ba74226532f8470ea0c Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 24 Nov 2022 12:44:05 +0100 Subject: [PATCH 281/282] comment about PRNG --- src/detail/urand.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detail/urand.hpp b/src/detail/urand.hpp index a785bcb53..b927585e5 100644 --- a/src/detail/urand.hpp +++ b/src/detail/urand.hpp @@ -21,7 +21,7 @@ namespace libcloudphxx { #if !defined(__NVCC__) // serial version using C++11's - using engine_t = std::mt19937; + using engine_t = std::mt19937; // TODO: if real_t = double, use std::mt19937_64 using dist_u01_t = std::uniform_real_distribution; using dist_normal01_t = std::normal_distribution; using dist_un_t = std::uniform_int_distribution; From 3de49594566757238bde73b63e921836ae035f48 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 20 Jan 2023 19:01:40 +0100 Subject: [PATCH 282/282] option to set rd bin edges in opts_init --- include/libcloudph++/lgrngn/opts_init.hpp | 5 +- src/impl/particles_impl_dist_analysis.ipp | 126 +++++++++++++--------- 2 files changed, 76 insertions(+), 55 deletions(-) diff --git a/include/libcloudph++/lgrngn/opts_init.hpp b/include/libcloudph++/lgrngn/opts_init.hpp index c012d8823..f3df59742 100644 --- a/include/libcloudph++/lgrngn/opts_init.hpp +++ b/include/libcloudph++/lgrngn/opts_init.hpp @@ -126,7 +126,7 @@ namespace libcloudphxx // SGS mixing length profile [m] std::vector SGS_mix_len; - real_t rd_min; // minimal dry radius of droplets (works only for init from spectrum) + real_t rd_min, rd_max; // min/max dry radius of droplets [m] bool no_ccn_at_init; // if true, no ccn / SD are put at the start of the simulation @@ -230,7 +230,8 @@ namespace libcloudphxx rlx_timescale(1), rlx_sd_per_bin(0), supstp_rlx(1), - rd_min(0.), + rd_min(-1), // negative means that rd_min will be automatically detected + rd_max(-1), diag_incloud_time(false), no_ccn_at_init(false), open_side_walls(false), diff --git a/src/impl/particles_impl_dist_analysis.ipp b/src/impl/particles_impl_dist_analysis.ipp index 74cafaafd..ac63c9740 100644 --- a/src/impl/particles_impl_dist_analysis.ipp +++ b/src/impl/particles_impl_dist_analysis.ipp @@ -20,17 +20,10 @@ namespace libcloudphxx const real_t dt ) { - // probing the spectrum to find rd_min-rd_max range - // when analysing distro for source, multiplier takes into account that - // the distribution is assumed to represent number of particles created per unit of time! - // TODO: document that - - // values to start the search - real_t rd_min = config.rd_min_init, rd_max = config.rd_max_init; - - bool found_optimal_range = false; - while (!found_optimal_range) + if(opts_init.rd_min >= 0 && opts_init.rd_max >= 0) // user-defined bin edges { + real_t rd_min = opts_init.rd_min, rd_max = opts_init.rd_max; + multiplier = log(rd_max / rd_min) / sd_conc * dt @@ -41,24 +34,46 @@ namespace libcloudphxx log_rd_min = log(rd_min); log_rd_max = log(rd_max); + } + else if (opts_init.rd_min < 0 && opts_init.rd_max < 0) // automatic detection of bin edges + { + // probing the spectrum to find rd_min-rd_max range + // when analysing distro for source, multiplier takes into account that + // the distribution is assumed to represent number of particles created per unit of time! + // TODO: document that + + // values to start the search + real_t rd_min = config.rd_min_init, rd_max = config.rd_max_init; + + bool found_optimal_range = false; + while (!found_optimal_range) + { + multiplier = log(rd_max / rd_min) + / sd_conc + * dt + * (n_dims == 0 + ? dv[0] + : (opts_init.dx * opts_init.dy * opts_init.dz) + ); - impl::n_t - n_min = n_of_lnrd_stp(log_rd_min) * multiplier, - n_max = n_of_lnrd_stp(log_rd_max) * multiplier; + log_rd_min = log(rd_min); + log_rd_max = log(rd_max); - if (rd_min == config.rd_min_init && n_min != 0) - throw std::runtime_error(detail::formatter() << "Initial dry radii distribution is non-zero (" << n_min << ") for rd_min_init (" << config.rd_min_init <<")"); - if (rd_max == config.rd_max_init && n_max != 0) - throw std::runtime_error(detail::formatter() << "Initial dry radii distribution is non-zero (" << n_max << ") for rd_max_init (" << config.rd_max_init <<")"); + impl::n_t + n_min = n_of_lnrd_stp(log_rd_min) * multiplier, + n_max = n_of_lnrd_stp(log_rd_max) * multiplier; - if (n_min == 0) rd_min *= 1.01; - else if (n_max == 0) rd_max /= 1.01; - else found_optimal_range = true; + if (rd_min == config.rd_min_init && n_min != 0) + throw std::runtime_error(detail::formatter() << "Initial dry radii distribution is non-zero (" << n_min << ") for rd_min_init (" << config.rd_min_init <<")"); + if (rd_max == config.rd_max_init && n_max != 0) + throw std::runtime_error(detail::formatter() << "Initial dry radii distribution is non-zero (" << n_max << ") for rd_max_init (" << config.rd_max_init <<")"); + + if (n_min == 0) rd_min *= 1.01; + else if (n_max == 0) rd_max /= 1.01; + else found_optimal_range = true; + } } -#if !defined(__NVCC__) - using std::max; -#endif - log_rd_min = max(log_rd_min, real_t(log(opts_init.rd_min))); // user-defined lower limit for rd + else assert(false && "opts_init.rd_min * opts_init.rd_max < 0"); }; template @@ -66,37 +81,42 @@ namespace libcloudphxx const common::unary_function &n_of_lnrd_stp ) { - // TODO: add same sanity check as above - // how does brent algorithm work for functions with multiple minima?? - std::pair init_distr_max; // [ln(position of distribution's maximum), -function value at maximum] - boost::uintmax_t n_iter = config.n_iter; - init_distr_max = boost::math::tools::brent_find_minima(detail::eval_and_mul(n_of_lnrd_stp, -1), log(config.rd_min_init), log(config.rd_max_init), 200, n_iter); // bits = 200 to make algorithm choose max precision available + if(opts_init.rd_min >= 0 && opts_init.rd_max >= 0) // user-defined bin edges + { + log_rd_min = log(opts_init.rd_min); + log_rd_max = log(opts_init.rd_max); + } + else if (opts_init.rd_min < 0 && opts_init.rd_max < 0) // automatic detection of bin edges + { + // TODO: add same sanity check as above + // how does brent algorithm work for functions with multiple minima?? + std::pair init_distr_max; // [ln(position of distribution's maximum), -function value at maximum] + boost::uintmax_t n_iter = config.n_iter; + init_distr_max = boost::math::tools::brent_find_minima(detail::eval_and_mul(n_of_lnrd_stp, -1), log(config.rd_min_init), log(config.rd_max_init), 200, n_iter); // bits = 200 to make algorithm choose max precision available - real_t init_dist_bound_value = -init_distr_max.second / config.threshold; // value of the distribution at which we bind it - n_iter = config.n_iter; - // TODO: it could be written more clearly by creating an object detail::eval_and_oper(*n_of_lnrd_stp, -init_dist_bound_value, 1), but for some reason it doesnt give the correct values - log_rd_min = - common::detail::toms748_solve( - detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value), - real_t(log(config.rd_min_init)), init_distr_max.first, - detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value)(real_t(log(config.rd_min_init))), - detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value)(init_distr_max.first), - config.eps_tolerance, n_iter - ); + real_t init_dist_bound_value = -init_distr_max.second / config.threshold; // value of the distribution at which we bind it + n_iter = config.n_iter; + // TODO: it could be written more clearly by creating an object detail::eval_and_oper(*n_of_lnrd_stp, -init_dist_bound_value, 1), but for some reason it doesnt give the correct values + log_rd_min = + common::detail::toms748_solve( + detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value), + real_t(log(config.rd_min_init)), init_distr_max.first, + detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value)(real_t(log(config.rd_min_init))), + detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value)(init_distr_max.first), + config.eps_tolerance, n_iter + ); - n_iter = config.n_iter; - log_rd_max = - common::detail::toms748_solve( - detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value), - init_distr_max.first, real_t(log(config.rd_max_init)), - detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value)(init_distr_max.first), - detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value)(real_t(log(config.rd_max_init))), - config.eps_tolerance, n_iter - ); -#if !defined(__NVCC__) - using std::max; -#endif - log_rd_min = max(log_rd_min, real_t(log(opts_init.rd_min))); // user-defined lower limit for rd + n_iter = config.n_iter; + log_rd_max = + common::detail::toms748_solve( + detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value), + init_distr_max.first, real_t(log(config.rd_max_init)), + detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value)(init_distr_max.first), + detail::eval_and_add(n_of_lnrd_stp, -init_dist_bound_value)(real_t(log(config.rd_max_init))), + config.eps_tolerance, n_iter + ); + } + else assert(false && "opts_init.rd_min * opts_init.rd_max < 0"); } }; };