Skip to content

Commit

Permalink
fix race condition
Browse files Browse the repository at this point in the history
  • Loading branch information
@atmyers
atmyers committed Sep 11, 2024
1 parent 66d67cc commit 8322804
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 45 deletions.
47 changes: 32 additions & 15 deletions Source/Particles/AddPlasmaUtilities.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,22 +93,40 @@ amrex::Real compute_scale_fac_area (const amrex::GpuArray<amrex::Real, AMREX_SPA
}

/*
This struct encapsulates several data structures needed for using the parser during plasma
These structs encapsulates several data structures needed for using the parser during plasma
injection.
*/
struct PlasmaParserHelper
struct PlasmaParserWrapper
{
PlasmaParserHelper (std::size_t a_num_user_int_attribs,
std::size_t a_num_user_real_attribs,
const amrex::Vector< std::unique_ptr<amrex::Parser> >& a_user_int_attrib_parser,
const amrex::Vector< std::unique_ptr<amrex::Parser> >& a_user_real_attrib_parser);
PlasmaParserWrapper (std::size_t a_num_user_int_attribs,
std::size_t a_num_user_real_attribs,
const amrex::Vector< std::unique_ptr<amrex::Parser> >& a_user_int_attrib_parser,
const amrex::Vector< std::unique_ptr<amrex::Parser> >& a_user_real_attrib_parser);

amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > m_user_int_attrib_parserexec_pinned;
amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > m_user_real_attrib_parserexec_pinned;
};

struct PlasmaParserHelper
{
template <typename SoAType>
void prepareData (SoAType& a_soa, std::size_t old_size,
const std::vector<std::string>& a_user_int_attribs,
const std::vector<std::string>& a_user_real_attribs,
std::map<std::string,int>& a_particle_icomps,
std::map<std::string,int>& a_particle_comps) {
PlasmaParserHelper (SoAType& a_soa, std::size_t old_size,
const std::vector<std::string>& a_user_int_attribs,
const std::vector<std::string>& a_user_real_attribs,
std::map<std::string,int>& a_particle_icomps,
std::map<std::string,int>& a_particle_comps,
const PlasmaParserWrapper& wrapper) {
m_wrapper_ptr = &wrapper;
m_pa_user_int_pinned.resize(a_user_int_attribs.size());
m_pa_user_real_pinned.resize(a_user_real_attribs.size());

#ifdef AMREX_USE_GPU
m_d_pa_user_int.resize(a_user_int_attribs.size());
m_d_pa_user_real.resize(a_user_real_attribs.size());
m_d_user_int_attrib_parserexec.resize(a_user_int_attribs.size());
m_d_user_real_attrib_parserexec.resize(a_user_real_attribs.size());
#endif

for (std::size_t ia = 0; ia < a_user_int_attribs.size(); ++ia) {
m_pa_user_int_pinned[ia] = a_soa.GetIntData(a_particle_icomps[a_user_int_attribs[ia]]).data() + old_size;
}
Expand All @@ -121,9 +139,9 @@ struct PlasmaParserHelper
m_pa_user_int_pinned.end(), m_d_pa_user_int.begin());
amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_pa_user_real_pinned.begin(),
m_pa_user_real_pinned.end(), m_d_pa_user_real.begin());
amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_user_int_attrib_parserexec_pinned.begin(),
amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, wrapper.m_user_int_attrib_parserexec_pinned.begin(),
m_user_int_attrib_parserexec_pinned.end(), m_d_user_int_attrib_parserexec.begin());
amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_user_real_attrib_parserexec_pinned.begin(),
amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, wrapper.m_user_real_attrib_parserexec_pinned.begin(),
m_user_real_attrib_parserexec_pinned.end(), m_d_user_real_attrib_parserexec.begin());
#endif
}
Expand All @@ -133,8 +151,6 @@ struct PlasmaParserHelper
amrex::ParserExecutor<7> const* getUserIntParserExecData ();
amrex::ParserExecutor<7> const* getUserRealParserExecData ();

amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > m_user_int_attrib_parserexec_pinned;
amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > m_user_real_attrib_parserexec_pinned;
amrex::Gpu::PinnedVector<int*> m_pa_user_int_pinned;
amrex::Gpu::PinnedVector<amrex::ParticleReal*> m_pa_user_real_pinned;

Expand All @@ -146,6 +162,7 @@ struct PlasmaParserHelper
amrex::Gpu::DeviceVector< amrex::ParserExecutor<7> > m_d_user_int_attrib_parserexec;
amrex::Gpu::DeviceVector< amrex::ParserExecutor<7> > m_d_user_real_attrib_parserexec;
#endif
const PlasmaParserWrapper* m_wrapper_ptr;
};

#endif /*WARPX_ADDPLASMAUTILITIES_H_*/
21 changes: 6 additions & 15 deletions Source/Particles/AddPlasmaUtilities.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,23 +107,14 @@ bool find_overlap_flux (const amrex::RealBox& tile_realbox, const amrex::RealBox
return no_overlap;
}

PlasmaParserHelper::PlasmaParserHelper (const std::size_t a_num_user_int_attribs,
const std::size_t a_num_user_real_attribs,
const amrex::Vector< std::unique_ptr<amrex::Parser> >& a_user_int_attrib_parser,
const amrex::Vector< std::unique_ptr<amrex::Parser> >& a_user_real_attrib_parser)
PlasmaParserWrapper::PlasmaParserWrapper (const std::size_t a_num_user_int_attribs,
const std::size_t a_num_user_real_attribs,
const amrex::Vector< std::unique_ptr<amrex::Parser> >& a_user_int_attrib_parser,
const amrex::Vector< std::unique_ptr<amrex::Parser> >& a_user_real_attrib_parser)

{
m_user_int_attrib_parserexec_pinned.resize(a_num_user_int_attribs);
m_user_real_attrib_parserexec_pinned.resize(a_num_user_real_attribs);
m_pa_user_int_pinned.resize(a_num_user_int_attribs);
m_pa_user_real_pinned.resize(a_num_user_real_attribs);

#ifdef AMREX_USE_GPU
m_d_pa_user_int.resize(a_num_user_int_attribs);
m_d_pa_user_real.resize(a_num_user_real_attribs);
m_d_user_int_attrib_parserexec.resize(a_num_user_int_attribs);
m_d_user_real_attrib_parserexec.resize(a_num_user_real_attribs);
#endif

for (std::size_t ia = 0; ia < a_num_user_int_attribs; ++ia) {
m_user_int_attrib_parserexec_pinned[ia] = a_user_int_attrib_parser[ia]->compile<7>();
Expand Down Expand Up @@ -153,14 +144,14 @@ amrex::ParserExecutor<7> const* PlasmaParserHelper::getUserIntParserExecData ()
#ifdef AMREX_USE_GPU
return m_d_user_int_attrib_parserexec.dataPtr();
#else
return m_user_int_attrib_parserexec_pinned.dataPtr();
return m_wrapper_ptr->m_user_int_attrib_parserexec_pinned.dataPtr();
#endif
}

amrex::ParserExecutor<7> const* PlasmaParserHelper::getUserRealParserExecData () {
#ifdef AMREX_USE_GPU
return m_d_user_real_attrib_parserexec.dataPtr();
#else
return m_user_real_attrib_parserexec_pinned.dataPtr();
return m_wrapper_ptr->m_user_real_attrib_parserexec_pinned.dataPtr();
#endif
}
31 changes: 16 additions & 15 deletions Source/Particles/PhysicalParticleContainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -993,6 +993,13 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int
const bool radially_weighted = plasma_injector.radially_weighted;
#endif

auto n_user_int_attribs = static_cast<int>(m_user_int_attribs.size());
auto n_user_real_attribs = static_cast<int>(m_user_real_attribs.size());
PlasmaParserWrapper plasma_parser_wrapper (m_user_int_attribs.size(),
m_user_real_attribs.size(),
m_user_int_attrib_parser,
m_user_real_attrib_parser);

MFItInfo info;
if (do_tiling && Gpu::notInLaunchRegion()) {
info.EnableTiling(tile_size);
Expand Down Expand Up @@ -1113,14 +1120,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int
}
uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size;

auto n_user_int_attribs = static_cast<int>(m_user_int_attribs.size());
auto n_user_real_attribs = static_cast<int>(m_user_real_attribs.size());
PlasmaParserHelper plasma_parser_helper (m_user_int_attribs.size(),
m_user_real_attribs.size(),
m_user_int_attrib_parser,
m_user_real_attrib_parser);

plasma_parser_helper.prepareData(soa, old_size, m_user_int_attribs, m_user_real_attribs, particle_icomps, particle_comps);
PlasmaParserHelper plasma_parser_helper (soa, old_size, m_user_int_attribs, m_user_real_attribs, particle_icomps, particle_comps, plasma_parser_wrapper);
int** pa_user_int_data = plasma_parser_helper.getUserIntDataPtrs();
ParticleReal** pa_user_real_data = plasma_parser_helper.getUserRealDataPtrs();
amrex::ParserExecutor<7> const* user_int_parserexec_data = plasma_parser_helper.getUserIntParserExecData();
Expand Down Expand Up @@ -1446,6 +1446,13 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector,
const bool radially_weighted = plasma_injector.radially_weighted;
#endif

auto n_user_int_attribs = static_cast<int>(m_user_int_attribs.size());
auto n_user_real_attribs = static_cast<int>(m_user_real_attribs.size());
PlasmaParserWrapper plasma_parser_wrapper (m_user_int_attribs.size(),
m_user_real_attribs.size(),
m_user_int_attrib_parser,
m_user_real_attrib_parser);

MFItInfo info;
if (do_tiling && Gpu::notInLaunchRegion()) {
info.EnableTiling(tile_size);
Expand Down Expand Up @@ -1547,13 +1554,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector,
}
uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size;

auto n_user_int_attribs = static_cast<int>(m_user_int_attribs.size());
auto n_user_real_attribs = static_cast<int>(m_user_real_attribs.size());
PlasmaParserHelper plasma_parser_helper (m_user_int_attribs.size(),
m_user_real_attribs.size(),
m_user_int_attrib_parser,
m_user_real_attrib_parser);
plasma_parser_helper.prepareData(soa, old_size, m_user_int_attribs, m_user_real_attribs, particle_icomps, particle_comps);
PlasmaParserHelper plasma_parser_helper (soa, old_size, m_user_int_attribs, m_user_real_attribs, particle_icomps, particle_comps, plasma_parser_wrapper);
int** pa_user_int_data = plasma_parser_helper.getUserIntDataPtrs();
ParticleReal** pa_user_real_data = plasma_parser_helper.getUserRealDataPtrs();
amrex::ParserExecutor<7> const* user_int_parserexec_data = plasma_parser_helper.getUserIntParserExecData();
Expand Down

0 comments on commit 8322804

Please sign in to comment.