diff --git a/src/qvi-bbuff-rmi.h b/src/qvi-bbuff-rmi.h index a087036..474ae58 100644 --- a/src/qvi-bbuff-rmi.h +++ b/src/qvi-bbuff-rmi.h @@ -104,7 +104,7 @@ template<> inline void qvi_bbuff_rmi_pack_type_picture( std::string &picture, - const qvi_hwloc_bitmap_s & + const qvi_hwloc_bitmap & ) { picture += "c"; } @@ -484,7 +484,7 @@ qvi_bbuff_rmi_pack_item( inline int qvi_bbuff_rmi_pack_item( qvi_bbuff *buff, - const qvi_hwloc_bitmap_s &bitmap + const qvi_hwloc_bitmap &bitmap ) { return qvi_bbuff_rmi_pack_item_impl(buff, bitmap.cdata()); } @@ -781,7 +781,7 @@ qvi_bbuff_rmi_unpack_item( */ inline int qvi_bbuff_rmi_unpack_item( - qvi_hwloc_bitmap_s &bitmap, + qvi_hwloc_bitmap &bitmap, byte_t *buffpos, size_t *bytes_written ) { diff --git a/src/qvi-group-omp.h b/src/qvi-group-omp.h index 1e12883..1abca33 100644 --- a/src/qvi-group-omp.h +++ b/src/qvi-group-omp.h @@ -25,7 +25,7 @@ #include "qvi-omp.h" struct qvi_group_omp : public qvi_group { -protected: +private: /** Task associated with this group. */ qvi_task *m_task = nullptr; /** Underlying group instance. */ diff --git a/src/qvi-hwloc.cc b/src/qvi-hwloc.cc index 7598beb..d112ee8 100644 --- a/src/qvi-hwloc.cc +++ b/src/qvi-hwloc.cc @@ -1248,7 +1248,7 @@ int qvi_hwloc_get_devices_in_bitmap( qvi_hwloc_t *hwl, qv_hw_obj_type_t dev_type, - const qvi_hwloc_bitmap_s &bitmap, + const qvi_hwloc_bitmap &bitmap, qvi_hwloc_dev_list_t &devs ) { return get_devices_in_cpuset(hwl, dev_type, bitmap.cdata(), devs); diff --git a/src/qvi-hwloc.h b/src/qvi-hwloc.h index d273dba..17c06bc 100644 --- a/src/qvi-hwloc.h +++ b/src/qvi-hwloc.h @@ -419,19 +419,19 @@ qvi_hwloc_supported_devices(void); /** * hwloc bitmap object. */ -struct qvi_hwloc_bitmap_s { +struct qvi_hwloc_bitmap { private: /** Internal bitmap. */ hwloc_bitmap_t m_data = nullptr; public: /** Default constructor. */ - qvi_hwloc_bitmap_s(void) + qvi_hwloc_bitmap(void) { const int rc = qvi_hwloc_bitmap_calloc(&m_data); if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error(); } /** Construct via hwloc_const_bitmap_t. */ - explicit qvi_hwloc_bitmap_s(hwloc_const_bitmap_t bitmap) + explicit qvi_hwloc_bitmap(hwloc_const_bitmap_t bitmap) { int rc = qvi_hwloc_bitmap_calloc(&m_data); if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error(); @@ -439,7 +439,7 @@ struct qvi_hwloc_bitmap_s { if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error(); } /** Copy constructor. */ - qvi_hwloc_bitmap_s(const qvi_hwloc_bitmap_s &src) + qvi_hwloc_bitmap(const qvi_hwloc_bitmap &src) { int rc = qvi_hwloc_bitmap_calloc(&m_data); if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error(); @@ -447,20 +447,20 @@ struct qvi_hwloc_bitmap_s { if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error(); } /** Destructor. */ - ~qvi_hwloc_bitmap_s(void) + ~qvi_hwloc_bitmap(void) { qvi_hwloc_bitmap_delete(&m_data); } /** Equality operator. */ bool operator==( - const qvi_hwloc_bitmap_s &x + const qvi_hwloc_bitmap &x ) const { return hwloc_bitmap_compare(cdata(), x.cdata()) == 0; } /** Assignment operator. */ void - operator=(const qvi_hwloc_bitmap_s &src) + operator=(const qvi_hwloc_bitmap &src) { const int rc = qvi_hwloc_bitmap_copy(src.m_data, m_data); if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error(); @@ -494,13 +494,13 @@ struct qvi_hwloc_bitmap_s { /** * Vector of cpuset objects. */ -using qvi_hwloc_cpusets_t = std::vector; +using qvi_hwloc_cpusets_t = std::vector; typedef struct qvi_hwloc_device_s { /** Device type. */ qv_hw_obj_type_t type = QV_HW_OBJ_LAST; /** Device affinity. */ - qvi_hwloc_bitmap_s affinity; + qvi_hwloc_bitmap affinity; /** Vendor ID. */ int vendor_id = QVI_HWLOC_DEVICE_INVALID_ID; /** System Management Interface ID. */ @@ -531,7 +531,7 @@ int qvi_hwloc_get_devices_in_bitmap( qvi_hwloc_t *hwl, qv_hw_obj_type_t dev_type, - const qvi_hwloc_bitmap_s &bitmap, + const qvi_hwloc_bitmap &bitmap, qvi_hwloc_dev_list_t &devs ); diff --git a/src/qvi-hwpool.cc b/src/qvi-hwpool.cc index 7e1c983..86e2595 100644 --- a/src/qvi-hwpool.cc +++ b/src/qvi-hwpool.cc @@ -23,13 +23,13 @@ qvi_hwpool_res::hints(void) return m_hints; } -qvi_hwloc_bitmap_s & +qvi_hwloc_bitmap & qvi_hwpool_res::affinity(void) { return m_affinity; } -const qvi_hwloc_bitmap_s & +const qvi_hwloc_bitmap & qvi_hwpool_res::affinity(void) const { return m_affinity; @@ -256,7 +256,7 @@ qvi_hwpool::initialize( return add_devices_with_affinity(hwloc); } -const qvi_hwloc_bitmap_s & +const qvi_hwloc_bitmap & qvi_hwpool::cpuset(void) const { return m_cpu.affinity(); diff --git a/src/qvi-hwpool.h b/src/qvi-hwpool.h index b63c34e..f38ded6 100644 --- a/src/qvi-hwpool.h +++ b/src/qvi-hwpool.h @@ -27,7 +27,7 @@ struct qvi_hwpool_res { /** Resource hint flags. */ qv_scope_create_hints_t m_hints = QV_SCOPE_CREATE_HINT_NONE; /** The resource's affinity encoded as a bitmap. */ - qvi_hwloc_bitmap_s m_affinity; + qvi_hwloc_bitmap m_affinity; public: /** Returns the resource's create hints. */ qv_scope_create_hints_t @@ -35,12 +35,12 @@ struct qvi_hwpool_res { /** * Returns a reference to the resource's affinity encoded by a bitmap. */ - qvi_hwloc_bitmap_s & + qvi_hwloc_bitmap & affinity(void); /** * Returns a const reference to the resource's affinity encoded by a bitmap. */ - const qvi_hwloc_bitmap_s & + const qvi_hwloc_bitmap & affinity(void) const; }; @@ -72,7 +72,7 @@ struct qvi_hwpool_dev : qvi_hwpool_res { /** Device type. */ qv_hw_obj_type_t m_type = QV_HW_OBJ_LAST; /** The bitmap encoding CPU affinity. */ - qvi_hwloc_bitmap_s m_affinity; + qvi_hwloc_bitmap m_affinity; /** Device ID (ordinal). */ int m_id = QVI_HWLOC_DEVICE_INVALID_ID; /** The PCI bus ID. */ @@ -164,7 +164,7 @@ struct qvi_hwpool { /** * Returns a const reference to the hardware pool's cpuset. */ - const qvi_hwloc_bitmap_s & + const qvi_hwloc_bitmap & cpuset(void) const; /** * Returns a const reference to the hardware pool's devices. diff --git a/src/qvi-hwsplit.cc b/src/qvi-hwsplit.cc index 5e81d42..6202573 100644 --- a/src/qvi-hwsplit.cc +++ b/src/qvi-hwsplit.cc @@ -128,7 +128,7 @@ pool_release_cpus_by_cpuset( /** Maintains a mapping between IDs to device information. */ using id2devs_t = std::multimap; -qvi_hwsplit_s::qvi_hwsplit_s( +qvi_hwsplit::qvi_hwsplit( qv_scope_t *parent, uint_t group_size, uint_t split_size, @@ -146,7 +146,7 @@ qvi_hwsplit_s::qvi_hwsplit_s( // appropriate vector sizing. } -qvi_hwsplit_s::~qvi_hwsplit_s(void) +qvi_hwsplit::~qvi_hwsplit(void) { for (auto &hwpool : m_hwpools) { qvi_delete(&hwpool); @@ -154,7 +154,7 @@ qvi_hwsplit_s::~qvi_hwsplit_s(void) } void -qvi_hwsplit_s::reserve(void) +qvi_hwsplit::reserve(void) { m_group_tids.resize(m_group_size); m_hwpools.resize(m_group_size); @@ -162,20 +162,44 @@ qvi_hwsplit_s::reserve(void) m_affinities.resize(m_group_size); } -qvi_hwloc_bitmap_s -qvi_hwsplit_s::cpuset(void) const +qvi_hwloc_bitmap +qvi_hwsplit::cpuset(void) const { // This shouldn't happen. assert(m_hwpools.size() != 0); return m_hwpools[0]->cpuset(); } +std::vector & +qvi_hwsplit::tids(void) +{ + return m_group_tids; +} + +std::vector & +qvi_hwsplit::colors(void) +{ + return m_colors; +} + +std::vector & +qvi_hwsplit::hwpools(void) +{ + return m_hwpools; +} + +qvi_hwloc_cpusets_t & +qvi_hwsplit::affinities(void) +{ + return m_affinities; +} + int -qvi_hwsplit_s::split_cpuset( +qvi_hwsplit::split_cpuset( qvi_hwloc_cpusets_t &result ) const { // The cpuset that we are going to split. - const qvi_hwloc_bitmap_s base_cpuset = cpuset(); + const qvi_hwloc_bitmap base_cpuset = cpuset(); // Pointer to my hwloc instance. qvi_hwloc_t *const hwloc = qvi_rmi_client_hwloc(m_rmi); // Holds the host's split cpusets. @@ -195,7 +219,7 @@ qvi_hwsplit_s::split_cpuset( } int -qvi_hwsplit_s::osdev_cpusets( +qvi_hwsplit::osdev_cpusets( qvi_hwloc_cpusets_t &result ) const { // Get the number of devices we have available in the provided scope. @@ -217,7 +241,7 @@ qvi_hwsplit_s::osdev_cpusets( } int -qvi_hwsplit_s::primary_cpusets( +qvi_hwsplit::primary_cpusets( qvi_hwloc_cpusets_t &result ) const { // We were provided a real host resource type that we have to split. Or @@ -236,7 +260,7 @@ qvi_hwsplit_s::primary_cpusets( } qvi_map_fn_t -qvi_hwsplit_s::affinity_preserving_policy(void) const +qvi_hwsplit::affinity_preserving_policy(void) const { switch (m_split_at_type) { // For split() @@ -250,7 +274,7 @@ qvi_hwsplit_s::affinity_preserving_policy(void) const /** Releases all devices contained in the provided split aggregate. */ int -qvi_hwsplit_s::release_devices(void) +qvi_hwsplit::release_devices(void) { int rc = QV_SUCCESS; for (auto &hwpool : m_hwpools) { @@ -266,7 +290,7 @@ qvi_hwsplit_s::release_devices(void) // TODO(skg) Plenty of opportunity for optimization. // TODO(skg) Move lots of logic to map int -qvi_hwsplit_s::split_devices_user_defined(void) +qvi_hwsplit::split_devices_user_defined(void) { // Release devices from the hardware pools because // they will be redistributed in the next step. @@ -328,7 +352,7 @@ qvi_hwsplit_s::split_devices_user_defined(void) * Affinity preserving device splitting. */ int -qvi_hwsplit_s::split_devices_affinity_preserving(void) +qvi_hwsplit::split_devices_affinity_preserving(void) { // Release devices from the hardware pools because // they will be redistributed in the next step. @@ -402,7 +426,7 @@ apply_cpuset_mapping( * User-defined split. */ int -qvi_hwsplit_s::split_user_defined(void) +qvi_hwsplit::split_user_defined(void) { // Split the base cpuset into the requested number of pieces. qvi_hwloc_cpusets_t cpusets; @@ -425,7 +449,7 @@ qvi_hwsplit_s::split_user_defined(void) } int -qvi_hwsplit_s::split_affinity_preserving_pass1(void) +qvi_hwsplit::split_affinity_preserving_pass1(void) { // cpusets used for first mapping pass. qvi_hwloc_cpusets_t cpusets{}; @@ -456,7 +480,7 @@ qvi_hwsplit_s::split_affinity_preserving_pass1(void) */ // TODO(skg) This needs more work. int -qvi_hwsplit_s::split_affinity_preserving(void) +qvi_hwsplit::split_affinity_preserving(void) { int rc = split_affinity_preserving_pass1(); if (rc != QV_SUCCESS) return rc; @@ -491,7 +515,7 @@ clamp_colors( * Splits aggregate scope data. */ int -qvi_hwsplit_s::split(void) +qvi_hwsplit::split(void) { int rc = QV_SUCCESS; bool auto_split = false; @@ -541,7 +565,7 @@ qvi_hwsplit_s::split(void) return rc; } -qvi_hwsplit_coll_s::qvi_hwsplit_coll_s( +qvi_hwsplit_coll::qvi_hwsplit_coll( qv_scope_t *parent, uint_t npieces, int color, @@ -550,8 +574,8 @@ qvi_hwsplit_coll_s::qvi_hwsplit_coll_s( , m_color(color) { const qvi_group *const pgroup = m_parent->group(); - if (pgroup->rank() == qvi_hwsplit_coll_s::s_rootid) { - m_hwsplit = qvi_hwsplit_s( + if (pgroup->rank() == qvi_hwsplit_coll::rootid) { + m_hwsplit = qvi_hwsplit( m_parent, pgroup->size(), npieces, split_at_type ); } @@ -559,7 +583,7 @@ qvi_hwsplit_coll_s::qvi_hwsplit_coll_s( template int -qvi_hwsplit_coll_s::scatter_values( +qvi_hwsplit_coll::scatter_values( const std::vector &values, TYPE *value ) { @@ -570,7 +594,7 @@ qvi_hwsplit_coll_s::scatter_values( qvi_group *const group = m_parent->group(); std::vector txbuffs(0); - if (group->rank() == s_rootid) { + if (group->rank() == rootid) { const uint_t group_size = group->size(); txbuffs.resize(group_size); // Pack the values. @@ -584,7 +608,7 @@ qvi_hwsplit_coll_s::scatter_values( if (qvi_unlikely(rc != QV_SUCCESS)) goto out; } - rc = group->scatter(txbuffs.data(), s_rootid, &rxbuff); + rc = group->scatter(txbuffs.data(), rootid, &rxbuff); if (qvi_unlikely(rc != QV_SUCCESS)) goto out; *value = *(TYPE *)rxbuff->data(); @@ -602,14 +626,14 @@ qvi_hwsplit_coll_s::scatter_values( template int -qvi_hwsplit_coll_s::bcast_value( +qvi_hwsplit_coll::bcast_value( TYPE *value ) { static_assert(std::is_trivially_copyable::value, ""); qvi_group *const group = m_parent->group(); std::vector values; - if (group->rank() == s_rootid) { + if (group->rank() == rootid) { values.resize(group->size()); std::fill(values.begin(), values.end(), *value); } @@ -618,7 +642,7 @@ qvi_hwsplit_coll_s::bcast_value( template int -qvi_hwsplit_coll_s::gather_values( +qvi_hwsplit_coll::gather_values( TYPE invalue, std::vector &outvals ) { @@ -638,10 +662,10 @@ qvi_hwsplit_coll_s::gather_values( // Gather the values to the root. bool shared = false; qvi_bbuff **bbuffs = nullptr; - rc = group->gather(txbuff, s_rootid, &shared, &bbuffs); + rc = group->gather(txbuff, rootid, &shared, &bbuffs); if (qvi_unlikely(rc != QV_SUCCESS)) goto out; // The root fills in the output. - if (group->rank() == s_rootid) { + if (group->rank() == rootid) { outvals.resize(group_size); // Unpack the values. for (uint_t i = 0; i < group_size; ++i) { @@ -649,7 +673,7 @@ qvi_hwsplit_coll_s::gather_values( } } out: - if (!shared || (shared && (group->rank() == s_rootid))) { + if (!shared || (shared && (group->rank() == rootid))) { if (bbuffs) { for (uint_t i = 0; i < group_size; ++i) { qvi_bbuff_delete(&bbuffs[i]); @@ -666,7 +690,7 @@ qvi_hwsplit_coll_s::gather_values( } int -qvi_hwsplit_coll_s::gather_hwpools( +qvi_hwsplit_coll::gather_hwpools( qvi_hwpool *txpool, std::vector &rxpools ) { @@ -679,10 +703,10 @@ qvi_hwsplit_coll_s::gather_hwpools( // Gather the values to the root. bool shared = false; qvi_bbuff **bbuffs = nullptr; - rc = group->gather(&txbuff, s_rootid, &shared, &bbuffs); + rc = group->gather(&txbuff, rootid, &shared, &bbuffs); if (rc != QV_SUCCESS) goto out; - if (group->rank() == s_rootid) { + if (group->rank() == rootid) { rxpools.resize(group_size); // Unpack the hwpools. for (uint_t i = 0; i < group_size; ++i) { @@ -693,7 +717,7 @@ qvi_hwsplit_coll_s::gather_hwpools( } } out: - if (!shared || (shared && (group->rank() == s_rootid))) { + if (!shared || (shared && (group->rank() == rootid))) { if (bbuffs) { for (uint_t i = 0; i < group_size; ++i) { qvi_bbuff_delete(&bbuffs[i]); @@ -709,26 +733,26 @@ qvi_hwsplit_coll_s::gather_hwpools( } int -qvi_hwsplit_coll_s::gather(void) +qvi_hwsplit_coll::gather(void) { - int rc = gather_values(qvi_task::mytid(), m_hwsplit.m_group_tids); + int rc = gather_values(qvi_task::mytid(), m_hwsplit.tids()); if (qvi_unlikely(rc != QV_SUCCESS)) return rc; - rc = gather_values(m_color, m_hwsplit.m_colors); + rc = gather_values(m_color, m_hwsplit.colors()); if (qvi_unlikely(rc != QV_SUCCESS)) return rc; // Note that the result hwpools are copies, so we can modify them freely. - rc = gather_hwpools(m_parent->hwpool(), m_hwsplit.m_hwpools); + rc = gather_hwpools(m_parent->hwpool(), m_hwsplit.hwpools()); if (qvi_unlikely(rc != QV_SUCCESS)) return rc; const int myrank = m_parent->group()->rank(); const uint_t group_size = m_parent->group()->size(); - if (myrank == qvi_hwsplit_coll_s::s_rootid) { - m_hwsplit.m_affinities.resize(group_size); + if (myrank == qvi_hwsplit_coll::rootid) { + m_hwsplit.affinities().resize(group_size); for (uint_t tid = 0; tid < group_size; ++tid) { hwloc_cpuset_t cpuset = nullptr; rc = m_parent->group()->task()->bind_top(&cpuset); if (qvi_unlikely(rc != QV_SUCCESS)) break; // - rc = m_hwsplit.m_affinities[tid].set(cpuset); + rc = m_hwsplit.affinities()[tid].set(cpuset); // Clean up. qvi_hwloc_bitmap_delete(&cpuset); if (qvi_unlikely(rc != QV_SUCCESS)) break; @@ -738,7 +762,7 @@ qvi_hwsplit_coll_s::gather(void) } int -qvi_hwsplit_coll_s::scatter_hwpools( +qvi_hwsplit_coll::scatter_hwpools( const std::vector &pools, qvi_hwpool **pool ) { @@ -748,7 +772,7 @@ qvi_hwsplit_coll_s::scatter_hwpools( qvi_group *const group = m_parent->group(); - if (group->rank() == s_rootid) { + if (group->rank() == rootid) { const uint_t group_size = group->size(); txbuffs.resize(group_size); // Pack the hwpools. @@ -762,7 +786,7 @@ qvi_hwsplit_coll_s::scatter_hwpools( if (rc != QV_SUCCESS) goto out; } - rc = group->scatter(txbuffs.data(), s_rootid, &rxbuff); + rc = group->scatter(txbuffs.data(), rootid, &rxbuff); if (rc != QV_SUCCESS) goto out; rc = qvi_bbuff_rmi_unpack(rxbuff->data(), pool); @@ -778,23 +802,23 @@ qvi_hwsplit_coll_s::scatter_hwpools( } int -qvi_hwsplit_coll_s::scatter( +qvi_hwsplit_coll::scatter( int *colorp, qvi_hwpool **result ) { - const int rc = scatter_values(m_hwsplit.m_colors, colorp); + const int rc = scatter_values(m_hwsplit.colors(), colorp); if (qvi_unlikely(rc != QV_SUCCESS)) return rc; - return scatter_hwpools(m_hwsplit.m_hwpools, result); + return scatter_hwpools(m_hwsplit.hwpools(), result); } int -qvi_hwsplit_coll_s::barrier(void) +qvi_hwsplit_coll::barrier(void) { return m_parent->group()->barrier(); } int -qvi_hwsplit_coll_s::split( +qvi_hwsplit_coll::split( int *colorp, qvi_hwpool **result ) { @@ -809,7 +833,7 @@ qvi_hwsplit_coll_s::split( if (qvi_unlikely(rc != QV_SUCCESS)) return rc; // The root does this calculation. int rc2 = QV_SUCCESS; - if (m_parent->group()->rank() == s_rootid) { + if (m_parent->group()->rank() == rootid) { rc2 = m_hwsplit.split(); } // Wait for the split information. Explicitly barrier here in case the diff --git a/src/qvi-hwsplit.h b/src/qvi-hwsplit.h index 0caab61..c24371f 100644 --- a/src/qvi-hwsplit.h +++ b/src/qvi-hwsplit.h @@ -29,8 +29,8 @@ * qvi_hwsplit_coll_s. Typically, collective operations will fill in a * this structure, but that isn't a requirement. */ -struct qvi_hwsplit_s { -//private: +struct qvi_hwsplit { +private: /** A pointer to my RMI. */ qvi_rmi_client_t *m_rmi = nullptr; /** The base hardware pool we are splitting. */ @@ -69,16 +69,16 @@ struct qvi_hwsplit_s { qvi_hwloc_cpusets_t m_affinities; public: /** Constructor. */ - qvi_hwsplit_s(void) = default; + qvi_hwsplit(void) = default; /** Constructor. */ - qvi_hwsplit_s( + qvi_hwsplit( qv_scope_t *parent, uint_t group_size, uint_t split_size, qv_hw_obj_type_t split_at_type ); /** Destructor. */ - ~qvi_hwsplit_s(void); + ~qvi_hwsplit(void); /** * Resizes the relevant containers to make * room for |group size| number of elements. @@ -90,8 +90,20 @@ struct qvi_hwsplit_s { * among the aggregate, but other resources may be distributed differently. * For example, some hardware pools may have GPUs, while others may not. */ - qvi_hwloc_bitmap_s + qvi_hwloc_bitmap cpuset(void) const; + /** Returns a reference to the group TIDs. */ + std::vector & + tids(void); + /** Returns a reference to the group colors. */ + std::vector & + colors(void); + /** Returns a reference to the group hardware pools. */ + std::vector & + hwpools(void); + /** Returns a reference to the group task affinities. */ + qvi_hwloc_cpusets_t & + affinities(void); /** * Performs a straightforward splitting of the provided cpuset: * split the provided base cpuset into split_size distinct pieces. @@ -153,12 +165,8 @@ struct qvi_hwsplit_s { * split operations requiring aggregated resource knowledge AND coordination * between tasks in the parent scope to perform a split. */ -struct qvi_hwsplit_coll_s { - /** - * The root task ID used for collective operations. - * We use 0 as the root because 0 will always exist. - */ - static constexpr int s_rootid = 0; +struct qvi_hwsplit_coll { +private: /** Points to the parent scope that we are splitting. */ qv_scope_t *m_parent = nullptr; /** My color. */ @@ -167,9 +175,15 @@ struct qvi_hwsplit_coll_s { * Stores group-global hardware split information brought together by * collective operations across the members in the parent scope. */ - qvi_hwsplit_s m_hwsplit; + qvi_hwsplit m_hwsplit; +public: + /** + * The root task ID used for collective operations. + * We use 0 as the root because 0 will always exist. + */ + static constexpr int rootid = 0; /** Constructor. */ - qvi_hwsplit_coll_s(void) = delete; + qvi_hwsplit_coll(void) = delete; /** Constructor. */ /** * Hardware resources will be split based on the provided split parameters: @@ -179,7 +193,7 @@ struct qvi_hwsplit_coll_s { * maybe_obj_type: Potentially the object type that we are splitting at. This * value influences how the splitting algorithms perform their mapping. */ - qvi_hwsplit_coll_s( + qvi_hwsplit_coll( qv_scope_t *parent, uint_t npieces, int color, diff --git a/src/qvi-nvml.cc b/src/qvi-nvml.cc index 8b3ba39..04d09eb 100644 --- a/src/qvi-nvml.cc +++ b/src/qvi-nvml.cc @@ -22,7 +22,7 @@ int qvi_hwloc_nvml_get_device_cpuset_by_pci_bus_id( qvi_hwloc_t *hwl, const std::string &uuid, - qvi_hwloc_bitmap_s &cpuset + qvi_hwloc_bitmap &cpuset ) { #ifndef CUDAToolkit_FOUND QVI_UNUSED(hwl); diff --git a/src/qvi-nvml.h b/src/qvi-nvml.h index 3df2095..be0feff 100644 --- a/src/qvi-nvml.h +++ b/src/qvi-nvml.h @@ -25,7 +25,7 @@ int qvi_hwloc_nvml_get_device_cpuset_by_pci_bus_id( qvi_hwloc_t *hwl, const std::string &uuid, - qvi_hwloc_bitmap_s &cpuset + qvi_hwloc_bitmap &cpuset ); #ifdef __cplusplus diff --git a/src/qvi-rsmi.cc b/src/qvi-rsmi.cc index 98cfd7d..244b3cb 100644 --- a/src/qvi-rsmi.cc +++ b/src/qvi-rsmi.cc @@ -23,7 +23,7 @@ int qvi_hwloc_rsmi_get_device_cpuset_by_device_id( qvi_hwloc_t *hwl, uint32_t devid, - qvi_hwloc_bitmap_s &cpuset + qvi_hwloc_bitmap &cpuset ) { #ifndef ROCmSMI_FOUND QVI_UNUSED(hwl); diff --git a/src/qvi-rsmi.h b/src/qvi-rsmi.h index 814307c..ac3d165 100644 --- a/src/qvi-rsmi.h +++ b/src/qvi-rsmi.h @@ -24,7 +24,7 @@ int qvi_hwloc_rsmi_get_device_cpuset_by_device_id( qvi_hwloc_t *hwl, uint32_t devid, - qvi_hwloc_bitmap_s &cpuset + qvi_hwloc_bitmap &cpuset ); #endif diff --git a/src/qvi-scope.cc b/src/qvi-scope.cc index b89e805..1fda238 100644 --- a/src/qvi-scope.cc +++ b/src/qvi-scope.cc @@ -232,7 +232,7 @@ qv_scope::split( qvi_group *group = nullptr; qv_scope_t *ichild = nullptr; // Split the hardware resources based on the provided split parameters. - qvi_hwsplit_coll_s chwsplit( + qvi_hwsplit_coll chwsplit( this, npieces, color, maybe_obj_type ); rc = chwsplit.split(&colorp, &hwpool); @@ -275,7 +275,7 @@ qv_scope::thsplit( const uint_t group_size = k; // Construct the hardware split. - qvi_hwsplit_s hwsplit(this, group_size, npieces, maybe_obj_type); + qvi_hwsplit hwsplit(this, group_size, npieces, maybe_obj_type); // Eagerly make room for the group member information. hwsplit.reserve(); // Since this is called by a single task, get its ID and associated @@ -289,15 +289,15 @@ qv_scope::thsplit( if (rc != QV_SUCCESS) return rc; for (uint_t i = 0; i < group_size; ++i) { // Store requested colors in aggregate. - hwsplit.m_colors[i] = kcolors[i]; + hwsplit.colors()[i] = kcolors[i]; // Since the parent hardware pool is the resource we are splitting and // agg_split_* calls expect |group_size| elements, replicate by dups. - rc = qvi_dup(*m_hwpool, &hwsplit.m_hwpools[i]); + rc = qvi_dup(*m_hwpool, &hwsplit.hwpools()[i]); if (rc != QV_SUCCESS) break; // Since this is called by a single task, replicate its task ID, too. - hwsplit.m_group_tids[i] = taskid; + hwsplit.tids()[i] = taskid; // Same goes for the task's affinity. - hwsplit.m_affinities[i].set(task_affinity); + hwsplit.affinities()[i].set(task_affinity); } // Cleanup: we don't need task_affinity anymore. qvi_hwloc_bitmap_delete(&task_affinity); @@ -316,7 +316,7 @@ qv_scope::thsplit( for (uint_t i = 0; i < group_size; ++i) { // Copy out, since the hardware pools in splitagg will get freed. qvi_hwpool *hwpool = nullptr; - rc = qvi_dup(*hwsplit.m_hwpools[i], &hwpool); + rc = qvi_dup(*hwsplit.hwpools()[i], &hwpool); if (rc != QV_SUCCESS) break; // Create and initialize the new scope. qv_scope_t *child = nullptr; diff --git a/src/qvi-task.cc b/src/qvi-task.cc index 6b67cb8..976cabf 100644 --- a/src/qvi-task.cc +++ b/src/qvi-task.cc @@ -46,7 +46,7 @@ qvi_task::init_bind_stack(void) ); if (qvi_unlikely(rc != QV_SUCCESS)) return rc; - m_stack.push(qvi_hwloc_bitmap_s(current_bind)); + m_stack.push(qvi_hwloc_bitmap(current_bind)); qvi_hwloc_bitmap_delete(¤t_bind); return rc; } @@ -90,7 +90,7 @@ qvi_task::bind_push( hwloc_const_cpuset_t cpuset ) { // Copy input bitmap because we don't want to directly modify it. - qvi_hwloc_bitmap_s bitmap_copy(cpuset); + qvi_hwloc_bitmap bitmap_copy(cpuset); // Change policy const int rc = qvi_rmi_set_cpubind( m_rmi, mytid(), bitmap_copy.cdata() diff --git a/src/qvi-task.h b/src/qvi-task.h index 954d56b..2606974 100644 --- a/src/qvi-task.h +++ b/src/qvi-task.h @@ -20,7 +20,7 @@ #include "qvi-common.h" #include "qvi-hwloc.h" -using qvi_task_bind_stack_t = std::stack; +using qvi_task_bind_stack_t = std::stack; struct qvi_task { private: