Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup scope and hardware pool code. #265

Merged
merged 1 commit into from
Jul 30, 2024
Merged

Cleanup scope and hardware pool code. #265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/quo-vadis-pthread.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ qv_pthread_scopes_free(
return QV_ERR_INVLD_ARG;
}
try {
qv_scope_s::thdel(&scopes, nscopes);
qv_scope_s::thdestroy(&scopes, nscopes);
return QV_SUCCESS;
}
qvi_catch_and_return();
Expand Down
6 changes: 3 additions & 3 deletions src/quo-vadis.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ qv_scope_free(
return QV_ERR_INVLD_ARG;
}
try {
qv_scope_s::del(&scope);
qv_scope_s::destroy(&scope);
return QV_SUCCESS;
}
qvi_catch_and_return();
Expand All @@ -98,7 +98,7 @@ qv_scope_nobjs(
return QV_ERR_INVLD_ARG;
}
try {
*nobjs = scope->nobjects(obj);
*nobjs = scope->hwpool_nobjects(obj);
return QV_SUCCESS;
}
qvi_catch_and_return();
Expand Down Expand Up @@ -142,7 +142,7 @@ qv_scope_barrier(
return QV_ERR_INVLD_ARG;
}
try {
return scope->barrier();
return scope->group_barrier();
}
qvi_catch_and_return();
}
Expand Down
136 changes: 12 additions & 124 deletions src/qvi-hwpool.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,132 +13,26 @@
* Hardware Resource Pool
*/

// TODOs
// * Resource reference counting.
// * Need to deal with resource unavailability.
// * Split and attach devices properly.
// * Have bitmap scratch pad that is initialized once, then destroyed? This
// approach may be an nice allocation optimization, but in heavily threaded
// code may be a bottleneck.


// Notes:
// * Does it make sense attempting resource exclusivity? Why not just let the
// users get what they ask for and hope that the abstractions that we provide do
// a good enough job most of the time. Making the user deal with resource
// exhaustion and retries (which will eventually be the case with
// QV_RES_UNAVAILABLE) is error prone and often frustrating.
//
// * Reference Counting: we should probably still implement a rudimentary
// reference counting system, but perhaps not for enforcing resource
// exclusivity. Rather we could use this information to guide a collection of
// resource allocators that would use resource availability for their pool
// management strategies.

// A Straightforward Reference Counting Approach: Maintain an array of integers
// with length number of cpuset bits. As each resource (bitmap) is obtained,
// increment the internal counter of each corresponding position. When a
// resource is released, decrement in a similar way. If a location in the array
// is zero, then the resource is not in use. For devices, we can take a similar
// approach using the device IDs instead of the bit positions.

#include "qvi-hwpool.h"
#include "qvi-bbuff-rmi.h"
#include "qvi-utils.h"

#if 0
/**
*
*/
static int
cpus_available(
hwloc_const_cpuset_t which,
hwloc_const_cpuset_t from,
bool *avail
) {
// TODO(skg) Cache storage for calculation?
hwloc_cpuset_t tcpus = nullptr;
int rc = qvi_hwloc_bitmap_calloc(&tcpus);
if (rc != QV_SUCCESS) return rc;

int hrc = hwloc_bitmap_and(tcpus, which, from);
if (hrc != 0) {
rc = QV_ERR_HWLOC;
}
if (rc == QV_SUCCESS) {
*avail = cpusets_equal(tcpus, which);
}
else {
*avail = false;
}
qvi_hwloc_bitmap_free(&tcpus);
return rc;
}
#endif

/**
* Example:
* obcpuset 0110 0101
* request 1000 1010
* obcpuset' 1110 1111
*/
#if 0
static int
pool_obtain_cpus_by_cpuset(
qvi_hwpool_s *pool,
hwloc_const_cpuset_t request
) {
#if 0
int hwrc = hwloc_bitmap_or(
pool->obcpuset,
pool->obcpuset,
request
);
return (hwrc == 0 ? QV_SUCCESS : QV_ERR_HWLOC);
#endif
QVI_UNUSED(pool);
QVI_UNUSED(request);
return QV_SUCCESS;
}
#endif

/**
* Example:
* obcpuset 0110 0101
* release 0100 0100
* obcpuset' 0010 0001
*/
#if 0
static int
pool_release_cpus_by_cpuset(
qvi_hwpool_s *pool,
hwloc_const_cpuset_t release
) {
int hwrc = hwloc_bitmap_andnot(
pool->obcpuset,
pool->obcpuset,
release
);
return (hwrc == 0 ? QV_SUCCESS : QV_ERR_HWLOC);
}
#endif

qv_scope_create_hints_t
qvi_hwpool_res_s::hints(void)
{
return m_hints;
}

qvi_hwloc_bitmap_s &
qvi_hwpool_cpu_s::cpuset(void)
qvi_hwpool_res_s::affinity(void)
{
return m_cpuset;
return m_affinity;
}

const qvi_hwloc_bitmap_s &
qvi_hwpool_cpu_s::cpuset(void) const
qvi_hwpool_res_s::affinity(void) const
{
return m_cpuset;
return m_affinity;
}

int
Expand All @@ -149,7 +43,7 @@ qvi_hwpool_cpu_s::packinto(
const int rc = qvi_bbuff_rmi_pack_item(buff, m_hints);
if (qvi_unlikely(rc != QV_SUCCESS)) return rc;
// Pack cpuset.
return qvi_bbuff_rmi_pack_item(buff, m_cpuset);
return qvi_bbuff_rmi_pack_item(buff, m_affinity);
}

int
Expand All @@ -168,7 +62,7 @@ qvi_hwpool_cpu_s::unpack(
buffpos += bw;
// Unpack bitmap.
rc = qvi_bbuff_rmi_unpack_item(
cpu.m_cpuset, buffpos, &bw
cpu.m_affinity, buffpos, &bw
);
if (qvi_unlikely(rc != QV_SUCCESS)) goto out;
total_bw += bw;
Expand Down Expand Up @@ -209,7 +103,7 @@ int
qvi_hwpool_dev_s::id(
qv_device_id_type_t format,
char **result
) {
) const {
int rc = QV_SUCCESS, nw = 0;
switch (format) {
case (QV_DEVICE_ID_UUID):
Expand All @@ -233,12 +127,6 @@ qvi_hwpool_dev_s::id(
return rc;
}

const qvi_hwloc_bitmap_s &
qvi_hwpool_dev_s::affinity(void) const
{
return m_affinity;
}

int
qvi_hwpool_dev_s::packinto(
qvi_bbuff_t *buff
Expand Down Expand Up @@ -327,7 +215,7 @@ qvi_hwpool_s::add_devices_with_affinity(
for (const auto devt : qvi_hwloc_supported_devices()) {
qvi_hwloc_dev_list_t devs;
rc = qvi_hwloc_get_devices_in_bitmap(
hwloc, devt, m_cpu.cpuset(), devs
hwloc, devt, m_cpu.affinity(), devs
);
if (qvi_unlikely(rc != QV_SUCCESS)) return rc;
for (const auto &dev : devs) {
Expand All @@ -339,7 +227,7 @@ qvi_hwpool_s::add_devices_with_affinity(
}

int
qvi_hwpool_s::new_hwpool(
qvi_hwpool_s::create(
qvi_hwloc_t *hwloc,
hwloc_const_cpuset_t cpuset,
qvi_hwpool_s **opool
Expand All @@ -362,7 +250,7 @@ qvi_hwpool_s::initialize(
qvi_hwloc_t *hwloc,
hwloc_const_bitmap_t cpuset
) {
const int rc = m_cpu.cpuset().set(cpuset);
const int rc = m_cpu.affinity().set(cpuset);
if (qvi_unlikely(rc != QV_SUCCESS)) return rc;
// Add devices with affinity to the hardware pool.
return add_devices_with_affinity(hwloc);
Expand All @@ -371,7 +259,7 @@ qvi_hwpool_s::initialize(
const qvi_hwloc_bitmap_s &
qvi_hwpool_s::cpuset(void) const
{
return m_cpu.cpuset();
return m_cpu.affinity();
}

const qvi_hwpool_devs_t &
Expand All @@ -388,7 +276,7 @@ qvi_hwpool_s::nobjects(
) {
if (qvi_hwloc_obj_type_is_host_resource(obj_type)) {
return qvi_hwloc_get_nobjs_in_cpuset(
hwloc, obj_type, m_cpu.cpuset().cdata(), result
hwloc, obj_type, m_cpu.affinity().cdata(), result
);
}
*result = m_devs.count(obj_type);
Expand Down
40 changes: 15 additions & 25 deletions src/qvi-hwpool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,33 +26,29 @@ struct qvi_hwpool_res_s {
protected:
/** Resource hint flags. */
qv_scope_create_hints_t m_hints = QV_SCOPE_CREATE_HINT_NONE;
/** The resource's affinity encoded as a bitmap. */
qvi_hwloc_bitmap_s m_affinity;
public:
/** Returns the resource's create hints. */
qv_scope_create_hints_t
hints(void);
};

/**
* Defines a hardware pool CPU. A CPU here may have multiple
* processing units (PUs), which are defined in the CPU's cpuset.
*/
struct qvi_hwpool_cpu_s : qvi_hwpool_res_s {
private:
/** The cpuset of the CPU's PUs. */
qvi_hwloc_bitmap_s m_cpuset;
public:
/**
* Returns a reference to the
* CPU's resources encoded by a bitmap.
* Returns a reference to the resource's affinity encoded by a bitmap.
*/
qvi_hwloc_bitmap_s &
cpuset(void);
affinity(void);
/**
* Returns a const reference to the
* CPU's resources encoded by a bitmap.
* Returns a const reference to the resource's affinity encoded by a bitmap.
*/
const qvi_hwloc_bitmap_s &
cpuset(void) const;
affinity(void) const;
};

/**
* Defines a hardware pool CPU. A CPU here may have multiple
* processing units (PUs), which are defined as the CPU's affinity.
*/
struct qvi_hwpool_cpu_s : qvi_hwpool_res_s {
/** Packs the instance into the provided buffer. */
int
packinto(
Expand Down Expand Up @@ -109,13 +105,7 @@ struct qvi_hwpool_dev_s : qvi_hwpool_res_s {
id(
qv_device_id_type_t format,
char **result
);
/**
* Returns a const reference to the
* device's affinity encoded by a bitmap.
*/
const qvi_hwloc_bitmap_s &
affinity(void) const;
) const;
/** Packs the instance into the provided buffer. */
int
packinto(
Expand Down Expand Up @@ -157,7 +147,7 @@ struct qvi_hwpool_s {
* on the affinity encoded in the provided cpuset.
*/
static int
new_hwpool(
create(
qvi_hwloc_t *hwloc,
hwloc_const_cpuset_t cpuset,
qvi_hwpool_s **opool
Expand Down
Loading
Loading