From 487a05561a8e3e01a8a6f91acd2f66f40f10c88b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20K=2E=20Guti=C3=A9rrez?= Date: Thu, 18 Jul 2024 23:04:38 -0600 Subject: [PATCH] Checkpoint first cut at pthread group support. (#233) Still needs plenty of work. Signed-off-by: Samuel K. Gutierrez --- src/CMakeLists.txt | 5 + src/quo-vadis-pthread.cc | 28 +++- src/qvi-common.h | 1 + src/qvi-group-omp.h | 9 ++ src/qvi-group-pthread.cc | 54 +++++++ src/qvi-group-pthread.h | 123 ++++++++++++++++ src/qvi-group.cc | 38 +++++ src/qvi-group.h | 13 +- src/qvi-pthread.cc | 21 +++ src/qvi-pthread.h | 182 ++++++++++++++++++++++++ src/qvi-scope.cc | 60 ++++---- src/qvi-scope.h | 40 +++--- src/qvi-utils.h | 24 ++++ tests/internal/CMakeLists.txt | 12 -- tests/internal/test-mpi-scope-ksplit.cc | 147 ------------------- tests/test-pthread-split.c | 8 +- 16 files changed, 544 insertions(+), 221 deletions(-) create mode 100644 src/qvi-group-pthread.cc create mode 100644 src/qvi-group-pthread.h create mode 100644 src/qvi-group.cc create mode 100644 src/qvi-pthread.cc create mode 100644 src/qvi-pthread.h delete mode 100644 tests/internal/test-mpi-scope-ksplit.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 194f1c2..d0d7d79 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,7 +29,9 @@ add_library( qvi-rmi.h qvi-bbuff-rmi.h qvi-task.h + qvi-pthread.h qvi-group.h + qvi-group-pthread.h qvi-map.h qvi-scope.h qvi-log.cc @@ -41,6 +43,9 @@ add_library( qvi-hwpool.cc qvi-rmi.cc qvi-task.cc + qvi-group.cc + qvi-pthread.cc + qvi-group-pthread.cc qvi-map.cc qvi-scope.cc quo-vadis.cc diff --git a/src/quo-vadis-pthread.cc b/src/quo-vadis-pthread.cc index baa50d2..dad6baf 100644 --- a/src/quo-vadis-pthread.cc +++ b/src/quo-vadis-pthread.cc @@ -19,11 +19,11 @@ #include "qvi-common.h" // IWYU pragma: keep #include "quo-vadis-pthread.h" +#include "qvi-pthread.h" +#include "qvi-group-pthread.h" #include "qvi-scope.h" #include "qvi-utils.h" -typedef void *(*qvi_pthread_routine_fun_ptr_t)(void *); - struct qvi_pthread_args_s { qv_scope_t *scope = nullptr; qvi_pthread_routine_fun_ptr_t th_routine = nullptr; @@ -66,7 +66,7 @@ qv_pthread_scope_split( return QV_ERR_INVLD_ARG; } try { - return qvi_scope_ksplit( + return qvi_scope_thsplit( scope, npieces, color_array, nthreads, QV_HW_OBJ_LAST, subscope ); @@ -86,7 +86,7 @@ qv_pthread_scope_split_at( return QV_ERR_INVLD_ARG; } try { - return qvi_scope_ksplit_at( + return qvi_scope_thsplit_at( scope, type, color_array, nthreads, subscopes ); } @@ -103,12 +103,26 @@ qv_pthread_create( ) { // Memory will be freed in qv_pthread_routine to avoid memory leaks. qvi_pthread_args_s *arg_ptr = nullptr; - const int rc = qvi_new(&arg_ptr, scope, thread_routine, arg); + int rc = qvi_new(&arg_ptr, scope, thread_routine, arg); // Since this is meant to behave similarly to // pthread_create(), return a reasonable errno. if (qvi_unlikely(rc != QV_SUCCESS)) return ENOMEM; + // TODO(skg) Cleanup. + qvi_pthread_group_pthread_create_args_s *cargs = nullptr; + rc = qvi_new(&cargs); + if (qvi_unlikely(rc != QV_SUCCESS)) { + qvi_delete(&arg_ptr); + return ENOMEM; + } + // TODO(skg) Cleanup. + auto gt = dynamic_cast(qvi_scope_group_get(scope)); + cargs->group = gt->thgroup; + cargs->th_routine = qvi_pthread_routine; + cargs->th_routine_argp = arg_ptr; - return pthread_create(thread, attr, qvi_pthread_routine, arg_ptr); + return pthread_create( + thread, attr, qvi_pthread_group_s::call_first_from_pthread_create, cargs + ); } int @@ -120,7 +134,7 @@ qv_pthread_scopes_free( return QV_ERR_INVLD_ARG; } try { - qvi_scope_kfree(&scopes, nscopes); + qvi_scope_thfree(&scopes, nscopes); return QV_SUCCESS; } qvi_catch_and_return(); diff --git a/src/qvi-common.h b/src/qvi-common.h index ea7b1ac..baf1425 100644 --- a/src/qvi-common.h +++ b/src/qvi-common.h @@ -47,6 +47,7 @@ #include "qvi-log.h" #include #include +#include #include #include #include diff --git a/src/qvi-group-omp.h b/src/qvi-group-omp.h index 41aec2a..5c735ca 100644 --- a/src/qvi-group-omp.h +++ b/src/qvi-group-omp.h @@ -63,6 +63,15 @@ struct qvi_group_omp_s : public qvi_group_s { qvi_group_s **child ); + virtual int + thsplit( + int, + qvi_group_s ** + ) { + // TODO(skg) Need to test this. + return QV_ERR_NOT_SUPPORTED; + } + virtual int split( int color, diff --git a/src/qvi-group-pthread.cc b/src/qvi-group-pthread.cc new file mode 100644 index 0000000..317218f --- /dev/null +++ b/src/qvi-group-pthread.cc @@ -0,0 +1,54 @@ +/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2020-2024 Triad National Security, LLC + * All rights reserved. + * + * This file is part of the quo-vadis project. See the LICENSE file at the + * top-level directory of this distribution. + */ + +/** + * @file qvi-group-pthread.cc + */ + +#include "qvi-group-pthread.h" +#include "qvi-utils.h" + +int +qvi_group_pthread_s::self( + qvi_group_t **child +) { + constexpr int group_size = 1; + qvi_group_pthread_t *ichild = nullptr; + int rc = qvi_new(&ichild); + if (rc != QV_SUCCESS) goto out; + // Create a group containing a single thread. + rc = qvi_new(&ichild->thgroup, group_size); +out: + if (rc != QV_SUCCESS) { + qvi_delete(&ichild); + } + *child = ichild; + return rc; +} + +int +qvi_group_pthread_s::split( + int, + int, + qvi_group_t ** +) { + // TODO(skg) + return QV_ERR_NOT_SUPPORTED; +} + +qvi_zgroup_pthread_s::qvi_zgroup_pthread_s( + int group_size +) { + int rc = qvi_new(&thgroup, group_size); + if (rc != QV_SUCCESS) throw qvi_runtime_error(); +} + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */ diff --git a/src/qvi-group-pthread.h b/src/qvi-group-pthread.h new file mode 100644 index 0000000..71bffbe --- /dev/null +++ b/src/qvi-group-pthread.h @@ -0,0 +1,123 @@ +/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2020-2024 Triad National Security, LLC + * All rights reserved. + * + * This file is part of the quo-vadis project. See the LICENSE file at the + * top-level directory of this distribution. + */ + +/** + * @file qvi-group-pthread.h + */ + +#ifndef QVI_GROUP_PTHREAD_H +#define QVI_GROUP_PTHREAD_H + +#include "qvi-common.h" +#include "qvi-group.h" +#include "qvi-pthread.h" + +struct qvi_group_pthread_s : public qvi_group_s { + /** Underlying group instance. */ + qvi_pthread_group_t *thgroup = nullptr; + /** Constructor. */ + qvi_group_pthread_s(void) = default; + /** Destructor. */ + virtual ~qvi_group_pthread_s(void) + { + qvi_delete(&thgroup); + } + + virtual qvi_task_t * + task(void) + { + return thgroup->task(); + } + + virtual int + rank(void) + { + return thgroup->rank(); + } + + virtual int + size(void) + { + return thgroup->size(); + } + + virtual int + barrier(void) + { + return thgroup->barrier(); + } + + virtual int + make_intrinsic( + qv_scope_intrinsic_t + ) { + // Nothing to do. + return QV_SUCCESS; + } + + virtual int + self( + qvi_group_s **child + ); + + virtual int + thsplit( + int, + qvi_group_s ** + ) { + // TODO(skg) Need to test this. + return QV_ERR_NOT_SUPPORTED; + } + + virtual int + split( + int color, + int key, + qvi_group_s **child + ); + + virtual int + gather( + qvi_bbuff_t *txbuff, + int root, + bool *shared, + qvi_bbuff_t ***rxbuffs + ) { + return thgroup->gather_bbuffs( + txbuff, root, shared, rxbuffs + ); + } + + virtual int + scatter( + qvi_bbuff_t **txbuffs, + int root, + qvi_bbuff_t **rxbuff + ) { + return thgroup->scatter_bbuffs( + txbuffs, root, rxbuff + ); + } +}; +typedef qvi_group_pthread_s qvi_group_pthread_t; + +struct qvi_zgroup_pthread_s : public qvi_group_pthread_s { + /** Default constructor. */ + qvi_zgroup_pthread_s(void) = delete; + /** Constructor. */ + qvi_zgroup_pthread_s( + int group_size + ); +}; + +#endif + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */ diff --git a/src/qvi-group.cc b/src/qvi-group.cc new file mode 100644 index 0000000..9e4b4f0 --- /dev/null +++ b/src/qvi-group.cc @@ -0,0 +1,38 @@ +/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2021-2024 Triad National Security, LLC + * All rights reserved. + * + * This file is part of the quo-vadis project. See the LICENSE file at the + * top-level directory of this distribution. + */ + +/** + * @file qvi-group.cc + */ + +#include "qvi-group.h" +#include "qvi-group-pthread.h" +#include "qvi-utils.h" + +int +qvi_group_s::thsplit( + int nthreads, + qvi_group_s **child +) { + qvi_group_pthread_t *ichild = nullptr; + int rc = qvi_new(&ichild); + if (rc != QV_SUCCESS) goto out; + + rc = qvi_new(&ichild->thgroup, nthreads); +out: + if (rc != QV_SUCCESS) { + qvi_delete(&ichild); + } + *child = ichild; + return rc; +} + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */ diff --git a/src/qvi-group.h b/src/qvi-group.h index 78303ea..734d4f8 100644 --- a/src/qvi-group.h +++ b/src/qvi-group.h @@ -27,8 +27,9 @@ using qvi_group_id_t = uint64_t; /** * Virtual base group class. */ -struct qvi_group_s { +struct qvi_group_s : qvi_refc_s { protected: + // TODO(skg) Remove from base. /** Task associated with this group. */ qvi_task_t *m_task = nullptr; public: @@ -44,7 +45,7 @@ struct qvi_group_s { qvi_delete(&m_task); } /** Returns pointer to the caller's task information. */ - qvi_task_t * + virtual qvi_task_t * task(void) { return m_task; @@ -71,6 +72,14 @@ struct qvi_group_s { self( qvi_group_s **child ) = 0; + /** + * Creates a new thread group by splitting off of the caller's group. + */ + virtual int + thsplit( + int nthreads, + qvi_group_s **child + ); /** * Creates new groups by splitting this group based on color, key. * Returns the appropriate newly created child group to the caller. diff --git a/src/qvi-pthread.cc b/src/qvi-pthread.cc new file mode 100644 index 0000000..db3bd12 --- /dev/null +++ b/src/qvi-pthread.cc @@ -0,0 +1,21 @@ +/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2024 Triad National Security, LLC + * All rights reserved. + * + * This file is part of the quo-vadis project. See the LICENSE file at the + * top-level directory of this distribution. + */ + +/** + * @file qvi-pthread.cc + */ + +#include "qvi-pthread.h" +#include "qvi-bbuff.h" +#include "qvi-utils.h" +#include "qvi-task.h" + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */ diff --git a/src/qvi-pthread.h b/src/qvi-pthread.h new file mode 100644 index 0000000..c6bdaf7 --- /dev/null +++ b/src/qvi-pthread.h @@ -0,0 +1,182 @@ +/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2024 Triad National Security, LLC + * All rights reserved. + * + * This file is part of the quo-vadis project. See the LICENSE file at the + * top-level directory of this distribution. + */ + +/** + * @file qvi-pthread.h + */ + +#ifndef QVI_PTHREAD_H +#define QVI_PTHREAD_H + +#include "qvi-common.h" +#include "qvi-task.h" // IWYU pragma: keep +#include "qvi-utils.h" + +// TODO(skg) Rename +typedef void *(*qvi_pthread_routine_fun_ptr_t)(void *); + +struct qvi_pthread_group_s; +typedef struct qvi_pthread_group_s qvi_pthread_group_t; + +struct qvi_pthread_group_pthread_create_args_s { + qvi_pthread_group_t *group = nullptr; + qvi_pthread_routine_fun_ptr_t th_routine = nullptr; + void *th_routine_argp = nullptr; +}; + +struct qvi_pthread_group_s { +private: + /** Group size. */ + int m_size = 0; + /** Holds the thread TIDs in this group. */ + std::vector m_tids; + /** Holds tid to rank mapping. */ + std::map m_tid2rank; + /** Holds tid to task mapping. */ + std::map m_tid2task; + /** Used for mutexy things. */ + std::mutex m_mutex; + /** Used for barrier things. */ + pthread_barrier_t m_barrier; +public: + /** Constructor. */ + qvi_pthread_group_s(void) = delete; + /** Constructor. */ + qvi_pthread_group_s( + int group_size + ) : m_size(group_size) + { + const int rc = pthread_barrier_init(&m_barrier, NULL, group_size); + if (qvi_unlikely(rc != 0)) throw qvi_runtime_error(); + } + /** */ + static void * + call_first_from_pthread_create( + void *arg + ) { + // TODO(skg) Cleanup. + auto args = (qvi_pthread_group_pthread_create_args_s *)arg; + auto group = args->group; + auto thread_routine = args->th_routine; + auto th_routine_argp = args->th_routine_argp; + // Let the threads add their TIDs to the vector. + { + std::lock_guard guard(group->m_mutex); + group->m_tids.push_back(qvi_gettid()); + } + // Make sure they all contribute before continuing. + pthread_barrier_wait(&group->m_barrier); + // Elect one thread to be the worker. + bool worker = false; + { + std::lock_guard guard(group->m_mutex); + worker = group->m_tids.at(0) == qvi_gettid(); + } + // The worker populates the TID to rank mapping, while the others wait. + if (worker) { + std::sort(group->m_tids.begin(), group->m_tids.end()); + + for (int i = 0; i < group->m_size; ++i) { + const pid_t tid = group->m_tids[i]; + group->m_tid2rank.insert({tid, i}); + } + pthread_barrier_wait(&group->m_barrier); + } + else { + pthread_barrier_wait(&group->m_barrier); + } + // Everyone can now create their task and populate the mapping table. + { + std::lock_guard guard(group->m_mutex); + qvi_task_t *task = nullptr; + const int rc = qvi_new(&task); + if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error(); + group->m_tid2task.insert({qvi_gettid(), task}); + } + // Make sure they all finish before continuing. + pthread_barrier_wait(&group->m_barrier); + // Free the provided argument container. + qvi_delete(&args); + // Finally, call the specified thread routine. + return thread_routine(th_routine_argp); + } + /** Destructor. */ + ~qvi_pthread_group_s(void) + { + for (auto &tt : m_tid2task) { + qvi_delete(&tt.second); + } + pthread_barrier_destroy(&m_barrier); + } + + int + size(void) + { + return m_size; + } + + int + rank(void) + { + std::lock_guard guard(m_mutex); + return m_tid2rank.at(qvi_gettid()); + } + + qvi_task_t * + task(void) + { + std::lock_guard guard(m_mutex); + return m_tid2task.at(qvi_gettid()); + } + + int + barrier(void) + { + pthread_barrier_wait(&m_barrier); + return QV_SUCCESS; + } + + int + create_from_split( + int, + int, + qvi_pthread_group_s ** + ) { + // TODO(skg) + return QV_ERR_NOT_SUPPORTED; + } + + int + gather_bbuffs( + qvi_bbuff_t *, + int, + bool *, + qvi_bbuff_t *** + ) { + // TODO(skg) + return QV_ERR_NOT_SUPPORTED; + } + + int + scatter_bbuffs( + qvi_bbuff_t **, + int, + qvi_bbuff_t ** + ) { + // TODO(skg) + return QV_ERR_NOT_SUPPORTED; + } +}; +typedef struct qvi_pthread_group_s qvi_pthread_group_t; + +#endif + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */ diff --git a/src/qvi-scope.cc b/src/qvi-scope.cc index da0358d..0d9ae7c 100644 --- a/src/qvi-scope.cc +++ b/src/qvi-scope.cc @@ -38,7 +38,7 @@ struct qv_scope_s { ~qv_scope_s(void) { qvi_delete(&hwpool); - qvi_delete(&group); + group->release(); } }; @@ -521,7 +521,7 @@ qvi_scope_free( } void -qvi_scope_kfree( +qvi_scope_thfree( qv_scope_t ***kscopes, uint_t k ) { @@ -1119,9 +1119,8 @@ qvi_scope_split( return rc; } - int -qvi_scope_ksplit( +qvi_scope_thsplit( qv_scope_t *parent, uint_t npieces, int *kcolors, @@ -1142,10 +1141,11 @@ qvi_scope_ksplit( // Since this is called by a single task, get its ID and associated hardware // affinity here, and replicate them in the following loop that populates // splitagg. No point in doing this in a loop. - const pid_t taskid = qvi_task_s::mytid(); + const pid_t taskid = qvi_task_t::mytid(); hwloc_cpuset_t task_affinity = nullptr; rc = qvi_rmi_task_get_cpubind( - parent->group->task()->rmi(), taskid, &task_affinity + parent->group->task()->rmi(), + taskid, &task_affinity ); if (rc != QV_SUCCESS) return rc; // Now populate the relevant data before attempting a split. @@ -1175,71 +1175,79 @@ qvi_scope_ksplit( // Now populate the children. qv_scope_t **ikchildren = new qv_scope_t*[group_size]; + qvi_group_t *thgroup = nullptr; + // Split off from our parent group. This call is called from a context in + // which a process is splitting its resources across threads, so create a + // new thread group for each child. + rc = parent->group->thsplit(group_size, &thgroup); + if (rc != QV_SUCCESS) { + qvi_scope_thfree(&ikchildren, group_size); + return rc; + } + for (uint_t i = 0; i < group_size; ++i) { - // Split off from our parent group. This call is usually called from a - // context in which a process is splitting its resources across its - // threads, so create a new self group for each child. - qvi_group_t *group = nullptr; - rc = parent->group->self(&group); - if (rc != QV_SUCCESS) break; // Create and initialize the new scope. qv_scope_t *child = nullptr; rc = qvi_scope_new(&child); if (rc != QV_SUCCESS) { - qvi_delete(&group); + qvi_delete(&thgroup); break; } // Copy out, since the hardware pools in splitagg will get freed. qvi_hwpool_s *hwpool = nullptr; rc = qvi_dup(*splitagg.hwpools[i], &hwpool); if (rc != QV_SUCCESS) { - qvi_delete(&group); + qvi_delete(&thgroup); qvi_scope_free(&child); break; } - rc = scope_init(child, group, hwpool); + rc = scope_init(child, thgroup, hwpool); if (rc != QV_SUCCESS) { qvi_delete(&hwpool); - qvi_delete(&group); + qvi_delete(&thgroup); qvi_scope_free(&child); break; } + thgroup->retain(); ikchildren[i] = child; } if (rc != QV_SUCCESS) { - qvi_scope_kfree(&ikchildren, k); + qvi_scope_thfree(&ikchildren, k); } + // Subtract one to account for the parent's implicit retain during + // construct. + thgroup->release(); *kchildren = ikchildren; return rc; } int -qvi_scope_split_at( +qvi_scope_thsplit_at( qv_scope_t *parent, qv_hw_obj_type_t type, - int color, - qv_scope_t **child + int *kgroup_ids, + uint_t k, + qv_scope_t ***kchildren ) { int nobj = 0; int rc = qvi_scope_nobjs(parent, type, &nobj); if (rc != QV_SUCCESS) return rc; - return qvi_scope_split(parent, nobj, color, type, child); + return qvi_scope_thsplit(parent, nobj, kgroup_ids, k, type, kchildren); } int -qvi_scope_ksplit_at( +qvi_scope_split_at( qv_scope_t *parent, qv_hw_obj_type_t type, - int *kgroup_ids, - uint_t k, - qv_scope_t ***kchildren + int color, + qv_scope_t **child ) { int nobj = 0; int rc = qvi_scope_nobjs(parent, type, &nobj); if (rc != QV_SUCCESS) return rc; - return qvi_scope_ksplit(parent, nobj, kgroup_ids, k, type, kchildren); + return qvi_scope_split(parent, nobj, color, type, child); } int diff --git a/src/qvi-scope.h b/src/qvi-scope.h index 8f13085..519c187 100644 --- a/src/qvi-scope.h +++ b/src/qvi-scope.h @@ -38,10 +38,10 @@ qvi_scope_free( ); /** - * Frees scope resources and container. + * Frees scope resources and container created by qvi_scope_thsplit*. */ void -qvi_scope_kfree( +qvi_scope_thfree( qv_scope_t ***kscopes, uint_t k ); @@ -98,28 +98,22 @@ qvi_scope_barrier( qv_scope_t *scope ); -/** - * - */ int -qvi_scope_split( +qvi_scope_thsplit( qv_scope_t *parent, - int ncolors, - int color, + uint_t npieces, + int *kcolors, + uint_t k, qv_hw_obj_type_t maybe_obj_type, - qv_scope_t **child + qv_scope_t ***kchildren ); -/** - * - */ int -qvi_scope_ksplit( +qvi_scope_thsplit_at( qv_scope_t *parent, - uint_t npieces, - int *kcolors, + qv_hw_obj_type_t type, + int *kgroup_ids, uint_t k, - qv_hw_obj_type_t maybe_obj_type, qv_scope_t ***kchildren ); @@ -127,10 +121,11 @@ qvi_scope_ksplit( * */ int -qvi_scope_split_at( +qvi_scope_split( qv_scope_t *parent, - qv_hw_obj_type_t type, - int group_id, + int ncolors, + int color, + qv_hw_obj_type_t maybe_obj_type, qv_scope_t **child ); @@ -138,12 +133,11 @@ qvi_scope_split_at( * */ int -qvi_scope_ksplit_at( +qvi_scope_split_at( qv_scope_t *parent, qv_hw_obj_type_t type, - int *kgroup_ids, - uint_t k, - qv_scope_t ***kchildren + int group_id, + qv_scope_t **child ); /** diff --git a/src/qvi-utils.h b/src/qvi-utils.h index f130360..caf0577 100644 --- a/src/qvi-utils.h +++ b/src/qvi-utils.h @@ -21,6 +21,30 @@ #ifdef __cplusplus +/** + * Reference counting base class that provides retain/release semantics. + */ +struct qvi_refc_s { +private: + mutable std::atomic refc = {1}; +public: + virtual ~qvi_refc_s(void) = default; + + void + retain(void) const + { + ++refc; + } + + void + release(void) const + { + if (--refc == 0) { + delete this; + } + } +}; + /** * Constructs a new object of a given type. *t will be valid if successful, * undefined otherwise. Returns QV_SUCCESS if successful. diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 98c2eae..737eb30 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -84,18 +84,6 @@ set_target_properties( ################################################################################ ################################################################################ -if(MPI_FOUND) - add_executable( - test-mpi-scope-ksplit - test-mpi-scope-ksplit.cc - ) - - target_link_libraries( - test-mpi-scope-ksplit - quo-vadis-mpi - ) -endif() - # Set core test properties. set_tests_properties( hwloc diff --git a/tests/internal/test-mpi-scope-ksplit.cc b/tests/internal/test-mpi-scope-ksplit.cc deleted file mode 100644 index 08fec7a..0000000 --- a/tests/internal/test-mpi-scope-ksplit.cc +++ /dev/null @@ -1,147 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2020-2024 Triad National Security, LLC - * All rights reserved. - * - * This file is part of the quo-vadis project. See the LICENSE file at the - * top-level directory of this distribution. - */ - -/** - * @file test-mpi-scope-ksplit.cc - */ - -#include "mpi.h" -#include "quo-vadis-mpi.h" -#include "qvi-scope.h" -#include "qvi-test-common.h" - -int -main(void) -{ - cstr_t ers = NULL; - MPI_Comm comm = MPI_COMM_WORLD; - - /* Initialization */ - int rc = MPI_Init(nullptr, nullptr); - if (rc != MPI_SUCCESS) { - ers = "MPI_Init() failed"; - qvi_test_panic("%s (rc=%d)", ers, rc); - } - - int wsize = 0; - rc = MPI_Comm_size(comm, &wsize); - if (rc != MPI_SUCCESS) { - ers = "MPI_Comm_size() failed"; - qvi_test_panic("%s (rc=%d)", ers, rc); - } - - int wrank = 0; - rc = MPI_Comm_rank(comm, &wrank); - if (rc != MPI_SUCCESS) { - ers = "MPI_Comm_rank() failed"; - qvi_test_panic("%s (rc=%d)", ers, rc); - } - // Helps with flushing MPI process output. - setbuf(stdout, NULL); - - qv_scope_t *base_scope = nullptr; - rc = qv_mpi_scope_get( - comm, QV_SCOPE_USER, &base_scope - ); - if (rc != QV_SUCCESS) { - ers = "qv_scope_get() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - int ncores = 0; - rc = qv_scope_nobjs( - base_scope, QV_HW_OBJ_CORE, &ncores - ); - if (rc != QV_SUCCESS) { - ers = "qv_scope_nobjs() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - // Test internal APIs - const uint_t npieces = ncores / 2; - std::vector colors(npieces * 2); - std::fill_n( - colors.begin(), colors.size(), - QV_SCOPE_SPLIT_AFFINITY_PRESERVING - ); - //std::iota(colors.begin(), colors.end(), 0); - const uint_t k = colors.size(); - - qvi_log_info("Testing ksplit()"); - - qv_scope_t **subscopes = nullptr; - rc = qvi_scope_ksplit( - base_scope, npieces, colors.data(), - k, QV_HW_OBJ_LAST, &subscopes - ); - if (rc != QV_SUCCESS) { - ers = "qvi_scope_ksplit() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - for (uint_t i = 0; i < k; ++i) { - qvi_test_scope_report( - subscopes[i], std::to_string(i).c_str() - ); - qvi_test_bind_push( - subscopes[i] - ); - qvi_test_bind_pop( - subscopes[i] - ); - } - // Done with all the subscopes, so clean-up everything. - qvi_scope_kfree(&subscopes, k); - - qvi_log_info("Testing ksplit_at()"); - // Test qvi_scope_ksplit_at(). - rc = qvi_scope_ksplit_at( - base_scope, QV_HW_OBJ_PU, colors.data(), k, &subscopes - ); - if (rc != QV_SUCCESS) { - ers = "qvi_scope_ksplit_at() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - for (uint_t i = 0; i < k; ++i) { - qvi_test_scope_report( - subscopes[i], std::to_string(i).c_str() - ); - qvi_test_bind_push( - subscopes[i] - ); - qvi_test_bind_pop( - subscopes[i] - ); - } - // Done with all the subscopes, so clean-up everything. - qvi_scope_kfree(&subscopes, k); - - rc = qv_scope_free(base_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_free() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - rc = MPI_Finalize(); - if (rc != MPI_SUCCESS) { - ers = "MPI_Finalize() failed"; - qvi_test_panic("%s (rc=%d)", ers, rc); - } - - if (wrank == 0) { - qvi_log_info("Test Passed"); - } - - return EXIT_SUCCESS; -} - -/* - * vim: ft=cpp ts=4 sts=4 sw=4 expandtab - */ diff --git a/tests/test-pthread-split.c b/tests/test-pthread-split.c index 55576b0..7423b81 100644 --- a/tests/test-pthread-split.c +++ b/tests/test-pthread-split.c @@ -108,7 +108,7 @@ main(void) pthread_t thid[nthreads]; pthread_attr_t *attr = NULL; - for (int i = 0 ; i < nthreads; i ++) { + for (int i = 0 ; i < nthreads; ++i) { const int ptrc = qv_pthread_create( &thid[i], attr, thread_work, &thargs[i], th_scopes[i] ); @@ -119,7 +119,7 @@ main(void) } void *ret = NULL; - for (int i = 0 ; i < nthreads; i ++){ + for (int i = 0 ; i < nthreads; i ++) { if (pthread_join(thid[i], &ret) != 0) { perror("pthread_create() error"); exit(3); @@ -159,7 +159,7 @@ main(void) } pthread_t thid2[nthreads]; - for(int i = 0 ; i < nthreads; i ++){ + for(int i = 0 ; i < nthreads; ++i) { const int ptrc = qv_pthread_create( &thid2[i], attr, thread_work, &thargs2[i], th_scopes[i] ); @@ -169,7 +169,7 @@ main(void) } } - for (int i = 0 ; i < nthreads; i ++) { + for (int i = 0 ; i < nthreads; ++i) { if (pthread_join(thid2[i], &ret) != 0) { perror("pthread_create() error"); exit(3);