From c70d13ca23aef309548d5dde3500fc935da72ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20K=2E=20Guti=C3=A9rrez?= Date: Thu, 11 Jul 2024 16:58:08 -0600 Subject: [PATCH] Differentiate between Pthread and OpenMP Infrastructure. (#210) Signed-off-by: Samuel K. Gutierrez --- include/CMakeLists.txt | 25 +- include/quo-vadis-omp.h | 43 +++ ...quo-vadis-thread.h => quo-vadis-pthread.h} | 12 +- src/CMakeLists.txt | 64 +++- src/quo-vadis-omp.cc | 43 +++ ...o-vadis-thread.cc => quo-vadis-pthread.cc} | 23 +- src/{qvi-group-thread.cc => qvi-group-omp.cc} | 20 +- src/{qvi-group-thread.h => qvi-group-omp.h} | 30 +- src/{qvi-thread.cc => qvi-omp.cc} | 277 +++++++----------- src/{qvi-thread.h => qvi-omp.h} | 54 ++-- tests/CMakeLists.txt | 42 +-- tests/test-mpi-threads-layout.c | 2 +- tests/test-mpi-threads.c | 2 +- tests/{test-threads.c => test-omp.c} | 4 +- tests/test-openmp.c | 32 -- tests/test-progress-thread.c | 2 +- tests/test-pthread-split.c | 2 +- 17 files changed, 344 insertions(+), 333 deletions(-) create mode 100644 include/quo-vadis-omp.h rename include/{quo-vadis-thread.h => quo-vadis-pthread.h} (88%) create mode 100644 src/quo-vadis-omp.cc rename src/{quo-vadis-thread.cc => quo-vadis-pthread.cc} (81%) rename src/{qvi-group-thread.cc => qvi-group-omp.cc} (76%) rename src/{qvi-group-thread.h => qvi-group-omp.h} (70%) rename src/{qvi-thread.cc => qvi-omp.cc} (50%) rename src/{qvi-thread.h => qvi-omp.h} (59%) rename tests/{test-threads.c => test-omp.c} (98%) delete mode 100644 tests/test-openmp.c diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 67949798..4f2c5d96 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2022 Triad National Security, LLC +# Copyright (c) 2020-2024 Triad National Security, LLC # All rights reserved. # # Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC @@ -14,18 +14,27 @@ install( FILES quo-vadis.h quo-vadis-process.h - quo-vadis-thread.h + quo-vadis-pthread.h DESTINATION include ) +if(OpenMP_C_FOUND) + install( + FILES + quo-vadis-omp.h + DESTINATION + include + ) +endif() + if(MPI_FOUND) -install( - FILES - quo-vadis-mpi.h - DESTINATION - include -) + install( + FILES + quo-vadis-mpi.h + DESTINATION + include + ) endif() # vim: ts=4 sts=4 sw=4 expandtab diff --git a/include/quo-vadis-omp.h b/include/quo-vadis-omp.h new file mode 100644 index 00000000..7cf6fea5 --- /dev/null +++ b/include/quo-vadis-omp.h @@ -0,0 +1,43 @@ +/* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2022-2024 Triad National Security, LLC + * All rights reserved. + * + * Copyright (c) 2022-2024 Inria + * All rights reserved. + * + * Copyright (c) 2022-2024 Bordeaux INP + * All rights reserved. + * + * This file is part of the quo-vadis project. See the LICENSE file at the + * top-level directory of this distribution. + */ + +/** + * @file quo-vadis-omp.h + */ + +#ifndef QUO_VADIS_OMP_H +#define QUO_VADIS_OMP_H + +#include "quo-vadis.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int +qv_omp_scope_get( + qv_scope_intrinsic_t iscope, + qv_scope_t **scope +); + +#ifdef __cplusplus +} +#endif + +#endif + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */ diff --git a/include/quo-vadis-thread.h b/include/quo-vadis-pthread.h similarity index 88% rename from include/quo-vadis-thread.h rename to include/quo-vadis-pthread.h index d985af15..ceab5c2f 100644 --- a/include/quo-vadis-thread.h +++ b/include/quo-vadis-pthread.h @@ -14,11 +14,11 @@ */ /** - * @file quo-vadis-thread.h + * @file quo-vadis-pthread.h */ -#ifndef QUO_VADIS_THREAD_H -#define QUO_VADIS_THREAD_H +#ifndef QUO_VADIS_PTHREAD_H +#define QUO_VADIS_PTHREAD_H #include "quo-vadis.h" #include @@ -47,12 +47,6 @@ qv_pthread_create( qv_scope_t *scope ); -int -qv_thread_scope_get( - qv_scope_intrinsic_t iscope, - qv_scope_t **scope -); - int qv_thread_scope_split_at( qv_scope_t *scope, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4d8a185b..194f1c2f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,6 +16,7 @@ add_library( quo-vadis-core OBJECT ../include/quo-vadis.h + ../include/quo-vadis-pthread.h qvi-common.h qvi-macros.h qvi-log.h @@ -43,6 +44,7 @@ add_library( qvi-map.cc qvi-scope.cc quo-vadis.cc + quo-vadis-pthread.cc ) set_target_properties( @@ -89,20 +91,6 @@ set( quo-vadis-process.cc ) -# TODO(skg) We need to fix this. OpenMP builds versus Pthread. -if(OpenMP_C_FOUND) -set( - QUO_VADIS_SOURCE - ${QUO_VADIS_SOURCE} - qvi-thread.h - qvi-group-thread.h - ../include/quo-vadis-thread.h - qvi-thread.cc - qvi-group-thread.cc - quo-vadis-thread.cc -) -endif() - add_library( quo-vadis SHARED @@ -110,12 +98,58 @@ add_library( ) if(OpenMP_C_FOUND) + add_library( + quo-vadis-omp + SHARED + qvi-omp.h + qvi-group-omp.h + ../include/quo-vadis-omp.h + qvi-omp.cc + qvi-group-omp.cc + quo-vadis-omp.cc + ) + set_target_properties( - quo-vadis + quo-vadis-omp PROPERTIES COMPILE_FLAGS ${OpenMP_C_FLAGS} LINK_FLAGS ${OpenMP_C_FLAGS} ) + + target_include_directories( + quo-vadis-omp + PUBLIC + $ + ${MPI_C_INCLUDE_DIRS} + ) + + target_link_libraries( + quo-vadis-omp + quo-vadis-core + ${MPI_C_LIBRARIES} + ) + + install( + TARGETS + quo-vadis-omp + DESTINATION + lib + ) + + # pkg-config ############################################################### + set(QV_PKG_LIBRARIES "-lquo-vadis-omp") + configure_file( + "${CMAKE_SOURCE_DIR}/pkgconfig/pkg-config.pc.in" + "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-omp.pc" + @ONLY + ) + + install( + FILES + "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-omp.pc" + DESTINATION + lib/pkgconfig + ) endif() target_include_directories( diff --git a/src/quo-vadis-omp.cc b/src/quo-vadis-omp.cc new file mode 100644 index 00000000..9d42d5b5 --- /dev/null +++ b/src/quo-vadis-omp.cc @@ -0,0 +1,43 @@ +/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2022-2024 Triad National Security, LLC + * All rights reserved. + * + * Copyright (c) 2022-2024 Inria + * All rights reserved. + * + * Copyright (c) 2022-2024 Bordeaux INP + * All rights reserved. + * + * This file is part of the quo-vadis project. See the LICENSE file at the + * top-level directory of this distribution. + */ + +/** + * @file quo-vadis-thread.cc + */ + +#include "qvi-common.h" // IWYU pragma: keep +#include "quo-vadis-omp.h" +#include "qvi-group-omp.h" +#include "qvi-scope.h" +#include "qvi-utils.h" + +int +qv_omp_scope_get( + qv_scope_intrinsic_t iscope, + qv_scope_t **scope +) { + // Create the base process group. + qvi_group_omp_s *zgroup = nullptr; + const int rc = qvi_new(&zgroup); + if (rc != QV_SUCCESS) { + *scope = nullptr; + return rc; + } + return qvi_scope_get(zgroup, iscope, scope); +} + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */ diff --git a/src/quo-vadis-thread.cc b/src/quo-vadis-pthread.cc similarity index 81% rename from src/quo-vadis-thread.cc rename to src/quo-vadis-pthread.cc index c8c5c1ad..a866c465 100644 --- a/src/quo-vadis-thread.cc +++ b/src/quo-vadis-pthread.cc @@ -14,32 +14,13 @@ */ /** - * @file quo-vadis-thread.cc + * @file quo-vadis-pthread.cc */ #include "qvi-common.h" // IWYU pragma: keep -#include "qvi-group-thread.h" -#include "quo-vadis-thread.h" +#include "quo-vadis-pthread.h" #include "qvi-scope.h" #include "qvi-utils.h" -#ifdef OPENMP_FOUND -#include -#endif - -int -qv_thread_scope_get( - qv_scope_intrinsic_t iscope, - qv_scope_t **scope -) { - // Create the base process group. - qvi_group_thread_s *zgroup = nullptr; - const int rc = qvi_new(&zgroup); - if (rc != QV_SUCCESS) { - *scope = nullptr; - return rc; - } - return qvi_scope_get(zgroup, iscope, scope); -} int qv_thread_scope_split( diff --git a/src/qvi-group-thread.cc b/src/qvi-group-omp.cc similarity index 76% rename from src/qvi-group-thread.cc rename to src/qvi-group-omp.cc index d743a940..66d0a439 100644 --- a/src/qvi-group-thread.cc +++ b/src/qvi-group-omp.cc @@ -14,30 +14,30 @@ */ /** - * @file qvi-group-thread.cc + * @file qvi-group-omp.cc */ -#include "qvi-group-thread.h" +#include "qvi-group-omp.h" #include "qvi-utils.h" int -qvi_group_thread_s::make_intrinsic( +qvi_group_omp_s::make_intrinsic( qv_scope_intrinsic_t ) { // NOTE: the provided scope doesn't affect how // we create the thread group, so we ignore it. - return qvi_thread_group_create(&th_group); + return qvi_omp_group_create(&th_group); } int -qvi_group_thread_s::self( +qvi_group_omp_s::self( qvi_group_t **child ) { - qvi_group_thread_t *ichild = nullptr; + qvi_group_omp_t *ichild = nullptr; int rc = qvi_new(&ichild); if (rc != QV_SUCCESS) goto out; // Create a group containing a single thread - rc = qvi_thread_group_create_single(&ichild->th_group); + rc = qvi_omp_group_create_single(&ichild->th_group); out: if (rc != QV_SUCCESS) { qvi_delete(&ichild); @@ -47,16 +47,16 @@ qvi_group_thread_s::self( } int -qvi_group_thread_s::split( +qvi_group_omp_s::split( int color, int key, qvi_group_t **child ) { - qvi_group_thread_t *ichild = nullptr; + qvi_group_omp_t *ichild = nullptr; int rc = qvi_new(&ichild); if (rc != QV_SUCCESS) goto out; - rc = qvi_thread_group_create_from_split( + rc = qvi_omp_group_create_from_split( th_group, color, key, &ichild->th_group ); out: diff --git a/src/qvi-group-thread.h b/src/qvi-group-omp.h similarity index 70% rename from src/qvi-group-thread.h rename to src/qvi-group-omp.h index 93f70c21..8a402660 100644 --- a/src/qvi-group-thread.h +++ b/src/qvi-group-omp.h @@ -14,43 +14,43 @@ */ /** - * @file qvi-group-thread.h + * @file qvi-group-omp.h */ -#ifndef QVI_GROUP_THREAD_H -#define QVI_GROUP_THREAD_H +#ifndef QVI_GROUP_OMP_H +#define QVI_GROUP_OMP_H #include "qvi-common.h" #include "qvi-group.h" -#include "qvi-thread.h" +#include "qvi-omp.h" -struct qvi_group_thread_s : public qvi_group_s { +struct qvi_group_omp_s : public qvi_group_s { /** Underlying group instance. */ - qvi_thread_group_t *th_group = nullptr; + qvi_omp_group_t *th_group = nullptr; /** Constructor. */ - qvi_group_thread_s(void) = default; + qvi_group_omp_s(void) = default; /** Destructor. */ - virtual ~qvi_group_thread_s(void) + virtual ~qvi_group_omp_s(void) { - qvi_thread_group_free(&th_group); + qvi_omp_group_free(&th_group); } virtual int id(void) { - return qvi_thread_group_id(th_group); + return qvi_omp_group_id(th_group); } virtual int size(void) { - return qvi_thread_group_size(th_group); + return qvi_omp_group_size(th_group); } virtual int barrier(void) { - return qvi_thread_group_barrier(th_group); + return qvi_omp_group_barrier(th_group); } virtual int @@ -77,7 +77,7 @@ struct qvi_group_thread_s : public qvi_group_s { qvi_bbuff_t ***rxbuffs, int *shared ) { - return qvi_thread_group_gather_bbuffs( + return qvi_omp_group_gather_bbuffs( th_group, txbuff, root, rxbuffs, shared ); } @@ -88,12 +88,12 @@ struct qvi_group_thread_s : public qvi_group_s { int root, qvi_bbuff_t **rxbuff ) { - return qvi_thread_group_scatter_bbuffs( + return qvi_omp_group_scatter_bbuffs( th_group, txbuffs, root, rxbuff ); } }; -typedef qvi_group_thread_s qvi_group_thread_t; +typedef qvi_group_omp_s qvi_group_omp_t; #endif diff --git a/src/qvi-thread.cc b/src/qvi-omp.cc similarity index 50% rename from src/qvi-thread.cc rename to src/qvi-omp.cc index f361f01c..7fbd2531 100644 --- a/src/qvi-thread.cc +++ b/src/qvi-omp.cc @@ -14,45 +14,67 @@ */ /** - * @file qvi-thread.cc + * @file qvi-omp.cc */ -#include "qvi-thread.h" +#include "qvi-omp.h" #include "qvi-bbuff.h" #include "qvi-utils.h" -#ifdef OPENMP_FOUND #include -#endif -static int -qvi_thread_omp_get_num_threads(void) -{ -#ifdef OPENMP_FOUND - return omp_get_num_threads(); -#else - return 1; -#endif -} +/** + * Internal data structure for rank and size computing. + */ +struct qvi_omp_color_key_rank_s { + int color = 0; + int key = 0; + int rank = 0; -static int -qvi_thread_omp_get_thread_num(void) -{ -#ifdef OPENMP_FOUND - return omp_get_thread_num(); -#else - return 0; -#endif -} + static bool + by_color( + const qvi_omp_color_key_rank_s &a, + const qvi_omp_color_key_rank_s &b + ) { + return a.color < b.color; + } + + static bool + by_key( + const qvi_omp_color_key_rank_s &a, + const qvi_omp_color_key_rank_s &b + ) { + // If colors are the same, sort by key. + return a.color == b.color && a.key < b.key; + } + + static bool + by_rank( + const qvi_omp_color_key_rank_s &a, + const qvi_omp_color_key_rank_s &b + ) { + // If colors and keys are the same, sort by rank. + return a.color == b.color && a.key == b.key && a.rank < b.rank; + } +}; + +struct qvi_omp_subgroup_info_s { + /** Number of sub-groups created from split. */ + int ngroups = 0; + /** Number of members in this sub-group. */ + int size = 0; + /** My rank in this sub-group. */ + int rank = 0; +}; -struct qvi_thread_group_s { +struct qvi_omp_group_s { /** Group size. */ int size = 0; /** ID (rank) in group: this ID is unique to each thread. */ int rank = 0; /** Constructor. */ - qvi_thread_group_s(void) = delete; + qvi_omp_group_s(void) = delete; /** Constructor. */ - qvi_thread_group_s( + qvi_omp_group_s( int group_size, int group_rank ) : size(group_size) @@ -60,121 +82,71 @@ struct qvi_thread_group_s { }; void -qvi_thread_group_free( - qvi_thread_group_t **group +qvi_omp_group_free( + qvi_omp_group_t **group ) { #pragma omp barrier qvi_delete(group); } static int -qvi_thread_group_create_size( - qvi_thread_group_t **group, +qvi_omp_group_create_size( + qvi_omp_group_t **group, int size ) { - return qvi_new(group, size, qvi_thread_omp_get_thread_num()); + return qvi_new(group, size, omp_get_thread_num()); } int -qvi_thread_group_create( - qvi_thread_group_t **group +qvi_omp_group_create( + qvi_omp_group_t **group ) { - return qvi_thread_group_create_size( - group, qvi_thread_omp_get_num_threads() + return qvi_omp_group_create_size( + group, omp_get_num_threads() ); } int -qvi_thread_group_create_single( - qvi_thread_group_t **group +qvi_omp_group_create_single( + qvi_omp_group_t **group ) { - return qvi_thread_group_create_size(group, 1); + return qvi_omp_group_create_size(group, 1); } int -qvi_thread_group_id( - const qvi_thread_group_t *group +qvi_omp_group_id( + const qvi_omp_group_t *group ) { return group->rank; } int -qvi_thread_group_size( - const qvi_thread_group_t *group +qvi_omp_group_size( + const qvi_omp_group_t *group ) { return group->size; } int -qvi_thread_group_barrier( - qvi_thread_group_t * +qvi_omp_group_barrier( + qvi_omp_group_t * ) { return QV_SUCCESS; } -#ifdef OPENMP_FOUND -/** - * Internal data structure for rank and size computing. - */ -struct qvi_thread_color_key_rank_s { - int color = 0; - int key = 0; - int rank = 0; -}; - -/** - * - */ -struct qvi_thread_subgroup_info_s { - /** Number of sub-groups created from split. */ - int num_sgrp = 0; - /** Number of members in this sub-group. */ - int sgrp_size = 0; - /** My rank in this sub-group. */ - int sgrp_rank = 0; -}; - -static bool -ckr_compare_by_color( - const qvi_thread_color_key_rank_s &a, - const qvi_thread_color_key_rank_s &b -) { - return a.color < b.color; -} - -static bool -ckr_compare_by_key( - const qvi_thread_color_key_rank_s &a, - const qvi_thread_color_key_rank_s &b -) { - // If colors are the same, sort by key. - return a.color == b.color && a.key < b.key; -} - -static bool -ckr_compare_by_rank( - const qvi_thread_color_key_rank_s &a, - const qvi_thread_color_key_rank_s &b -) { - // If colors and keys are the same, sort by rank. - return a.color == b.color && a.key == b.key && a.rank < b.rank; -} - -#endif - static int qvi_get_subgroup_info( - qvi_thread_group_t *parent, + qvi_omp_group_t *parent, int color, int key, - qvi_thread_subgroup_info_s *sginfo + qvi_omp_subgroup_info_s *sginfo ) { const int size = parent->size; const int rank = parent->rank; - qvi_thread_color_key_rank_s *ckrs = nullptr; + qvi_omp_color_key_rank_s *ckrs = nullptr; #pragma omp single copyprivate(ckrs) - ckrs = new qvi_thread_color_key_rank_s[size]; + ckrs = new qvi_omp_color_key_rank_s[size]; // Gather colors and keys from ALL threads. ckrs[rank].color = color; ckrs[rank].key = key; @@ -189,9 +161,9 @@ qvi_get_subgroup_info( // Sort the color/key/rank array. First according to color, then by key, // but in the same color realm. If color and key are identical, sort by // the rank from given group. - std::sort(ckrs, ckrs + size, ckr_compare_by_color); - std::sort(ckrs, ckrs + size, ckr_compare_by_key); - std::sort(ckrs, ckrs + size, ckr_compare_by_rank); + std::sort(ckrs, ckrs + size, qvi_omp_color_key_rank_s::by_color); + std::sort(ckrs, ckrs + size, qvi_omp_color_key_rank_s::by_key); + std::sort(ckrs, ckrs + size, qvi_omp_color_key_rank_s::by_rank); // Calculate the number of distinct colors provided. std::set color_set; for (int i = 0; i < size; ++i) { @@ -200,7 +172,7 @@ qvi_get_subgroup_info( ncolors = color_set.size(); } // The number of distinct colors is the number of subgroups. - sginfo->num_sgrp = ncolors; + sginfo->ngroups = ncolors; // Compute my sub-group size and sub-group rank. int group_rank = 0; int group_size = 0; @@ -210,131 +182,111 @@ qvi_get_subgroup_info( const int current_color = ckrs[i].color; for (int j = i; j < size && current_color == ckrs[j].color; ++j) { if (ckrs[j].rank == rank) { - sginfo->sgrp_rank = group_rank; + sginfo->rank = group_rank; } group_size++; group_rank++; } - sginfo->sgrp_size = group_size; + sginfo->size = group_size; break; } // Barrier to sync for array deletion. #pragma omp barrier // Only one task deletes. - if (parent->rank == 0) { - delete[] ckrs; - } + #pragma omp single + delete[] ckrs; return QV_SUCCESS; } int -qvi_thread_group_create_from_split( - qvi_thread_group_t *parent, +qvi_omp_group_create_from_split( + qvi_omp_group_t *parent, int color, int key, - qvi_thread_group_t **child + qvi_omp_group_t **child ) { -#ifndef OPENMP_FOUND - // TODO(skg) This should return self. - *child = nullptr; - return QV_SUCCESS; -#else - qvi_thread_subgroup_info_s sginfo; + qvi_omp_group_t *ichild = nullptr; + + qvi_omp_subgroup_info_s sginfo; int rc = qvi_get_subgroup_info( parent, color, key, &sginfo ); - if (rc != QV_SUCCESS) { - *child = nullptr; - return QV_ERR_SPLIT; - } + if (rc != QV_SUCCESS) goto out; - qvi_thread_group_t *ichild = nullptr; - rc = qvi_new(&ichild, sginfo.sgrp_size, sginfo.sgrp_rank); + rc = qvi_new(&ichild, sginfo.size, sginfo.rank); +out: if (rc != QV_SUCCESS) { - *child = nullptr; - return rc; + qvi_delete(&ichild); } *child = ichild; - return rc; } -#endif int -qvi_thread_group_gather_bbuffs( - qvi_thread_group_t *group, +qvi_omp_group_gather_bbuffs( + qvi_omp_group_t *group, qvi_bbuff_t *txbuff, - int root, + int, qvi_bbuff_t ***rxbuffs, int *shared_alloc ) { - QVI_UNUSED(root); - const int send_count = (int)qvi_bbuff_size(txbuff); const int group_size = group->size; - const int group_id = group->rank; - int rc = QV_SUCCESS; + const int group_id = group->rank; + qvi_bbuff_t **bbuffs = nullptr; - // Zero initialize array of pointers to nullptr. -#pragma omp single copyprivate(bbuffs) //shared bbuffs allocation + #pragma omp single copyprivate(bbuffs) bbuffs = new qvi_bbuff_t *[group_size](); - rc = qvi_bbuff_new(&bbuffs[group_id]); + int rc = qvi_bbuff_new(&bbuffs[group_id]); if (rc != QV_SUCCESS) goto out; - rc = qvi_bbuff_append(bbuffs[group_id], qvi_bbuff_data(txbuff), send_count); + rc = qvi_bbuff_append( + bbuffs[group_id], qvi_bbuff_data(txbuff), qvi_bbuff_size(txbuff) + ); if (rc != QV_SUCCESS) goto out; -#pragma omp barrier // Need to ensure that all threads have contributed to bbuffs - + // Need to ensure that all threads have contributed to bbuffs. + #pragma omp barrier out: if (rc != QV_SUCCESS) { -#pragma omp single - { - if (bbuffs) { - for (int i = 0; i < group_size; ++i) { - qvi_bbuff_free(&bbuffs[i]); + #pragma omp single + { + if (bbuffs) { + for (int i = 0; i < group_size; ++i) { + qvi_bbuff_free(&bbuffs[i]); + } + delete[] bbuffs; } - delete[] bbuffs; } - } - bbuffs = nullptr; + bbuffs = nullptr; } *rxbuffs = bbuffs; *shared_alloc = 1; - //for(int i = 0 ; i < group_size ; i++) - //fprintf(stdout,"===== [%i] bbuff[%i] à %p\n",group_id,i,bbuffs[i]); return rc; } int -qvi_thread_group_scatter_bbuffs( - qvi_thread_group_t *group, +qvi_omp_group_scatter_bbuffs( + qvi_omp_group_t *group, qvi_bbuff_t **txbuffs, - int root, + int, qvi_bbuff_t **rxbuff ) { - QVI_UNUSED(root); - const int group_id = group->rank; - qvi_bbuff_t ***tmp = nullptr; - - /* GM: Oh man, that is UGLY */ - /* Surely, some nice OpenMP magic will fix this mess */ + qvi_bbuff_t ***tmp = nullptr; #pragma omp single copyprivate(tmp) tmp = new qvi_bbuff_t**(); #pragma omp master *tmp = txbuffs; #pragma omp barrier - - //fprintf(stdout,">>>>>>>>>>>>>>>>>>>>>>> SCATTER [%i] tmp @ %p txbuffs @ %p\n",group_id,*tmp,(*tmp)[group_id]); - + const int group_id = group->rank; qvi_bbuff_t *inbuff = (*tmp)[group_id]; - size_t data_size = qvi_bbuff_size(inbuff); + const size_t data_size = qvi_bbuff_size(inbuff); void *data = qvi_bbuff_data(inbuff); qvi_bbuff_t *mybbuff = nullptr; int rc = qvi_bbuff_new(&mybbuff); if (rc == QV_SUCCESS) { - rc = qvi_bbuff_append(mybbuff, data, data_size); + rc = qvi_bbuff_append(mybbuff, data, data_size); } #pragma omp barrier #pragma omp single @@ -344,7 +296,6 @@ qvi_thread_group_scatter_bbuffs( qvi_bbuff_free(&mybbuff); } *rxbuff = mybbuff; - //fprintf(stdout,">>>>>>>>>>>>>>>>>>>>>>> SCATTER [%i] OUT with RC %i\n",group_id,rc); return rc; } diff --git a/src/qvi-thread.h b/src/qvi-omp.h similarity index 59% rename from src/qvi-thread.h rename to src/qvi-omp.h index 4565c7b4..eaed7a32 100644 --- a/src/qvi-thread.h +++ b/src/qvi-omp.h @@ -2,8 +2,12 @@ /* * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. - * Copyright (c) 2022 Inria. All rights reserved. - * Copyright (c) 2022 Bordeaux INP. All rights reserved. + * + * Copyright (c) 2022 Inria + * All rights reserved. + * + * Copyright (c) 2022 Bordeaux INP + * All rights reserved. * * This file is part of the quo-vadis project. See the LICENSE file at the * top-level directory of this distribution. @@ -13,14 +17,14 @@ * @file qvi-thread.h */ -#ifndef QVI_THREAD_H -#define QVI_THREAD_H +#ifndef QVI_OMP_H +#define QVI_OMP_H #include "qvi-common.h" // Forward declarations. -struct qvi_thread_group_s; -typedef struct qvi_thread_group_s qvi_thread_group_t; +struct qvi_omp_group_s; +typedef struct qvi_omp_group_s qvi_omp_group_t; #if 0 /** @@ -50,46 +54,46 @@ typedef struct qv_layout_s { #endif void -qvi_thread_group_free( - qvi_thread_group_t **group +qvi_omp_group_free( + qvi_omp_group_t **group ); int -qvi_thread_group_size( - const qvi_thread_group_t *group +qvi_omp_group_size( + const qvi_omp_group_t *group ); int -qvi_thread_group_id( - const qvi_thread_group_t *group +qvi_omp_group_id( + const qvi_omp_group_t *group ); int -qvi_thread_group_barrier( - qvi_thread_group_t *group +qvi_omp_group_barrier( + qvi_omp_group_t *group ); int -qvi_thread_group_create( - qvi_thread_group_t **group +qvi_omp_group_create( + qvi_omp_group_t **group ); int -qvi_thread_group_create_single( - qvi_thread_group_t **group +qvi_omp_group_create_single( + qvi_omp_group_t **group ); int -qvi_thread_group_create_from_split( - qvi_thread_group_t *parent, +qvi_omp_group_create_from_split( + qvi_omp_group_t *parent, int color, int key, - qvi_thread_group_t **child + qvi_omp_group_t **child ); int -qvi_thread_group_gather_bbuffs( - qvi_thread_group_t *group, +qvi_omp_group_gather_bbuffs( + qvi_omp_group_t *group, qvi_bbuff_t *txbuff, int root, qvi_bbuff_t ***rxbuffs, @@ -97,8 +101,8 @@ qvi_thread_group_gather_bbuffs( ); int -qvi_thread_group_scatter_bbuffs( - qvi_thread_group_t *group, +qvi_omp_group_scatter_bbuffs( + qvi_omp_group_t *group, qvi_bbuff_t **txbuffs, int root, qvi_bbuff_t **rxbuff diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 14759ceb..1a8934cd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,49 +47,33 @@ set_tests_properties( ################################################################################ if (OPENMP_FOUND) add_executable( - test-threads - test-threads.c + test-omp + test-omp.c ) target_link_libraries( - test-threads + test-omp quo-vadis + quo-vadis-omp OpenMP::OpenMP_C ) - add_executable( - test-openmp - test-openmp.c - ) - - target_link_libraries( - test-openmp - OpenMP::OpenMP_C - ) - - set_target_properties( - test-threads - test-openmp - PROPERTIES LINKER_LANGUAGE C - ) - add_test( NAME - threads + omp COMMAND bash -c "export QV_PORT=\"55999\" && \ ( ${CMAKE_SOURCE_DIR}/tests/exec-timeout.sh ../src/quo-vadisd 5 & ) && \ - ${CMAKE_CURRENT_BINARY_DIR}/test-threads" + ${CMAKE_CURRENT_BINARY_DIR}/test-omp" ) - # Set threads test properties. - # TODO(skg) ASAN detects a leak sometimes, so remove from core for now. - # set_tests_properties( - # threads - # PROPERTIES - # TIMEOUT 60 - # LABELS "threads" - # ) + # Set omp test properties. + set_tests_properties( + omp + PROPERTIES + TIMEOUT 60 + LABELS "omp" + ) endif() ################################################################################ diff --git a/tests/test-mpi-threads-layout.c b/tests/test-mpi-threads-layout.c index c9612614..cc6c1504 100644 --- a/tests/test-mpi-threads-layout.c +++ b/tests/test-mpi-threads-layout.c @@ -21,7 +21,7 @@ */ #include "quo-vadis-mpi.h" -#include "quo-vadis-thread.h" +#include "quo-vadis-pthread.h" #include "qvi-test-common.h" #include diff --git a/tests/test-mpi-threads.c b/tests/test-mpi-threads.c index 3d9fb301..fa23f54c 100644 --- a/tests/test-mpi-threads.c +++ b/tests/test-mpi-threads.c @@ -21,7 +21,7 @@ */ #include "quo-vadis-mpi.h" -#include "quo-vadis-thread.h" +#include "quo-vadis-pthread.h" #include "qvi-test-common.h" diff --git a/tests/test-threads.c b/tests/test-omp.c similarity index 98% rename from tests/test-threads.c rename to tests/test-omp.c index 2deace20..24462820 100644 --- a/tests/test-threads.c +++ b/tests/test-omp.c @@ -5,7 +5,7 @@ */ #include "qvi-test-common.h" -#include "quo-vadis-thread.h" +#include "quo-vadis-omp.h" #include typedef struct { @@ -72,7 +72,7 @@ scopei_base( scopei *sinfo ) { char *ers = NULL; - const int rc = qv_thread_scope_get( + const int rc = qv_omp_scope_get( QV_SCOPE_PROCESS, &sinfo->scope ); if (rc != QV_SUCCESS) { diff --git a/tests/test-openmp.c b/tests/test-openmp.c deleted file mode 100644 index 33a38277..00000000 --- a/tests/test-openmp.c +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2022-2023 Triad National Security, LLC - * All rights reserved. - * - * This file is part of the quo-vadis project. See the LICENSE file at the - * top-level directory of this distribution. - */ - -/** - * @file test-openmp.c - * - * Used to verify OpenMP support. - */ - -#include -#include -#include - -int -main(void) -{ - printf("# Starting OpenMP test\n"); - - printf("# OpenMP Test Max Threads=%d\n ", omp_get_max_threads()); - - return EXIT_SUCCESS; -} - -/* - * vim: ft=cpp ts=4 sts=4 sw=4 expandtab - */ diff --git a/tests/test-progress-thread.c b/tests/test-progress-thread.c index 78319012..c6caed61 100644 --- a/tests/test-progress-thread.c +++ b/tests/test-progress-thread.c @@ -2,7 +2,7 @@ #include #include #include "quo-vadis-mpi.h" -#include "quo-vadis-thread.h" +#include "quo-vadis-pthread.h" #include "qvi-test-common.h" /* diff --git a/tests/test-pthread-split.c b/tests/test-pthread-split.c index 009183c8..bab6374a 100644 --- a/tests/test-pthread-split.c +++ b/tests/test-pthread-split.c @@ -2,7 +2,7 @@ #include #include #include "quo-vadis-mpi.h" -#include "quo-vadis-thread.h" +#include "quo-vadis-pthread.h" #include "qvi-test-common.h" #include