diff --git a/CMakeLists.txt b/CMakeLists.txt index 58e0449c..1128a453 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2023 Triad National Security, LLC +# Copyright (c) 2020-2024 Triad National Security, LLC # All rights reserved. # # Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC diff --git a/cmake/QVFortranSupport.cmake b/cmake/QVFortranSupport.cmake index 322878ea..bf570cd3 100644 --- a/cmake/QVFortranSupport.cmake +++ b/cmake/QVFortranSupport.cmake @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023 Triad National Security, LLC +# Copyright (c) 2022-2024 Triad National Security, LLC # All rights reserved. # # This file is part of the quo-vadis project. See the LICENSE file at the @@ -32,6 +32,8 @@ if(QV_FORTRAN_SUPPORT) FortranCInterface_VERIFY() # Make sure we found a Fortran compiler. if(NOT CMAKE_Fortran_COMPILER STREQUAL "") + # TODO(skg) Improve + set(CMAKE_FORTRAN_FLAGS "${CMAKE_FORTRAN_FLAGS} -Wall -Wextra -pedantic") set(QV_FORTRAN_HAPPY TRUE) set( CMAKE_Fortran_MODULE_DIRECTORY diff --git a/include/quo-vadis-mpi.h b/include/quo-vadis-mpi.h index 9216eaeb..3ba8d8c6 100644 --- a/include/quo-vadis-mpi.h +++ b/include/quo-vadis-mpi.h @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2020-2022 Triad National Security, LLC + * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. * * Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC @@ -34,8 +34,8 @@ extern "C" { */ int qv_mpi_context_create( - qv_context_t **ctx, - MPI_Comm comm + MPI_Comm comm, + qv_context_t **ctx ); /** diff --git a/include/quo-vadis.h b/include/quo-vadis.h index 066d1410..71184936 100644 --- a/include/quo-vadis.h +++ b/include/quo-vadis.h @@ -223,7 +223,7 @@ qv_scope_nobjs( qv_context_t *ctx, qv_scope_t *scope, qv_hw_obj_type_t obj, - int *n + int *nobjs ); /** diff --git a/src/fortran/quo-vadis-mpif.f90 b/src/fortran/quo-vadis-mpif.f90 index 89d53c92..54f4fc79 100644 --- a/src/fortran/quo-vadis-mpif.f90 +++ b/src/fortran/quo-vadis-mpif.f90 @@ -1,5 +1,5 @@ ! -! Copyright (c) 2013-2022 Triad National Security, LLC +! Copyright (c) 2013-2024 Triad National Security, LLC ! All rights reserved. ! ! This file is part of the quo-vadis project. See the LICENSE file at the @@ -12,7 +12,7 @@ module quo_vadis_mpif interface integer(c_int) & - function qv_mpi_context_create_c(ctx, comm) & + function qv_mpi_context_create_c(comm, ctx) & bind(c, name='qvi_mpi_context_create_f2c') use, intrinsic :: iso_c_binding, only: c_ptr, c_int implicit none @@ -41,13 +41,13 @@ end function qv_mpi_context_free_c contains - subroutine qv_mpi_context_create(ctx, comm, info) + subroutine qv_mpi_context_create(comm, ctx, info) use, intrinsic :: iso_c_binding, only: c_ptr, c_int implicit none type(c_ptr), intent(out) :: ctx integer, value :: comm integer(c_int), intent(out) :: info - info = qv_mpi_context_create_c(ctx, comm) + info = qv_mpi_context_create_c(comm, ctx) end subroutine qv_mpi_context_create subroutine qv_mpi_scope_comm_dup(ctx, scope, comm, info) diff --git a/src/quo-vadis-mpi.cc b/src/quo-vadis-mpi.cc index 48339b82..70c5ab83 100644 --- a/src/quo-vadis-mpi.cc +++ b/src/quo-vadis-mpi.cc @@ -33,11 +33,11 @@ extern "C" { int qvi_mpi_context_create_f2c( - qv_context_t **ctx, - MPI_Fint comm + MPI_Fint comm, + qv_context_t **ctx ) { MPI_Comm c_comm = MPI_Comm_f2c(comm); - return qv_mpi_context_create(ctx, c_comm); + return qv_mpi_context_create(c_comm, ctx); } int @@ -58,8 +58,8 @@ qvi_mpi_scope_comm_dup_f2c( int qv_mpi_context_create( - qv_context_t **ctx, - MPI_Comm comm + MPI_Comm comm, + qv_context_t **ctx ) { if (!ctx || comm == MPI_COMM_NULL) { return QV_ERR_INVLD_ARG; diff --git a/src/quo-vadis-process.cc b/src/quo-vadis-process.cc index a7faa1b8..613b2ce2 100644 --- a/src/quo-vadis-process.cc +++ b/src/quo-vadis-process.cc @@ -18,16 +18,6 @@ #include "qvi-context.h" #include "qvi-zgroup-process.h" -int -qv_process_context_free( - qv_context_t *ctx -) { - if (!ctx) return QV_ERR_INVLD_ARG; - delete ctx->zgroup; - qvi_context_free(&ctx); - return QV_SUCCESS; -} - int qv_process_context_create( qv_context_t **ctx @@ -76,6 +66,16 @@ qv_process_context_create( return rc; } +int +qv_process_context_free( + qv_context_t *ctx +) { + if (!ctx) return QV_ERR_INVLD_ARG; + delete ctx->zgroup; + qvi_context_free(&ctx); + return QV_SUCCESS; +} + /* * vim: ft=cpp ts=4 sts=4 sw=4 expandtab */ diff --git a/src/quo-vadis.cc b/src/quo-vadis.cc index cc6775ae..b54f2550 100644 --- a/src/quo-vadis.cc +++ b/src/quo-vadis.cc @@ -123,13 +123,13 @@ qv_scope_nobjs( qv_context_t *ctx, qv_scope_t *scope, qv_hw_obj_type_t obj, - int *n + int *nobjs ) { - if (!ctx || !scope || !n) { + if (!ctx || !scope || !nobjs) { return QV_ERR_INVLD_ARG; } - return qvi_scope_nobjs(scope, obj, n); + return qvi_scope_nobjs(scope, obj, nobjs); } int @@ -191,25 +191,23 @@ qv_scope_barrier( return qvi_scope_barrier(scope); } +// TODO(skg) Add Fortran interface. int -qv_scope_split( +qv_scope_create( qv_context_t *ctx, qv_scope_t *scope, - int npieces, - int color, + qv_hw_obj_type_t type, + int nobjs, + qv_scope_create_hint_t hint, qv_scope_t **subscope ) { - if (!ctx || !scope || (npieces <= 0) | !subscope) { + if (!ctx || !scope || (nobjs < 0) || !subscope) { return QV_ERR_INVLD_ARG; } qv_scope_t *isubscope = nullptr; - // We use the sentinel value QV_HW_OBJ_LAST to differentiate between calls - // from split() and split_at(). Since this call doesn't have a hardware type - // argument, we use QV_HW_OBJ_LAST as the hardware type. - int rc = qvi_scope_split( - scope, npieces, color, - QV_HW_OBJ_LAST, &isubscope + int rc = qvi_scope_create( + scope, type, nobjs, hint, &isubscope ); if (rc != QV_SUCCESS) { qvi_scope_free(&isubscope); @@ -219,20 +217,24 @@ qv_scope_split( } int -qv_scope_split_at( +qv_scope_split( qv_context_t *ctx, qv_scope_t *scope, - qv_hw_obj_type_t type, - int group_id, + int npieces, + int color, qv_scope_t **subscope ) { - if (!ctx || !scope || !subscope) { + if (!ctx || !scope || (npieces <= 0) | !subscope) { return QV_ERR_INVLD_ARG; } qv_scope_t *isubscope = nullptr; - int rc = qvi_scope_split_at( - scope, type, group_id, &isubscope + // We use the sentinel value QV_HW_OBJ_LAST to differentiate between calls + // from split() and split_at(). Since this call doesn't have a hardware type + // argument, we use QV_HW_OBJ_LAST as the hardware type. + int rc = qvi_scope_split( + scope, npieces, color, + QV_HW_OBJ_LAST, &isubscope ); if (rc != QV_SUCCESS) { qvi_scope_free(&isubscope); @@ -241,23 +243,21 @@ qv_scope_split_at( return rc; } -// TODO(skg) Add Fortran interface. int -qv_scope_create( +qv_scope_split_at( qv_context_t *ctx, qv_scope_t *scope, qv_hw_obj_type_t type, - int nobjs, - qv_scope_create_hint_t hint, + int group_id, qv_scope_t **subscope ) { - if (!ctx || !scope || (nobjs < 0) || !subscope) { + if (!ctx || !scope || !subscope) { return QV_ERR_INVLD_ARG; } qv_scope_t *isubscope = nullptr; - int rc = qvi_scope_create( - scope, type, nobjs, hint, &isubscope + int rc = qvi_scope_split_at( + scope, type, group_id, &isubscope ); if (rc != QV_SUCCESS) { qvi_scope_free(&isubscope); diff --git a/tests/test-mpi-api.c b/tests/test-mpi-api.c index 6e3136b7..d06822f8 100644 --- a/tests/test-mpi-api.c +++ b/tests/test-mpi-api.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2022-2023 Triad National Security, LLC + * Copyright (c) 2022-2024 Triad National Security, LLC * All rights reserved. * * This file is part of the quo-vadis project. See the LICENSE file at the @@ -54,7 +54,7 @@ main( } qv_context_t *ctx = NULL; - rc = qv_mpi_context_create(&ctx, comm); + rc = qv_mpi_context_create(comm, &ctx); if (rc != QV_SUCCESS) { ers = "qv_scope_free() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-fortapi.f90 b/tests/test-mpi-fortapi.f90 index 09beae4a..01c64231 100644 --- a/tests/test-mpi-fortapi.f90 +++ b/tests/test-mpi-fortapi.f90 @@ -1,5 +1,5 @@ ! -! Copyright (c) 2013-2022 Triad National Security, LLC +! Copyright (c) 2013-2024 Triad National Security, LLC ! All rights reserved. ! ! This file is part of the quo-vadis project. See the LICENSE file at the @@ -53,7 +53,7 @@ program mpi_fortapi print *, 'cwsize', cwsize end if - call qv_mpi_context_create(ctx, MPI_COMM_WORLD, info) + call qv_mpi_context_create(MPI_COMM_WORLD, ctx, info) if (info .ne. QV_SUCCESS) then error stop end if diff --git a/tests/test-mpi-getdev.c b/tests/test-mpi-getdev.c index 6217f824..193b411f 100644 --- a/tests/test-mpi-getdev.c +++ b/tests/test-mpi-getdev.c @@ -88,7 +88,7 @@ main( /* Create a QV context */ qv_context_t *ctx; - rc = qv_mpi_context_create(&ctx, comm); + rc = qv_mpi_context_create(comm, &ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-init.c b/tests/test-mpi-init.c index b49e6769..348aea1a 100644 --- a/tests/test-mpi-init.c +++ b/tests/test-mpi-init.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2020-2023 Triad National Security, LLC + * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. * * Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC @@ -46,7 +46,7 @@ main( } qv_context_t *ctx = NULL; - rc = qv_mpi_context_create(&ctx, comm); + rc = qv_mpi_context_create(comm, &ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-phases.c b/tests/test-mpi-phases.c index ced0916d..1eb28a85 100644 --- a/tests/test-mpi-phases.c +++ b/tests/test-mpi-phases.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2020-2023 Triad National Security, LLC + * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. * * Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC @@ -70,7 +70,7 @@ main( /* Create a QV context */ qv_context_t *ctx; - rc = qv_mpi_context_create(&ctx, comm); + rc = qv_mpi_context_create(comm, &ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-pthreads-layout.c b/tests/test-mpi-pthreads-layout.c index 486a56c3..5952f97a 100644 --- a/tests/test-mpi-pthreads-layout.c +++ b/tests/test-mpi-pthreads-layout.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2020-2023 Triad National Security, LLC + * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. * * Copyright (c) 2020 Lawrence Livermore National Security, LLC @@ -91,7 +91,7 @@ main(void) } qv_context_t *mpi_ctx; - rc = qv_mpi_context_create(&mpi_ctx, comm); + rc = qv_mpi_context_create(comm, &mpi_ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-scope-create.c b/tests/test-mpi-scope-create.c index e9a24bb8..8f907e02 100644 --- a/tests/test-mpi-scope-create.c +++ b/tests/test-mpi-scope-create.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2020-2023 Triad National Security, LLC + * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. * * Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC @@ -133,7 +133,7 @@ main( /* Create a QV context */ qv_context_t *ctx; - rc = qv_mpi_context_create(&ctx, comm); + rc = qv_mpi_context_create(comm, &ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-scopes-affinity-preserving.c b/tests/test-mpi-scopes-affinity-preserving.c index 3981c9ec..16734d13 100644 --- a/tests/test-mpi-scopes-affinity-preserving.c +++ b/tests/test-mpi-scopes-affinity-preserving.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2022-2023 Triad National Security, LLC + * Copyright (c) 2022-2024 Triad National Security, LLC * All rights reserved. * * This file is part of the quo-vadis project. See the LICENSE file at the @@ -49,7 +49,7 @@ main( } qv_context_t *ctx; - rc = qv_mpi_context_create(&ctx, comm); + rc = qv_mpi_context_create(comm, &ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-scopes.c b/tests/test-mpi-scopes.c index 171f38a1..f7aef77c 100644 --- a/tests/test-mpi-scopes.c +++ b/tests/test-mpi-scopes.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2020-2023 Triad National Security, LLC + * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. * * Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC @@ -61,7 +61,7 @@ main( setbuf(stdout, NULL); qv_context_t *ctx; - rc = qv_mpi_context_create(&ctx, comm); + rc = qv_mpi_context_create(comm, &ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-threads-layout.c b/tests/test-mpi-threads-layout.c index b2ae8f0a..c9612614 100644 --- a/tests/test-mpi-threads-layout.c +++ b/tests/test-mpi-threads-layout.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2020-2023 Triad National Security, LLC + * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. * * Copyright (c) 2020 Lawrence Livermore National Security, LLC @@ -22,7 +22,6 @@ #include "quo-vadis-mpi.h" #include "quo-vadis-thread.h" -#include "qvi-macros.h" #include "qvi-test-common.h" #include @@ -63,7 +62,7 @@ main(void) } qv_context_t *mpi_ctx; - rc = qv_mpi_context_create(&mpi_ctx, comm); + rc = qv_mpi_context_create(comm, &mpi_ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-mpi-threads.c b/tests/test-mpi-threads.c index 32155f81..92306161 100644 --- a/tests/test-mpi-threads.c +++ b/tests/test-mpi-threads.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2020-2023 Triad National Security, LLC + * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. * * Copyright (c) 2020 Lawrence Livermore National Security, LLC @@ -20,7 +20,6 @@ * @file test-mpi-threads.c */ -#include "qvi-macros.h" #include "quo-vadis-mpi.h" #include "quo-vadis-thread.h" @@ -151,7 +150,7 @@ main(void) } qv_context_t *mpi_ctx; - rc = qv_mpi_context_create(&mpi_ctx, comm); + rc = qv_mpi_context_create(comm, &mpi_ctx); if (rc != QV_SUCCESS) { ers = "qv_mpi_context_create() failed"; qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); diff --git a/tests/test-progress-thread.c b/tests/test-progress-thread.c index 59e9bc32..464fa260 100644 --- a/tests/test-progress-thread.c +++ b/tests/test-progress-thread.c @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ #include #include #include "quo-vadis-mpi.h" @@ -10,45 +11,45 @@ */ typedef struct { - qv_context_t *ctx; - qv_scope_t *scope; - void *(*thread_routine)(void *); - void *arg; + qv_context_t *ctx; + qv_scope_t *scope; + void *(*thread_routine)(void *); + void *arg; } qv_thread_args_t; void *qv_thread_routine(void * arg) { - qv_thread_args_t *qvp = (qv_thread_args_t *) arg; - // printf("qv_thread_routine: ctx=%p scope=%p\n", qvp->ctx, qvp->scope); + qv_thread_args_t *qvp = (qv_thread_args_t *) arg; + // printf("qv_thread_routine: ctx=%p scope=%p\n", qvp->ctx, qvp->scope); - int rc = qv_bind_push(qvp->ctx, qvp->scope); - if (rc != QV_SUCCESS) { - char const *ers = "qv_bind_push() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } + int rc = qv_bind_push(qvp->ctx, qvp->scope); + if (rc != QV_SUCCESS) { + char const *ers = "qv_bind_push() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } - qvp->thread_routine(qvp->arg); + qvp->thread_routine(qvp->arg); - /* Memory allocated in qv_pthread_create */ - free(qvp); + /* Memory allocated in qv_pthread_create */ + free(qvp); - pthread_exit(NULL); + pthread_exit(NULL); } int qv_pthread_create(pthread_t *thread, pthread_attr_t *attr, - void *(*thread_routine)(void *arg), void *arg, - qv_context_t *ctx, qv_scope_t *scope) + void *(*thread_routine)(void *arg), void *arg, + qv_context_t *ctx, qv_scope_t *scope) { - /* Memory will be freed in qv_thread_routine to avoid - a memory leak */ - qv_thread_args_t *qv_thargs = malloc(sizeof(qv_thread_args_t)); - qv_thargs->ctx = ctx; - qv_thargs->scope = scope; - qv_thargs->thread_routine = thread_routine; - qv_thargs->arg = arg; - - // printf("qv_pthread_create: ctx=%p scope=%p\n", ctx, scope); - return pthread_create(thread, attr, qv_thread_routine, qv_thargs); + /* Memory will be freed in qv_thread_routine to avoid + a memory leak */ + qv_thread_args_t *qv_thargs = malloc(sizeof(qv_thread_args_t)); + qv_thargs->ctx = ctx; + qv_thargs->scope = scope; + qv_thargs->thread_routine = thread_routine; + qv_thargs->arg = arg; + + // printf("qv_pthread_create: ctx=%p scope=%p\n", ctx, scope); + return pthread_create(thread, attr, qv_thread_routine, qv_thargs); } @@ -80,280 +81,284 @@ int qv_pthread_create(pthread_t *thread, pthread_attr_t *attr, void *thread_work(void *arg) { - qv_context_t *ctx = (qv_context_t *) arg; + qv_context_t *ctx = (qv_context_t *) arg; - char *binds; - char const *ers = NULL; - int rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); + char *binds; + char const *ers = NULL; + int rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); - if (rc != QV_SUCCESS) { - ers = "qv_bind_get_list_as_string() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } + if (rc != QV_SUCCESS) { + ers = "qv_bind_get_list_as_string() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } - printf("Progress thread running on %s\n", binds); - free(binds); + printf("Progress thread running on %s\n", binds); + free(binds); - return NULL; + return NULL; } int main(int argc, char *argv[]) { - char const *ers = NULL; - MPI_Comm comm = MPI_COMM_WORLD; - - int rc = MPI_Init(&argc, &argv); - if (rc != MPI_SUCCESS) { - ers = "MPI_Init() failed"; - qvi_test_panic("%s (rc=%d)", ers, rc); - } - - int wsize; - rc = MPI_Comm_size(comm, &wsize); - if (rc != MPI_SUCCESS) { - ers = "MPI_Comm_size() failed"; - qvi_test_panic("%s (rc=%d)", ers, rc); - } - - int wrank; - rc = MPI_Comm_rank(comm, &wrank); - if (rc != MPI_SUCCESS) { - ers = "MPI_Comm_rank() failed"; - qvi_test_panic("%s (rc=%d)", ers, rc); - } - - qv_context_t *ctx; - rc = qv_mpi_context_create(&ctx, comm); - if (rc != QV_SUCCESS) { - ers = "qv_mpi_context_create() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - qv_scope_t *user_scope; - rc = qv_scope_get(ctx, QV_SCOPE_USER, &user_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_get() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - /* Split user scope evenly across tasks */ - qv_scope_t *task_scope; - rc = qv_scope_split(ctx, user_scope, wsize, wrank, &task_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_split() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - /* Push into my task scope */ - rc = qv_bind_push(ctx, task_scope); - if (rc != QV_SUCCESS) { - ers = "qv_bind_push() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - /* Where did I end up? */ - char *binds; - rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); - if (rc != QV_SUCCESS) { - ers = "qv_bind_get_list_as_string() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - printf("[%d] Split: running on %s\n", wrank, binds); - free(binds); - - /* - * Todo: - * Before calling the library with the progress threads, - * how could we set aside cores/pus for the progress threads - * and not use those resources for the application's work? - * Currently, we could get around this by implementing - * hints in qv_scope_create as detailed in this file. - * Another way of doing this is by allowing the creation of - * intrinsic *named* scopes that can be set by the application - * and used by the external library. - * For example, I could create an "SMT top" scope and an - * "SMT bottom" scope... - * - * Todo: - * Bug in scope create: asking/releasing of cores not honoring - * resources that have already been used without releasing. - * See test-mpi-scope-create. See GitHub Issue #4. - * We could also create a hint QV_SCOPE_CREATE_EXCLUSIVE - * to not give out these resources until the scope is - * released. - */ - - int ncores; - rc = qv_scope_nobjs(ctx, task_scope, QV_HW_OBJ_CORE, &ncores); - if (rc != QV_SUCCESS) { - ers = "qv_scope_nobjs() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - qv_scope_t *wk_scope; - rc = qv_scope_create(ctx, task_scope, QV_HW_OBJ_CORE, ncores-1, 0, - &wk_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_create() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - qv_scope_t *ut_scope; - rc = qv_scope_create(ctx, task_scope, QV_HW_OBJ_CORE, 1, 0, - &ut_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_create() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - /* Test work scope */ - rc = qv_bind_push(ctx, wk_scope); - if (rc != QV_SUCCESS) { - ers = "qv_bind_push() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); - if (rc != QV_SUCCESS) { - ers = "qv_bind_get_list_as_string() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - printf("[%d] Work scope: running on %s\n", wrank, binds); - free(binds); - rc = qv_bind_pop(ctx); - if (rc != QV_SUCCESS) { - ers = "qv_bind_pop() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - /* Test utility scope */ - rc = qv_bind_push(ctx, ut_scope); - if (rc != QV_SUCCESS) { - ers = "qv_bind_push() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); - if (rc != QV_SUCCESS) { - ers = "qv_bind_get_list_as_string() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - printf("[%d] Utility scope: running on %s\n", wrank, binds); - free(binds); - rc = qv_bind_pop(ctx); - if (rc != QV_SUCCESS) { - ers = "qv_bind_pop() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - /* Clean up for now */ - rc = qv_scope_free(ctx, ut_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_free() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - rc = qv_scope_free(ctx, wk_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_free() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - - /*************************************** - * Emulate a progress thread scenario - * within a library external to the application - ***************************************/ - - /* Where do I get the qv_context from? since - this is likely a library with its own interface - and we don't want to modify the library's interface. - Shall I create a new context? */ - qv_context_t *ctx2; - rc = qv_mpi_context_create(&ctx2, comm); - if (rc != QV_SUCCESS) { - ers = "qv_mpi_context_create() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - /* Get scope from which to derive the progress thread. - Since this would be called externally to the application - using utility threads, then we can only derive the - scope from the intrinsic scopes. - I'd like to try QV_SCORE_SYSTEM and if - there's nothing available, then use QV_PROCESS_SCOPE */ - qv_scope_t *base_scope; - rc = qv_scope_get(ctx2, MY_INTRINSIC_SCOPE, &base_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_get() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - /* Test we have PUs to use in the base scope */ - int npus; - rc = qv_scope_nobjs(ctx2, base_scope, QV_HW_OBJ_PU, &npus); - if (rc != QV_SUCCESS) { - ers = "qv_scope_nobjs() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - printf("[%d] Base scope: npus=%d\n", wrank, npus); - - /* Create the progress thread scope */ - qv_scope_t *pt_scope = user_scope; - if (npus > 0) { - rc = qv_scope_create(ctx2, base_scope, QV_HW_OBJ_PU, 1, 0, &pt_scope); + char const *ers = NULL; + MPI_Comm comm = MPI_COMM_WORLD; + + int rc = MPI_Init(&argc, &argv); + if (rc != MPI_SUCCESS) { + ers = "MPI_Init() failed"; + qvi_test_panic("%s (rc=%d)", ers, rc); + } + + int wsize; + rc = MPI_Comm_size(comm, &wsize); + if (rc != MPI_SUCCESS) { + ers = "MPI_Comm_size() failed"; + qvi_test_panic("%s (rc=%d)", ers, rc); + } + + int wrank; + rc = MPI_Comm_rank(comm, &wrank); + if (rc != MPI_SUCCESS) { + ers = "MPI_Comm_rank() failed"; + qvi_test_panic("%s (rc=%d)", ers, rc); + } + + qv_context_t *ctx; + rc = qv_mpi_context_create(comm, &ctx); + if (rc != QV_SUCCESS) { + ers = "qv_mpi_context_create() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + qv_scope_t *user_scope; + rc = qv_scope_get(ctx, QV_SCOPE_USER, &user_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_get() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + /* Split user scope evenly across tasks */ + qv_scope_t *task_scope; + rc = qv_scope_split(ctx, user_scope, wsize, wrank, &task_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_split() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + /* Push into my task scope */ + rc = qv_bind_push(ctx, task_scope); + if (rc != QV_SUCCESS) { + ers = "qv_bind_push() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + /* Where did I end up? */ + char *binds; + rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); + if (rc != QV_SUCCESS) { + ers = "qv_bind_get_list_as_string() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + printf("[%d] Split: running on %s\n", wrank, binds); + free(binds); + + /* + * Todo: + * Before calling the library with the progress threads, + * how could we set aside cores/pus for the progress threads + * and not use those resources for the application's work? + * Currently, we could get around this by implementing + * hints in qv_scope_create as detailed in this file. + * Another way of doing this is by allowing the creation of + * intrinsic *named* scopes that can be set by the application + * and used by the external library. + * For example, I could create an "SMT top" scope and an + * "SMT bottom" scope... + * + * Todo: + * Bug in scope create: asking/releasing of cores not honoring + * resources that have already been used without releasing. + * See test-mpi-scope-create. See GitHub Issue #4. + * We could also create a hint QV_SCOPE_CREATE_EXCLUSIVE + * to not give out these resources until the scope is + * released. + */ + + int ncores; + rc = qv_scope_nobjs(ctx, task_scope, QV_HW_OBJ_CORE, &ncores); + if (rc != QV_SUCCESS) { + ers = "qv_scope_nobjs() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + qv_scope_t *wk_scope; + rc = qv_scope_create(ctx, task_scope, QV_HW_OBJ_CORE, ncores-1, 0, + &wk_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_create() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + qv_scope_t *ut_scope; + rc = qv_scope_create(ctx, task_scope, QV_HW_OBJ_CORE, 1, 0, + &ut_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_create() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + /* Test work scope */ + rc = qv_bind_push(ctx, wk_scope); + if (rc != QV_SUCCESS) { + ers = "qv_bind_push() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); if (rc != QV_SUCCESS) { - ers = "qv_scope_create() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + ers = "qv_bind_get_list_as_string() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); } - } - - pthread_t thid; - void *args = ctx2; - if (qv_pthread_create(&thid, NULL, thread_work, args, - ctx2, pt_scope) != 0) { - perror("pthread_create() error"); - exit(1); - } - - void *ret; - if (pthread_join(thid, &ret) != 0) { - perror("pthread_create() error"); - exit(3); - } - printf("Thread finished with '%s'\n", (char *)ret); - - - /* Clean up */ - rc = qv_scope_free(ctx2, pt_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_free() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - rc = qv_scope_free(ctx2, base_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_free() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - - /*************************************** - * Back to the application - ***************************************/ - - rc = qv_scope_free(ctx, task_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_free() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - rc = qv_scope_free(ctx, user_scope); - if (rc != QV_SUCCESS) { - ers = "qv_scope_free() failed"; - qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); - } - - MPI_Finalize(); - - return EXIT_SUCCESS; + printf("[%d] Work scope: running on %s\n", wrank, binds); + free(binds); + rc = qv_bind_pop(ctx); + if (rc != QV_SUCCESS) { + ers = "qv_bind_pop() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + /* Test utility scope */ + rc = qv_bind_push(ctx, ut_scope); + if (rc != QV_SUCCESS) { + ers = "qv_bind_push() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); + if (rc != QV_SUCCESS) { + ers = "qv_bind_get_list_as_string() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + printf("[%d] Utility scope: running on %s\n", wrank, binds); + free(binds); + rc = qv_bind_pop(ctx); + if (rc != QV_SUCCESS) { + ers = "qv_bind_pop() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + /* Clean up for now */ + rc = qv_scope_free(ctx, ut_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + rc = qv_scope_free(ctx, wk_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + + /*************************************** + * Emulate a progress thread scenario + * within a library external to the application + ***************************************/ + + /* Where do I get the qv_context from? since + this is likely a library with its own interface + and we don't want to modify the library's interface. + Shall I create a new context? */ + qv_context_t *ctx2; + rc = qv_mpi_context_create(comm, &ctx2); + if (rc != QV_SUCCESS) { + ers = "qv_mpi_context_create() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + /* Get scope from which to derive the progress thread. + Since this would be called externally to the application + using utility threads, then we can only derive the + scope from the intrinsic scopes. + I'd like to try QV_SCORE_SYSTEM and if + there's nothing available, then use QV_PROCESS_SCOPE */ + qv_scope_t *base_scope; + rc = qv_scope_get(ctx2, MY_INTRINSIC_SCOPE, &base_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_get() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + /* Test we have PUs to use in the base scope */ + int npus; + rc = qv_scope_nobjs(ctx2, base_scope, QV_HW_OBJ_PU, &npus); + if (rc != QV_SUCCESS) { + ers = "qv_scope_nobjs() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + printf("[%d] Base scope: npus=%d\n", wrank, npus); + + /* Create the progress thread scope */ + qv_scope_t *pt_scope = user_scope; + if (npus > 0) { + rc = qv_scope_create(ctx2, base_scope, QV_HW_OBJ_PU, 1, 0, &pt_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_create() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + } + + pthread_t thid; + void *args = ctx2; + if (qv_pthread_create(&thid, NULL, thread_work, args, + ctx2, pt_scope) != 0) { + perror("pthread_create() error"); + exit(1); + } + + void *ret; + if (pthread_join(thid, &ret) != 0) { + perror("pthread_create() error"); + exit(3); + } + printf("Thread finished with '%s'\n", (char *)ret); + + + /* Clean up */ + rc = qv_scope_free(ctx2, pt_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + rc = qv_scope_free(ctx2, base_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + + /*************************************** + * Back to the application + ***************************************/ + + rc = qv_scope_free(ctx, task_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + rc = qv_scope_free(ctx, user_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + MPI_Finalize(); + + return EXIT_SUCCESS; } + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */