From effed42ef1d91ff156fa227f972aa7459eff14ba Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 19 Sep 2024 10:59:08 +0300 Subject: [PATCH 01/40] Fix atlas_test_array when hic devices are present but no acc::devices --- src/tests/array/test_array.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/array/test_array.cc b/src/tests/array/test_array.cc index 290e47e0e..9f651fb13 100644 --- a/src/tests/array/test_array.cc +++ b/src/tests/array/test_array.cc @@ -20,6 +20,7 @@ #endif #include "hic/hic.h" +#include "atlas/parallel/acc/acc.h" using namespace atlas::array; @@ -555,7 +556,7 @@ CASE("test_wrap") { EXPECT(view(2) == 19); } -static int devices() { +static int hic_devices() { static int devices_ = [](){ int n = 0; auto err = hicGetDeviceCount(&n); @@ -573,7 +574,7 @@ CASE("test_acc_map") { EXPECT_NO_THROW(ds->allocateDevice()); if( ds->deviceAllocated() ) { EXPECT_NO_THROW(ds->accMap()); - EXPECT_EQ(ds->accMapped(), std::min(devices(),1)); + EXPECT_EQ(ds->accMapped(), std::min(std::min(acc::devices(),hic_devices()),1)); } else { Log::warning() << "WARNING: Array could not be allocated on device, so acc_map could not be tested" << std::endl; From 9af40a3312a11c9883fe167167fa2401a0149b28 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 19 Sep 2024 19:11:19 +0300 Subject: [PATCH 02/40] Update ACC support to use Fortran runtime functions, supporting Cray compilers --- cmake/features/ACC.cmake | 41 ++++------- cmake/project_summary.cmake | 4 +- src/atlas/array/gridtools/GridToolsArray.cc | 2 +- .../array/gridtools/GridToolsDataStore.h | 2 +- src/atlas/parallel/acc/acc.cc | 18 ++++- src/atlas_acc_support/CMakeLists.txt | 31 +++----- src/atlas_acc_support/atlas_acc.F90 | 70 +++++++++++++++++++ .../{atlas_acc_map_data.c => atlas_acc.cc} | 5 +- .../{atlas_acc_map_data.h => atlas_acc.h} | 12 +--- src/atlas_f/CMakeLists.txt | 12 ++-- src/sandbox/fortran_acc_fields/CMakeLists.txt | 4 +- src/tests/acc/CMakeLists.txt | 14 +--- .../fctest_unified_memory_with_openacc_cxx.cc | 2 +- src/tests/array/test_array.cc | 16 +---- src/tests/field/CMakeLists.txt | 16 ++--- src/tests/field/test_field_acc.cc | 6 +- 16 files changed, 137 insertions(+), 118 deletions(-) create mode 100644 src/atlas_acc_support/atlas_acc.F90 rename src/atlas_acc_support/{atlas_acc_map_data.c => atlas_acc.cc} (99%) rename src/atlas_acc_support/{atlas_acc_map_data.h => atlas_acc.h} (75%) diff --git a/cmake/features/ACC.cmake b/cmake/features/ACC.cmake index 056b2c812..273512006 100644 --- a/cmake/features/ACC.cmake +++ b/cmake/features/ACC.cmake @@ -1,40 +1,23 @@ ### OpenACC -if( atlas_HAVE_ATLAS_FIELD ) +if( atlas_HAVE_ATLAS_FIELD AND HAVE_GPU ) -set( ATLAS_ACC_CAPABLE FALSE ) -if( HAVE_GPU ) - if( CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC" ) - set( ATLAS_ACC_CAPABLE TRUE ) - else() - find_package(OpenACC COMPONENTS C Fortran) - if(OpenACC_Fortran_FOUND AND OpenACC_C_FOUND) - set( ATLAS_ACC_CAPABLE TRUE ) - endif() + if( DEFINED ATLAS_ENABLE_ACC ) + set( ENABLE_ACC ${ATLAS_ENABLE_ACC} ) endif() -endif() - -ecbuild_add_option( FEATURE ACC - DESCRIPTION "OpenACC capable data structures" - CONDITION ATLAS_ACC_CAPABLE ) - -if( atlas_HAVE_ACC ) - if( CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC" ) - #set( ACC_Fortran_FLAGS -acc -ta=tesla,nordc ) - set( ACC_Fortran_FLAGS "-acc=gpu;-gpu=gvmode,lineinfo,fastmath,rdc" ) - set( ACC_C_FLAGS ${ACC_Fortran_FLAGS} ) - find_program( ACC_C_COMPILER NAMES pgcc HINTS ${PGI_DIR} ${NVPHC_DIR} ENV PGI_DIR NVHPC_DIR PATH_SUFFIXES bin ) - if( NOT ACC_C_COMPILER ) - ecbuild_error( "Could not find OpenACC capable C compiler" ) + if( ENABLE_ACC ) + if( NOT HAVE_FORTRAN ) + enable_language(Fortran) endif() - else() - set( ACC_Fortran_FLAGS ${OpenACC_Fortran_FLAGS} ) - set( ACC_C_FLAGS ${OpenACC_C_FLAGS} ) + find_package( OpenACC COMPONENTS Fortran CXX ) endif() -endif() + ecbuild_add_option( FEATURE ACC + DESCRIPTION "OpenACC capable data structures" + CONDITION OpenACC_Fortran_FOUND ) else() + set( HAVE_ACC 0 ) set( atlas_HAVE_ACC 0 ) -endif() +endif() diff --git a/cmake/project_summary.cmake b/cmake/project_summary.cmake index 62ead6584..b5bfa6e3c 100644 --- a/cmake/project_summary.cmake +++ b/cmake/project_summary.cmake @@ -42,9 +42,7 @@ endif() if( atlas_HAVE_ACC ) ecbuild_info( "ACC" ) - ecbuild_info( " ACC_C_COMPILER : [${ACC_C_COMPILER}]" ) - ecbuild_info( " ACC_C_FLAGS : [${ACC_C_FLAGS}]" ) - ecbuild_info( " ACC_Fortran_FLAGS : [${ACC_Fortran_FLAGS}]" ) + ecbuild_info( " OpenACC_Fortran_FLAGS : [${OpenACC_Fortran_FLAGS}]" ) endif() diff --git a/src/atlas/array/gridtools/GridToolsArray.cc b/src/atlas/array/gridtools/GridToolsArray.cc index ac660742a..572c62385 100644 --- a/src/atlas/array/gridtools/GridToolsArray.cc +++ b/src/atlas/array/gridtools/GridToolsArray.cc @@ -25,7 +25,7 @@ #include "atlas/runtime/Log.h" #if ATLAS_HAVE_ACC -#include "atlas_acc_support/atlas_acc_map_data.h" +#include "atlas_acc_support/atlas_acc.h" #endif //------------------------------------------------------------------------------ diff --git a/src/atlas/array/gridtools/GridToolsDataStore.h b/src/atlas/array/gridtools/GridToolsDataStore.h index 8fb604640..058f726e5 100644 --- a/src/atlas/array/gridtools/GridToolsDataStore.h +++ b/src/atlas/array/gridtools/GridToolsDataStore.h @@ -15,7 +15,7 @@ #include "atlas/array/gridtools/GridToolsTraits.h" #if ATLAS_HAVE_ACC -#include "atlas_acc_support/atlas_acc_map_data.h" +#include "atlas_acc_support/atlas_acc.h" #endif //------------------------------------------------------------------------------ diff --git a/src/atlas/parallel/acc/acc.cc b/src/atlas/parallel/acc/acc.cc index 8a600aea5..c73bdea2c 100644 --- a/src/atlas/parallel/acc/acc.cc +++ b/src/atlas/parallel/acc/acc.cc @@ -13,7 +13,20 @@ #include "atlas/library/defines.h" #if ATLAS_HAVE_ACC -#include "atlas_acc_support/atlas_acc_map_data.h" +#include "hic/hic.h" +#include "atlas_acc_support/atlas_acc.h" +static int hic_devices() { + static int devices_ = [](){ + int n = 0; + auto err = hicGetDeviceCount(&n); + if (err != hicSuccess) { + n = 0; + static_cast(hicGetLastError()); + } + return n; + }(); + return devices_; +} #endif namespace atlas::acc { @@ -21,6 +34,9 @@ namespace atlas::acc { int devices() { #if ATLAS_HAVE_ACC static int num_devices = [](){ + if (hic_devices() == 0) { + return 0; + } auto devicetype = atlas_acc_get_device_type(); int _num_devices = atlas_acc_get_num_devices(); if (_num_devices == 1 && devicetype == atlas_acc_device_host) { diff --git a/src/atlas_acc_support/CMakeLists.txt b/src/atlas_acc_support/CMakeLists.txt index 7bcf8ac06..7bf818b1d 100644 --- a/src/atlas_acc_support/CMakeLists.txt +++ b/src/atlas_acc_support/CMakeLists.txt @@ -8,30 +8,15 @@ if( atlas_HAVE_ACC ) - if( NOT (CMAKE_C_COMPILER_ID MATCHES ${CMAKE_Fortran_COMPILER_ID}) ) - add_custom_command( - OUTPUT ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so ${CMAKE_CURRENT_BINARY_DIR}/atlas_acc_map_data.c.o - COMMAND ${ACC_C_COMPILER} ${ACC_C_FLAGS} ${ACC_C_INCLUDE} -fPIC -o ${CMAKE_CURRENT_BINARY_DIR}/atlas_acc_map_data.c.o - -c ${CMAKE_CURRENT_SOURCE_DIR}/atlas_acc_map_data.c - COMMAND mkdir -p ${CMAKE_BINARY_DIR}/lib - COMMAND ${ACC_C_COMPILER} ${ACC_C_FLAGS} -shared -o ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so - ${CMAKE_CURRENT_BINARY_DIR}/atlas_acc_map_data.c.o - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/atlas_acc_map_data.c - COMMENT "Building atlas_acc_support with ${ACC_C_COMPILER}" - ) - add_custom_target( build-atlas_acc_support ALL DEPENDS ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so ) - add_library( atlas_acc_support SHARED IMPORTED GLOBAL ) - set_property( TARGET atlas_acc_support PROPERTY IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so ) - set_property( TARGET atlas_acc_support PROPERTY IMPORTED_NO_SONAME TRUE ) - set_property( TARGET atlas_acc_support PROPERTY IMPORTED_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR} ) - add_dependencies( atlas_acc_support build-atlas_acc_support ) - install( FILES ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so DESTINATION ${INSTALL_LIB_DIR}/ ) - + if( CMAKE_CXX_COMPILER_ID MATCHES NVHPC ) + if( NOT TARGET OpenACC::OpenACC_CXX ) + ecbuild_error("ERROR: OpenACC::OpenACC_CXX TARGET not found") + endif() + ecbuild_add_library( TARGET atlas_acc_support SOURCES atlas_acc.cc ) + target_link_libraries( atlas_acc_support PRIVATE OpenACC::OpenACC_CXX ) else() - - ecbuild_add_library( TARGET atlas_acc_support SOURCES atlas_acc_map_data.c ) - target_compile_options( atlas_acc_support PRIVATE ${ACC_C_FLAGS} ) - target_link_libraries( atlas_acc_support PRIVATE ${ACC_C_FLAGS} ) + ecbuild_add_library( TARGET atlas_acc_support SOURCES atlas_acc.F90 ) + target_link_libraries( atlas_acc_support PRIVATE OpenACC::OpenACC_Fortran ) endif() endif() diff --git a/src/atlas_acc_support/atlas_acc.F90 b/src/atlas_acc_support/atlas_acc.F90 new file mode 100644 index 000000000..bb73b5237 --- /dev/null +++ b/src/atlas_acc_support/atlas_acc.F90 @@ -0,0 +1,70 @@ +module atlas_acc +use openacc +implicit none +private + +public :: atlas_acc_get_num_devices +public :: atlas_acc_map_data +public :: atlas_acc_unmap_data +public :: atlas_acc_is_present +public :: atlas_acc_get_device_type +public :: atlas_acc_deviceptr + +contains + +function atlas_acc_get_num_devices() bind(C,name="atlas_acc_get_num_devices") result(num_devices) + use, intrinsic :: iso_c_binding, only : c_int + integer(c_int) :: num_devices + integer(acc_device_kind) :: devicetype + + devicetype = acc_get_device_type() + num_devices = acc_get_num_devices(devicetype) +end function + +subroutine atlas_acc_map_data(data_arg, data_dev, bytes) bind(C,name="atlas_acc_map_data") + use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t + type(*), dimension(*) :: data_arg + type(c_ptr), value :: data_dev + integer(c_size_t), value :: bytes + call acc_map_data(data_arg, data_dev, bytes) +end subroutine + +subroutine atlas_acc_unmap_data(data_arg) bind(C,name="atlas_acc_unmap_data") + use, intrinsic :: iso_c_binding, only : c_ptr + type(*), dimension(*) :: data_arg + call acc_unmap_data(data_arg) +end subroutine + +function atlas_acc_is_present(data_arg, bytes) bind(C,name="atlas_acc_is_present") result(is_present) + use, intrinsic :: iso_c_binding, only : c_size_t, c_ptr, c_char, c_int + integer(c_int) :: is_present + logical :: lpresent + type(c_ptr), value :: data_arg + integer(c_size_t), value :: bytes + character(kind=c_char), pointer :: data_f(:) + call c_f_pointer(data_arg, data_f,[bytes]) + lpresent = acc_is_present(data_f) + is_present = 0 + if (lpresent) is_present = 1 +end function + +function atlas_acc_deviceptr(data_arg) bind(C,name="atlas_acc_deviceptr") result(deviceptr) + use, intrinsic :: iso_c_binding, only : c_ptr + type(*), dimension(*) :: data_arg + type(c_ptr):: deviceptr + deviceptr = acc_deviceptr(data_arg) +end function + +function atlas_acc_get_device_type() bind(C,name="atlas_acc_get_device_type") result(devicetype) + use, intrinsic :: iso_c_binding, only : c_int + integer(c_int) :: devicetype + integer(acc_device_kind) :: acc_devicetype + acc_devicetype = acc_get_device_type() + if (acc_devicetype == acc_device_host .or. acc_devicetype == acc_device_none) then + devicetype = 0 + else + devicetype = 1 + endif +end function + +end module diff --git a/src/atlas_acc_support/atlas_acc_map_data.c b/src/atlas_acc_support/atlas_acc.cc similarity index 99% rename from src/atlas_acc_support/atlas_acc_map_data.c rename to src/atlas_acc_support/atlas_acc.cc index 826579686..8cc58981a 100644 --- a/src/atlas_acc_support/atlas_acc_map_data.c +++ b/src/atlas_acc_support/atlas_acc.cc @@ -19,7 +19,9 @@ #include #include -#include "atlas_acc_map_data.h" +#include "atlas_acc.h" + +extern "C" { void atlas_acc_map_data(void* cpu_ptr, void* gpu_ptr, unsigned long bytes) { acc_map_data(cpu_ptr, gpu_ptr, bytes); @@ -125,3 +127,4 @@ const char* atlas_acc_info_str() { int atlas_acc_get_num_devices() { return acc_get_num_devices(acc_get_device_type()); } +} diff --git a/src/atlas_acc_support/atlas_acc_map_data.h b/src/atlas_acc_support/atlas_acc.h similarity index 75% rename from src/atlas_acc_support/atlas_acc_map_data.h rename to src/atlas_acc_support/atlas_acc.h index ba5990968..37c928b95 100644 --- a/src/atlas_acc_support/atlas_acc_map_data.h +++ b/src/atlas_acc_support/atlas_acc.h @@ -10,6 +10,7 @@ #pragma once +#include #ifdef __cplusplus extern "C" { #endif @@ -19,18 +20,11 @@ typedef enum { atlas_acc_device_not_host = 1 } atlas_acc_device_t; -void atlas_acc_map_data(void* cpu_ptr, void* gpu_ptr, unsigned long bytes); +void atlas_acc_map_data(void* cpu_ptr, void* gpu_ptr, size_t bytes); void atlas_acc_unmap_data(void* cpu_ptr); -int atlas_acc_is_present(void* cpu_ptr, unsigned long bytes); +int atlas_acc_is_present(void* cpu_ptr, size_t bytes); void* atlas_acc_deviceptr(void* cpu_ptr); atlas_acc_device_t atlas_acc_get_device_type(); - -int atlas_acc_devices(); - -const char* atlas_acc_version_str(); - -const char* atlas_acc_info_str(); - int atlas_acc_get_num_devices(); #ifdef __cplusplus diff --git a/src/atlas_f/CMakeLists.txt b/src/atlas_f/CMakeLists.txt index 2e9a97e31..53067f222 100644 --- a/src/atlas_f/CMakeLists.txt +++ b/src/atlas_f/CMakeLists.txt @@ -278,11 +278,13 @@ ecbuild_add_library( TARGET atlas_f ) if( HAVE_ACC ) - target_link_options( atlas_f INTERFACE - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> ) + if( CMAKE_Fortran_COMPILER_ID MATCHES NVHPC ) + target_link_options( atlas_f INTERFACE + $<$:SHELL:-acc=gpu> + $<$:SHELL:-acc=gpu> + $<$:SHELL:-acc=gpu> + $<$:SHELL:-acc=gpu> ) + endif() endif() fckit_target_preprocess_fypp( atlas_f diff --git a/src/sandbox/fortran_acc_fields/CMakeLists.txt b/src/sandbox/fortran_acc_fields/CMakeLists.txt index a3b49ad08..733960140 100644 --- a/src/sandbox/fortran_acc_fields/CMakeLists.txt +++ b/src/sandbox/fortran_acc_fields/CMakeLists.txt @@ -10,10 +10,8 @@ if( atlas_HAVE_ACC ) ecbuild_add_executable( TARGET atlas-acc-fields SOURCES atlas-acc-fields.F90 - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran LINKER_LANGUAGE Fortran NOINSTALL ) - target_compile_options( atlas-acc-fields PUBLIC ${ACC_Fortran_FLAGS} ) - target_link_libraries( atlas-acc-fields ${ACC_Fortran_FLAGS} ) endif() diff --git a/src/tests/acc/CMakeLists.txt b/src/tests/acc/CMakeLists.txt index 0e323611f..c827dc65a 100644 --- a/src/tests/acc/CMakeLists.txt +++ b/src/tests/acc/CMakeLists.txt @@ -6,33 +6,25 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. -if( HAVE_CUDA AND HAVE_TESTS AND HAVE_FCTEST AND HAVE_ACC ) +if( HAVE_GPU AND HAVE_TESTS AND HAVE_FCTEST AND HAVE_ACC ) - string (REPLACE ";" " " ACC_Fortran_FLAGS_STR "${ACC_Fortran_FLAGS}") - - - set_source_files_properties( fctest_unified_memory_with_openacc.F90 PROPERTIES COMPILE_FLAGS ${ACC_Fortran_FLAGS_STR} ) add_fctest( TARGET atlas_test_unified_memory_with_openacc SOURCES fctest_unified_memory_with_openacc.F90 fctest_unified_memory_with_openacc_cxx.cc - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran hic LINKER_LANGUAGE Fortran ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 ) - target_link_libraries( atlas_test_unified_memory_with_openacc ${ACC_Fortran_FLAGS} hic ) - set_tests_properties( atlas_test_unified_memory_with_openacc PROPERTIES LABELS "gpu;acc") add_fctest( TARGET atlas_test_connectivity_openacc SOURCES fctest_connectivity_openacc.F90 - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran LINKER_LANGUAGE Fortran ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 ) - target_link_libraries( atlas_test_connectivity_openacc ${ACC_Fortran_FLAGS} ) - set_target_properties( atlas_test_connectivity_openacc PROPERTIES COMPILE_FLAGS "${ACC_Fortran_FLAGS_STR}" ) set_tests_properties ( atlas_test_connectivity_openacc PROPERTIES LABELS "gpu;acc") endif() diff --git a/src/tests/acc/fctest_unified_memory_with_openacc_cxx.cc b/src/tests/acc/fctest_unified_memory_with_openacc_cxx.cc index eb251572b..4ded2d78e 100644 --- a/src/tests/acc/fctest_unified_memory_with_openacc_cxx.cc +++ b/src/tests/acc/fctest_unified_memory_with_openacc_cxx.cc @@ -12,6 +12,6 @@ extern "C" { void allocate_unified_impl(double** a, int N) { - hicMallocManaged(a, N * sizeof(double)); + HIC_CALL(hicMallocManaged(a, N * sizeof(double))); } } diff --git a/src/tests/array/test_array.cc b/src/tests/array/test_array.cc index 9f651fb13..63f630c98 100644 --- a/src/tests/array/test_array.cc +++ b/src/tests/array/test_array.cc @@ -19,7 +19,6 @@ #include "atlas/array/gridtools/GridToolsMakeView.h" #endif -#include "hic/hic.h" #include "atlas/parallel/acc/acc.h" using namespace atlas::array; @@ -556,25 +555,12 @@ CASE("test_wrap") { EXPECT(view(2) == 19); } -static int hic_devices() { - static int devices_ = [](){ - int n = 0; - auto err = hicGetDeviceCount(&n); - if (err != hicSuccess) { - n = 0; - static_cast(hicGetLastError()); - } - return n; - }(); - return devices_; -} - CASE("test_acc_map") { Array* ds = Array::create(2, 3, 4); EXPECT_NO_THROW(ds->allocateDevice()); if( ds->deviceAllocated() ) { EXPECT_NO_THROW(ds->accMap()); - EXPECT_EQ(ds->accMapped(), std::min(std::min(acc::devices(),hic_devices()),1)); + EXPECT_EQ(ds->accMapped(), std::min(acc::devices(),1)); } else { Log::warning() << "WARNING: Array could not be allocated on device, so acc_map could not be tested" << std::endl; diff --git a/src/tests/field/CMakeLists.txt b/src/tests/field/CMakeLists.txt index 3382e576b..dc91d6f65 100644 --- a/src/tests/field/CMakeLists.txt +++ b/src/tests/field/CMakeLists.txt @@ -27,13 +27,11 @@ ecbuild_add_test( TARGET atlas_test_field_foreach ecbuild_add_test( TARGET atlas_test_field_acc SOURCES test_field_acc.cc - LIBS atlas + LIBS atlas OpenACC::OpenACC_CXX ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} - CONDITION atlas_HAVE_ACC + CONDITION atlas_HAVE_ACC AND OpenACC_CXX_Found ) if( TEST atlas_test_field_acc ) - target_compile_options( atlas_test_field_acc PRIVATE "${ACC_C_FLAGS}") - target_link_options( atlas_test_field_acc PRIVATE "${ACC_C_FLAGS}") set_tests_properties ( atlas_test_field_acc PROPERTIES LABELS "gpu;acc") endif() @@ -73,14 +71,11 @@ if( HAVE_FCTEST ) CONDITION atlas_HAVE_ACC LINKER_LANGUAGE Fortran SOURCES fctest_field_gpu.F90 external_acc_routine.F90 - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 ) if( TARGET atlas_fctest_field_device ) - target_compile_options( atlas_fctest_field_device PUBLIC ${ACC_Fortran_FLAGS} ) - target_link_libraries( atlas_fctest_field_device ${ACC_Fortran_FLAGS} ) - target_link_options( atlas_fctest_field_device PRIVATE "${ACC_Fortran_FLAGS}") set_tests_properties ( atlas_fctest_field_device PROPERTIES LABELS "gpu;acc") endif() @@ -88,14 +83,11 @@ if( HAVE_FCTEST ) CONDITION atlas_HAVE_ACC LINKER_LANGUAGE Fortran SOURCES fctest_field_wrap_gpu.F90 external_acc_routine.F90 - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 ) if( TARGET atlas_fctest_field_wrap_device ) - target_compile_options( atlas_fctest_field_wrap_device PUBLIC ${ACC_Fortran_FLAGS} ) - target_link_libraries( atlas_fctest_field_wrap_device ${ACC_Fortran_FLAGS} ) - target_link_options( atlas_fctest_field_wrap_device PRIVATE "${ACC_Fortran_FLAGS}") set_tests_properties ( atlas_fctest_field_wrap_device PROPERTIES LABELS "gpu;acc") endif() diff --git a/src/tests/field/test_field_acc.cc b/src/tests/field/test_field_acc.cc index e998f5dda..8dad81fbd 100644 --- a/src/tests/field/test_field_acc.cc +++ b/src/tests/field/test_field_acc.cc @@ -35,10 +35,10 @@ CASE("test_acc") { *c_ptr = 5; int* d_ptr; - hicMalloc(&d_ptr, sizeof(int)); + HIC_CALL(hicMalloc(&d_ptr, sizeof(int))); acc_map_data(c_ptr, d_ptr, sizeof(int)); - hicMemcpy(d_ptr, c_ptr, sizeof(int), hicMemcpyHostToDevice); + HIC_CALL(hicMemcpy(d_ptr, c_ptr, sizeof(int), hicMemcpyHostToDevice)); #pragma acc kernels present(c_ptr) { @@ -47,7 +47,7 @@ CASE("test_acc") { EXPECT_EQ( *c_ptr, 5. ); - hicMemcpy(c_ptr, d_ptr, sizeof(int), hicMemcpyDeviceToHost); + HIC_CALL(hicMemcpy(c_ptr, d_ptr, sizeof(int), hicMemcpyDeviceToHost)); EXPECT_EQ( *c_ptr, 2. ); } From cb0270c9cb50cd43faa038d04e9ff25637edb79a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 24 Sep 2024 15:46:17 +0000 Subject: [PATCH 03/40] Make OpenACC link options cleaner --- cmake/features/ACC.cmake | 4 ++++ src/atlas/CMakeLists.txt | 10 +++++----- src/atlas_f/CMakeLists.txt | 14 ++++++-------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cmake/features/ACC.cmake b/cmake/features/ACC.cmake index 273512006..a1537142e 100644 --- a/cmake/features/ACC.cmake +++ b/cmake/features/ACC.cmake @@ -15,6 +15,10 @@ if( atlas_HAVE_ATLAS_FIELD AND HAVE_GPU ) ecbuild_add_option( FEATURE ACC DESCRIPTION "OpenACC capable data structures" CONDITION OpenACC_Fortran_FOUND ) + if( HAVE_ACC ) + set( ACC_LINK_OPTIONS ${OpenACC_Fortran_FLAGS} ) + endif() + else() set( HAVE_ACC 0 ) diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 47c82b031..50091b3e8 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -1001,12 +1001,12 @@ ecbuild_add_library( TARGET atlas ) -if( HAVE_ACC ) +if( HAVE_ACC AND CMAKE_Fortran_COMPILER_ID MATCHES NVHPC ) target_link_options( atlas INTERFACE - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> ) + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> ) endif() target_compile_features( atlas PUBLIC cxx_std_17 ) diff --git a/src/atlas_f/CMakeLists.txt b/src/atlas_f/CMakeLists.txt index 53067f222..4fc068f9c 100644 --- a/src/atlas_f/CMakeLists.txt +++ b/src/atlas_f/CMakeLists.txt @@ -277,14 +277,12 @@ ecbuild_add_library( TARGET atlas_f $ ) -if( HAVE_ACC ) - if( CMAKE_Fortran_COMPILER_ID MATCHES NVHPC ) - target_link_options( atlas_f INTERFACE - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> - $<$:SHELL:-acc=gpu> ) - endif() +if( HAVE_ACC AND CMAKE_Fortran_COMPILER_ID MATCHES NVHPC ) + target_link_options( atlas_f INTERFACE + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> ) endif() fckit_target_preprocess_fypp( atlas_f From e1276d1ce4b34d1be6de608a67ce32901c583483 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 9 Jan 2024 23:08:01 +0100 Subject: [PATCH 04/40] Fortran API: atlas_Grid%name() --- src/atlas/grid/detail/grid/Grid.cc | 10 +++++++++- src/atlas/grid/detail/grid/Grid.h | 1 + src/atlas_f/grid/atlas_Grid_module.F90 | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/atlas/grid/detail/grid/Grid.cc b/src/atlas/grid/detail/grid/Grid.cc index e9750b3d7..e2ec13d69 100644 --- a/src/atlas/grid/detail/grid/Grid.cc +++ b/src/atlas/grid/detail/grid/Grid.cc @@ -163,7 +163,7 @@ Grid::Spec* atlas__grid__Grid__spec(Grid* This) { } void atlas__grid__Grid__uid(const Grid* This, char*& uid, int& size) { - ATLAS_ASSERT(This != nullptr, "Cannot access uninitialised atlas_Projection"); + ATLAS_ASSERT(This != nullptr, "Cannot access uninitialised atlas_Grid"); eckit::MD5 md5; This->hash(md5); std::string s = This->uid(); @@ -172,6 +172,14 @@ void atlas__grid__Grid__uid(const Grid* This, char*& uid, int& size) { std::strncpy(uid, s.c_str(), size + 1); } +void atlas__grid__Grid__name(const Grid* This, char*& name, int& size) { + ATLAS_ASSERT(This != nullptr, "Cannot access uninitialised atlas_Grid"); + std::string s = This->name(); + size = static_cast(s.size()); + name = new char[size + 1]; + std::strncpy(name, s.c_str(), size + 1); +} + Grid::Domain::Implementation* atlas__grid__Grid__lonlat_bounding_box(const Grid* This) { Grid::Domain::Implementation* lonlatboundingbox; { diff --git a/src/atlas/grid/detail/grid/Grid.h b/src/atlas/grid/detail/grid/Grid.h index 35fe9eeed..db652cf82 100644 --- a/src/atlas/grid/detail/grid/Grid.h +++ b/src/atlas/grid/detail/grid/Grid.h @@ -183,6 +183,7 @@ class GridObserver { //---------------------------------------------------------------------------------------------------------------------- extern "C" { +void atlas__grid__Grid__name(const Grid* This, char*& uid, int& size); idx_t atlas__grid__Grid__size(Grid* This); Grid::Spec* atlas__grid__Grid__spec(Grid* This); void atlas__grid__Grid__uid(const Grid* This, char*& uid, int& size); diff --git a/src/atlas_f/grid/atlas_Grid_module.F90 b/src/atlas_f/grid/atlas_Grid_module.F90 index 39168f74d..d5a50d62d 100644 --- a/src/atlas_f/grid/atlas_Grid_module.F90 +++ b/src/atlas_f/grid/atlas_Grid_module.F90 @@ -54,6 +54,7 @@ module atlas_Grid_module !------------------------------------------------------------------------------ contains + procedure :: name => atlas_Grid__name procedure :: size => atlas_Grid__size procedure :: spec => atlas_Grid__spec procedure :: uid @@ -751,6 +752,19 @@ function atlas_grid_ShiftedLat__ctor_int64(nlon,nlat) result(this) ! ----------------------------------------------------------------------------- ! Structured members +function atlas_Grid__name(this) result(name) + use atlas_grid_Grid_c_binding + use fckit_c_interop_module, only : c_ptr_to_string, c_ptr_free + use, intrinsic :: iso_c_binding, only : c_ptr + class(atlas_Grid), intent(in) :: this + character(len=:), allocatable :: name + type(c_ptr) :: name_c_str + integer :: size + call atlas__grid__Grid__name(this%CPTR_PGIBUG_A, name_c_str, size ) + name = c_ptr_to_string(name_c_str) + call c_ptr_free(name_c_str) +end function + function atlas_Grid__size(this) result(npts) use, intrinsic :: iso_c_binding, only: c_int use atlas_grid_Grid_c_binding From 9d27e46ede99f528806cd9d146c3986eec8addda Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 11 Jan 2024 16:39:47 +0100 Subject: [PATCH 05/40] Fortran API: computing stencils for StructuredGrid --- doc/CMakeLists.txt | 4 + doc/example-fortran/CMakeLists.txt | 10 + .../structured-grid-stencils/CMakeLists.txt | 12 + .../structured-grid-stencils/program.F90 | 208 +++++++++++ src/atlas/CMakeLists.txt | 2 + .../detail/grid/StencilComputerInterface.cc | 48 +++ .../detail/grid/StencilComputerInterface.h | 27 ++ src/atlas_f/CMakeLists.txt | 7 +- src/atlas_f/atlas_module.F90 | 5 + .../grid/atlas_StencilComputer_module.F90 | 339 ++++++++++++++++++ 10 files changed, 661 insertions(+), 1 deletion(-) create mode 100644 doc/example-fortran/CMakeLists.txt create mode 100644 doc/example-fortran/structured-grid-stencils/CMakeLists.txt create mode 100644 doc/example-fortran/structured-grid-stencils/program.F90 create mode 100644 src/atlas/grid/detail/grid/StencilComputerInterface.cc create mode 100644 src/atlas/grid/detail/grid/StencilComputerInterface.h create mode 100644 src/atlas_f/grid/atlas_StencilComputer_module.F90 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 6c7fa7d3a..05c754c4b 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -6,6 +6,10 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +if(atlas_HAVE_FORTRAN) + add_subdirectory(example-fortran) +endif() + if(atlas_HAVE_DOCS) add_subdirectory(user-guide) diff --git a/doc/example-fortran/CMakeLists.txt b/doc/example-fortran/CMakeLists.txt new file mode 100644 index 000000000..564a289f0 --- /dev/null +++ b/doc/example-fortran/CMakeLists.txt @@ -0,0 +1,10 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +add_subdirectory(structured-grid-stencils) + diff --git a/doc/example-fortran/structured-grid-stencils/CMakeLists.txt b/doc/example-fortran/structured-grid-stencils/CMakeLists.txt new file mode 100644 index 000000000..c3a221662 --- /dev/null +++ b/doc/example-fortran/structured-grid-stencils/CMakeLists.txt @@ -0,0 +1,12 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +ecbuild_add_executable( TARGET example-fortran-structured-grid-stencils + OUTPUT_NAME program NOINSTALL + SOURCES program.F90 + LIBS atlas_f ) diff --git a/doc/example-fortran/structured-grid-stencils/program.F90 b/doc/example-fortran/structured-grid-stencils/program.F90 new file mode 100644 index 000000000..179fcd51a --- /dev/null +++ b/doc/example-fortran/structured-grid-stencils/program.F90 @@ -0,0 +1,208 @@ +! (C) Copyright 2023 ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +! ------------------------------------------------------------------------------------------------ +! Example program on how to access and use StructuredColumns functionspace +! Also includes computation of stencils around a given coordinate +! ------------------------------------------------------------------------------------------------ + +subroutine run + +use, intrinsic :: iso_fortran_env, only : real64, real32 +use atlas_module, only : & + & atlas_StructuredGrid ,& + & atlas_functionspace_StructuredColumns ,& + & atlas_Field, & + & ATLAS_KIND_IDX, & + & atlas_StructuredGrid_ComputeNorth ,& + & atlas_StructuredGrid_ComputeWest ,& + & atlas_StructuredGrid_ComputeStencil ,& + & atlas_StructuredGrid_Stencil, & + & atlas_real + +use fckit_mpi_module, only : fckit_mpi + +! ------------------------------------------------ +implicit none + +character(len=:), allocatable :: gridname +type(atlas_StructuredGrid) :: grid +type(atlas_functionspace_StructuredColumns) :: functionspace +real(real64) :: zlon, zlat +integer(ATLAS_KIND_IDX) :: jlon,jlat, jgp +real(real64), pointer :: xy(:,:) +integer :: log_rank + +! ------------------------------------------------ + +log_rank = 1 ! First partition has rank=0 !!! +if (fckit_mpi%size() == 1) then + log_rank = 0 +endif + +gridname = "O320" + +! Create grid and write some properties +grid = atlas_StructuredGrid(gridname) + +if (fckit_mpi%rank() == log_rank) then + write(0,*) "grid%name() : ", grid%name() + write(0,*) "grid%size() : ", grid%size() + write(0,*) "grid%ny() : ", grid%ny() + write(0,*) "grid%nxmax() : ", grid%nxmax() +endif + +! Create functionspace of grid, distributed across MPI tasks with default partitioner, and with halo of 2 wide +! The default partitioner is "equal_regions" +functionspace = atlas_functionspace_StructuredColumns(grid, halo=2) + +! Latitude bounds without and with halos of this partition +if (fckit_mpi%rank() == log_rank) then + write(0,*) "functionspace%size_owned() : ", functionspace%size_owned() + write(0,*) "functionspace%size() : ", functionspace%size() + write(0,'(A,I0,A,I0)') " halo points between index ", functionspace%size_owned()+1 , " and " , functionspace%size() + write(0,*) "functionspace%j_begin(), functionspace%j_end() : ", & + & functionspace%j_begin(), functionspace%j_end() + write(0,*) "functionspace%j_begin_halo(), functionspace%j_end_halo() : ", & + & functionspace%j_begin_halo(), functionspace%j_end_halo() +endif + +! Longitude bounds without and with halos of this partition for the first latitude +jlat = functionspace%j_begin() +if (fckit_mpi%rank() == log_rank) then + write(0,*) "functionspace%i_begin(jlat), functionspace%i_end(jlat) : ", functionspace%i_begin(jlat), & + & functionspace%i_end(jlat) + write(0,*) "functionspace%i_begin_halo(jlat), functionspace%i_end_halo(jlat) : ", functionspace%i_begin_halo(jlat),& + & functionspace%i_end_halo(jlat) +endif + +! Access i,j indices of grid points +block + type(atlas_Field) :: field_index_i, field_index_j + integer(ATLAS_KIND_IDX), pointer :: index_i(:), index_j(:) + field_index_i = functionspace%index_i() + field_index_j = functionspace%index_j() + call field_index_i%data(index_i) + call field_index_j%data(index_j) + if (fckit_mpi%rank() == log_rank) then + write(0,*) "i,j of first partition point :", index_i(1), index_j(1) + endif + call field_index_i%final() + call field_index_j%final() +end block + +! Access to xy coordinates +block + type(atlas_Field) :: field_xy + field_xy = functionspace%xy() + call field_xy%data(xy) + call field_xy%final() +end block + +! Creating a horizontal field and perform halo exchange +block + type(atlas_Field) :: field + real(real32), pointer :: view(:) + field = functionspace%create_field(name="myfield", kind=atlas_real(real32)) + call field%data(view) + if (fckit_mpi%rank() == log_rank) then + write(0,*) "shape( horizontal field )", shape(view) + endif + view(1:functionspace%size_owned()) = 1. + view(functionspace%size_owned()+1:functionspace%size()) = 0 + call field%set_dirty() ! This marks that halos are in "dirty" state + call field%halo_exchange() ! Only exchanges halos when halos are marked as "dirty" + if (fckit_mpi%rank() == log_rank) then + write(0,*) "halo exhange success : ", all(view == 1.) + endif + call field%final() +end block + +! Creating a horizontal/vertical field +block + type(atlas_Field) :: field + real(real32), pointer :: view(:,:) + field = functionspace%create_field(name="myfield", kind=atlas_real(real32), levels=10) + call field%data(view) + if (fckit_mpi%rank() == log_rank) then + write(0,*) "shape( horizontal/vertical field )", shape(view) + endif + call field%final() +end block + +! Set a coordinate somewhere south-east of the first grid point, but still within this partition +jgp = 1 +zlon = xy(1,jgp) + 0.1 +zlat = xy(2,jgp) - 0.1 + +! Compute nearest points to the north-west of a coordinate +block + type(atlas_StructuredGrid_ComputeNorth) :: compute_north + type(atlas_StructuredGrid_ComputeWest) :: compute_west + compute_north = atlas_StructuredGrid_ComputeNorth(grid, halo=2) + compute_west = atlas_StructuredGrid_ComputeWest(grid, halo=2) + + jlat = compute_north%execute(zlat) + jlon = compute_west%execute(zlon, jlat) + jgp = functionspace%index(jlon,jlat) ! gridpoint index in 1-dimensional array + + if (fckit_mpi%rank() == log_rank) then + write(0,'(A,F6.2,A,I0)') "compute_north%execute(y=",zlat,") : ", jlat + write(0,'(A,F6.2,A,I0,A,I0)') "compute_west%execute(x=",zlon,", j=", jlat,") : ", jlon + write(0,'(A,I0,A,F6.2,A,F6.2,A)') "gridpoint north-west: jpg=",jgp," (lon,lat)=(", xy(1,jgp), ",", xy(2,jgp), ")" + endif + call compute_west%final() + call compute_north%final() +end block + +! Stencil computer of 4x4 stencil around a coordinate +block + type(atlas_StructuredGrid_ComputeStencil) :: compute_stencil + type(atlas_StructuredGrid_Stencil) :: stencil + call compute_stencil%setup(grid, stencil_width=4) + call compute_stencil%execute(zlon,zlat,stencil) + if (fckit_mpi%rank() == log_rank) then + write(0,'(A,A,dt)') 'stencil:', new_line('a'), stencil + endif + if (fckit_mpi%rank() == log_rank) then + write(0,'(A,A,dt)') 'stencil gridpoints:' + do jlat=1,stencil%width + do jlon=1,stencil%width + write(0,'(I8)',advance='no') functionspace%index(stencil%i(jlon,jlat),stencil%j(jlat)) + enddo + write(0,'(A)') "" + enddo + + write(0,'(A,A,dt)') 'stencil coordinates:' + do jlat=1,stencil%width + jgp = functionspace%index(stencil%i(1,jlat),stencil%j(jlat)) + write(0,'(A, F6.2,A)', advance='no') " lat : ", xy(2,jgp), " --- lon : " + do jlon=1,stencil%width + jgp = functionspace%index(stencil%i(jlon,jlat),stencil%j(jlat)) + write(0,'(F8.2)',advance='no') xy(1,jgp) + enddo + write(0,'(A)') "" + enddo + + endif + call compute_stencil%final() +end block + +call functionspace%final() +call grid%final() +end subroutine + +! ------------------------------------------------------------------------------------------------ + +program main +use atlas_module, only : atlas_library +implicit none +call atlas_library%initialize() +call run() +call atlas_library%finalize() +end program \ No newline at end of file diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 50091b3e8..12f0db620 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -175,6 +175,8 @@ grid/detail/grid/RegionalVariableResolution.h grid/detail/grid/RegionalVariableResolution.cc grid/detail/grid/Healpix.h grid/detail/grid/Healpix.cc +grid/detail/grid/StencilComputerInterface.h +grid/detail/grid/StencilComputerInterface.cc grid/detail/tiles/Tiles.cc grid/detail/tiles/Tiles.h diff --git a/src/atlas/grid/detail/grid/StencilComputerInterface.cc b/src/atlas/grid/detail/grid/StencilComputerInterface.cc new file mode 100644 index 000000000..4e1df4765 --- /dev/null +++ b/src/atlas/grid/detail/grid/StencilComputerInterface.cc @@ -0,0 +1,48 @@ +/* + * (C) Copyright 2023 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "StencilComputerInterface.h" + +extern "C" { + atlas::grid::ComputeNorth* atlas__grid__ComputeNorth__new(const atlas::StructuredGrid::Implementation* grid, int halo) { + return new atlas::grid::ComputeNorth{atlas::StructuredGrid{grid}, halo}; + } + + void atlas__grid__ComputeNorth__delete(atlas::grid::ComputeNorth* This) { + delete This; + } + + atlas::idx_t atlas__grid__ComputeNorth__execute_real32(const atlas::grid::ComputeNorth* This, float y) { + return This->operator()(y); + } + + atlas::idx_t atlas__grid__ComputeNorth__execute_real64(const atlas::grid::ComputeNorth* This, double y) { + return This->operator()(y); + } + + + atlas::grid::ComputeWest* atlas__grid__ComputeWest__new(const atlas::StructuredGrid::Implementation* grid, int halo) { + return new atlas::grid::ComputeWest{atlas::StructuredGrid{grid}, halo}; + } + + void atlas__grid__ComputeWest__delete(atlas::grid::ComputeWest* This) { + delete This; + } + + atlas::idx_t atlas__grid__ComputeWest__execute_real32(const atlas::grid::ComputeWest* This, float x, atlas::idx_t j) { + return This->operator()(x, j); + } + + atlas::idx_t atlas__grid__ComputeWest__execute_real64(const atlas::grid::ComputeWest* This, double x, atlas::idx_t j) { + return This->operator()(x, j); + } + + +} diff --git a/src/atlas/grid/detail/grid/StencilComputerInterface.h b/src/atlas/grid/detail/grid/StencilComputerInterface.h new file mode 100644 index 000000000..87e4172d2 --- /dev/null +++ b/src/atlas/grid/detail/grid/StencilComputerInterface.h @@ -0,0 +1,27 @@ +/* + * (C) Copyright 2023 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include "atlas/grid/StencilComputer.h" +#include "atlas/grid/StructuredGrid.h" + +extern "C" { + atlas::grid::ComputeNorth* atlas__grid__ComputeNorth__new(const atlas::StructuredGrid::Implementation* grid, int halo); + void atlas__grid__ComputeNorth__delete(atlas::grid::ComputeNorth* This); + atlas::idx_t atlas__grid__ComputeNorth__execute_real32(const atlas::grid::ComputeNorth* This, float y); + atlas::idx_t atlas__grid__ComputeNorth__execute_real64(const atlas::grid::ComputeNorth* This, double y); + + atlas::grid::ComputeWest* atlas__grid__ComputeWest__new(const atlas::StructuredGrid::Implementation* grid, int halo); + void atlas__grid__ComputeWest__delete(atlas::grid::ComputeWest* This); + atlas::idx_t atlas__grid__ComputeWest__execute_real32(const atlas::grid::ComputeWest* This, float x, atlas::idx_t j); + atlas::idx_t atlas__grid__ComputeWest__execute_real64(const atlas::grid::ComputeWest* This, double x, atlas::idx_t j); + +} diff --git a/src/atlas_f/CMakeLists.txt b/src/atlas_f/CMakeLists.txt index 4fc068f9c..60f55318f 100644 --- a/src/atlas_f/CMakeLists.txt +++ b/src/atlas_f/CMakeLists.txt @@ -49,7 +49,7 @@ function(generate_fortran_bindings output filename) OUTPUT ${outfile} COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/c2f.py ${CMAKE_CURRENT_SOURCE_DIR}/${filename} -o ${outfile} -m ${_PAR_MODULE} - -t '{"idx_t":"${F_IDX}","gidx_t":"${F_GIDX}"}' + -t '{"idx_t":"${F_IDX}","gidx_t":"${F_GIDX}","atlas::idx_t":"${F_IDX}","atlas::gidx_t":"${F_GIDX}"}' DEPENDS ${filename} ) set_source_files_properties(${outfile} PROPERTIES GENERATED TRUE) endfunction() @@ -138,6 +138,10 @@ generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/redistribution/detail/Redist MODULE atlas_redistribution_c_binding OUTPUT redistribution_c_binding.f90) +generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/grid/detail/grid/StencilComputerInterface.h + MODULE atlas_grid_StencilComputer_c_binding + OUTPUT grid_StencilComputer_c_binding.f90 ) + if( atlas_HAVE_ATLAS_NUMERICS ) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/numerics/Nabla.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/numerics/Nabla.h) @@ -231,6 +235,7 @@ ecbuild_add_library( TARGET atlas_f grid/atlas_GridDistribution_module.F90 grid/atlas_Vertical_module.F90 grid/atlas_Partitioner_module.F90 + grid/atlas_StencilComputer_module.F90 mesh/atlas_MeshBuilder_module.F90 mesh/atlas_MeshGenerator_module.F90 mesh/atlas_Mesh_module.F90 diff --git a/src/atlas_f/atlas_module.F90 b/src/atlas_f/atlas_module.F90 index e82c13851..e21890629 100644 --- a/src/atlas_f/atlas_module.F90 +++ b/src/atlas_f/atlas_module.F90 @@ -89,6 +89,11 @@ module atlas_module & atlas_RegionalGrid use atlas_Vertical_module, only :& & atlas_Vertical +use atlas_StencilComputer_module, only: & + & atlas_StructuredGrid_ComputeNorth, & + & atlas_StructuredGrid_ComputeWest, & + & atlas_StructuredGrid_ComputeStencil, & + & atlas_StructuredGrid_Stencil use atlas_functionspace_EdgeColumns_module, only: & & atlas_functionspace_EdgeColumns use atlas_functionspace_CellColumns_module, only: & diff --git a/src/atlas_f/grid/atlas_StencilComputer_module.F90 b/src/atlas_f/grid/atlas_StencilComputer_module.F90 new file mode 100644 index 000000000..8ec0a1b0f --- /dev/null +++ b/src/atlas_f/grid/atlas_StencilComputer_module.F90 @@ -0,0 +1,339 @@ +! (C) Copyright 2013 ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +#include "atlas/atlas_f.h" + +module atlas_StencilComputer_module + +use fckit_shared_object_module, only: fckit_shared_object, fckit_c_deleter +use atlas_kinds_module, only : ATLAS_KIND_IDX +use, intrinsic :: iso_c_binding, only : c_ptr, c_int + +implicit none + +private :: fckit_shared_object, fckit_c_deleter +private :: c_ptr, c_int +private :: ATLAS_KIND_IDX + +public :: atlas_StructuredGrid_ComputeNorth +public :: atlas_StructuredGrid_ComputeWest +public :: atlas_StructuredGrid_ComputeStencil +public :: atlas_StructuredGrid_Stencil + +private + +!------------------------------------------------------------------------------ +TYPE, extends(fckit_shared_object) :: atlas_StructuredGrid_ComputeNorth + +! Purpose : +! ------- +! *atlas_StructuredGrid_ComputeNorth* : To compute latitude index north of latitude + +! Methods : +! ------- + +! Author : +! ------ +! 9-Jan-2024 Willem Deconinck *ECMWF* + +!------------------------------------------------------------------------------ +contains + + procedure, private :: atlas_StructuredGrid_ComputeNorth__execute_real32 + procedure, private :: atlas_StructuredGrid_ComputeNorth__execute_real64 + generic :: execute => atlas_StructuredGrid_ComputeNorth__execute_real32, & + & atlas_StructuredGrid_ComputeNorth__execute_real64 + +#if FCKIT_FINAL_NOT_INHERITING + final :: atlas_StructuredGrid_ComputeNorth__final_auto +#endif + +END TYPE atlas_StructuredGrid_ComputeNorth + +interface atlas_StructuredGrid_ComputeNorth + module procedure atlas_StructuredGrid_ComputeNorth__ctor +end interface + +!------------------------------------------------------------------------------ + +!------------------------------------------------------------------------------ +TYPE, extends(fckit_shared_object) :: atlas_StructuredGrid_ComputeWest + +! Purpose : +! ------- +! *atlas_StructuredGrid_ComputeWest* : To compute longitude index west of longitude at given latitude index + +! Methods : +! ------- + +! Author : +! ------ +! 9-Jan-2024 Willem Deconinck *ECMWF* + +!------------------------------------------------------------------------------ +contains + + procedure, private :: atlas_StructuredGrid_ComputeWest__execute_real32 + procedure, private :: atlas_StructuredGrid_ComputeWest__execute_real64 + generic :: execute => atlas_StructuredGrid_ComputeWest__execute_real32, & + & atlas_StructuredGrid_ComputeWest__execute_real64 + +#if FCKIT_FINAL_NOT_INHERITING + final :: atlas_StructuredGrid_ComputeWest__final_auto +#endif + +END TYPE atlas_StructuredGrid_ComputeWest + +interface atlas_StructuredGrid_ComputeWest + module procedure atlas_StructuredGrid_ComputeWest__ctor +end interface + +!------------------------------------------------------------------------------ + +integer(c_int), parameter, private :: STENCIL_MAX_WIDTH = 6 !-> maximum 6x6 stencil + +TYPE atlas_StructuredGrid_Stencil + integer(ATLAS_KIND_IDX) :: width + integer(ATLAS_KIND_IDX) :: j_begin + integer(ATLAS_KIND_IDX) :: i_begin(STENCIL_MAX_WIDTH) ! on stack +contains + procedure, pass :: write => atlas_StructuredGrid_Stencil__write + generic, public :: write(FORMATTED) => write + procedure, public :: i => atlas_StructuredGrid_Stencil__i + procedure, public :: j => atlas_StructuredGrid_Stencil__j +END TYPE atlas_StructuredGrid_Stencil + +TYPE :: atlas_StructuredGrid_ComputeStencil + integer(c_int) :: halo + integer(ATLAS_KIND_IDX) :: stencil_width + integer(ATLAS_KIND_IDX) :: stencil_offset + type(atlas_StructuredGrid_ComputeNorth) :: compute_north + type(atlas_StructuredGrid_ComputeWest) :: compute_west +contains + procedure :: setup => atlas_StructuredGrid_ComputeStencil__setup + procedure :: execute => atlas_StructuredGrid_ComputeStencil__execute_real64 + + procedure :: assignment_operator => atlas_StructuredGrid_ComputeStencil__assignment + generic, public :: assignment(=) => assignment_operator + + procedure :: final => atlas_StructuredGrid_ComputeStencil__final +END TYPE + +! Better not use, use setup member function instead ! +!interface atlas_StructuredGrid_ComputeStencil +! module procedure atlas_StructuredGrid_ComputeStencil__ctor +!end interface + + +!======================================================== +contains +!======================================================== + + +! ----------------------------------------------------------------------------- +! Destructor + +#if FCKIT_FINAL_NOT_INHERITING +ATLAS_FINAL subroutine atlas_StructuredGrid_ComputeNorth__final_auto(this) + type(atlas_StructuredGrid_ComputeNorth), intent(inout) :: this +#if FCKIT_FINAL_NOT_PROPAGATING + call this%final() +#endif + FCKIT_SUPPRESS_UNUSED( this ) +end subroutine +#endif + +#if FCKIT_FINAL_NOT_INHERITING +ATLAS_FINAL subroutine atlas_StructuredGrid_ComputeWest__final_auto(this) + type(atlas_StructuredGrid_ComputeWest), intent(inout) :: this +#if FCKIT_FINAL_NOT_PROPAGATING + call this%final() +#endif + FCKIT_SUPPRESS_UNUSED( this ) +end subroutine +#endif + + +! ----------------------------------------------------------------------------- +! Constructors + +function atlas_StructuredGrid_ComputeNorth__ctor(grid, halo) result(this) + use, intrinsic :: iso_c_binding, only : c_int + use fckit_c_interop_module, only: c_str + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeNorth__new, atlas__grid__ComputeNorth__delete + use atlas_grid_module, only : atlas_StructuredGrid + implicit none + type(atlas_StructuredGrid_ComputeNorth) :: this + type(atlas_StructuredGrid), intent(in) :: grid + integer(c_int), intent(in) :: halo + call this%reset_c_ptr( atlas__grid__ComputeNorth__new(grid%CPTR_PGIBUG_B, halo), & + & fckit_c_deleter(atlas__grid__ComputeNorth__delete) ) + call this%return() +end function + +function atlas_StructuredGrid_ComputeWest__ctor(grid, halo) result(this) + use, intrinsic :: iso_c_binding, only : c_int + use fckit_c_interop_module, only: c_str + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeWest__new, atlas__grid__ComputeWest__delete + use atlas_grid_module, only : atlas_StructuredGrid + implicit none + type(atlas_StructuredGrid_ComputeWest) :: this + type(atlas_StructuredGrid), intent(in) :: grid + integer(c_int), intent(in) :: halo + call this%reset_c_ptr( atlas__grid__ComputeWest__new(grid%CPTR_PGIBUG_B, halo), & + & fckit_c_deleter(atlas__grid__ComputeWest__delete) ) + call this%return() +end function + +subroutine atlas_StructuredGrid_ComputeStencil__setup(this, grid, stencil_width) + use, intrinsic :: iso_c_binding, only : c_double + use atlas_grid_module, only : atlas_StructuredGrid + implicit none + class(atlas_StructuredGrid_ComputeStencil) :: this + type(atlas_StructuredGrid), intent(in) :: grid + integer(ATLAS_KIND_IDX), intent(in) :: stencil_width + this%stencil_width = stencil_width + this%halo = (stencil_width + 1) / 2 + this%stencil_offset = stencil_width - floor(real(stencil_width,c_double) / 2._c_double + 1._c_double, ATLAS_KIND_IDX) + this%compute_north = atlas_StructuredGrid_ComputeNorth(grid, this%halo) + this%compute_west = atlas_StructuredGrid_ComputeWest(grid, this%halo) +end subroutine + + + +! ---------------------------------------------------------------------------------------- + +function atlas_StructuredGrid_ComputeNorth__execute_real32(this, y) result(index) + use, intrinsic :: iso_c_binding, only : c_float + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeNorth__execute_real32 + implicit none + integer(ATLAS_KIND_IDX) :: index + class(atlas_StructuredGrid_ComputeNorth), intent(in) :: this + real(c_float), intent(in) :: y + index = atlas__grid__ComputeNorth__execute_real32(this%CPTR_PGIBUG_B, y) + 1 +end function + +function atlas_StructuredGrid_ComputeNorth__execute_real64(this, y) result(index) + use, intrinsic :: iso_c_binding, only : c_double + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeNorth__execute_real64 + implicit none + integer(ATLAS_KIND_IDX) :: index + class(atlas_StructuredGrid_ComputeNorth), intent(in) :: this + real(c_double), intent(in) :: y + index = atlas__grid__ComputeNorth__execute_real64(this%CPTR_PGIBUG_B, y) + 1 +end function +! ---------------------------------------------------------------------------------------- + +function atlas_StructuredGrid_ComputeWest__execute_real32(this, x, j) result(index) + use, intrinsic :: iso_c_binding, only : c_float + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeWest__execute_real32 + implicit none + integer(ATLAS_KIND_IDX) :: index + class(atlas_StructuredGrid_ComputeWest), intent(in) :: this + real(c_float), intent(in) :: x + integer(ATLAS_KIND_IDX), intent(in) :: j + index = atlas__grid__ComputeWest__execute_real32(this%CPTR_PGIBUG_B, x, j-int(1,ATLAS_KIND_IDX)) + 1 +end function + +function atlas_StructuredGrid_ComputeWest__execute_real64(this, x, j) result(index) + use, intrinsic :: iso_c_binding, only : c_double + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeWest__execute_real64 + implicit none + integer(ATLAS_KIND_IDX) :: index + class(atlas_StructuredGrid_ComputeWest), intent(in) :: this + real(c_double), intent(in) :: x + integer(ATLAS_KIND_IDX), intent(in) :: j + index = atlas__grid__ComputeWest__execute_real64(this%CPTR_PGIBUG_B, x, j-int(1,ATLAS_KIND_IDX)) + 1 +end function +! ---------------------------------------------------------------------------------------- + + +subroutine atlas_StructuredGrid_ComputeStencil__execute_real64(this, x, y, stencil) + use, intrinsic :: iso_c_binding, only : c_double + implicit none + class(atlas_StructuredGrid_ComputeStencil), intent(in) :: this + real(c_double), intent(in) :: x, y + type(atlas_StructuredGrid_Stencil), intent(inout) :: stencil + + integer(ATLAS_KIND_IDX) :: jj + stencil%width = this%stencil_width + + stencil%j_begin = this%compute_north%execute(y) - this%stencil_offset + do jj = 1_ATLAS_KIND_IDX, this%stencil_width + stencil%i_begin(jj) = this%compute_west%execute(x, stencil%j_begin + jj - 1) - this%stencil_offset + enddo +end subroutine +! ---------------------------------------------------------------------------------------- + +subroutine atlas_StructuredGrid_Stencil__write (stencil, unit, iotype, v_list, iostat, iomsg) + implicit none + class(atlas_StructuredGrid_Stencil), intent(in) :: stencil + INTEGER, INTENT(IN) :: unit + CHARACTER(*), INTENT(IN) :: iotype + INTEGER, INTENT(IN) :: v_list(:) + INTEGER, INTENT(OUT) :: iostat + CHARACTER(*), INTENT(INOUT) :: iomsg + integer(ATLAS_KIND_IDX) :: jlat, jlon + do jlat = 1, stencil%width + write(unit,'(a,I0,a)',advance='no',IOSTAT=iostat) " j: ", stencil%j(jlat), " i = " + do jlon = 1, stencil%width + write(unit,'(I3)',advance='no',IOSTAT=iostat) stencil%i(jlon,jlat) + enddo + write(0,'(a)',IOSTAT=iostat) new_line('a') + enddo +end subroutine + +function atlas_StructuredGrid_Stencil__j(this, j_index) result(j) + integer(ATLAS_KIND_IDX) :: j + class(atlas_StructuredGrid_Stencil), intent(in) :: this + integer(ATLAS_KIND_IDX) :: j_index + j = this%j_begin + (j_index-1) +end function + +function atlas_StructuredGrid_Stencil__i(this, i_index, j_index) result(i) + integer(ATLAS_KIND_IDX) :: i + class(atlas_StructuredGrid_Stencil), intent(in) :: this + integer(ATLAS_KIND_IDX) :: i_index + integer(ATLAS_KIND_IDX) :: j_index + i = this%i_begin(j_index) + (i_index-1) +end function + +! ---------------------------------------------------------------------------------------- + +function atlas_StructuredGrid_ComputeStencil__ctor(grid, stencil_width) result(this) + use atlas_grid_module, only : atlas_StructuredGrid + implicit none + type(atlas_StructuredGrid_ComputeStencil) :: this + type(atlas_StructuredGrid), intent(in) :: grid + integer(ATLAS_KIND_IDX), intent(in) :: stencil_width + call this%setup(grid, stencil_width) +end function + + +subroutine atlas_StructuredGrid_ComputeStencil__assignment(this, other) +implicit none +class(atlas_StructuredGrid_ComputeStencil), intent(inout) :: this +class(atlas_StructuredGrid_ComputeStencil), intent(in) :: other +call this%final() +write(0,*) "owners = ", other%compute_north%owners() +this%compute_north = other%compute_north +this%compute_west = other%compute_west +this%stencil_width = other%stencil_width +this%stencil_offset = other%stencil_offset +this%halo = other%halo +end subroutine + +subroutine atlas_StructuredGrid_ComputeStencil__final(this) + class(atlas_StructuredGrid_ComputeStencil), intent(inout) :: this + call this%compute_north%final() + call this%compute_west%final() +end subroutine + +! ---------------------------------------------------------------------------------------- + +end module atlas_StencilComputer_module From 873c50e88e2e1d23c17daf5aeee1f3fce14d7844 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 25 Jan 2024 09:20:37 +0100 Subject: [PATCH 06/40] Fortran: Use class(atlas_StructuredGrid) instead of type(atlas_StructuredGrid) --- doc/example-fortran/structured-grid-stencils/program.F90 | 8 +++++--- src/atlas_f/grid/atlas_StencilComputer_module.F90 | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/example-fortran/structured-grid-stencils/program.F90 b/doc/example-fortran/structured-grid-stencils/program.F90 index 179fcd51a..6517ad3a4 100644 --- a/doc/example-fortran/structured-grid-stencils/program.F90 +++ b/doc/example-fortran/structured-grid-stencils/program.F90 @@ -15,7 +15,7 @@ subroutine run use, intrinsic :: iso_fortran_env, only : real64, real32 use atlas_module, only : & - & atlas_StructuredGrid ,& + & atlas_ReducedGaussianGrid ,& & atlas_functionspace_StructuredColumns ,& & atlas_Field, & & ATLAS_KIND_IDX, & @@ -31,7 +31,7 @@ subroutine run implicit none character(len=:), allocatable :: gridname -type(atlas_StructuredGrid) :: grid +type(atlas_ReducedGaussianGrid) :: grid type(atlas_functionspace_StructuredColumns) :: functionspace real(real64) :: zlon, zlat integer(ATLAS_KIND_IDX) :: jlon,jlat, jgp @@ -48,7 +48,9 @@ subroutine run gridname = "O320" ! Create grid and write some properties -grid = atlas_StructuredGrid(gridname) +!grid = atlas_StructuredGrid(gridname) +grid = atlas_ReducedGaussianGrid([20,24,28,32,36,40,44,48,52,56,60,64,& + 64,60,56,52,48,44,40,36,32,28,24,20]) if (fckit_mpi%rank() == log_rank) then write(0,*) "grid%name() : ", grid%name() diff --git a/src/atlas_f/grid/atlas_StencilComputer_module.F90 b/src/atlas_f/grid/atlas_StencilComputer_module.F90 index 8ec0a1b0f..e3bb5966f 100644 --- a/src/atlas_f/grid/atlas_StencilComputer_module.F90 +++ b/src/atlas_f/grid/atlas_StencilComputer_module.F90 @@ -169,7 +169,7 @@ function atlas_StructuredGrid_ComputeNorth__ctor(grid, halo) result(this) use atlas_grid_module, only : atlas_StructuredGrid implicit none type(atlas_StructuredGrid_ComputeNorth) :: this - type(atlas_StructuredGrid), intent(in) :: grid + class(atlas_StructuredGrid), intent(in) :: grid integer(c_int), intent(in) :: halo call this%reset_c_ptr( atlas__grid__ComputeNorth__new(grid%CPTR_PGIBUG_B, halo), & & fckit_c_deleter(atlas__grid__ComputeNorth__delete) ) @@ -183,7 +183,7 @@ function atlas_StructuredGrid_ComputeWest__ctor(grid, halo) result(this) use atlas_grid_module, only : atlas_StructuredGrid implicit none type(atlas_StructuredGrid_ComputeWest) :: this - type(atlas_StructuredGrid), intent(in) :: grid + class(atlas_StructuredGrid), intent(in) :: grid integer(c_int), intent(in) :: halo call this%reset_c_ptr( atlas__grid__ComputeWest__new(grid%CPTR_PGIBUG_B, halo), & & fckit_c_deleter(atlas__grid__ComputeWest__delete) ) @@ -195,7 +195,7 @@ subroutine atlas_StructuredGrid_ComputeStencil__setup(this, grid, stencil_width) use atlas_grid_module, only : atlas_StructuredGrid implicit none class(atlas_StructuredGrid_ComputeStencil) :: this - type(atlas_StructuredGrid), intent(in) :: grid + class(atlas_StructuredGrid), intent(in) :: grid integer(ATLAS_KIND_IDX), intent(in) :: stencil_width this%stencil_width = stencil_width this%halo = (stencil_width + 1) / 2 @@ -309,7 +309,7 @@ function atlas_StructuredGrid_ComputeStencil__ctor(grid, stencil_width) result(t use atlas_grid_module, only : atlas_StructuredGrid implicit none type(atlas_StructuredGrid_ComputeStencil) :: this - type(atlas_StructuredGrid), intent(in) :: grid + class(atlas_StructuredGrid), intent(in) :: grid integer(ATLAS_KIND_IDX), intent(in) :: stencil_width call this%setup(grid, stencil_width) end function From b5cbccef8ef1395989a540522bb005927cfbb401 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 4 Oct 2024 17:52:40 +0200 Subject: [PATCH 07/40] Fix use of ATLAS_ENABLE_CUDA ON/OFF --- CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f89d8fa6..0b59e607f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,13 @@ ecbuild_add_option( FEATURE ECKIT_DEVELOP DESCRIPTION "Used to enable new features or API depending on eckit develop branch, not yet in a tagged release" DEFAULT OFF ) +if (DEFINED ATLAS_ENABLE_CUDA AND NOT DEFINED HIC_ENABLE_CUDA ) + set( HIC_ENABLE_CUDA ${ATLAS_ENABLE_CUDA} ) +endif() +if (DEFINED ATLAS_ENABLE_HIP AND NOT DEFINED HIC_ENABLE_HIP ) + set( HIC_ENABLE_HIP ${ATLAS_ENABLE_HIP} ) +endif() + add_subdirectory( hic ) find_package(hic) From b8988b1246d83e828559bfbc40ac8e8c51ec893a Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 6 Sep 2023 19:47:58 +0200 Subject: [PATCH 08/40] Add capability to concatenate two fieldsets --- src/atlas/field/FieldSet.cc | 12 ++++++++++++ src/atlas/field/FieldSet.h | 3 +++ src/atlas_f/field/atlas_FieldSet_module.fypp | 13 +++++++++++-- src/tests/field/fctest_field.F90 | 13 +++++++++++-- src/tests/field/test_fieldset.cc | 17 +++++++++++++++++ 5 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/atlas/field/FieldSet.cc b/src/atlas/field/FieldSet.cc index e8ba510ce..d9ee713f9 100644 --- a/src/atlas/field/FieldSet.cc +++ b/src/atlas/field/FieldSet.cc @@ -102,6 +102,12 @@ Field FieldSetImpl::add(const Field& field) { return field; } +void FieldSetImpl::add(const FieldSet& fieldset) { + for( auto& field : fieldset) { + add(field); + } +} + bool FieldSetImpl::has(const std::string& name) const { return index_.count(name); } @@ -204,6 +210,12 @@ void atlas__FieldSet__add_field(FieldSetImpl* This, FieldImpl* field) { This->add(field); } +void atlas__FieldSet__add_fieldset(FieldSetImpl* This, FieldSetImpl* fieldset) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldSet"); + ATLAS_ASSERT(fieldset != nullptr, "Reason: Use of uninitialised atlas_FieldSet"); + This->add(fieldset); +} + int atlas__FieldSet__has_field(const FieldSetImpl* This, char* name) { ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldSet"); return This->has(std::string(name)); diff --git a/src/atlas/field/FieldSet.h b/src/atlas/field/FieldSet.h index fbcd065d5..7e110b121 100644 --- a/src/atlas/field/FieldSet.h +++ b/src/atlas/field/FieldSet.h @@ -117,6 +117,7 @@ class FieldSetImpl : public util::Object { const std::vector& field_names() const; Field add(const Field&); + void add(const FieldSet&); bool has(const std::string& name) const; @@ -153,6 +154,7 @@ extern "C" { FieldSetImpl* atlas__FieldSet__new(char* name); void atlas__FieldSet__delete(FieldSetImpl* This); void atlas__FieldSet__add_field(FieldSetImpl* This, FieldImpl* field); +void atlas__FieldSet__add_fieldset(FieldSetImpl* This, FieldSetImpl* fieldset); int atlas__FieldSet__has_field(const FieldSetImpl* This, char* name); const char* atlas__FieldSet__name(FieldSetImpl* This); idx_t atlas__FieldSet__size(const FieldSetImpl* This); @@ -239,6 +241,7 @@ class FieldSet : DOXYGEN_HIDE(public util::ObjectHandle) { const std::vector& field_names() const { return get()->field_names(); } Field add(const Field& field) { return get()->add(field); } + void add(const FieldSet& fieldset) { return get()->add(fieldset); } bool has(const std::string& name) const { return get()->has(name); } diff --git a/src/atlas_f/field/atlas_FieldSet_module.fypp b/src/atlas_f/field/atlas_FieldSet_module.fypp index e337c958c..36a2a38da 100644 --- a/src/atlas_f/field/atlas_FieldSet_module.fypp +++ b/src/atlas_f/field/atlas_FieldSet_module.fypp @@ -51,7 +51,9 @@ contains procedure, private :: field_by_name procedure, private :: field_by_idx_int procedure, private :: field_by_idx_long - procedure, public :: add + procedure, public :: add_field + procedure, public :: add_fieldset + generic :: add => add_field, add_fieldset generic :: field => field_by_name, field_by_idx_int, field_by_idx_long procedure, public :: set_dirty @@ -213,7 +215,7 @@ function FieldSet__name(this) result(fieldset_name) fieldset_name = c_ptr_to_string(fieldset_name_c_str) end function FieldSet__name -subroutine add(this,field) +subroutine add_field(this,field) use atlas_fieldset_c_binding use atlas_Field_module, only: atlas_Field class(atlas_FieldSet), intent(in) :: this @@ -221,6 +223,13 @@ subroutine add(this,field) call atlas__FieldSet__add_field(this%CPTR_PGIBUG_A, field%CPTR_PGIBUG_A) end subroutine +subroutine add_fieldset(this,fset) + use atlas_fieldset_c_binding + class(atlas_FieldSet), intent(inout) :: this + class(atlas_FieldSet), intent(in) :: fset + call atlas__FieldSet__add_fieldset(this%CPTR_PGIBUG_A, fset%CPTR_PGIBUG_A) +end subroutine + function has(this,name) result(flag) use, intrinsic :: iso_c_binding, only: c_int use fckit_c_interop_module, only: c_str diff --git a/src/tests/field/fctest_field.F90 b/src/tests/field/fctest_field.F90 index 8ea98e09d..d934b4886 100644 --- a/src/tests/field/fctest_field.F90 +++ b/src/tests/field/fctest_field.F90 @@ -139,8 +139,8 @@ module fcta_Field_fixture TEST( test_fieldset ) implicit none - type(atlas_FieldSet) :: fieldset - type(atlas_Field) :: field + type(atlas_FieldSet) :: fieldset, fieldset_2 + type(atlas_Field) :: field, field_2 write(*,*) "test_fieldset starting" @@ -163,6 +163,15 @@ module fcta_Field_fixture FCTEST_CHECK_EQUAL( field%name(), "field_1" ) field = fieldset%field(3) FCTEST_CHECK_EQUAL( field%name(), "field_2" ) + + fieldset_2 = atlas_FieldSet() + call fieldset_2%add(fieldset) + field_2 = fieldset_2%field("field_0") + call field_2%rename("field_00") + FCTEST_CHECK(fieldset%has("field_00")) + field = fieldset%field("field_00") + FCTEST_CHECK_EQUAL(field%name(), "field_00") + call fieldset%final() write(0,*) "test_fieldset end" diff --git a/src/tests/field/test_fieldset.cc b/src/tests/field/test_fieldset.cc index e8762e657..474fd5b5b 100644 --- a/src/tests/field/test_fieldset.cc +++ b/src/tests/field/test_fieldset.cc @@ -56,7 +56,24 @@ CASE("test_rename") { EXPECT(!fieldset.has("field_1")); EXPECT_EQ(fieldset.field(1).name(),std::string("")); EXPECT(fieldset.has("[1]")); +} +CASE("test_fieldset_concatenation") { + FieldSet fieldset_1; + FieldSet fieldset_2; + auto field_1 = fieldset_1.add(Field("0", make_datatype(), array::make_shape(10,4))); + auto field_1_v = array::make_view(field_1); + field_1_v(1,1) = 2.; + + fieldset_2.add(fieldset_1); + auto field_2 = fieldset_2.field("0"); + field_2.rename(""); + auto field_2_v = array::make_view(field_2); + field_2_v(1,1) = 1.; + + EXPECT(!fieldset_2.has("")); + EXPECT_EQ(fieldset_2.field(0).name(), ""); + EXPECT_EQ(field_1_v(1,1), 1.); } CASE("test_duplicate_name_throws") { From 380eecde3c605750e888466eb4b5864f8932ee13 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 23 Aug 2023 09:17:46 +0000 Subject: [PATCH 09/40] Add access to nproma for BlockStructuredColumns in Fortran --- .../detail/BlockStructuredColumnsInterface.cc | 3 +++ .../detail/BlockStructuredColumnsInterface.h | 1 + .../atlas_functionspace_BlockStructuredColumns_module.F90 | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.cc b/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.cc index 988bc1c86..492a6105b 100644 --- a/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.cc +++ b/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.cc @@ -175,6 +175,9 @@ idx_t atlas__fs__BStructuredColumns__block_begin(const detail::BlockStructuredCo idx_t atlas__fs__BStructuredColumns__block_size(const detail::BlockStructuredColumns* This, idx_t jblk) { return This->block_size(jblk); } +idx_t atlas__fs__BStructuredColumns__nproma(const detail::BlockStructuredColumns* This) { + return This->nproma(); +} idx_t atlas__fs__BStructuredColumns__nblks(const detail::BlockStructuredColumns* This) { return This->nblks(); } diff --git a/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.h b/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.h index 7428afdf1..5a660132c 100644 --- a/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.h +++ b/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.h @@ -102,6 +102,7 @@ idx_t atlas__fs__BStructuredColumns__i_end_halo(const detail::BlockStructuredCol idx_t atlas__fs__BStructuredColumns__levels(const detail::BlockStructuredColumns* This); idx_t atlas__fs__BStructuredColumns__block_begin(const detail::BlockStructuredColumns* This, idx_t jblk); idx_t atlas__fs__BStructuredColumns__block_size(const detail::BlockStructuredColumns* This, idx_t jblk); +idx_t atlas__fs__BStructuredColumns__nproma(const detail::BlockStructuredColumns* This); idx_t atlas__fs__BStructuredColumns__nblks(const detail::BlockStructuredColumns* This); field::FieldImpl* atlas__fs__BStructuredColumns__xy(const detail::BlockStructuredColumns* This); diff --git a/src/atlas_f/functionspace/atlas_functionspace_BlockStructuredColumns_module.F90 b/src/atlas_f/functionspace/atlas_functionspace_BlockStructuredColumns_module.F90 index f25f76321..de3770e28 100644 --- a/src/atlas_f/functionspace/atlas_functionspace_BlockStructuredColumns_module.F90 +++ b/src/atlas_f/functionspace/atlas_functionspace_BlockStructuredColumns_module.F90 @@ -89,6 +89,7 @@ module atlas_functionspace_BlockStructuredColumns_module procedure :: levels procedure :: block_begin procedure :: block_size + procedure :: nproma procedure :: nblks procedure :: xy @@ -554,6 +555,13 @@ function block_size(this,j) result(i) i = atlas__fs__BStructuredColumns__block_size(this%CPTR_PGIBUG_A,j-1) end function +function nproma(this) result(i) + use atlas_functionspace_BlockStructuredColumns_c_binding + integer(ATLAS_KIND_IDX) :: i + class(atlas_functionspace_BlockStructuredColumns), intent(in) :: this + i = atlas__fs__BStructuredColumns__nproma(this%CPTR_PGIBUG_A) +end function + function nblks(this) result(i) use atlas_functionspace_BlockStructuredColumns_c_binding integer(ATLAS_KIND_IDX) :: i From e5823e08ab2db107d7a2cd9f71e26e855fd8cbc9 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 5 Sep 2023 18:41:49 +0200 Subject: [PATCH 10/40] Add FieldObserver::onFieldDestruction --- src/atlas/field/detail/FieldImpl.cc | 3 +++ src/atlas/field/detail/FieldImpl.h | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/atlas/field/detail/FieldImpl.cc b/src/atlas/field/detail/FieldImpl.cc index 78d929f23..3d655926d 100644 --- a/src/atlas/field/detail/FieldImpl.cc +++ b/src/atlas/field/detail/FieldImpl.cc @@ -94,6 +94,9 @@ FieldImpl::FieldImpl(const std::string& name, array::Array* array) } FieldImpl::~FieldImpl() { + for (FieldObserver* observer : field_observers_) { + observer->onFieldDestruction(*this); + } array_->detach(); if (array_->owners() == 0) { for (auto& f : callback_on_destruction_) { diff --git a/src/atlas/field/detail/FieldImpl.h b/src/atlas/field/detail/FieldImpl.h index cdd9c308f..24af72d2c 100644 --- a/src/atlas/field/detail/FieldImpl.h +++ b/src/atlas/field/detail/FieldImpl.h @@ -257,7 +257,8 @@ class FieldObserver { } } - virtual void onFieldRename(FieldImpl&) = 0; + virtual void onFieldRename(FieldImpl&) {} + virtual void onFieldDestruction(FieldImpl&) {} }; //---------------------------------------------------------------------------------------------------------------------- From 955b688eb463d2ac5aae395ffb2f19d2e6c1db4a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 25 Mar 2022 22:42:44 +0000 Subject: [PATCH 11/40] Introducing MultiField --- src/atlas/CMakeLists.txt | 2 + src/atlas/field/MultiField.cc | 207 +++++++++++++++++++++++ src/atlas/field/MultiField.h | 149 ++++++++++++++++ src/tests/field/CMakeLists.txt | 5 + src/tests/field/test_multifield_ifs.cc | 224 +++++++++++++++++++++++++ 5 files changed, 587 insertions(+) create mode 100644 src/atlas/field/MultiField.cc create mode 100644 src/atlas/field/MultiField.h create mode 100644 src/tests/field/test_multifield_ifs.cc diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 12f0db620..55aadeb6d 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -474,6 +474,8 @@ field/FieldSet.cc field/FieldSet.h field/MissingValue.cc field/MissingValue.h +field/MultiField.cc +field/MultiField.h field/State.cc field/State.h field/detail/FieldImpl.cc diff --git a/src/atlas/field/MultiField.cc b/src/atlas/field/MultiField.cc new file mode 100644 index 000000000..9220226af --- /dev/null +++ b/src/atlas/field/MultiField.cc @@ -0,0 +1,207 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +// file deepcode ignore CppMemoryLeak: static pointers for global registry are OK and will be cleaned up at end + +#include "MultiField.h" + +#include +#include +#include +#include + +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/Mutex.h" + +#include "atlas/field/Field.h" +#include "atlas/grid/Grid.h" +#include "atlas/mesh/Mesh.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + +namespace atlas { +namespace field { + +namespace { + +static eckit::Mutex* local_mutex = nullptr; +static std::map* m = nullptr; +static pthread_once_t once = PTHREAD_ONCE_INIT; + +static void init() { + local_mutex = new eckit::Mutex(); + m = new std::map(); +} + +template +void load_builder() { + MultiFieldCreatorBuilder("tmp"); +} + +void force_link() { + static struct Link { Link() = default; } link; + [](const Link&) {}(link); // disable unused warnings +} + +} // namespace + +void MultiField::initialize(const std::string& generator, const eckit::Parametrisation& params) { + std::unique_ptr FieldPool_generator(MultiFieldCreatorFactory::build(generator, params)); + FieldPool_generator->generate(*this, params); +} + +array::Array& MultiField::allocate(array::DataType datatype, array::ArraySpec&& spec) { + array_.reset(array::Array::create(datatype, std::move(spec))); + return *array_; +} + +//------------------------------------------------------------------------------------------------------ + +MultiField::MultiField() = default; + +MultiField::MultiField(const std::string& generator, const eckit::Parametrisation& params) { + initialize(generator, params); +} + +const util::Metadata& MultiField::metadata() const { + return metadata_; +} + +util::Metadata& MultiField::metadata() { + return metadata_; +} + +std::vector MultiField::field_names() const { + std::vector ret; + if (fields_.size()) { + ret.reserve(fields_.size()); + } + + for (auto it = field_index_.begin(); it != field_index_.end(); ++it) { + ret.push_back(it->first); + } + return ret; +} + +//----------------------------------------------------------------------------- + +MultiFieldCreator::MultiFieldCreator(const eckit::Parametrisation&) {} + +MultiFieldCreator::~MultiFieldCreator() = default; + +MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& name, const eckit::Parametrisation& param) { + pthread_once(&once, init); + + eckit::AutoLock lock(local_mutex); + + force_link(); + + std::map::const_iterator j = m->find(name); + + Log::debug() << "Looking for MultiFieldCreatorFactory [" << name << "]" << std::endl; + + if (j == m->end()) { + Log::error() << "No MultiFieldCreatorFactory for [" << name << "]" << std::endl; + Log::error() << "FieldPoolFactories are:" << std::endl; + for (j = m->begin(); j != m->end(); ++j) { + Log::error() << " " << (*j).first << std::endl; + } + throw_Exception(std::string("No MultiFieldCreatorFactory called ") + name, Here()); + } + + return (*j).second->make(param); +} + +void MultiFieldCreatorFactory::list(std::ostream& out) { + pthread_once(&once, init); + + eckit::AutoLock lock(local_mutex); + + force_link(); + + const char* sep = ""; + for (std::map::const_iterator j = m->begin(); j != m->end(); ++j) { + out << sep << (*j).first; + sep = ", "; + } +} + +bool MultiFieldCreatorFactory::has(const std::string& name) { + pthread_once(&once, init); + + eckit::AutoLock lock(local_mutex); + + force_link(); + + return (m->find(name) != m->end()); +} + +MultiFieldCreatorFactory::MultiFieldCreatorFactory(const std::string& name): name_(name) { + pthread_once(&once, init); + + eckit::AutoLock lock(local_mutex); + + ATLAS_ASSERT(m->find(name) == m->end()); + (*m)[name] = this; +} + +MultiFieldCreatorFactory::~MultiFieldCreatorFactory() { + eckit::AutoLock lock(local_mutex); + m->erase(name_); +} + +//----------------------------------------------------------------------------- +// C wrapper interfaces to C++ routines + +extern "C" { + +MultiField* atlas__FieldPool__new() { + return new MultiField; +} + +void atlas__FieldPool__initialize(MultiField* This, const char* generator, const eckit::Parametrisation* params) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); + This->initialize(std::string(generator), *params); +} + +void atlas__FieldPool__delete(MultiField* This) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); + delete This; +} + +int atlas__FieldPool__has(MultiField* This, const char* name) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); + return This->has(name); +} + +FieldImpl* atlas__FieldPool__field_by_name(MultiField* This, const char* name) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); + return This->field(std::string(name)).get(); +} + +FieldImpl* atlas__FieldPool__field_by_index(MultiField* This, idx_t index) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); + return This->field(index).get(); +} + +idx_t atlas__FieldPool__size(const MultiField* This) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); + return This->size(); +} + +util::Metadata* atlas__FieldPool__metadata(MultiField* This) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); + return &This->metadata(); +} +} +//----------------------------------------------------------------------------- + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiField.h b/src/atlas/field/MultiField.h new file mode 100644 index 000000000..963d16b13 --- /dev/null +++ b/src/atlas/field/MultiField.h @@ -0,0 +1,149 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Willem Deconinck +/// @date June 2015 + +#pragma once + +#include + +#include "atlas/array/Array.h" +#include "atlas/field/Field.h" +#include "atlas/util/Config.h" +#include "atlas/util/Metadata.h" +#include "atlas/util/Object.h" + +namespace eckit { +class Parametrisation; +} + +namespace atlas { +namespace field { + +/** + * \brief MultiField class that owns a collection of fields that are co-allocated + * + * Fields can only be described by parametrisation during the construction. + * Once setup, no additional fields can be added. + * + * Fields have to all be of same memory layout and data type + */ +class MultiField : public util::Object { +public: // methods + //-- Constructors + MultiField(); + + MultiField(const std::string& generator, const eckit::Parametrisation& = util::Config()); + + //-- Accessors + + const Field& field(const std::string& name) const { return fields_[field_index_.at(name)]; } + Field& field(const std::string& name) { return fields_[field_index_.at(name)]; } + bool has(const std::string& name) const { return (field_index_.find(name) != field_index_.end()); } + std::vector field_names() const; + + const Field& field(const idx_t idx) const { return fields_[idx]; } + Field& field(const idx_t idx) { return fields_[idx]; } + idx_t size() const { return static_cast(fields_.size()); } + + const Field& operator[](const idx_t idx) const { return fields_[idx]; } + Field& operator[](const idx_t idx) { return fields_[idx]; } + + const Field& operator[](const std::string& name) const { return field(name); } + Field& operator[](const std::string& name) { return field(name); } + + const util::Metadata& metadata() const; + util::Metadata& metadata(); + + // -- Modifiers + + void initialize(const std::string& generator, const eckit::Parametrisation& = util::Config()); + + array::Array& allocate(array::DataType datatype, array::ArraySpec&&); + + /// @brief Implicit conversion to Array + operator const array::Array&() const { return *array_; } + operator array::Array&() { return *array_; } + + /// @brief Access contained Array + const array::Array& array() const { return *array_; } + array::Array& array() { return *array_; } + + +public: // temporary public for prototyping + std::map field_index_; + std::vector fields_; + std::unique_ptr array_; + util::Metadata metadata_; +}; + +//------------------------------------------------------------------------------------------------------ + +class MultiFieldCreator : public util::Object { +public: + MultiFieldCreator(const eckit::Parametrisation& = util::Config()); + + virtual ~MultiFieldCreator(); + + virtual void generate(MultiField&, const eckit::Parametrisation& = util::Config()) const = 0; +}; + +//------------------------------------------------------------------------------------------------------ + +class MultiFieldCreatorFactory { +public: + /*! + * \brief build FieldPoolCreator with options specified in parametrisation + * \return mesh generator + */ + static MultiFieldCreator* build(const std::string& FieldPool_generator, + const eckit::Parametrisation& = util::Config()); + + /*! + * \brief list all registered field creators + */ + static void list(std::ostream&); + static bool has(const std::string& name); + +private: + virtual MultiFieldCreator* make(const eckit::Parametrisation& = util::Config()) = 0; + + std::string name_; + +protected: + MultiFieldCreatorFactory(const std::string&); + virtual ~MultiFieldCreatorFactory(); +}; + +template +class MultiFieldCreatorBuilder : public MultiFieldCreatorFactory { + virtual MultiFieldCreator* make(const eckit::Parametrisation& param = util::Config()) { return new T(param); } + +public: + MultiFieldCreatorBuilder(const std::string& name): MultiFieldCreatorFactory(name) {} +}; + +// ------------------------------------------------------------------------------------ + +// C wrapper interfaces to C++ routines +extern "C" { +MultiField* atlas__FieldPool__new(); +void atlas__FieldPool__initialize(MultiField* This, const char* generator, const eckit::Parametrisation* params); +void atlas__FieldPool__delete(MultiField* This); +int atlas__FieldPool__has(MultiField* This, const char* name); +FieldImpl* atlas__FieldPool__field_by_name(MultiField* This, const char* name); +FieldImpl* atlas__FieldPool__field_by_index(MultiField* This, idx_t index); +idx_t atlas__FieldPool__size(const MultiField* This); +util::Metadata* atlas__FieldPool__metadata(MultiField* This); +} + +} // namespace field +} // namespace atlas diff --git a/src/tests/field/CMakeLists.txt b/src/tests/field/CMakeLists.txt index dc91d6f65..e103c2886 100644 --- a/src/tests/field/CMakeLists.txt +++ b/src/tests/field/CMakeLists.txt @@ -23,6 +23,11 @@ ecbuild_add_test( TARGET atlas_test_field_foreach LIBS atlas OMP 4 ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} + +ecbuild_add_test( TARGET atlas_test_multifield_ifs + SOURCES test_multifield_ifs.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) ecbuild_add_test( TARGET atlas_test_field_acc diff --git a/src/tests/field/test_multifield_ifs.cc b/src/tests/field/test_multifield_ifs.cc new file mode 100644 index 000000000..e458fe1a2 --- /dev/null +++ b/src/tests/field/test_multifield_ifs.cc @@ -0,0 +1,224 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include + +#include "eckit/log/JSON.h" +#include "eckit/parser/JSONParser.h" + +#include "atlas/array/ArrayView.h" +#include "atlas/array/DataType.h" +#include "atlas/array/MakeView.h" +#include "atlas/field/Field.h" +#include "atlas/field/MultiField.h" +#include "atlas/grid/Grid.h" +#include "atlas/mesh/Mesh.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + +#include "tests/AtlasTestEnvironment.h" + +using namespace atlas::field; +using namespace atlas::field; + +namespace atlas { +namespace test { + +// ------------------------------------------------------------------- +// Example IFS MultiField creato + +// --- Declaration (in .h file) +class MultiFieldCreatorIFS : public MultiFieldCreator { +public: + MultiFieldCreatorIFS(const eckit::Parametrisation& p = util::Config()): MultiFieldCreator(p) {} + ~MultiFieldCreatorIFS() override = default; + void generate(MultiField& fieldpool, const eckit::Parametrisation& p = util::Config()) const override; +}; + +// --- Implementation (in .cc file) +void MultiFieldCreatorIFS::generate(MultiField& multifield, const eckit::Parametrisation& p) const { + const eckit::LocalConfiguration* params = dynamic_cast(&p); + if (!params) { + throw_Exception("Parametrisation has to be of atlas::util::Config type"); + } + + + long nproma = 0; + long ngptot = 0; + ; + long nfld = 0; + ; + long nlev = 0; + ; + long nblk = 0; + ; + + if (!p.get("ngptot", ngptot)) { + std::string grid_uid; + if (p.get("grid", grid_uid)) { + ngptot = Grid(grid_uid).size(); + } + else { + throw_Exception("Could not find 'ngptot' in Parametrisation"); + } + } + + if (!p.get("nproma", nproma)) { + throw_Exception("Could not find 'nproma' in Parametrisation"); + } + + if (!p.get("nlev", nlev)) { + throw_Exception("Could not find 'nlev' in Parametrisation"); + } + + array::DataType datatype = array::DataType::create(); + std::string datatype_str; + if (p.get("datatype", datatype_str)) { + datatype = array::DataType(datatype_str); + } + else { + array::DataType::kind_t kind(array::DataType::kind()); + p.get("kind", kind); + if (!array::DataType::kind_valid(kind)) { + std::stringstream msg; + msg << "Could not create field. kind parameter unrecognized"; + throw_Exception(msg.str()); + } + datatype = array::DataType(kind); + } + + nblk = std::ceil(static_cast(ngptot) / static_cast(nproma)); + + std::vector fields; + params->get("fields", fields); + nfld = fields.size(); + + auto multiarray_spec = array::make_shape(nblk, nfld, nlev, nproma); + auto& multiarray = multifield.allocate(datatype, multiarray_spec); + + auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2), multiarray.shape(3)); + auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2), multiarray.stride(3)); + auto field_array_spec = array::ArraySpec(field_shape, field_strides); + + for (size_t i = 0; i < fields.size(); ++i) { + std::string name; + fields[i].get("name", name); + Field field; + constexpr auto all = array::Range::all(); + if (datatype.kind() == array::DataType::KIND_REAL64) { + auto slice = array::make_view(multiarray).slice(all, i, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + auto slice = array::make_view(multiarray).slice(all, i, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + ATLAS_NOTIMPLEMENTED; + } + field.set_levels(nlev); + multifield.fields_.emplace_back(field); + multifield.field_index_[field.name()] = i; + } +} + +// Register in factory +MultiFieldCreatorBuilder __MultiFieldCreatorIFS("MultiFieldCreatorIFS"); + +// =================================================================== +// BEGIN TESTS +// =================================================================== + + +CASE("multifield_generator") { + EXPECT(MultiFieldCreatorFactory::has("MultiFieldCreatorIFS")); + std::unique_ptr MultiFieldCreator(MultiFieldCreatorFactory::build("MultiFieldCreatorIFS")); +} + + +CASE("multifield_create") { + int nproma = 16; + int nlev = 100; + int ngptot = 2000; + + auto json = [&]() -> std::string { + util::Config p; + p.set("ngptot", ngptot); + p.set("nproma", nproma); + p.set("nlev", nlev); + p.set("datatype", array::DataType::real64().str()); + + std::vector fields(3); + fields[0].set("name", "temperature"); + fields[1].set("name", "pressure"); + fields[2].set("name", "density"); + + p.set("fields", fields); + + // We can also translate parameters to a json: + std::stringstream json; + eckit::JSON js(json); + js << p; + std::string json_str = json.str(); + Log::info() << "json = " << json_str << std::endl; + return json_str; + }; + + + // And we can create back parameters from json: + std::stringstream json_stream; + json_stream << json(); + util::Config config(json_stream); + + MultiField multifield("MultiFieldCreatorIFS", config); + + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("pressure")); + EXPECT(multifield.has("density")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("pressure") << std::endl; + Log::info() << multifield.field("density") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto pres = array::make_view(multifield.field("pressure")); + auto dens = array::make_view(multifield.field("density")); + + temp(1, 2, 3) = 4; + pres(5, 6, 7) = 8; + dens(9, 10, 11) = 12; + + EXPECT_EQ(temp.stride(2), 1); + EXPECT_EQ(temp.stride(1), nproma); + EXPECT_EQ(temp.stride(0), nproma * nlev * multifield.size()); + + // Underlying array of MultiField + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(3), 1); + EXPECT_EQ(multiarray.stride(2), nproma); + EXPECT_EQ(multiarray.stride(1), nproma * nlev); + EXPECT_EQ(multiarray.stride(0), nproma * nlev * multifield.size()); + + EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); + EXPECT_EQ(multiarray(5, 1, 6, 7), 8.); + EXPECT_EQ(multiarray(9, 2, 10, 11), 12.); + } +} + +//----------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} From 8a9245e31d51d472061afd6ba112987ae6c79542 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 28 Mar 2022 15:51:11 +0100 Subject: [PATCH 12/40] Support for vector fields in multifield --- src/tests/field/test_multifield_ifs.cc | 175 +++++++++++++++++-------- 1 file changed, 123 insertions(+), 52 deletions(-) diff --git a/src/tests/field/test_multifield_ifs.cc b/src/tests/field/test_multifield_ifs.cc index e458fe1a2..8ea78cd67 100644 --- a/src/tests/field/test_multifield_ifs.cc +++ b/src/tests/field/test_multifield_ifs.cc @@ -50,25 +50,14 @@ void MultiFieldCreatorIFS::generate(MultiField& multifield, const eckit::Paramet throw_Exception("Parametrisation has to be of atlas::util::Config type"); } - long nproma = 0; long ngptot = 0; - ; - long nfld = 0; - ; - long nlev = 0; - ; - long nblk = 0; - ; + long nfld = 0; + long nlev = 0; + long nblk = 0; if (!p.get("ngptot", ngptot)) { - std::string grid_uid; - if (p.get("grid", grid_uid)) { - ngptot = Grid(grid_uid).size(); - } - else { - throw_Exception("Could not find 'ngptot' in Parametrisation"); - } + throw_Exception("Could not find 'ngptot' in Parametrisation"); } if (!p.get("nproma", nproma)) { @@ -99,34 +88,69 @@ void MultiFieldCreatorIFS::generate(MultiField& multifield, const eckit::Paramet std::vector fields; params->get("fields", fields); - nfld = fields.size(); + nfld = fields.size(); + size_t nvar_tot = 0; + for (const auto& field_config : fields) { + size_t nvar = 1; + field_config.get("nvar", nvar); + nvar_tot += nvar; + } - auto multiarray_spec = array::make_shape(nblk, nfld, nlev, nproma); + auto multiarray_spec = array::make_shape(nblk, nvar_tot, nlev, nproma); auto& multiarray = multifield.allocate(datatype, multiarray_spec); - auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2), multiarray.shape(3)); - auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2), multiarray.stride(3)); - auto field_array_spec = array::ArraySpec(field_shape, field_strides); - + size_t multiarray_field_idx = 0; for (size_t i = 0; i < fields.size(); ++i) { std::string name; fields[i].get("name", name); Field field; - constexpr auto all = array::Range::all(); - if (datatype.kind() == array::DataType::KIND_REAL64) { - auto slice = array::make_view(multiarray).slice(all, i, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else if (datatype.kind() == array::DataType::KIND_REAL32) { - auto slice = array::make_view(multiarray).slice(all, i, all, all); - field = Field(name, slice.data(), field_array_spec); + size_t field_vars = 1; + + if (fields[i].get("nvar", field_vars)) { + auto field_shape = + array::make_shape(multiarray.shape(0), field_vars, multiarray.shape(2), multiarray.shape(3)); + auto field_strides = multiarray.strides(); + auto field_array_spec = array::ArraySpec(field_shape, field_strides); + + constexpr auto all = array::Range::all(); + const auto range = array::Range(multiarray_field_idx, multiarray_field_idx + field_vars); + if (datatype.kind() == array::DataType::KIND_REAL64) { + auto slice = array::make_view(multiarray).slice(all, range, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + auto slice = array::make_view(multiarray).slice(all, range, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + ATLAS_NOTIMPLEMENTED; + } + field.set_variables(field_vars); } else { - ATLAS_NOTIMPLEMENTED; + auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2), multiarray.shape(3)); + auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2), multiarray.stride(3)); + auto field_array_spec = array::ArraySpec(field_shape, field_strides); + + constexpr auto all = array::Range::all(); + if (datatype.kind() == array::DataType::KIND_REAL64) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + ATLAS_NOTIMPLEMENTED; + } } field.set_levels(nlev); multifield.fields_.emplace_back(field); + ATLAS_ASSERT(multifield.field_index_.find(field.name()) == multifield.field_index_.end(), + "Field with name already exists!"); multifield.field_index_[field.name()] = i; + multiarray_field_idx += field_vars; } } @@ -154,63 +178,110 @@ CASE("multifield_create") { p.set("ngptot", ngptot); p.set("nproma", nproma); p.set("nlev", nlev); - p.set("datatype", array::DataType::real64().str()); + p.set("datatype", array::make_datatype().str()); + p.set("fields", [] { + std::vector fields(5); + fields[0].set("name", "temperature"); + + fields[1].set("name", "pressure"); - std::vector fields(3); - fields[0].set("name", "temperature"); - fields[1].set("name", "pressure"); - fields[2].set("name", "density"); + fields[2].set("name", "density"); - p.set("fields", fields); + fields[3].set("name", "clv"); + fields[3].set("nvar", 5); + + fields[4].set("name", "wind_u"); + return fields; + }()); // We can also translate parameters to a json: std::stringstream json; eckit::JSON js(json); js << p; - std::string json_str = json.str(); - Log::info() << "json = " << json_str << std::endl; - return json_str; + return json.str(); }; // And we can create back parameters from json: + Log::info() << "json = " << json() << std::endl; std::stringstream json_stream; json_stream << json(); util::Config config(json_stream); MultiField multifield("MultiFieldCreatorIFS", config); + const auto nblk = multifield.array().shape(0); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 5); + + EXPECT_EQ(multifield.size(), 5); EXPECT(multifield.has("temperature")); EXPECT(multifield.has("pressure")); EXPECT(multifield.has("density")); + EXPECT(multifield.has("clv")); + EXPECT(multifield.has("wind_u")); Log::info() << multifield.field("temperature") << std::endl; Log::info() << multifield.field("pressure") << std::endl; Log::info() << multifield.field("density") << std::endl; + Log::info() << multifield.field("clv") << std::endl; + Log::info() << multifield.field("wind_u") << std::endl; - auto temp = array::make_view(multifield.field("temperature")); - auto pres = array::make_view(multifield.field("pressure")); - auto dens = array::make_view(multifield.field("density")); + auto temp = array::make_view(multifield.field("temperature")); + auto pres = array::make_view(multifield.field("pressure")); + auto dens = array::make_view(multifield.field("density")); + auto clv = array::make_view(multifield.field("clv")); // note rank 4 + auto wind_u = array::make_view(multifield.field("wind_u")); - temp(1, 2, 3) = 4; - pres(5, 6, 7) = 8; - dens(9, 10, 11) = 12; + // or + { + auto temp = array::make_view(multifield[0]); + auto pres = array::make_view(multifield[1]); + auto dens = array::make_view(multifield[2]); + auto clv = array::make_view(multifield[3]); // note rank 4 + auto wind_u = array::make_view(multifield[4]); + } + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma * nlev; + auto level_stride = nproma; + auto nproma_stride = 1; + + temp(1, 2, 3) = 4; + pres(5, 6, 7) = 8; + dens(9, 10, 11) = 12; + clv(13, 2, 14, 15) = 16; + wind_u(17, 18, 3) = 19; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), level_stride); + EXPECT_EQ(temp.stride(2), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nlev * nproma); + + EXPECT_EQ(clv.stride(0), block_stride); + EXPECT_EQ(clv.stride(1), field_stride); + EXPECT_EQ(clv.stride(2), level_stride); + EXPECT_EQ(clv.stride(3), nproma_stride); + + EXPECT_EQ(clv.size(), nblk * 5 * nlev * nproma); - EXPECT_EQ(temp.stride(2), 1); - EXPECT_EQ(temp.stride(1), nproma); - EXPECT_EQ(temp.stride(0), nproma * nlev * multifield.size()); // Underlying array of MultiField { auto multiarray = array::make_view(multifield); - EXPECT_EQ(multiarray.stride(3), 1); - EXPECT_EQ(multiarray.stride(2), nproma); - EXPECT_EQ(multiarray.stride(1), nproma * nlev); - EXPECT_EQ(multiarray.stride(0), nproma * nlev * multifield.size()); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), level_stride); + EXPECT_EQ(multiarray.stride(3), nproma_stride); EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); EXPECT_EQ(multiarray(5, 1, 6, 7), 8.); EXPECT_EQ(multiarray(9, 2, 10, 11), 12.); + EXPECT_EQ(multiarray(13, 5, 14, 15), 16.); + EXPECT_EQ(multiarray(17, 8, 18, 3), 19.); + + + EXPECT_EQ(multiarray.size(), nblk * (nfld - 1 + 5) * nlev * nproma); } } From f2a2f5445fb6ca2a74d3fc4db73812e2a54df673 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 4 Apr 2022 15:56:40 +0100 Subject: [PATCH 13/40] Use util::Factory for MultiFieldCreatorFactory --- src/atlas/field/MultiField.cc | 142 ++++------------------------------ src/atlas/field/MultiField.h | 43 +++------- 2 files changed, 27 insertions(+), 158 deletions(-) diff --git a/src/atlas/field/MultiField.cc b/src/atlas/field/MultiField.cc index 9220226af..aeeac5fcb 100644 --- a/src/atlas/field/MultiField.cc +++ b/src/atlas/field/MultiField.cc @@ -8,8 +8,6 @@ * nor does it submit to any jurisdiction. */ -// file deepcode ignore CppMemoryLeak: static pointers for global registry are OK and will be cleaned up at end - #include "MultiField.h" #include @@ -30,31 +28,20 @@ namespace atlas { namespace field { namespace { - -static eckit::Mutex* local_mutex = nullptr; -static std::map* m = nullptr; -static pthread_once_t once = PTHREAD_ONCE_INIT; - -static void init() { - local_mutex = new eckit::Mutex(); - m = new std::map(); -} - -template -void load_builder() { - MultiFieldCreatorBuilder("tmp"); -} - void force_link() { - static struct Link { Link() = default; } link; - [](const Link&) {}(link); // disable unused warnings + static struct Link { + Link() { + ; + // For static linking add here something like + // MultiFieldCreatorBuilder(); + } + } link; } - } // namespace -void MultiField::initialize(const std::string& generator, const eckit::Parametrisation& params) { - std::unique_ptr FieldPool_generator(MultiFieldCreatorFactory::build(generator, params)); - FieldPool_generator->generate(*this, params); +void MultiField::initialize(const std::string& creator, const eckit::Parametrisation& params) { + std::unique_ptr MultiField_creator(MultiFieldCreatorFactory::build(creator, params)); + MultiField_creator->generate(*this, params); } array::Array& MultiField::allocate(array::DataType datatype, array::ArraySpec&& spec) { @@ -66,8 +53,8 @@ array::Array& MultiField::allocate(array::DataType datatype, array::ArraySpec&& MultiField::MultiField() = default; -MultiField::MultiField(const std::string& generator, const eckit::Parametrisation& params) { - initialize(generator, params); +MultiField::MultiField(const std::string& creator, const eckit::Parametrisation& params) { + initialize(creator, params); } const util::Metadata& MultiField::metadata() const { @@ -96,111 +83,12 @@ MultiFieldCreator::MultiFieldCreator(const eckit::Parametrisation&) {} MultiFieldCreator::~MultiFieldCreator() = default; -MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& name, const eckit::Parametrisation& param) { - pthread_once(&once, init); - - eckit::AutoLock lock(local_mutex); - - force_link(); - - std::map::const_iterator j = m->find(name); - - Log::debug() << "Looking for MultiFieldCreatorFactory [" << name << "]" << std::endl; - - if (j == m->end()) { - Log::error() << "No MultiFieldCreatorFactory for [" << name << "]" << std::endl; - Log::error() << "FieldPoolFactories are:" << std::endl; - for (j = m->begin(); j != m->end(); ++j) { - Log::error() << " " << (*j).first << std::endl; - } - throw_Exception(std::string("No MultiFieldCreatorFactory called ") + name, Here()); - } - - return (*j).second->make(param); -} - -void MultiFieldCreatorFactory::list(std::ostream& out) { - pthread_once(&once, init); - - eckit::AutoLock lock(local_mutex); - +MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& builder, const eckit::Parametrisation& config) { force_link(); - - const char* sep = ""; - for (std::map::const_iterator j = m->begin(); j != m->end(); ++j) { - out << sep << (*j).first; - sep = ", "; - } + auto factory = get(builder); + return factory->make(config); } -bool MultiFieldCreatorFactory::has(const std::string& name) { - pthread_once(&once, init); - - eckit::AutoLock lock(local_mutex); - - force_link(); - - return (m->find(name) != m->end()); -} - -MultiFieldCreatorFactory::MultiFieldCreatorFactory(const std::string& name): name_(name) { - pthread_once(&once, init); - - eckit::AutoLock lock(local_mutex); - - ATLAS_ASSERT(m->find(name) == m->end()); - (*m)[name] = this; -} - -MultiFieldCreatorFactory::~MultiFieldCreatorFactory() { - eckit::AutoLock lock(local_mutex); - m->erase(name_); -} - -//----------------------------------------------------------------------------- -// C wrapper interfaces to C++ routines - -extern "C" { - -MultiField* atlas__FieldPool__new() { - return new MultiField; -} - -void atlas__FieldPool__initialize(MultiField* This, const char* generator, const eckit::Parametrisation* params) { - ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); - This->initialize(std::string(generator), *params); -} - -void atlas__FieldPool__delete(MultiField* This) { - ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); - delete This; -} - -int atlas__FieldPool__has(MultiField* This, const char* name) { - ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); - return This->has(name); -} - -FieldImpl* atlas__FieldPool__field_by_name(MultiField* This, const char* name) { - ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); - return This->field(std::string(name)).get(); -} - -FieldImpl* atlas__FieldPool__field_by_index(MultiField* This, idx_t index) { - ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); - return This->field(index).get(); -} - -idx_t atlas__FieldPool__size(const MultiField* This) { - ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); - return This->size(); -} - -util::Metadata* atlas__FieldPool__metadata(MultiField* This) { - ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldPool"); - return &This->metadata(); -} -} //----------------------------------------------------------------------------- } // namespace field diff --git a/src/atlas/field/MultiField.h b/src/atlas/field/MultiField.h index 963d16b13..9b03c5b03 100644 --- a/src/atlas/field/MultiField.h +++ b/src/atlas/field/MultiField.h @@ -18,6 +18,7 @@ #include "atlas/array/Array.h" #include "atlas/field/Field.h" #include "atlas/util/Config.h" +#include "atlas/util/Factory.h" #include "atlas/util/Metadata.h" #include "atlas/util/Object.h" @@ -98,52 +99,32 @@ class MultiFieldCreator : public util::Object { //------------------------------------------------------------------------------------------------------ -class MultiFieldCreatorFactory { + +class MultiFieldCreatorFactory : public util::Factory { public: - /*! - * \brief build FieldPoolCreator with options specified in parametrisation - * \return mesh generator - */ - static MultiFieldCreator* build(const std::string& FieldPool_generator, - const eckit::Parametrisation& = util::Config()); + static std::string className() { return "MultiFieldCreatorFactory"; } /*! - * \brief list all registered field creators + * \brief build MultiFieldCreator with options specified in parametrisation + * \return MutliField creator */ - static void list(std::ostream&); - static bool has(const std::string& name); - -private: - virtual MultiFieldCreator* make(const eckit::Parametrisation& = util::Config()) = 0; + static MultiFieldCreator* build(const std::string&, const eckit::Parametrisation& = util::NoConfig()); - std::string name_; + using Factory::Factory; -protected: - MultiFieldCreatorFactory(const std::string&); - virtual ~MultiFieldCreatorFactory(); +private: + virtual MultiFieldCreator* make(const eckit::Parametrisation&) = 0; }; template class MultiFieldCreatorBuilder : public MultiFieldCreatorFactory { - virtual MultiFieldCreator* make(const eckit::Parametrisation& param = util::Config()) { return new T(param); } + virtual MultiFieldCreator* make(const eckit::Parametrisation& param) override { return new T(param); } public: - MultiFieldCreatorBuilder(const std::string& name): MultiFieldCreatorFactory(name) {} + using MultiFieldCreatorFactory::MultiFieldCreatorFactory; }; // ------------------------------------------------------------------------------------ -// C wrapper interfaces to C++ routines -extern "C" { -MultiField* atlas__FieldPool__new(); -void atlas__FieldPool__initialize(MultiField* This, const char* generator, const eckit::Parametrisation* params); -void atlas__FieldPool__delete(MultiField* This); -int atlas__FieldPool__has(MultiField* This, const char* name); -FieldImpl* atlas__FieldPool__field_by_name(MultiField* This, const char* name); -FieldImpl* atlas__FieldPool__field_by_index(MultiField* This, idx_t index); -idx_t atlas__FieldPool__size(const MultiField* This); -util::Metadata* atlas__FieldPool__metadata(MultiField* This); -} - } // namespace field } // namespace atlas From 0b1d96726e532512c798cf72cd647e960f15e4e7 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 5 Apr 2022 22:30:50 +0100 Subject: [PATCH 14/40] MultiField with ArrayAllocator --- src/atlas/field/MultiField.cc | 51 ++------ src/atlas/field/MultiField.h | 160 ++++++++++++++++++++----- src/tests/field/test_multifield_ifs.cc | 142 +++++++++------------- 3 files changed, 194 insertions(+), 159 deletions(-) diff --git a/src/atlas/field/MultiField.cc b/src/atlas/field/MultiField.cc index aeeac5fcb..463b3bc65 100644 --- a/src/atlas/field/MultiField.cc +++ b/src/atlas/field/MultiField.cc @@ -39,56 +39,27 @@ void force_link() { } } // namespace -void MultiField::initialize(const std::string& creator, const eckit::Parametrisation& params) { - std::unique_ptr MultiField_creator(MultiFieldCreatorFactory::build(creator, params)); - MultiField_creator->generate(*this, params); -} - -array::Array& MultiField::allocate(array::DataType datatype, array::ArraySpec&& spec) { - array_.reset(array::Array::create(datatype, std::move(spec))); - return *array_; -} - -//------------------------------------------------------------------------------------------------------ - -MultiField::MultiField() = default; - -MultiField::MultiField(const std::string& creator, const eckit::Parametrisation& params) { - initialize(creator, params); -} - -const util::Metadata& MultiField::metadata() const { - return metadata_; -} - -util::Metadata& MultiField::metadata() { - return metadata_; -} - -std::vector MultiField::field_names() const { - std::vector ret; - if (fields_.size()) { - ret.reserve(fields_.size()); - } - - for (auto it = field_index_.begin(); it != field_index_.end(); ++it) { - ret.push_back(it->first); - } - return ret; -} - //----------------------------------------------------------------------------- -MultiFieldCreator::MultiFieldCreator(const eckit::Parametrisation&) {} +MultiFieldCreator::MultiFieldCreator(const eckit::Configuration&) {} MultiFieldCreator::~MultiFieldCreator() = default; -MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& builder, const eckit::Parametrisation& config) { +MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& builder, const eckit::Configuration& config) { force_link(); auto factory = get(builder); return factory->make(config); } +atlas::field::MultiField::MultiField(const eckit::Configuration& config) { + std::string type; + if (!config.get("type", type)) { + ATLAS_THROW_EXCEPTION("Could not find \"type\" in configuration"); + } + std::unique_ptr creator(MultiFieldCreatorFactory::build(type, config)); + reset(creator->create(config)); +} + //----------------------------------------------------------------------------- } // namespace field diff --git a/src/atlas/field/MultiField.h b/src/atlas/field/MultiField.h index 9b03c5b03..773a098e3 100644 --- a/src/atlas/field/MultiField.h +++ b/src/atlas/field/MultiField.h @@ -14,13 +14,16 @@ #pragma once #include +#include #include "atlas/array/Array.h" #include "atlas/field/Field.h" +#include "atlas/field/FieldSet.h" #include "atlas/util/Config.h" #include "atlas/util/Factory.h" #include "atlas/util/Metadata.h" #include "atlas/util/Object.h" +#include "atlas/util/ObjectHandle.h" namespace eckit { class Parametrisation; @@ -37,69 +40,162 @@ namespace field { * * Fields have to all be of same memory layout and data type */ -class MultiField : public util::Object { + +namespace detail { +class ArrayAllocator { +public: + virtual array::Array* allocate(const array::ArraySpec&) = 0; + virtual void deallocate() = 0; + virtual ~ArrayAllocator() = default; +}; + +class NoArrayAllocator : public ArrayAllocator { +public: + virtual array::Array* allocate(const array::ArraySpec& arrayspec) override { return nullptr; } + virtual void deallocate() override {} +}; + +class StandardArrayAllocator : public ArrayAllocator { +public: + virtual array::Array* allocate(const array::ArraySpec& arrayspec) override { + array::ArraySpec spec(arrayspec); + return array::Array::create(std::move(spec)); + } + virtual void deallocate() override { + // Nothing to deallocate. + // The Array owns the allocated data. + } +}; +} // namespace detail + +class MultiFieldImpl : public util::Object { public: // methods - //-- Constructors - MultiField(); + //-- Constructors + + MultiFieldImpl() { allocator_ = std::unique_ptr(new detail::NoArrayAllocator()); } + + MultiFieldImpl(const array::ArraySpec& spec, detail::ArrayAllocator* allocator = nullptr) { + if (allocator) { + allocator_ = std::unique_ptr(allocator); + } + else { + allocator_ = std::unique_ptr(new detail::StandardArrayAllocator()); + } + ATLAS_ASSERT(allocator_); + array_ = std::unique_ptr(allocator_->allocate(spec)); + } + + virtual ~MultiFieldImpl() { allocator_->deallocate(); } - MultiField(const std::string& generator, const eckit::Parametrisation& = util::Config()); //-- Accessors - const Field& field(const std::string& name) const { return fields_[field_index_.at(name)]; } - Field& field(const std::string& name) { return fields_[field_index_.at(name)]; } - bool has(const std::string& name) const { return (field_index_.find(name) != field_index_.end()); } - std::vector field_names() const; + const Field& field(const std::string& name) const { return fieldset_.field(name); } + Field& field(const std::string& name) { return fieldset_.field(name); } + bool has(const std::string& name) const { return fieldset_.has(name); } + std::vector field_names() const { return fieldset_.field_names(); } - const Field& field(const idx_t idx) const { return fields_[idx]; } - Field& field(const idx_t idx) { return fields_[idx]; } - idx_t size() const { return static_cast(fields_.size()); } + const Field& field(const idx_t idx) const { return fieldset_[idx]; } + Field& field(const idx_t idx) { return fieldset_[idx]; } + idx_t size() const { return fieldset_.size(); } - const Field& operator[](const idx_t idx) const { return fields_[idx]; } - Field& operator[](const idx_t idx) { return fields_[idx]; } + const Field& operator[](const idx_t idx) const { return fieldset_[idx]; } + Field& operator[](const idx_t idx) { return fieldset_[idx]; } - const Field& operator[](const std::string& name) const { return field(name); } - Field& operator[](const std::string& name) { return field(name); } + const Field& operator[](const std::string& name) const { return fieldset_.field(name); } + Field& operator[](const std::string& name) { return fieldset_.field(name); } - const util::Metadata& metadata() const; - util::Metadata& metadata(); + const util::Metadata& metadata() const { return metadata_; } + util::Metadata& metadata() { return metadata_; } // -- Modifiers - void initialize(const std::string& generator, const eckit::Parametrisation& = util::Config()); + /// @brief Implicit conversion to Array + operator const array::Array&() const { return array(); } + operator array::Array&() { return array(); } - array::Array& allocate(array::DataType datatype, array::ArraySpec&&); + operator const FieldSet&() const { return fieldset_; } - /// @brief Implicit conversion to Array - operator const array::Array&() const { return *array_; } - operator array::Array&() { return *array_; } + operator FieldSet&() { return fieldset_; } /// @brief Access contained Array - const array::Array& array() const { return *array_; } - array::Array& array() { return *array_; } + const array::Array& array() const { + ATLAS_ASSERT(array_); + return *array_; + } + array::Array& array() { + ATLAS_ASSERT(array_); + return *array_; + } + + /// @brief Access contained FieldSet + const FieldSet& fieldset() const { return fieldset_; } + FieldSet& fieldset() { return fieldset_; } public: // temporary public for prototyping - std::map field_index_; - std::vector fields_; + FieldSet fieldset_; std::unique_ptr array_; + std::unique_ptr allocator_; util::Metadata metadata_; }; + +class MultiField : public util::ObjectHandle { +public: // methods + //-- Constructors + using Handle::Handle; + + MultiField(const eckit::Configuration&); + + //-- Accessors + + const Field& field(const std::string& name) const { return get()->field(name); } + Field& field(const std::string& name) { return get()->field(name); } + bool has(const std::string& name) const { return get()->has(name); } + std::vector field_names() const { return get()->field_names(); } + + const Field& field(const idx_t idx) const { return get()->field(idx); } + Field& field(const idx_t idx) { return get()->field(idx); } + idx_t size() const { return get()->size(); } + + const Field& operator[](const idx_t idx) const { return get()->field(idx); } + Field& operator[](const idx_t idx) { return get()->field(idx); } + + const Field& operator[](const std::string& name) const { return get()->field(name); } + Field& operator[](const std::string& name) { return get()->field(name); } + + const util::Metadata& metadata() const { return get()->metadata(); } + util::Metadata& metadata() { return get()->metadata(); } + + // -- Modifiers + + /// @brief Implicit conversion to Array + operator const array::Array&() const { return get()->array(); } + operator array::Array&() { return get()->array(); } + + operator const FieldSet&() const { return get()->fieldset_; } + operator FieldSet&() { return get()->fieldset_; } + + /// @brief Access contained Array + const array::Array& array() const { return get()->array(); } + array::Array& array() { return get()->array(); } +}; + + //------------------------------------------------------------------------------------------------------ class MultiFieldCreator : public util::Object { public: - MultiFieldCreator(const eckit::Parametrisation& = util::Config()); + MultiFieldCreator(const eckit::Configuration& = util::Config()); virtual ~MultiFieldCreator(); - virtual void generate(MultiField&, const eckit::Parametrisation& = util::Config()) const = 0; + virtual MultiFieldImpl* create(const eckit::Configuration& = util::Config()) const = 0; }; //------------------------------------------------------------------------------------------------------ - class MultiFieldCreatorFactory : public util::Factory { public: static std::string className() { return "MultiFieldCreatorFactory"; } @@ -108,17 +204,17 @@ class MultiFieldCreatorFactory : public util::Factory * \brief build MultiFieldCreator with options specified in parametrisation * \return MutliField creator */ - static MultiFieldCreator* build(const std::string&, const eckit::Parametrisation& = util::NoConfig()); + static MultiFieldCreator* build(const std::string&, const eckit::Configuration& = util::NoConfig()); using Factory::Factory; private: - virtual MultiFieldCreator* make(const eckit::Parametrisation&) = 0; + virtual MultiFieldCreator* make(const eckit::Configuration&) = 0; }; template class MultiFieldCreatorBuilder : public MultiFieldCreatorFactory { - virtual MultiFieldCreator* make(const eckit::Parametrisation& param) override { return new T(param); } + virtual MultiFieldCreator* make(const eckit::Configuration& config) override { return new T(config); } public: using MultiFieldCreatorFactory::MultiFieldCreatorFactory; diff --git a/src/tests/field/test_multifield_ifs.cc b/src/tests/field/test_multifield_ifs.cc index 8ea78cd67..87db13e45 100644 --- a/src/tests/field/test_multifield_ifs.cc +++ b/src/tests/field/test_multifield_ifs.cc @@ -11,8 +11,7 @@ #include #include -#include "eckit/log/JSON.h" -#include "eckit/parser/JSONParser.h" +#include "eckit/config/YAMLConfiguration.h" #include "atlas/array/ArrayView.h" #include "atlas/array/DataType.h" @@ -20,7 +19,6 @@ #include "atlas/field/Field.h" #include "atlas/field/MultiField.h" #include "atlas/grid/Grid.h" -#include "atlas/mesh/Mesh.h" #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" @@ -38,44 +36,27 @@ namespace test { // --- Declaration (in .h file) class MultiFieldCreatorIFS : public MultiFieldCreator { public: - MultiFieldCreatorIFS(const eckit::Parametrisation& p = util::Config()): MultiFieldCreator(p) {} + MultiFieldCreatorIFS(const eckit::Configuration& config = util::Config()): MultiFieldCreator(config) {} ~MultiFieldCreatorIFS() override = default; - void generate(MultiField& fieldpool, const eckit::Parametrisation& p = util::Config()) const override; + MultiFieldImpl* create(const eckit::Configuration& = util::Config()) const override; }; // --- Implementation (in .cc file) -void MultiFieldCreatorIFS::generate(MultiField& multifield, const eckit::Parametrisation& p) const { - const eckit::LocalConfiguration* params = dynamic_cast(&p); - if (!params) { - throw_Exception("Parametrisation has to be of atlas::util::Config type"); - } - - long nproma = 0; - long ngptot = 0; - long nfld = 0; - long nlev = 0; +MultiFieldImpl* MultiFieldCreatorIFS::create(const eckit::Configuration& config) const { + long ngptot = config.getLong("ngptot"); + long nproma = config.getLong("nproma"); + long nlev = config.getLong("nlev"); long nblk = 0; - if (!p.get("ngptot", ngptot)) { - throw_Exception("Could not find 'ngptot' in Parametrisation"); - } - - if (!p.get("nproma", nproma)) { - throw_Exception("Could not find 'nproma' in Parametrisation"); - } - - if (!p.get("nlev", nlev)) { - throw_Exception("Could not find 'nlev' in Parametrisation"); - } array::DataType datatype = array::DataType::create(); std::string datatype_str; - if (p.get("datatype", datatype_str)) { + if (config.get("datatype", datatype_str)) { datatype = array::DataType(datatype_str); } else { array::DataType::kind_t kind(array::DataType::kind()); - p.get("kind", kind); + config.get("kind", kind); if (!array::DataType::kind_valid(kind)) { std::stringstream msg; msg << "Could not create field. kind parameter unrecognized"; @@ -86,18 +67,20 @@ void MultiFieldCreatorIFS::generate(MultiField& multifield, const eckit::Paramet nblk = std::ceil(static_cast(ngptot) / static_cast(nproma)); - std::vector fields; - params->get("fields", fields); - nfld = fields.size(); - size_t nvar_tot = 0; + auto fields = config.getSubConfigurations("fields"); + long nfld = 0; for (const auto& field_config : fields) { - size_t nvar = 1; + long nvar = 1; field_config.get("nvar", nvar); - nvar_tot += nvar; + nfld += nvar; } - auto multiarray_spec = array::make_shape(nblk, nvar_tot, nlev, nproma); - auto& multiarray = multifield.allocate(datatype, multiarray_spec); + auto multiarray_shape = array::make_shape(nblk, nfld, nlev, nproma); + + MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; + + auto& multiarray = multifield->array(); + auto& fieldset = multifield->fieldset(); size_t multiarray_field_idx = 0; for (size_t i = 0; i < fields.size(); ++i) { @@ -146,12 +129,13 @@ void MultiFieldCreatorIFS::generate(MultiField& multifield, const eckit::Paramet } } field.set_levels(nlev); - multifield.fields_.emplace_back(field); - ATLAS_ASSERT(multifield.field_index_.find(field.name()) == multifield.field_index_.end(), - "Field with name already exists!"); - multifield.field_index_[field.name()] = i; + ATLAS_ASSERT(not fieldset.has(field.name()), "Field with name \"" + field.name() + "\" already exists!"); + + fieldset.add(field); + multiarray_field_idx += field_vars; } + return multifield; } // Register in factory @@ -169,50 +153,37 @@ CASE("multifield_generator") { CASE("multifield_create") { - int nproma = 16; - int nlev = 100; - int ngptot = 2000; + using Value = float; + int nproma = 16; + int nlev = 100; + int ngptot = 2000; auto json = [&]() -> std::string { util::Config p; + p.set("type", "MultiFieldCreatorIFS"); p.set("ngptot", ngptot); p.set("nproma", nproma); p.set("nlev", nlev); - p.set("datatype", array::make_datatype().str()); - p.set("fields", [] { - std::vector fields(5); - fields[0].set("name", "temperature"); - - fields[1].set("name", "pressure"); - - fields[2].set("name", "density"); - - fields[3].set("name", "clv"); - fields[3].set("nvar", 5); - - fields[4].set("name", "wind_u"); - return fields; - }()); - - // We can also translate parameters to a json: - std::stringstream json; - eckit::JSON js(json); - js << p; - return json.str(); + p.set("datatype", array::make_datatype().str()); + p.set("fields", { + util::Config("name", "temperature"), // + util::Config("name", "pressure"), // + util::Config("name", "density"), // + util::Config("name", "clv")("nvar", 5), // + util::Config("name", "wind_u") // + }); + return p.json(); }; - - // And we can create back parameters from json: Log::info() << "json = " << json() << std::endl; - std::stringstream json_stream; - json_stream << json(); - util::Config config(json_stream); - MultiField multifield("MultiFieldCreatorIFS", config); + MultiField multifield{eckit::YAMLConfiguration{json()}}; const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); const auto nfld = multifield.size(); EXPECT_EQ(nfld, 5); + EXPECT_EQ(nvar, 9); EXPECT_EQ(multifield.size(), 5); EXPECT(multifield.has("temperature")); @@ -227,20 +198,17 @@ CASE("multifield_create") { Log::info() << multifield.field("clv") << std::endl; Log::info() << multifield.field("wind_u") << std::endl; - auto temp = array::make_view(multifield.field("temperature")); - auto pres = array::make_view(multifield.field("pressure")); - auto dens = array::make_view(multifield.field("density")); - auto clv = array::make_view(multifield.field("clv")); // note rank 4 - auto wind_u = array::make_view(multifield.field("wind_u")); + auto temp = array::make_view(multifield.field("temperature")); + auto pres = array::make_view(multifield.field("pressure")); + auto dens = array::make_view(multifield.field("density")); + auto clv = array::make_view(multifield.field("clv")); // note rank 4 + auto wind_u = array::make_view(multifield.field("wind_u")); - // or - { - auto temp = array::make_view(multifield[0]); - auto pres = array::make_view(multifield[1]); - auto dens = array::make_view(multifield[2]); - auto clv = array::make_view(multifield[3]); // note rank 4 - auto wind_u = array::make_view(multifield[4]); - } + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[1].name(), "pressure"); + EXPECT_EQ(multifield[2].name(), "density"); + EXPECT_EQ(multifield[3].name(), "clv"); + EXPECT_EQ(multifield[4].name(), "wind_u"); auto block_stride = multifield.array().stride(0); auto field_stride = nproma * nlev; @@ -266,9 +234,10 @@ CASE("multifield_create") { EXPECT_EQ(clv.size(), nblk * 5 * nlev * nproma); - // Underlying array of MultiField + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. { - auto multiarray = array::make_view(multifield); + auto multiarray = array::make_view(multifield); EXPECT_EQ(multiarray.stride(0), block_stride); EXPECT_EQ(multiarray.stride(1), field_stride); EXPECT_EQ(multiarray.stride(2), level_stride); @@ -280,8 +249,7 @@ CASE("multifield_create") { EXPECT_EQ(multiarray(13, 5, 14, 15), 16.); EXPECT_EQ(multiarray(17, 8, 18, 3), 19.); - - EXPECT_EQ(multiarray.size(), nblk * (nfld - 1 + 5) * nlev * nproma); + EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); } } From 40e801ec6bbc892aacfb35ca0b3e14153a94af02 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Thu, 31 Aug 2023 17:52:59 +0200 Subject: [PATCH 15/40] add Fortran interface and a unit test for multifield --- src/atlas/CMakeLists.txt | 2 + src/atlas/field/detail/MultiFieldInterface.cc | 133 ++++++++++++++++++ src/atlas/field/detail/MultiFieldInterface.h | 38 +++++ src/atlas_f/CMakeLists.txt | 4 + .../field/atlas_MultiField_module.fypp | 102 ++++++++++++++ src/tests/field/CMakeLists.txt | 18 ++- src/tests/field/fctest_multifield_ifs.F90 | 77 ++++++++++ 7 files changed, 369 insertions(+), 5 deletions(-) create mode 100644 src/atlas/field/detail/MultiFieldInterface.cc create mode 100644 src/atlas/field/detail/MultiFieldInterface.h create mode 100644 src/atlas_f/field/atlas_MultiField_module.fypp create mode 100644 src/tests/field/fctest_multifield_ifs.F90 diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 55aadeb6d..58fed83e8 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -482,6 +482,8 @@ field/detail/FieldImpl.cc field/detail/FieldImpl.h field/detail/FieldInterface.cc field/detail/FieldInterface.h +field/detail/MultiFieldInterface.cc +field/detail/MultiFieldInterface.h field/detail/MissingValue.cc field/detail/MissingValue.h ) diff --git a/src/atlas/field/detail/MultiFieldInterface.cc b/src/atlas/field/detail/MultiFieldInterface.cc new file mode 100644 index 000000000..685867685 --- /dev/null +++ b/src/atlas/field/detail/MultiFieldInterface.cc @@ -0,0 +1,133 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include + +#include "atlas/library/config.h" +#include "atlas/field/detail/MultiFieldInterface.h" +#include "atlas/runtime/Exception.h" + +namespace atlas { +namespace field { + +extern "C" { +MultiFieldImpl* atlas__MultiField__create(eckit::Configuration* config) { + ATLAS_ASSERT(config != nullptr); + long nproma = config->getLong("nproma"); + long nlev = config->getLong("nlev"); + long nblk = 0; + if (config->has("nblk")) { + nblk = config->getLong("nblk"); + } + else if (config->has("ngptot")) { + long ngptot = config->getLong("ngptot"); + nblk = std::ceil(static_cast(ngptot) / static_cast(nproma)); + } + else { + ATLAS_THROW_EXCEPTION("Configuration not found: ngptot or nblk"); + } + array::DataType datatype = array::DataType::create(); + std::string datatype_str; + if (config->get("datatype", datatype_str)) { + datatype = array::DataType(datatype_str); + } + else { + array::DataType::kind_t kind(array::DataType::kind()); + config->get("kind", kind); + if (!array::DataType::kind_valid(kind)) { + std::stringstream msg; + msg << "Could not create field. kind parameter unrecognized"; + throw_Exception(msg.str()); + } + datatype = array::DataType(kind); + } + + auto fields = config->getSubConfigurations("fields"); + long nfld = 0; + for (const auto& field_config : fields) { + long nvar = 1; + field_config.get("nvar", nvar); + nfld += nvar; + } + auto multiarray_shape = array::make_shape(nblk, nfld, nlev, nproma); + + MultiFieldImpl* field = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; + auto& multiarray = field->array(); + auto& fieldset = field->fieldset(); + + size_t multiarray_field_idx = 0; + for (size_t i = 0; i < fields.size(); ++i) { + std::string name; + fields[i].get("name", name); + Field field; + size_t field_vars = 1; + + if (fields[i].get("nvar", field_vars)) { + auto field_shape = + array::make_shape(multiarray.shape(0), field_vars, multiarray.shape(2), multiarray.shape(3)); + auto field_strides = multiarray.strides(); + auto field_array_spec = array::ArraySpec(field_shape, field_strides); + + constexpr auto all = array::Range::all(); + const auto range = array::Range(multiarray_field_idx, multiarray_field_idx + field_vars); + if (datatype.kind() == array::DataType::KIND_REAL64) { + auto slice = array::make_view(multiarray).slice(all, range, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + auto slice = array::make_view(multiarray).slice(all, range, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + ATLAS_NOTIMPLEMENTED; + } + field.set_variables(field_vars); + } + else { + auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2), multiarray.shape(3)); + auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2), multiarray.stride(3)); + auto field_array_spec = array::ArraySpec(field_shape, field_strides); + + constexpr auto all = array::Range::all(); + if (datatype.kind() == array::DataType::KIND_REAL64) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + ATLAS_NOTIMPLEMENTED; + } + } + field.set_levels(nlev); + // field.set_blocks(nblk); + ATLAS_ASSERT(not fieldset.has(field.name()), "Field with name \"" + field.name() + "\" already exists!"); + + fieldset.add(field); + + multiarray_field_idx += field_vars; + } + ATLAS_ASSERT(field); + return field; +} + +void atlas__MultiField__delete(MultiFieldImpl* This) { + delete This; +} + +} + +// ------------------------------------------------------------------ + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/detail/MultiFieldInterface.h b/src/atlas/field/detail/MultiFieldInterface.h new file mode 100644 index 000000000..ac3b1ae22 --- /dev/null +++ b/src/atlas/field/detail/MultiFieldInterface.h @@ -0,0 +1,38 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @file MultiFieldInterface.h +/// @author Willem Deconinck +/// @date Sep 2014 + +#pragma once + +#include "atlas/field/MultiField.h" + +namespace atlas { +namespace functionspace { +} +} // namespace atlas + +namespace atlas { +namespace field { + +//---------------------------------------------------------------------------------------------------------------------- + +// C wrapper interfaces to C++ routines +extern "C" { +MultiFieldImpl* atlas__MultiField__create(eckit::Configuration* config); +void atlas__MultiField__delete(MultiFieldImpl* This); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace field +} // namespace atlas diff --git a/src/atlas_f/CMakeLists.txt b/src/atlas_f/CMakeLists.txt index 60f55318f..eefbbf647 100644 --- a/src/atlas_f/CMakeLists.txt +++ b/src/atlas_f/CMakeLists.txt @@ -109,6 +109,9 @@ generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/field/State.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/field/detail/FieldInterface.h MODULE atlas_field_c_binding OUTPUT field_c_binding.f90 ) +generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/field/detail/MultiFieldInterface.h + MODULE atlas_multifield_c_binding + OUTPUT multifield_c_binding.f90 ) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/field/FieldSet.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/functionspace/detail/FunctionSpaceInterface.h MODULE atlas_functionspace_c_binding @@ -231,6 +234,7 @@ ecbuild_add_library( TARGET atlas_f field/atlas_FieldSet_module.fypp field/atlas_State_module.F90 field/atlas_Field_module.fypp + field/atlas_MultiField_module.fypp grid/atlas_Grid_module.F90 grid/atlas_GridDistribution_module.F90 grid/atlas_Vertical_module.F90 diff --git a/src/atlas_f/field/atlas_MultiField_module.fypp b/src/atlas_f/field/atlas_MultiField_module.fypp new file mode 100644 index 000000000..4d45836a5 --- /dev/null +++ b/src/atlas_f/field/atlas_MultiField_module.fypp @@ -0,0 +1,102 @@ +! (C) Copyright 2013 ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +#include "atlas/atlas_f.h" + +#:include "atlas/atlas_f.fypp" +#:include "internals/atlas_generics.fypp" + +module atlas_multifield_module + +use fckit_owned_object_module, only : fckit_owned_object +use atlas_Config_module, only: atlas_Config + +implicit none + +public :: atlas_MultiField + +private + +!------------------------------------------------------------------------------ +TYPE, extends(fckit_owned_object) :: atlas_MultiField + +! Purpose : +! ------- +! *MultiField* : Object containing field data and Metadata + +! Methods : +! ------- +! name : The name or tag this field was created with +! data : Return the field as a fortran array of specified shape +! Metadata : Return object that can contain a variety of Metadata + +! Author : +! ------ +! 20-Nov-2013 Willem Deconinck *ECMWF* +! 29-Aug-2023 Slavko Brdar *ECMWF* + +!------------------------------------------------------------------------------ +contains + +#if FCKIT_FINAL_NOT_INHERITING + final :: atlas_MultiField__final_auto +#endif + +END TYPE + +interface atlas_MultiField + module procedure atlas_MultiField__cptr + module procedure atlas_MultiField__create +end interface + +private :: fckit_owned_object +private :: atlas_Config + +!======================================================== +contains +!======================================================== + +!------------------------------------------------------------------------------- + +function atlas_MultiField__cptr(cptr) result(field) + use, intrinsic :: iso_c_binding, only : c_ptr + type(atlas_MultiField) :: field + type(c_ptr), intent(in) :: cptr + call field%reset_c_ptr( cptr ) + call field%return() +end function + +!------------------------------------------------------------------------------- + +function atlas_MultiField__create(params) result(field) + use atlas_multifield_c_binding + type(atlas_MultiField) :: field + class(atlas_Config), intent(in) :: params + field = atlas_MultiField__cptr( atlas__MultiField__create(params%CPTR_PGIBUG_B) ) + call field%return() +end function + +!------------------------------------------------------------------------------- + +#if FCKIT_FINAL_NOT_INHERITING +ATLAS_FINAL subroutine atlas_MultiField__final_auto(this) + type(atlas_MultiField), intent(inout) :: this +#if FCKIT_FINAL_DEBUGGING + write(0,*) "atlas_MultiField__final_auto" +#endif +#if FCKIT_FINAL_NOT_PROPAGATING + call this%final() +#endif + FCKIT_SUPPRESS_UNUSED( this ) +end subroutine +#endif + +!------------------------------------------------------------------------------- + +end module atlas_multifield_module + diff --git a/src/tests/field/CMakeLists.txt b/src/tests/field/CMakeLists.txt index e103c2886..dbf51ee37 100644 --- a/src/tests/field/CMakeLists.txt +++ b/src/tests/field/CMakeLists.txt @@ -23,11 +23,6 @@ ecbuild_add_test( TARGET atlas_test_field_foreach LIBS atlas OMP 4 ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} - -ecbuild_add_test( TARGET atlas_test_multifield_ifs - SOURCES test_multifield_ifs.cc - LIBS atlas - ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) ecbuild_add_test( TARGET atlas_test_field_acc @@ -40,8 +35,21 @@ if( TEST atlas_test_field_acc ) set_tests_properties ( atlas_test_field_acc PROPERTIES LABELS "gpu;acc") endif() +ecbuild_add_test( TARGET atlas_test_multifield_ifs + SOURCES test_multifield_ifs.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + if( HAVE_FCTEST ) + add_fctest( TARGET atlas_fctest_multifield_ifs + LINKER_LANGUAGE Fortran + SOURCES fctest_multifield_ifs.F90 + LIBS atlas_f + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} + ) + add_fctest( TARGET atlas_fctest_field LINKER_LANGUAGE Fortran SOURCES fctest_field.F90 diff --git a/src/tests/field/fctest_multifield_ifs.F90 b/src/tests/field/fctest_multifield_ifs.F90 new file mode 100644 index 000000000..c6ca36f47 --- /dev/null +++ b/src/tests/field/fctest_multifield_ifs.F90 @@ -0,0 +1,77 @@ +! (C) Copyright 2013 ECMWF. +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +! This File contains Unit Tests for testing the +! C++ / Fortran Interfaces to the Mesh Datastructure +! @author Willem Deconinck +! @author Slavko Brdar + +#include "fckit/fctest.h" + +! ----------------------------------------------------------------------------- + +module fcta_MultiField_fixture +use atlas_module +use atlas_multifield_module +use, intrinsic :: iso_c_binding +implicit none +end module + +! ----------------------------------------------------------------------------- + +TESTSUITE_WITH_FIXTURE(fctest_atlas_MultiField,fcta_MultiField_fixture) + +! ----------------------------------------------------------------------------- + +TESTSUITE_INIT + call atlas_library%initialise() +END_TESTSUITE_INIT + +! ----------------------------------------------------------------------------- + +TESTSUITE_FINALIZE + call atlas_library%finalise() +END_TESTSUITE_FINALIZE + +! ----------------------------------------------------------------------------- + +TEST( test_multifield ) + implicit none + + type(atlas_MultiField) :: mfield + type(atlas_config) :: config + + integer, parameter :: nproma = 16; + integer, parameter :: nlev = 100; + integer, parameter :: ngptot = 2000; + type(atlas_Config), dimension(5) :: field_configs + + config = atlas_Config() + call config%set("type", "MultiFieldCreatorIFS"); + call config%set("ngptot", ngptot); + call config%set("nproma", nproma); + call config%set("nlev", nlev); + call config%set("datatype", "real64"); + field_configs(1) = atlas_Config() + field_configs(2) = atlas_Config() + field_configs(3) = atlas_Config() + field_configs(4) = atlas_Config() + field_configs(5) = atlas_Config() + call field_configs(1)%set("name", "temperature") + call field_configs(2)%set("name", "pressure") + call field_configs(3)%set("name", "density") + call field_configs(4)%set("name", "clv") + call field_configs(4)%set("nvar", 5) + call field_configs(5)%set("name", "wind_u") + call config%set("fields", field_configs) + + mfield = atlas_MultiField(config) +END_TEST + +! ----------------------------------------------------------------------------- + +END_TESTSUITE From 39ee651c7770e87fcd509925a35990d9d071105a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 5 Sep 2023 18:43:10 +0200 Subject: [PATCH 16/40] Use MultiFieldArrayRegistry to avoid accidental deletion of multifield arrays --- src/atlas/field/MultiField.cc | 41 ++++- src/atlas/field/MultiField.h | 49 +----- src/atlas/field/detail/MultiFieldInterface.cc | 12 +- src/tests/field/test_multifield_ifs.cc | 165 +++++++++--------- 4 files changed, 137 insertions(+), 130 deletions(-) diff --git a/src/atlas/field/MultiField.cc b/src/atlas/field/MultiField.cc index 463b3bc65..f7a3ba88d 100644 --- a/src/atlas/field/MultiField.cc +++ b/src/atlas/field/MultiField.cc @@ -14,9 +14,7 @@ #include #include #include - -#include "eckit/thread/AutoLock.h" -#include "eckit/thread/Mutex.h" +#include #include "atlas/field/Field.h" #include "atlas/grid/Grid.h" @@ -41,6 +39,34 @@ void force_link() { //----------------------------------------------------------------------------- +class MultiFieldArrayRegistry : public field::FieldObserver { +private: + MultiFieldArrayRegistry() {} + +public: + static MultiFieldArrayRegistry& instance() { + static MultiFieldArrayRegistry inst; + return inst; + } + void onFieldDestruction(FieldImpl& field) override { + std::lock_guard guard(lock_); + map_.erase(&field); + } + + ~MultiFieldArrayRegistry() override = default; + + void add(Field& field, std::shared_ptr array) { + std::lock_guard guard(lock_); + map_.emplace(field.get(),array); + field->attachObserver(*this); + } + +public: + std::mutex lock_; + std::map> map_; + +}; + MultiFieldCreator::MultiFieldCreator(const eckit::Configuration&) {} MultiFieldCreator::~MultiFieldCreator() = default; @@ -51,7 +77,7 @@ MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& builder, c return factory->make(config); } -atlas::field::MultiField::MultiField(const eckit::Configuration& config) { +MultiField::MultiField(const eckit::Configuration& config) { std::string type; if (!config.get("type", type)) { ATLAS_THROW_EXCEPTION("Could not find \"type\" in configuration"); @@ -60,6 +86,13 @@ atlas::field::MultiField::MultiField(const eckit::Configuration& config) { reset(creator->create(config)); } +void MultiFieldImpl::add(Field& field) { + ATLAS_ASSERT(not fieldset_.has(field.name()), "Field with name \"" + field.name() + "\" already exists!"); + fieldset_.add(field); + MultiFieldArrayRegistry::instance().add(field,array_); + +} + //----------------------------------------------------------------------------- } // namespace field diff --git a/src/atlas/field/MultiField.h b/src/atlas/field/MultiField.h index 773a098e3..46d0adea0 100644 --- a/src/atlas/field/MultiField.h +++ b/src/atlas/field/MultiField.h @@ -41,51 +41,18 @@ namespace field { * Fields have to all be of same memory layout and data type */ -namespace detail { -class ArrayAllocator { -public: - virtual array::Array* allocate(const array::ArraySpec&) = 0; - virtual void deallocate() = 0; - virtual ~ArrayAllocator() = default; -}; - -class NoArrayAllocator : public ArrayAllocator { -public: - virtual array::Array* allocate(const array::ArraySpec& arrayspec) override { return nullptr; } - virtual void deallocate() override {} -}; - -class StandardArrayAllocator : public ArrayAllocator { -public: - virtual array::Array* allocate(const array::ArraySpec& arrayspec) override { - array::ArraySpec spec(arrayspec); - return array::Array::create(std::move(spec)); - } - virtual void deallocate() override { - // Nothing to deallocate. - // The Array owns the allocated data. - } -}; -} // namespace detail - class MultiFieldImpl : public util::Object { public: // methods //-- Constructors - MultiFieldImpl() { allocator_ = std::unique_ptr(new detail::NoArrayAllocator()); } - - MultiFieldImpl(const array::ArraySpec& spec, detail::ArrayAllocator* allocator = nullptr) { - if (allocator) { - allocator_ = std::unique_ptr(allocator); - } - else { - allocator_ = std::unique_ptr(new detail::StandardArrayAllocator()); - } - ATLAS_ASSERT(allocator_); - array_ = std::unique_ptr(allocator_->allocate(spec)); + MultiFieldImpl() { } + + MultiFieldImpl(const array::ArraySpec& spec) { + array::ArraySpec s(spec); + array_.reset(array::Array::create(std::move(s))); } - virtual ~MultiFieldImpl() { allocator_->deallocate(); } + virtual ~MultiFieldImpl() {} //-- Accessors @@ -132,11 +99,11 @@ class MultiFieldImpl : public util::Object { const FieldSet& fieldset() const { return fieldset_; } FieldSet& fieldset() { return fieldset_; } + void add(Field& field); public: // temporary public for prototyping FieldSet fieldset_; - std::unique_ptr array_; - std::unique_ptr allocator_; + std::shared_ptr array_; util::Metadata metadata_; }; diff --git a/src/atlas/field/detail/MultiFieldInterface.cc b/src/atlas/field/detail/MultiFieldInterface.cc index 685867685..a5e729b2a 100644 --- a/src/atlas/field/detail/MultiFieldInterface.cc +++ b/src/atlas/field/detail/MultiFieldInterface.cc @@ -59,9 +59,8 @@ MultiFieldImpl* atlas__MultiField__create(eckit::Configuration* config) { } auto multiarray_shape = array::make_shape(nblk, nfld, nlev, nproma); - MultiFieldImpl* field = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; - auto& multiarray = field->array(); - auto& fieldset = field->fieldset(); + MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; + auto& multiarray = multifield->array(); size_t multiarray_field_idx = 0; for (size_t i = 0; i < fields.size(); ++i) { @@ -111,14 +110,13 @@ MultiFieldImpl* atlas__MultiField__create(eckit::Configuration* config) { } field.set_levels(nlev); // field.set_blocks(nblk); - ATLAS_ASSERT(not fieldset.has(field.name()), "Field with name \"" + field.name() + "\" already exists!"); - fieldset.add(field); + multifield->add(field); multiarray_field_idx += field_vars; } - ATLAS_ASSERT(field); - return field; + ATLAS_ASSERT(multifield); + return multifield; } void atlas__MultiField__delete(MultiFieldImpl* This) { diff --git a/src/tests/field/test_multifield_ifs.cc b/src/tests/field/test_multifield_ifs.cc index 87db13e45..ce11afa35 100644 --- a/src/tests/field/test_multifield_ifs.cc +++ b/src/tests/field/test_multifield_ifs.cc @@ -80,7 +80,6 @@ MultiFieldImpl* MultiFieldCreatorIFS::create(const eckit::Configuration& config) MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; auto& multiarray = multifield->array(); - auto& fieldset = multifield->fieldset(); size_t multiarray_field_idx = 0; for (size_t i = 0; i < fields.size(); ++i) { @@ -129,9 +128,8 @@ MultiFieldImpl* MultiFieldCreatorIFS::create(const eckit::Configuration& config) } } field.set_levels(nlev); - ATLAS_ASSERT(not fieldset.has(field.name()), "Field with name \"" + field.name() + "\" already exists!"); - fieldset.add(field); + multifield->add(field); multiarray_field_idx += field_vars; } @@ -175,81 +173,92 @@ CASE("multifield_create") { return p.json(); }; - Log::info() << "json = " << json() << std::endl; - - MultiField multifield{eckit::YAMLConfiguration{json()}}; - - const auto nblk = multifield.array().shape(0); - const auto nvar = multifield.array().shape(1); - const auto nfld = multifield.size(); - EXPECT_EQ(nfld, 5); - EXPECT_EQ(nvar, 9); - - EXPECT_EQ(multifield.size(), 5); - EXPECT(multifield.has("temperature")); - EXPECT(multifield.has("pressure")); - EXPECT(multifield.has("density")); - EXPECT(multifield.has("clv")); - EXPECT(multifield.has("wind_u")); - - Log::info() << multifield.field("temperature") << std::endl; - Log::info() << multifield.field("pressure") << std::endl; - Log::info() << multifield.field("density") << std::endl; - Log::info() << multifield.field("clv") << std::endl; - Log::info() << multifield.field("wind_u") << std::endl; - - auto temp = array::make_view(multifield.field("temperature")); - auto pres = array::make_view(multifield.field("pressure")); - auto dens = array::make_view(multifield.field("density")); - auto clv = array::make_view(multifield.field("clv")); // note rank 4 - auto wind_u = array::make_view(multifield.field("wind_u")); - - EXPECT_EQ(multifield[0].name(), "temperature"); - EXPECT_EQ(multifield[1].name(), "pressure"); - EXPECT_EQ(multifield[2].name(), "density"); - EXPECT_EQ(multifield[3].name(), "clv"); - EXPECT_EQ(multifield[4].name(), "wind_u"); - - auto block_stride = multifield.array().stride(0); - auto field_stride = nproma * nlev; - auto level_stride = nproma; - auto nproma_stride = 1; - - temp(1, 2, 3) = 4; - pres(5, 6, 7) = 8; - dens(9, 10, 11) = 12; - clv(13, 2, 14, 15) = 16; - wind_u(17, 18, 3) = 19; - - EXPECT_EQ(temp.stride(0), block_stride); - EXPECT_EQ(temp.stride(1), level_stride); - EXPECT_EQ(temp.stride(2), nproma_stride); - EXPECT_EQ(temp.size(), nblk * nlev * nproma); - - EXPECT_EQ(clv.stride(0), block_stride); - EXPECT_EQ(clv.stride(1), field_stride); - EXPECT_EQ(clv.stride(2), level_stride); - EXPECT_EQ(clv.stride(3), nproma_stride); - - EXPECT_EQ(clv.size(), nblk * 5 * nlev * nproma); - - - // Advanced usage, to access underlying array. This should only be used - // in a driver and not be exposed to algorithms. - { - auto multiarray = array::make_view(multifield); - EXPECT_EQ(multiarray.stride(0), block_stride); - EXPECT_EQ(multiarray.stride(1), field_stride); - EXPECT_EQ(multiarray.stride(2), level_stride); - EXPECT_EQ(multiarray.stride(3), nproma_stride); - - EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); - EXPECT_EQ(multiarray(5, 1, 6, 7), 8.); - EXPECT_EQ(multiarray(9, 2, 10, 11), 12.); - EXPECT_EQ(multiarray(13, 5, 14, 15), 16.); - EXPECT_EQ(multiarray(17, 8, 18, 3), 19.); - - EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); + SECTION("Print configuration") { + Log::info() << "json = " << json() << std::endl; + } + + SECTION("test") { + MultiField multifield{eckit::YAMLConfiguration{json()}}; + + const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 5); + EXPECT_EQ(nvar, 9); + + EXPECT_EQ(multifield.size(), 5); + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("pressure")); + EXPECT(multifield.has("density")); + EXPECT(multifield.has("clv")); + EXPECT(multifield.has("wind_u")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("pressure") << std::endl; + Log::info() << multifield.field("density") << std::endl; + Log::info() << multifield.field("clv") << std::endl; + Log::info() << multifield.field("wind_u") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto pres = array::make_view(multifield.field("pressure")); + auto dens = array::make_view(multifield.field("density")); + auto clv = array::make_view(multifield.field("clv")); // note rank 4 + auto wind_u = array::make_view(multifield.field("wind_u")); + + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[1].name(), "pressure"); + EXPECT_EQ(multifield[2].name(), "density"); + EXPECT_EQ(multifield[3].name(), "clv"); + EXPECT_EQ(multifield[4].name(), "wind_u"); + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma * nlev; + auto level_stride = nproma; + auto nproma_stride = 1; + + temp(1, 2, 3) = 4; + pres(5, 6, 7) = 8; + dens(9, 10, 11) = 12; + clv(13, 2, 14, 15) = 16; + wind_u(17, 18, 3) = 19; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), level_stride); + EXPECT_EQ(temp.stride(2), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nlev * nproma); + + EXPECT_EQ(clv.stride(0), block_stride); + EXPECT_EQ(clv.stride(1), field_stride); + EXPECT_EQ(clv.stride(2), level_stride); + EXPECT_EQ(clv.stride(3), nproma_stride); + + EXPECT_EQ(clv.size(), nblk * 5 * nlev * nproma); + + + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), level_stride); + EXPECT_EQ(multiarray.stride(3), nproma_stride); + + EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); + EXPECT_EQ(multiarray(5, 1, 6, 7), 8.); + EXPECT_EQ(multiarray(9, 2, 10, 11), 12.); + EXPECT_EQ(multiarray(13, 5, 14, 15), 16.); + EXPECT_EQ(multiarray(17, 8, 18, 3), 19.); + + EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); + } + } + + SECTION("test registry") { + { + Field field = MultiField {eckit::YAMLConfiguration{json()}}.field("temperature"); + auto temp = array::make_view(field); + } } } From 83cd8cdd15fe54e19dae5a83b381b61d3fe6a526 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Wed, 6 Sep 2023 19:47:58 +0200 Subject: [PATCH 17/40] Improve and clean up MultiField --- src/atlas/CMakeLists.txt | 8 + src/atlas/field/MultiField.cc | 95 ++--- src/atlas/field/MultiField.h | 167 +++----- src/atlas/field/MultiFieldCreator.cc | 58 +++ src/atlas/field/MultiFieldCreator.h | 89 ++++ src/atlas/field/MultiFieldCreatorArray.cc | 150 +++++++ src/atlas/field/MultiFieldCreatorArray.h | 58 +++ src/atlas/field/MultiFieldCreatorIFS.cc | 230 +++++++++++ src/atlas/field/MultiFieldCreatorIFS.h | 63 +++ src/atlas/field/detail/MultiFieldImpl.cc | 32 ++ src/atlas/field/detail/MultiFieldImpl.h | 96 +++++ src/atlas/field/detail/MultiFieldInterface.cc | 125 ++---- src/atlas/field/detail/MultiFieldInterface.h | 5 + src/atlas_f/CMakeLists.txt | 2 +- ...odule.fypp => atlas_MultiField_module.F90} | 70 +++- src/tests/field/fctest_multifield_ifs.F90 | 187 ++++++++- src/tests/field/test_multifield_ifs.cc | 387 ++++++++++++------ 17 files changed, 1412 insertions(+), 410 deletions(-) create mode 100644 src/atlas/field/MultiFieldCreator.cc create mode 100644 src/atlas/field/MultiFieldCreator.h create mode 100644 src/atlas/field/MultiFieldCreatorArray.cc create mode 100644 src/atlas/field/MultiFieldCreatorArray.h create mode 100644 src/atlas/field/MultiFieldCreatorIFS.cc create mode 100644 src/atlas/field/MultiFieldCreatorIFS.h create mode 100644 src/atlas/field/detail/MultiFieldImpl.cc create mode 100644 src/atlas/field/detail/MultiFieldImpl.h rename src/atlas_f/field/{atlas_MultiField_module.fypp => atlas_MultiField_module.F90} (52%) diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 58fed83e8..ebecad3bb 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -476,12 +476,20 @@ field/MissingValue.cc field/MissingValue.h field/MultiField.cc field/MultiField.h +field/MultiFieldCreator.cc +field/MultiFieldCreator.h +field/MultiFieldCreatorIFS.cc +field/MultiFieldCreatorIFS.h +field/MultiFieldCreatorArray.cc +field/MultiFieldCreatorArray.h field/State.cc field/State.h field/detail/FieldImpl.cc field/detail/FieldImpl.h field/detail/FieldInterface.cc field/detail/FieldInterface.h +field/detail/MultiFieldImpl.cc +field/detail/MultiFieldImpl.h field/detail/MultiFieldInterface.cc field/detail/MultiFieldInterface.h field/detail/MissingValue.cc diff --git a/src/atlas/field/MultiField.cc b/src/atlas/field/MultiField.cc index f7a3ba88d..e646f17af 100644 --- a/src/atlas/field/MultiField.cc +++ b/src/atlas/field/MultiField.cc @@ -8,75 +8,24 @@ * nor does it submit to any jurisdiction. */ -#include "MultiField.h" +#include "atlas/field/MultiField.h" #include #include #include +#include #include #include -#include "atlas/field/Field.h" -#include "atlas/grid/Grid.h" -#include "atlas/mesh/Mesh.h" +#include "atlas/field/MultiFieldCreator.h" +#include "atlas/field/detail/MultiFieldImpl.h" #include "atlas/runtime/Exception.h" -#include "atlas/runtime/Log.h" namespace atlas { namespace field { -namespace { -void force_link() { - static struct Link { - Link() { - ; - // For static linking add here something like - // MultiFieldCreatorBuilder(); - } - } link; -} -} // namespace - //----------------------------------------------------------------------------- -class MultiFieldArrayRegistry : public field::FieldObserver { -private: - MultiFieldArrayRegistry() {} - -public: - static MultiFieldArrayRegistry& instance() { - static MultiFieldArrayRegistry inst; - return inst; - } - void onFieldDestruction(FieldImpl& field) override { - std::lock_guard guard(lock_); - map_.erase(&field); - } - - ~MultiFieldArrayRegistry() override = default; - - void add(Field& field, std::shared_ptr array) { - std::lock_guard guard(lock_); - map_.emplace(field.get(),array); - field->attachObserver(*this); - } - -public: - std::mutex lock_; - std::map> map_; - -}; - -MultiFieldCreator::MultiFieldCreator(const eckit::Configuration&) {} - -MultiFieldCreator::~MultiFieldCreator() = default; - -MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& builder, const eckit::Configuration& config) { - force_link(); - auto factory = get(builder); - return factory->make(config); -} - MultiField::MultiField(const eckit::Configuration& config) { std::string type; if (!config.get("type", type)) { @@ -86,13 +35,39 @@ MultiField::MultiField(const eckit::Configuration& config) { reset(creator->create(config)); } -void MultiFieldImpl::add(Field& field) { - ATLAS_ASSERT(not fieldset_.has(field.name()), "Field with name \"" + field.name() + "\" already exists!"); - fieldset_.add(field); - MultiFieldArrayRegistry::instance().add(field,array_); - +MultiField::MultiField(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) { + std::unique_ptr creator(MultiFieldCreatorFactory::build("MultiFieldCreatorArray")); + reset(creator->create(datatype, shape, var_names)); } +const Field& MultiField::field(const std::string& name) const { return get()->field(name); } +Field& MultiField::field(const std::string& name) { return get()->field(name); } +bool MultiField::has(const std::string& name) const { return get()->has(name); } +std::vector MultiField::field_names() const { return get()->field_names(); } + +const Field& MultiField::field(const idx_t idx) const { return get()->field(idx); } +Field& MultiField::field(const idx_t idx) { return get()->field(idx); } +idx_t MultiField::size() const { return get()->size(); } + +const Field& MultiField::operator[](const idx_t idx) const { return get()->field(idx); } +Field& MultiField::operator[](const idx_t idx) { return get()->field(idx); } + +const Field& MultiField::operator[](const std::string& name) const { return get()->field(name); } +Field& MultiField::operator[](const std::string& name) { return get()->field(name); } + +const util::Metadata& MultiField::metadata() const { return get()->metadata(); } +util::Metadata& MultiField::metadata() { return get()->metadata(); } + +MultiField::operator const array::Array&() const { return get()->array(); } +MultiField::operator array::Array&() { return get()->array(); } + +MultiField::operator const FieldSet&() const { return get()->fieldset_; } +MultiField::operator FieldSet&() { return get()->fieldset_; } + +const array::Array& MultiField::array() const { return get()->array(); } +array::Array& MultiField::array() { return get()->array(); } + //----------------------------------------------------------------------------- } // namespace field diff --git a/src/atlas/field/MultiField.h b/src/atlas/field/MultiField.h index 46d0adea0..2cc3c6903 100644 --- a/src/atlas/field/MultiField.h +++ b/src/atlas/field/MultiField.h @@ -29,6 +29,12 @@ namespace eckit { class Parametrisation; } +namespace atlas { +namespace field { +class MultiFieldImpl; +} +} + namespace atlas { namespace field { @@ -41,150 +47,83 @@ namespace field { * Fields have to all be of same memory layout and data type */ -class MultiFieldImpl : public util::Object { -public: // methods - //-- Constructors - - MultiFieldImpl() { } - - MultiFieldImpl(const array::ArraySpec& spec) { - array::ArraySpec s(spec); - array_.reset(array::Array::create(std::move(s))); - } - - virtual ~MultiFieldImpl() {} - - - //-- Accessors - - const Field& field(const std::string& name) const { return fieldset_.field(name); } - Field& field(const std::string& name) { return fieldset_.field(name); } - bool has(const std::string& name) const { return fieldset_.has(name); } - std::vector field_names() const { return fieldset_.field_names(); } - - const Field& field(const idx_t idx) const { return fieldset_[idx]; } - Field& field(const idx_t idx) { return fieldset_[idx]; } - idx_t size() const { return fieldset_.size(); } - - const Field& operator[](const idx_t idx) const { return fieldset_[idx]; } - Field& operator[](const idx_t idx) { return fieldset_[idx]; } - - const Field& operator[](const std::string& name) const { return fieldset_.field(name); } - Field& operator[](const std::string& name) { return fieldset_.field(name); } - - const util::Metadata& metadata() const { return metadata_; } - util::Metadata& metadata() { return metadata_; } - - // -- Modifiers - - /// @brief Implicit conversion to Array - operator const array::Array&() const { return array(); } - operator array::Array&() { return array(); } - - operator const FieldSet&() const { return fieldset_; } - - operator FieldSet&() { return fieldset_; } - - /// @brief Access contained Array - const array::Array& array() const { - ATLAS_ASSERT(array_); - return *array_; - } - array::Array& array() { - ATLAS_ASSERT(array_); - return *array_; - } - - /// @brief Access contained FieldSet - const FieldSet& fieldset() const { return fieldset_; } - FieldSet& fieldset() { return fieldset_; } - - void add(Field& field); - -public: // temporary public for prototyping - FieldSet fieldset_; - std::shared_ptr array_; - util::Metadata metadata_; -}; - - class MultiField : public util::ObjectHandle { public: // methods //-- Constructors using Handle::Handle; MultiField(const eckit::Configuration&); + MultiField(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names); //-- Accessors - const Field& field(const std::string& name) const { return get()->field(name); } - Field& field(const std::string& name) { return get()->field(name); } - bool has(const std::string& name) const { return get()->has(name); } - std::vector field_names() const { return get()->field_names(); } + const Field& field(const std::string& name) const; + Field& field(const std::string& name); + bool has(const std::string& name) const; + std::vector field_names() const; - const Field& field(const idx_t idx) const { return get()->field(idx); } - Field& field(const idx_t idx) { return get()->field(idx); } - idx_t size() const { return get()->size(); } + const Field& field(const idx_t idx) const; + Field& field(const idx_t idx); + idx_t size() const; - const Field& operator[](const idx_t idx) const { return get()->field(idx); } - Field& operator[](const idx_t idx) { return get()->field(idx); } + const Field& operator[](const idx_t idx) const; + Field& operator[](const idx_t idx); - const Field& operator[](const std::string& name) const { return get()->field(name); } - Field& operator[](const std::string& name) { return get()->field(name); } + const Field& operator[](const std::string& name) const; + Field& operator[](const std::string& name); - const util::Metadata& metadata() const { return get()->metadata(); } - util::Metadata& metadata() { return get()->metadata(); } + const util::Metadata& metadata() const; + util::Metadata& metadata(); // -- Modifiers /// @brief Implicit conversion to Array - operator const array::Array&() const { return get()->array(); } - operator array::Array&() { return get()->array(); } + operator const array::Array&() const; + operator array::Array&(); - operator const FieldSet&() const { return get()->fieldset_; } - operator FieldSet&() { return get()->fieldset_; } + operator const FieldSet&() const; + operator FieldSet&(); /// @brief Access contained Array - const array::Array& array() const { return get()->array(); } - array::Array& array() { return get()->array(); } + const array::Array& array() const; + array::Array& array(); + +private: + template + void create(const std::vector shape, const std::vector var_names); }; +/** + * \brief MultiFieldArrayRegistry + */ -//------------------------------------------------------------------------------------------------------ +class MultiFieldArrayRegistry : public field::FieldObserver { +private: + MultiFieldArrayRegistry() {} -class MultiFieldCreator : public util::Object { public: - MultiFieldCreator(const eckit::Configuration& = util::Config()); - - virtual ~MultiFieldCreator(); + static MultiFieldArrayRegistry& instance() { + static MultiFieldArrayRegistry inst; + return inst; + } + void onFieldDestruction(FieldImpl& field) override { + std::lock_guard guard(lock_); + map_.erase(&field); + } - virtual MultiFieldImpl* create(const eckit::Configuration& = util::Config()) const = 0; -}; + ~MultiFieldArrayRegistry() override = default; -//------------------------------------------------------------------------------------------------------ + void add(Field& field, std::shared_ptr array) { + std::lock_guard guard(lock_); + map_.emplace(field.get(), array); + field->attachObserver(*this); + } -class MultiFieldCreatorFactory : public util::Factory { public: - static std::string className() { return "MultiFieldCreatorFactory"; } - - /*! - * \brief build MultiFieldCreator with options specified in parametrisation - * \return MutliField creator - */ - static MultiFieldCreator* build(const std::string&, const eckit::Configuration& = util::NoConfig()); + std::mutex lock_; + std::map> map_; - using Factory::Factory; - -private: - virtual MultiFieldCreator* make(const eckit::Configuration&) = 0; -}; - -template -class MultiFieldCreatorBuilder : public MultiFieldCreatorFactory { - virtual MultiFieldCreator* make(const eckit::Configuration& config) override { return new T(config); } - -public: - using MultiFieldCreatorFactory::MultiFieldCreatorFactory; }; // ------------------------------------------------------------------------------------ diff --git a/src/atlas/field/MultiFieldCreator.cc b/src/atlas/field/MultiFieldCreator.cc new file mode 100644 index 000000000..72908f997 --- /dev/null +++ b/src/atlas/field/MultiFieldCreator.cc @@ -0,0 +1,58 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +// file deepcode ignore CppMemoryLeak: static pointers for global registry are OK and will be cleaned up at end + +#include "atlas/field/MultiFieldCreator.h" + +#include +#include + +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/Mutex.h" + +#include "atlas/field/MultiFieldCreatorIFS.h" +#include "atlas/field/MultiFieldCreatorArray.h" +#include "atlas/grid/Grid.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + + +namespace atlas { +namespace field { + + +namespace { + +void force_link() { + static struct Link { + Link() { + MultiFieldCreatorBuilder(); + MultiFieldCreatorBuilder(); + } + } link; +} + +} // namespace + +// ------------------------------------------------------------------ + +MultiFieldCreator::MultiFieldCreator() = default; + +MultiFieldCreator::~MultiFieldCreator() = default; + +MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& builder, const eckit::Configuration& config) { + force_link(); + auto factory = get(builder); + return factory->make(config); +} + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreator.h b/src/atlas/field/MultiFieldCreator.h new file mode 100644 index 000000000..84a0cf48e --- /dev/null +++ b/src/atlas/field/MultiFieldCreator.h @@ -0,0 +1,89 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#ifndef atlas_field_MultiFieldCreator_h +#define atlas_field_MultiFieldCreator_h + +#include + +#include "atlas/util/Object.h" + +#include "atlas/field/MultiField.h" + +namespace eckit { +class Configuration; +} + +//------------------------------------------------------------------------------------------------------ + +namespace atlas { +namespace field { + +//------------------------------------------------------------------------------------------------------ + +/*! + * \brief Base class for creating new multifields based on Configuration + * + * \details + * Example to create field[100][3] of default type double: + * \code{.cpp} + * FieldImpl* field = Field::create( + * Config + * ("creator","ArraySpec") // ArraySpec MultiFieldCreator + * ("shape",array::make_shape(100,3)) // Rank 2 field with indexing [100][3] + * ); + * \endcode + */ +class MultiFieldCreator : public util::Object { +public: + MultiFieldCreator(); + MultiFieldCreator(const eckit::Configuration& config); + + virtual ~MultiFieldCreator(); + + virtual MultiFieldImpl* create(const eckit::Configuration& config = util::Config()) const = 0; + virtual MultiFieldImpl* create(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) const = 0; +}; + +//------------------------------------------------------------------------------------------------------ + +class MultiFieldCreatorFactory : public util::Factory { +public: + static std::string className() { return "MultiFieldCreatorFactory"; } + + /*! + * \brief build MultiFieldCreator with options specified in parametrisation + * \return mesh generator + */ + static MultiFieldCreator* build(const std::string&, const eckit::Configuration& = util::Config()); + + using Factory::Factory; + +private: + virtual MultiFieldCreator* make() = 0; + virtual MultiFieldCreator* make(const eckit::Configuration&) = 0; +}; + +template +class MultiFieldCreatorBuilder : public MultiFieldCreatorFactory { + virtual MultiFieldCreator* make() { return new T(); } + virtual MultiFieldCreator* make(const eckit::Configuration& config) { return new T(config); } + +public: + using MultiFieldCreatorFactory::MultiFieldCreatorFactory; +}; + +//------------------------------------------------------------------------------------------------------ + +} // namespace field +} // namespace atlas + +#endif diff --git a/src/atlas/field/MultiFieldCreatorArray.cc b/src/atlas/field/MultiFieldCreatorArray.cc new file mode 100644 index 000000000..5ecc5c2aa --- /dev/null +++ b/src/atlas/field/MultiFieldCreatorArray.cc @@ -0,0 +1,150 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Slavko Brdar +/// @author Willem Deconinck +/// @date September 2024 + +#include +#include + +#include "atlas/array/ArrayDataStore.h" +#include "atlas/array/DataType.h" +#include "atlas/array/Range.h" +#include "atlas/field/MultiFieldCreatorArray.h" +#include "atlas/field/detail/MultiFieldImpl.h" +#include "atlas/grid/Grid.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + +namespace atlas { +namespace field { + +MultiFieldCreatorArray::MultiFieldCreatorArray() {} + +MultiFieldCreatorArray::MultiFieldCreatorArray(const eckit::Configuration&) {} + +MultiFieldCreatorArray::~MultiFieldCreatorArray() = default; + +MultiFieldImpl* MultiFieldCreatorArray::create(const eckit::Configuration& config) const { + array::DataType datatype = array::DataType::create(); + std::string datatype_str; + if (config.get("datatype", datatype_str)) { + datatype = array::DataType(datatype_str); + } + else { + array::DataType::kind_t kind(array::DataType::kind()); + config.get("kind", kind); + if (!array::DataType::kind_valid(kind)) { + std::stringstream msg; + msg << "Could not create field. kind parameter unrecognized"; + throw_Exception(msg.str()); + } + datatype = array::DataType(kind); + } + std::vector shape; + config.get("shape", shape); + const auto fields = config.getSubConfigurations("fields"); + int nflds = 0; + for (size_t i = 0; i < fields.size(); ++i) { + long nvar = 1; + fields[i].get("nvar", nvar); + nflds += nvar; + } + std::vector var_names; + var_names.resize(nflds); + for (size_t i = 0, cnt = 0; i < fields.size(); ++i) { + std::string name; + fields[i].get("name", name); + long nvar = 1; + fields[i].get("nvar", nvar); + if (nvar > 1) { + for (int ivar = 0; ivar < nvar; ivar++) { + std::stringstream ss; + ss << name << "_" << ivar; + var_names[cnt++] = ss.str(); + } + + } + else { + var_names[cnt++] = name; + } + } + return create(datatype, shape, var_names); +} + +MultiFieldImpl* MultiFieldCreatorArray::create(const array::DataType datatype, const std::vector& shape, const std::vector& var_names) const { + const int dim = shape.size(); + const int nvar = var_names.size(); + ATLAS_ASSERT(nvar > 0 && dim > 2, "MultiField must have at least one field name."); + + int varidx = -1; + for (int i = 0; i < dim; i++) { + if (shape[i] == -1) { + varidx = i; + break; + } + } + + array::ArrayShape multiarray_shape = shape; + multiarray_shape[varidx] = nvar; + + MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; + auto& multiarray = multifield->array(); + + array::ArrayShape field_shape; + field_shape.resize(dim - 1); + array::ArrayStrides field_strides; + field_strides.resize(dim - 1); + for (int i = 0, j = 0; i < dim; i++) { + if (i != varidx) { + field_shape[j] = multiarray.shape()[i]; + field_strides[j] = multiarray.strides()[i]; + ++j; + } + } + array::ArraySpec field_array_spec(field_shape, field_strides); + + for (size_t ifield = 0; ifield < nvar; ++ifield) { + idx_t start_index = multiarray.strides()[varidx] * ifield; + Field field; + if (datatype.kind() == array::DataType::KIND_REAL64) { + double* slice_begin = multiarray.data() + start_index; + field = Field(var_names[ifield], slice_begin, field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + float* slice_begin = multiarray.data() + start_index; + field = Field(var_names[ifield], slice_begin, field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_INT32) { + int* slice_begin = multiarray.data() + start_index; + field = Field(var_names[ifield], slice_begin, field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_INT64) { + long* slice_begin = multiarray.data() + start_index; + field = Field(var_names[ifield], slice_begin, field_array_spec); + } + else { + ATLAS_NOTIMPLEMENTED; + } + multifield->add(field); + } + Log::debug() << "Creating multifield of " << datatype.str() << " type" << std::endl; + return multifield; +} + +// ------------------------------------------------------------------ + +namespace { +static MultiFieldCreatorBuilder __MultiFieldCreatorArray("MultiFieldCreatorArray"); +} + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreatorArray.h b/src/atlas/field/MultiFieldCreatorArray.h new file mode 100644 index 000000000..c66f2c144 --- /dev/null +++ b/src/atlas/field/MultiFieldCreatorArray.h @@ -0,0 +1,58 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Slavko Brdar +/// @date September 2024 + +#pragma once + +#include "atlas/field/MultiFieldCreator.h" + +namespace eckit { +class Configuration; +} +namespace atlas { +namespace field { +class MultiFieldImpl; +} +} // namespace atlas + +namespace atlas { +namespace field { + +// ------------------------------------------------------------------ + +/*! + * \brief MultiField creator using datatype, shape, variable names as arguments + * \details + * shape argument contains -1 at the position which gets filled with variable names + * Example use: + * \code{.cpp} + * MultiFieldImpl* multifield = MultiField::create( + * datatype, + * shape, + * var_names + * ); + * \endcode + */ +class MultiFieldCreatorArray : public MultiFieldCreator { +public: + MultiFieldCreatorArray(); + MultiFieldCreatorArray(const eckit::Configuration& config); + ~MultiFieldCreatorArray() override; + MultiFieldImpl* create(const eckit::Configuration& config = util::Config()) const override; + MultiFieldImpl* create(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) const override; +}; + +// ------------------------------------------------------------------ + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreatorIFS.cc b/src/atlas/field/MultiFieldCreatorIFS.cc new file mode 100644 index 000000000..c90681694 --- /dev/null +++ b/src/atlas/field/MultiFieldCreatorIFS.cc @@ -0,0 +1,230 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Willem Deconinck +/// @author Slavko Brdar +/// @date June 2024 + +#include +#include + +#include "eckit/config/Parametrisation.h" + +#include "atlas/array/ArrayDataStore.h" +#include "atlas/array/DataType.h" +#include "atlas/field/MultiFieldCreatorIFS.h" +#include "atlas/field/detail/MultiFieldImpl.h" +#include "atlas/grid/Grid.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + +namespace atlas { +namespace field { + +MultiFieldCreatorIFS::MultiFieldCreatorIFS() {} + +MultiFieldCreatorIFS::MultiFieldCreatorIFS(const eckit::Configuration& config) {} + +MultiFieldCreatorIFS::~MultiFieldCreatorIFS() = default; + +MultiFieldImpl* MultiFieldCreatorIFS::create(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) const { + ATLAS_NOTIMPLEMENTED; + return nullptr; +} + +MultiFieldImpl* MultiFieldCreatorIFS::create(const eckit::Configuration& config) const { + long nproma; + config.get("nproma", nproma); + long nlev; + config.get("nlev", nlev); + long nblk = 0; + if (config.has("nblk")) { + config.get("nblk", nblk); + } + else if (config.has("ngptot")) { + long ngptot; + config.get("ngptot", ngptot); + nblk = std::ceil(static_cast(ngptot) / static_cast(nproma)); + } + else { + ATLAS_THROW_EXCEPTION("Configuration not found: ngptot or nblk"); + } + array::DataType datatype = array::DataType::create(); + std::string datatype_str; + if (config.get("datatype", datatype_str)) { + datatype = array::DataType(datatype_str); + } + else { + array::DataType::kind_t kind(array::DataType::kind()); + config.get("kind", kind); + if (!array::DataType::kind_valid(kind)) { + std::stringstream msg; + msg << "Could not create field. kind parameter unrecognized"; + throw_Exception(msg.str()); + } + datatype = array::DataType(kind); + } + + auto fields = config.getSubConfigurations("fields"); + long nfld = 0; + for (const auto& field_params : fields) { + long nvar = 1; + field_params.get("nvar", nvar); + nfld += nvar; + } + array::ArrayShape multiarray_shape = ( (nlev > 0 and nfld > 0) ? array::make_shape(nblk, nfld, nlev, nproma) : + ( (nlev > 0) ? array::make_shape(nblk, nlev, nproma) : ( (nfld > 0) ? array::make_shape(nblk, nfld, nproma) : + array::make_shape(nblk, nproma) ) ) ); + + MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; + auto& multiarray = multifield->array(); + + size_t multiarray_field_idx = 0; + for (size_t i = 0; i < fields.size(); ++i) { + std::string name; + fields[i].get("name", name); + Field field; + size_t field_vars = 1; + if (fields[i].get("nvar", field_vars)) { + array::ArrayShape field_shape = + ( (nlev > 0 and field_vars > 0) ? + array::make_shape(multiarray.shape(0), field_vars, multiarray.shape(2), multiarray.shape(3)) : + ( nlev > 0 ? + array::make_shape(multiarray.shape(0), multiarray.shape(1), multiarray.shape(2)) : + ( field_vars > 0 ? + array::make_shape(multiarray.shape(0), field_vars, multiarray.shape(2)) : + array::make_shape(multiarray.shape(0), multiarray.shape(1)) ) ) ); + array::ArrayShape multiarray_shape = + ( (nlev > 0 and field_vars > 0) ? array::make_shape(nblk, field_vars, nlev, nproma) : + ( (nlev > 0) ? array::make_shape(nblk, nlev, nproma) : ( (field_vars > 0) ? + array::make_shape(nblk, field_vars, nproma) : array::make_shape(nblk, nproma) ) ) ); + auto field_strides = multiarray.strides(); + auto field_array_spec = array::ArraySpec(field_shape, field_strides); + + constexpr auto all = array::Range::all(); + const auto range = array::Range(multiarray_field_idx, multiarray_field_idx + field_vars); + if (datatype.kind() == array::DataType::KIND_REAL64) { + if (nlev > 0 and field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, range, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (nlev > 0) { + auto slice = array::make_view(multiarray).slice(all, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, range, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + auto slice = array::make_view(multiarray).slice(all, all); + field = Field(name, slice.data(), field_array_spec); + } + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + if (nlev > 0 and field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, range, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (nlev > 0) { + auto slice = array::make_view(multiarray).slice(all, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, range, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + auto slice = array::make_view(multiarray).slice(all, all); + field = Field(name, slice.data(), field_array_spec); + } + } + else { + ATLAS_NOTIMPLEMENTED; + } + field.set_variables(field_vars); + } + else { + array::ArraySpec field_array_spec; + if (nlev > 0) { + auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2), multiarray.shape(3)); + auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2), multiarray.stride(3)); + field_array_spec = array::ArraySpec(field_shape, field_strides); + } + else if (field_vars > 0) { + auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2)); + auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2)); + field_array_spec = array::ArraySpec(field_shape, field_strides); + } + + constexpr auto all = array::Range::all(); + if (datatype.kind() == array::DataType::KIND_REAL64) { + if (nlev > 0 and field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (nlev > 0) { + auto slice = array::make_view(multiarray).slice(all, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + auto slice = array::make_view(multiarray).slice(all, all); + field = Field(name, slice.data(), field_array_spec); + } + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + if (nlev > 0 and field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (nlev > 0) { + auto slice = array::make_view(multiarray).slice(all, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + auto slice = array::make_view(multiarray).slice(all, all); + field = Field(name, slice.data(), field_array_spec); + } + } + else { + ATLAS_NOTIMPLEMENTED; + } + } + field.set_levels(nlev); + //field.set_blocks(nblk); + + multifield->add(field); + + multiarray_field_idx += field_vars; + } + std::string name; + config.get("name", name); + Log::debug() << "Creating IFS " << datatype.str() << " multifield: " << name << "[nblk=" << nblk << "][nvar=" << nfld + << "][nlev=" << nlev << "][nproma=" << nproma << "]\n"; + return multifield; +} + +// ------------------------------------------------------------------ + +namespace { +static MultiFieldCreatorBuilder __MultiFieldCreatorIFS("MultiFieldCreatorIFS"); +} + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreatorIFS.h b/src/atlas/field/MultiFieldCreatorIFS.h new file mode 100644 index 000000000..dddd7cd04 --- /dev/null +++ b/src/atlas/field/MultiFieldCreatorIFS.h @@ -0,0 +1,63 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Willem Deconinck +/// @date June 2015 + +#pragma once + +#include "atlas/field/MultiFieldCreator.h" + +namespace eckit { +class Configuration; +} +namespace atlas { +namespace field { +class MultiFieldImpl; +} +} // namespace atlas + +namespace atlas { +namespace field { + +// ------------------------------------------------------------------ + +/*! + * \brief MultiField creator using IFS parametrisation + * \details + * Ideally this class should belong to IFS. + * The only reference to IFS in Atlas::MultiField should be here. + * Example use: + * \code{.cpp} + * MultiFieldImpl* multifield = MultiField::create( + * Config + * ("creator","MultiFieldIFS") // MultiFieldIFS FieldCreator + * ("ngptot",ngptot) // Total number of grid points + * ("nproma",nproma) // Grouping of grid points for vectorlength + * ("nlev",nlev) // Number of levels + * ("nvar",nvar) // Number of variables + * ("kind",8) // Real kind in bytes + * ); + * \endcode + */ +class MultiFieldCreatorIFS : public MultiFieldCreator { +public: + MultiFieldCreatorIFS(); + MultiFieldCreatorIFS(const eckit::Configuration& config); + ~MultiFieldCreatorIFS() override; + MultiFieldImpl* create(const eckit::Configuration& config = util::Config()) const override; + MultiFieldImpl* create(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) const override; +}; + +// ------------------------------------------------------------------ + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/detail/MultiFieldImpl.cc b/src/atlas/field/detail/MultiFieldImpl.cc new file mode 100644 index 000000000..c3a6abb73 --- /dev/null +++ b/src/atlas/field/detail/MultiFieldImpl.cc @@ -0,0 +1,32 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +// file deepcode ignore CppMemoryLeak: static pointers for global registry are OK and will be cleaned up at end + +#include "atlas/field/MultiField.h" +#include "atlas/field/MultiFieldCreator.h" + +#include +#include +#include + +#include "atlas/field/detail/MultiFieldImpl.h" + +namespace atlas { +namespace field { + +void MultiFieldImpl::add(Field& field) { + ATLAS_ASSERT(not fieldset_.has(field.name()), "Field with name \"" + field.name() + "\" already exists!"); + fieldset_.add(field); + MultiFieldArrayRegistry::instance().add(field, array_); +} + +} +} diff --git a/src/atlas/field/detail/MultiFieldImpl.h b/src/atlas/field/detail/MultiFieldImpl.h new file mode 100644 index 000000000..4a1445e1d --- /dev/null +++ b/src/atlas/field/detail/MultiFieldImpl.h @@ -0,0 +1,96 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Willem Deconinck +/// @date June 2015 + +#pragma once + +#include +#include + +#include "atlas/array/Array.h" +#include "atlas/field/Field.h" +#include "atlas/field/FieldSet.h" +#include "atlas/util/Config.h" +#include "atlas/util/Metadata.h" +#include "atlas/util/Object.h" + +namespace atlas { +namespace field { + +class MultiFieldImpl : public util::Object { +public: // methods + //-- Constructors + + MultiFieldImpl() {} + + MultiFieldImpl(const array::ArraySpec& spec) { + array::ArraySpec s(spec); + array_.reset(array::Array::create(std::move(s))); + } + + virtual ~MultiFieldImpl() {} + + + //-- Accessors + + const Field& field(const std::string& name) const { return fieldset_.field(name); } + Field& field(const std::string& name) { return fieldset_.field(name); } + bool has(const std::string& name) const { return fieldset_.has(name); } + std::vector field_names() const { return fieldset_.field_names(); } + + const Field& field(const idx_t idx) const { return fieldset_[idx]; } + Field& field(const idx_t idx) { return fieldset_[idx]; } + idx_t size() const { return fieldset_.size(); } + + const Field& operator[](const idx_t idx) const { return fieldset_[idx]; } + Field& operator[](const idx_t idx) { return fieldset_[idx]; } + + const Field& operator[](const std::string& name) const { return fieldset_.field(name); } + Field& operator[](const std::string& name) { return fieldset_.field(name); } + + const util::Metadata& metadata() const { return metadata_; } + util::Metadata& metadata() { return metadata_; } + + // -- Modifiers + + /// @brief Implicit conversion to Array + operator const array::Array&() const { return array(); } + operator array::Array&() { return array(); } + + operator const FieldSet&() const { return fieldset_; } + + operator FieldSet&() { return fieldset_; } + + /// @brief Access contained Array + const array::Array& array() const { + ATLAS_ASSERT(array_); + return *array_; + } + array::Array& array() { + ATLAS_ASSERT(array_); + return *array_; + } + + /// @brief Access contained FieldSet + const FieldSet& fieldset() const { return fieldset_; } + FieldSet& fieldset() { return fieldset_; } + + void add(Field& field); + +public: // temporary public for prototyping + FieldSet fieldset_; + std::shared_ptr array_; + util::Metadata metadata_; +}; + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/detail/MultiFieldInterface.cc b/src/atlas/field/detail/MultiFieldInterface.cc index a5e729b2a..805f5e162 100644 --- a/src/atlas/field/detail/MultiFieldInterface.cc +++ b/src/atlas/field/detail/MultiFieldInterface.cc @@ -8,10 +8,13 @@ * nor does it submit to any jurisdiction. */ +#include #include #include #include "atlas/library/config.h" +#include "atlas/field/MultiField.h" +#include "atlas/field/detail/MultiFieldImpl.h" #include "atlas/field/detail/MultiFieldInterface.h" #include "atlas/runtime/Exception.h" @@ -21,108 +24,48 @@ namespace field { extern "C" { MultiFieldImpl* atlas__MultiField__create(eckit::Configuration* config) { ATLAS_ASSERT(config != nullptr); - long nproma = config->getLong("nproma"); - long nlev = config->getLong("nlev"); - long nblk = 0; - if (config->has("nblk")) { - nblk = config->getLong("nblk"); - } - else if (config->has("ngptot")) { - long ngptot = config->getLong("ngptot"); - nblk = std::ceil(static_cast(ngptot) / static_cast(nproma)); - } - else { - ATLAS_THROW_EXCEPTION("Configuration not found: ngptot or nblk"); - } - array::DataType datatype = array::DataType::create(); - std::string datatype_str; - if (config->get("datatype", datatype_str)) { - datatype = array::DataType(datatype_str); - } - else { - array::DataType::kind_t kind(array::DataType::kind()); - config->get("kind", kind); - if (!array::DataType::kind_valid(kind)) { - std::stringstream msg; - msg << "Could not create field. kind parameter unrecognized"; - throw_Exception(msg.str()); - } - datatype = array::DataType(kind); - } + auto multifield = new MultiField(*config); + ATLAS_ASSERT(multifield); + return multifield->get(); +} - auto fields = config->getSubConfigurations("fields"); - long nfld = 0; - for (const auto& field_config : fields) { - long nvar = 1; - field_config.get("nvar", nvar); - nfld += nvar; +MultiFieldImpl* atlas__MultiField__create_shape(int kind, int rank, int shapef[], const char* var_names, + size_t length, size_t size) { + array::ArrayShape shape; + shape.resize(rank); + array::ArrayStrides strides; + for (idx_t j = 0, jf = rank - 1; j < rank; ++j) { + shape[j] = shapef[jf--]; } - auto multiarray_shape = array::make_shape(nblk, nfld, nlev, nproma); - - MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; - auto& multiarray = multifield->array(); - - size_t multiarray_field_idx = 0; - for (size_t i = 0; i < fields.size(); ++i) { - std::string name; - fields[i].get("name", name); - Field field; - size_t field_vars = 1; - if (fields[i].get("nvar", field_vars)) { - auto field_shape = - array::make_shape(multiarray.shape(0), field_vars, multiarray.shape(2), multiarray.shape(3)); - auto field_strides = multiarray.strides(); - auto field_array_spec = array::ArraySpec(field_shape, field_strides); - - constexpr auto all = array::Range::all(); - const auto range = array::Range(multiarray_field_idx, multiarray_field_idx + field_vars); - if (datatype.kind() == array::DataType::KIND_REAL64) { - auto slice = array::make_view(multiarray).slice(all, range, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else if (datatype.kind() == array::DataType::KIND_REAL32) { - auto slice = array::make_view(multiarray).slice(all, range, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else { - ATLAS_NOTIMPLEMENTED; - } - field.set_variables(field_vars); - } - else { - auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2), multiarray.shape(3)); - auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2), multiarray.stride(3)); - auto field_array_spec = array::ArraySpec(field_shape, field_strides); - - constexpr auto all = array::Range::all(); - if (datatype.kind() == array::DataType::KIND_REAL64) { - auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else if (datatype.kind() == array::DataType::KIND_REAL32) { - auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else { - ATLAS_NOTIMPLEMENTED; - } - } - field.set_levels(nlev); - // field.set_blocks(nblk); - - multifield->add(field); - - multiarray_field_idx += field_vars; + std::vector var_names_str; + for (size_t jj = 0; jj < size; ++jj) { + char str[length + 1]; + ATLAS_ASSERT(snprintf(str, sizeof(str), "%s", var_names + jj * length ) >= 0); + std::string sstr(str); + sstr.erase(std::find_if(sstr.rbegin(), sstr.rend(), [](unsigned char ch) { + return !std::isspace(ch); + }).base(), sstr.end()); + var_names_str.push_back(sstr); } + + auto multifield = new MultiField(array::DataType{kind}, shape, var_names_str); ATLAS_ASSERT(multifield); - return multifield; + return multifield->get(); } void atlas__MultiField__delete(MultiFieldImpl* This) { delete This; } +int atlas__MultiField__size(MultiFieldImpl* This) { + return This->size(); +} + +FieldSetImpl* atlas__MultiField__fieldset(MultiFieldImpl* This) { + return This->fieldset().get(); +} + } // ------------------------------------------------------------------ diff --git a/src/atlas/field/detail/MultiFieldInterface.h b/src/atlas/field/detail/MultiFieldInterface.h index ac3b1ae22..c63d4f755 100644 --- a/src/atlas/field/detail/MultiFieldInterface.h +++ b/src/atlas/field/detail/MultiFieldInterface.h @@ -14,6 +14,7 @@ #pragma once +#include "atlas/field/FieldSet.h" #include "atlas/field/MultiField.h" namespace atlas { @@ -29,7 +30,11 @@ namespace field { // C wrapper interfaces to C++ routines extern "C" { MultiFieldImpl* atlas__MultiField__create(eckit::Configuration* config); +MultiFieldImpl* atlas__MultiField__create_shape(int kind, int rank, int shapef[], const char* var_names, + size_t length, size_t size); void atlas__MultiField__delete(MultiFieldImpl* This); +int atlas__MultiField__size(MultiFieldImpl* This); +FieldSetImpl* atlas__MultiField__fieldset(MultiFieldImpl* This); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/atlas_f/CMakeLists.txt b/src/atlas_f/CMakeLists.txt index eefbbf647..2fd47e7d8 100644 --- a/src/atlas_f/CMakeLists.txt +++ b/src/atlas_f/CMakeLists.txt @@ -234,7 +234,7 @@ ecbuild_add_library( TARGET atlas_f field/atlas_FieldSet_module.fypp field/atlas_State_module.F90 field/atlas_Field_module.fypp - field/atlas_MultiField_module.fypp + field/atlas_MultiField_module.F90 grid/atlas_Grid_module.F90 grid/atlas_GridDistribution_module.F90 grid/atlas_Vertical_module.F90 diff --git a/src/atlas_f/field/atlas_MultiField_module.fypp b/src/atlas_f/field/atlas_MultiField_module.F90 similarity index 52% rename from src/atlas_f/field/atlas_MultiField_module.fypp rename to src/atlas_f/field/atlas_MultiField_module.F90 index 4d45836a5..57168efaa 100644 --- a/src/atlas_f/field/atlas_MultiField_module.fypp +++ b/src/atlas_f/field/atlas_MultiField_module.F90 @@ -8,13 +8,12 @@ #include "atlas/atlas_f.h" -#:include "atlas/atlas_f.fypp" -#:include "internals/atlas_generics.fypp" - module atlas_multifield_module use fckit_owned_object_module, only : fckit_owned_object use atlas_Config_module, only: atlas_Config +use atlas_field_module, only: atlas_field, array_c_to_f +use atlas_fieldset_module, only: atlas_fieldset implicit none @@ -42,6 +41,10 @@ module atlas_multifield_module !------------------------------------------------------------------------------ contains + procedure, public :: MultiField__fieldset + procedure, public :: MultiField__size + generic :: fieldset => MultiField__fieldset + generic :: size => MultiField__size #if FCKIT_FINAL_NOT_INHERITING final :: atlas_MultiField__final_auto @@ -52,6 +55,7 @@ module atlas_multifield_module interface atlas_MultiField module procedure atlas_MultiField__cptr module procedure atlas_MultiField__create + module procedure atlas_MultiField__create_names end interface private :: fckit_owned_object @@ -83,6 +87,66 @@ function atlas_MultiField__create(params) result(field) !------------------------------------------------------------------------------- +function atlas_MultiField__create_names(kind, shape, field_names) result(field) + use, intrinsic :: iso_c_binding, only : c_char, c_int, c_int32_t, c_size_t + use atlas_multifield_c_binding + type(atlas_MultiField) :: field + integer(c_int), intent(in) :: kind + integer, intent(in) :: shape(:) + character(*), intent(in) :: field_names(:) + character(kind=c_char,len=:), allocatable :: flat_field_names + integer(c_size_t) :: length + integer(c_int32_t) :: ii + integer(c_int32_t), allocatable :: field_name_sizes(:) + + if (size(field_names) == 0 .or. size(shape) < 3) then + print *, "atlas_MultiField__create_names: must have at least one field name, and the size of shape", & + & " is minimum 3, e.g. [nproma,-1,nblk]" + stop -1 + end if + + length = len(field_names(1)) + allocate(field_name_sizes(size(field_names))) + field_name_sizes = len(field_names(:)) + + if (any(field_name_sizes /= length)) then + print *, "atlas_MultiField__create_names: field_names have to have same length in characters" + stop -1 + end if + + allocate(character(len=length*size(field_names) ) :: flat_field_names) + do ii = 1, size(field_names) + flat_field_names((ii-1)*length+1:ii*length) = field_names(ii) + enddo + + field = atlas_MultiField__cptr( atlas__MultiField__create_shape(kind, size(shape), shape, & + & flat_field_names, length, size(field_names,kind=c_size_t)) ) + call field%return() +end function + +!------------------------------------------------------------------------------- + +function MultiField__size(this) result(size) + use atlas_multifield_c_binding + class(atlas_MultiField), intent(in) :: this + integer :: size + size = atlas__MultiField__size(this%CPTR_PGIBUG_B) +end function + +!------------------------------------------------------------------------------- +function MultiField__fieldset(this) result(fset) + use, intrinsic :: iso_c_binding, only : c_ptr + use atlas_multifield_c_binding + class(atlas_MultiField), intent(in) :: this + type(c_ptr) :: fset_cptr + type(atlas_FieldSet) :: fset + fset_cptr = atlas__MultiField__fieldset(this%CPTR_PGIBUG_B) + fset = atlas_FieldSet( fset_cptr ) + call fset%return() +end function + +!------------------------------------------------------------------------------- + #if FCKIT_FINAL_NOT_INHERITING ATLAS_FINAL subroutine atlas_MultiField__final_auto(this) type(atlas_MultiField), intent(inout) :: this diff --git a/src/tests/field/fctest_multifield_ifs.F90 b/src/tests/field/fctest_multifield_ifs.F90 index c6ca36f47..e71798c02 100644 --- a/src/tests/field/fctest_multifield_ifs.F90 +++ b/src/tests/field/fctest_multifield_ifs.F90 @@ -23,7 +23,7 @@ module fcta_MultiField_fixture ! ----------------------------------------------------------------------------- -TESTSUITE_WITH_FIXTURE(fctest_atlas_MultiField,fcta_MultiField_fixture) +TESTSUITE_WITH_FIXTURE(fctest_atlas_MultiField, fcta_MultiField_fixture) ! ----------------------------------------------------------------------------- @@ -42,34 +42,185 @@ module fcta_MultiField_fixture TEST( test_multifield ) implicit none - type(atlas_MultiField) :: mfield - type(atlas_config) :: config + type(atlas_MultiField) :: mfield(2) + type(atlas_FieldSet) :: fieldset(2) + type(atlas_Field) :: field + type(atlas_config) :: config + integer, pointer :: fdata_int_2d(:,:) + real(c_float), pointer :: fdata_f2d(:,:), fdata_f3d(:,:,:) + real(c_double), pointer :: fdata_d3d(:,:,:) integer, parameter :: nproma = 16; - integer, parameter :: nlev = 100; + integer, parameter :: nlev = 1; integer, parameter :: ngptot = 2000; - type(atlas_Config), dimension(5) :: field_configs + integer, parameter :: nblk = (ngptot + nproma - 1) / nproma + type(atlas_Config), allocatable :: field_configs(:) + integer :: i + character(len=64), parameter, dimension(5) :: var_names = [ character(64) :: & + "temperature", "pressure", "density", "clv", "wind_u" ] config = atlas_Config() call config%set("type", "MultiFieldCreatorIFS"); call config%set("ngptot", ngptot); call config%set("nproma", nproma); - call config%set("nlev", nlev); + allocate(field_configs(size(var_names))) + do i = 1, size(var_names) + field_configs(i) = atlas_Config() + call field_configs(i)%set("name", trim(var_names(i))) + end do + call field_configs(4)%set("nvar", 4) ! clv has four subvariables + call config%set("fields", field_configs) + + call config%set("nlev", 0); ! surface fields + call config%set("datatype", "real32"); + mfield(1) = atlas_MultiField(config) + + call config%set("nlev", 4); ! fields are 3d call config%set("datatype", "real64"); - field_configs(1) = atlas_Config() - field_configs(2) = atlas_Config() - field_configs(3) = atlas_Config() - field_configs(4) = atlas_Config() - field_configs(5) = atlas_Config() - call field_configs(1)%set("name", "temperature") - call field_configs(2)%set("name", "pressure") - call field_configs(3)%set("name", "density") - call field_configs(4)%set("name", "clv") - call field_configs(4)%set("nvar", 5) - call field_configs(5)%set("name", "wind_u") + mfield(2) = atlas_MultiField(config) + + fieldset(1) = mfield(1)%fieldset() + FCTEST_CHECK_EQUAL(mfield(1)%size(), 5) + FCTEST_CHECK_EQUAL(fieldset(1)%size(), 5) + + fieldset(2) = atlas_FieldSet() + call fieldset(2)%add(mfield(1)%fieldset()) + field = fieldset(2)%field("density") + call field%data(fdata_f2d) + fdata_f2d(1,1) = 2. + call field%rename("dens") + + ! check data access directly though multifield + call fieldset(1)%data("dens", fdata_f2d) + fdata_f2d(1,1) = 3. + + ! check access to the renamed variable + field = fieldset(1)%field("dens") + call field%data(fdata_f2d) + FCTEST_CHECK_EQUAL(fdata_f2d(1,1), 3._c_float) + + ! check dimesionality + fieldset(2) = mfield(2)%fieldset() + call fieldset(2)%data("density", fdata_d3d) + fdata_d3d(1,1,1) = 4. + fieldset(2) = atlas_FieldSet() + call fieldset(2)%add(mfield(2)%fieldset()) + field = fieldset(2)%field("density") + call field%data(fdata_d3d) + FCTEST_CHECK_EQUAL(fdata_d3d(1,1,1), 4._c_double) +END_TEST + + +TEST( test_multifield_array_direct_constructor ) + implicit none + + type(atlas_MultiField) :: mfield(2) + type(atlas_FieldSet) :: fieldset(2), fset + type(atlas_Field) :: field + type(atlas_config) :: config + real(c_float), pointer :: fdata_f2d(:,:), fdata_f3d(:,:,:) + real(c_double), pointer :: fdata_d3d(:,:,:) + + integer, parameter :: nproma = 16; + integer, parameter :: nlev = 100; + integer, parameter :: ngptot = 2000; + integer, parameter :: nblk = (ngptot + nproma - 1) / nproma + integer :: i + character(len=64), parameter, dimension(5) :: var_names = [ character(64) :: & + "temperature ", "pressure", "density", "clv", "wind_u" ] + +return + + ! surface fields + mfield(1) = atlas_MultiField(atlas_real(c_float), [nproma, -1, nblk], var_names) + + ! 3d fields + mfield(2) = atlas_MultiField(atlas_real(c_double), [nproma, nlev, -1, nblk], var_names) + + FCTEST_CHECK_EQUAL(mfield(1)%size(), 5) + + fieldset(1) = mfield(1)%fieldset() + call fieldset(1)%data("density", fdata_f2d) + fdata_f2d(1,1) = 3. + fieldset(2) = mfield(2)%fieldset() + call fieldset(2)%data("density", fdata_d3d) + fdata_d3d(1,1,1) = 4. + + fset = atlas_FieldSet() + call fset%add(mfield(1)%fieldset()) + call fset%add(mfield(2)%fieldset()) + +END_TEST + + +TEST( test_multifield_array_config_constuctor ) + implicit none + + type(atlas_MultiField) :: mfield(2) + type(atlas_FieldSet) :: fieldset(2) + type(atlas_Field) :: field + type(atlas_config) :: config + integer, pointer :: fdata_int_2d(:,:) + real(c_float), pointer :: fdata_f2d(:,:), fdata_f3d(:,:,:) + real(c_double), pointer :: fdata_d3d(:,:,:) + + integer, parameter :: nproma = 16; + integer, parameter :: nlev = 1; + integer, parameter :: nblk = 200; + type(atlas_Config), allocatable :: field_configs(:) + integer :: i + character(len=64), parameter, dimension(5) :: var_names = [ character(64) :: & + "temperature", "pressure", "density", "clv", "wind_u" ] + + config = atlas_Config() + call config%set("type", "MultiFieldCreatorArray"); + allocate(field_configs(size(var_names))) + do i = 1, size(var_names) + field_configs(i) = atlas_Config() + call field_configs(i)%set("name", trim(var_names(i))) + end do + call field_configs(4)%set("nvar", 5) ! clv has four subvariables call config%set("fields", field_configs) - mfield = atlas_MultiField(config) + ! surface fields + call config%set("shape", [nproma, -1, nblk]); + call config%set("datatype", "real32"); + mfield(1) = atlas_MultiField(config) + + ! fields are 3d + call config%set("shape", [nproma, nlev, -1, nblk]); + call config%set("datatype", "real64"); + mfield(2) = atlas_MultiField(config) + + fieldset(1) = mfield(1)%fieldset() + FCTEST_CHECK_EQUAL(mfield(1)%size(), 9) + FCTEST_CHECK_EQUAL(fieldset(1)%size(), 9) + + fieldset(2) = atlas_FieldSet() + call fieldset(2)%add(mfield(1)%fieldset()) + field = fieldset(2)%field("density") + call field%data(fdata_f2d) + fdata_f2d(1,1) = 2. + call field%rename("dens") + + ! check data access directly though multifield + call fieldset(1)%data("dens", fdata_f2d) + fdata_f2d(1,1) = 3. + + ! check access to the renamed variable + field = fieldset(1)%field("dens") + call field%data(fdata_f2d) + FCTEST_CHECK_EQUAL(fdata_f2d(1,1), 3._c_float) + + ! check dimesionality + fieldset(2) = mfield(2)%fieldset() + call fieldset(2)%data("density", fdata_d3d) + fdata_d3d(1,1,1) = 4. + fieldset(2) = atlas_FieldSet() + call fieldset(2)%add(mfield(2)%fieldset()) + field = fieldset(2)%field("density") + call field%data(fdata_d3d) + FCTEST_CHECK_EQUAL(fdata_d3d(1,1,1), 4._c_double) END_TEST ! ----------------------------------------------------------------------------- diff --git a/src/tests/field/test_multifield_ifs.cc b/src/tests/field/test_multifield_ifs.cc index ce11afa35..e4539bc28 100644 --- a/src/tests/field/test_multifield_ifs.cc +++ b/src/tests/field/test_multifield_ifs.cc @@ -10,147 +10,40 @@ #include #include +#include #include "eckit/config/YAMLConfiguration.h" #include "atlas/array/ArrayView.h" -#include "atlas/array/DataType.h" #include "atlas/array/MakeView.h" #include "atlas/field/Field.h" #include "atlas/field/MultiField.h" -#include "atlas/grid/Grid.h" +#include "atlas/field/MultiFieldCreatorArray.h" +#include "atlas/field/detail/MultiFieldImpl.h" #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "tests/AtlasTestEnvironment.h" -using namespace atlas::field; using namespace atlas::field; namespace atlas { namespace test { -// ------------------------------------------------------------------- -// Example IFS MultiField creato - -// --- Declaration (in .h file) -class MultiFieldCreatorIFS : public MultiFieldCreator { -public: - MultiFieldCreatorIFS(const eckit::Configuration& config = util::Config()): MultiFieldCreator(config) {} - ~MultiFieldCreatorIFS() override = default; - MultiFieldImpl* create(const eckit::Configuration& = util::Config()) const override; -}; - -// --- Implementation (in .cc file) -MultiFieldImpl* MultiFieldCreatorIFS::create(const eckit::Configuration& config) const { - long ngptot = config.getLong("ngptot"); - long nproma = config.getLong("nproma"); - long nlev = config.getLong("nlev"); - long nblk = 0; - - - array::DataType datatype = array::DataType::create(); - std::string datatype_str; - if (config.get("datatype", datatype_str)) { - datatype = array::DataType(datatype_str); - } - else { - array::DataType::kind_t kind(array::DataType::kind()); - config.get("kind", kind); - if (!array::DataType::kind_valid(kind)) { - std::stringstream msg; - msg << "Could not create field. kind parameter unrecognized"; - throw_Exception(msg.str()); - } - datatype = array::DataType(kind); - } - - nblk = std::ceil(static_cast(ngptot) / static_cast(nproma)); - - auto fields = config.getSubConfigurations("fields"); - long nfld = 0; - for (const auto& field_config : fields) { - long nvar = 1; - field_config.get("nvar", nvar); - nfld += nvar; - } - - auto multiarray_shape = array::make_shape(nblk, nfld, nlev, nproma); - - MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; - - auto& multiarray = multifield->array(); - - size_t multiarray_field_idx = 0; - for (size_t i = 0; i < fields.size(); ++i) { - std::string name; - fields[i].get("name", name); - Field field; - size_t field_vars = 1; - - if (fields[i].get("nvar", field_vars)) { - auto field_shape = - array::make_shape(multiarray.shape(0), field_vars, multiarray.shape(2), multiarray.shape(3)); - auto field_strides = multiarray.strides(); - auto field_array_spec = array::ArraySpec(field_shape, field_strides); - - constexpr auto all = array::Range::all(); - const auto range = array::Range(multiarray_field_idx, multiarray_field_idx + field_vars); - if (datatype.kind() == array::DataType::KIND_REAL64) { - auto slice = array::make_view(multiarray).slice(all, range, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else if (datatype.kind() == array::DataType::KIND_REAL32) { - auto slice = array::make_view(multiarray).slice(all, range, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else { - ATLAS_NOTIMPLEMENTED; - } - field.set_variables(field_vars); - } - else { - auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2), multiarray.shape(3)); - auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2), multiarray.stride(3)); - auto field_array_spec = array::ArraySpec(field_shape, field_strides); - - constexpr auto all = array::Range::all(); - if (datatype.kind() == array::DataType::KIND_REAL64) { - auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else if (datatype.kind() == array::DataType::KIND_REAL32) { - auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); - field = Field(name, slice.data(), field_array_spec); - } - else { - ATLAS_NOTIMPLEMENTED; - } - } - field.set_levels(nlev); - - multifield->add(field); - - multiarray_field_idx += field_vars; - } - return multifield; +const std::vector make_shape(const std::initializer_list& list) { + return std::vector(list); } -// Register in factory -MultiFieldCreatorBuilder __MultiFieldCreatorIFS("MultiFieldCreatorIFS"); - -// =================================================================== -// BEGIN TESTS -// =================================================================== - - CASE("multifield_generator") { EXPECT(MultiFieldCreatorFactory::has("MultiFieldCreatorIFS")); - std::unique_ptr MultiFieldCreator(MultiFieldCreatorFactory::build("MultiFieldCreatorIFS")); + std::unique_ptr MultiFieldCreatorIFS(MultiFieldCreatorFactory::build("MultiFieldCreatorIFS")); + + EXPECT(MultiFieldCreatorFactory::has("MultiFieldCreatorArray")); + std::unique_ptr MultiFieldCreatorArray(MultiFieldCreatorFactory::build("MultiFieldCreatorArray")); } -CASE("multifield_create") { +CASE("multifield_ifs_create") { using Value = float; int nproma = 16; int nlev = 100; @@ -163,12 +56,12 @@ CASE("multifield_create") { p.set("nproma", nproma); p.set("nlev", nlev); p.set("datatype", array::make_datatype().str()); - p.set("fields", { - util::Config("name", "temperature"), // - util::Config("name", "pressure"), // - util::Config("name", "density"), // - util::Config("name", "clv")("nvar", 5), // - util::Config("name", "wind_u") // + p.set("fields", std::vector{ + util::Config("name", "temperature"), + util::Config("name", "pressure"), + util::Config("name", "density"), + util::Config("name", "clv")("nvar", 5), // 'clv' with 5 subvariables + util::Config("name", "wind_u") }); return p.json(); }; @@ -252,6 +145,14 @@ CASE("multifield_create") { EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); } + + // access FieldSet through MultiField + auto fieldset = multifield->fieldset(); + auto field_v = array::make_view(fieldset.field("temperature")); + EXPECT_EQ(fieldset.size(), 5); + EXPECT(fieldset.has("temperature")); + EXPECT(fieldset.has("wind_u")); + EXPECT_EQ(field_v(1,2,3), 4); } SECTION("test registry") { @@ -264,6 +165,246 @@ CASE("multifield_create") { //----------------------------------------------------------------------------- +CASE("multifield_array_create") { + using Value = float; + int nproma = 16; + int nlev = 100; + int ngptot = 2000; + + const int nblks = (ngptot + nproma - 1) / nproma; + const std::vector var_names = {"temperature", "pressure", "density", "clv", "wind_u"}; + + auto json = [&]() -> std::string { + util::Config p; + p.set("type", "MultiFieldCreatorArray"); + p.set("shape", {nblks, -1, nlev, nproma}); + p.set("datatype", array::make_datatype().str()); + p.set("fields", std::vector{ + util::Config("name", "temperature"), // + util::Config("name", "pressure"), // + util::Config("name", "density"), // + util::Config("name", "clv")("nvar", 5), // + util::Config("name", "wind_u") // + }); + return p.json(); + }; + + SECTION("test_MultiFieldArray_noconfig_3d") { + int nlev = 3; + const std::vector vshape = make_shape({nblks, -1, nlev, nproma}); + MultiField multifield(array::make_datatype(), vshape, var_names); + + const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); + nlev = multifield.array().shape(2); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 5); + EXPECT_EQ(nvar, 5); + + EXPECT_EQ(multifield.size(), 5); + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("clv")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("clv") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto clv = array::make_view(multifield.field("clv")); + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[3].name(), "clv"); + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma * nlev; + auto level_stride = nproma; + auto nproma_stride = 1; + + temp(1, 2, 3) = 4; + clv(13, 2, 14) = 16; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), level_stride); + EXPECT_EQ(temp.stride(2), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nlev * nproma); + + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), level_stride); + EXPECT_EQ(multiarray.stride(3), nproma_stride); + + EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); + EXPECT_EQ(multiarray(13, 3, 2, 14), 16.); + + EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); + } + + // access FieldSet through MultiField + auto fieldset = multifield->fieldset(); + auto field_v = array::make_view(fieldset.field("temperature")); + EXPECT_EQ(fieldset.size(), 5); + EXPECT(fieldset.has("temperature")); + EXPECT(fieldset.has("wind_u")); + EXPECT_EQ(field_v(1,2,3), 4); + } + + SECTION("test_MultiFieldArray_noconfig_2d") { + const std::vector vshape = make_shape({nblks, -1, nproma}); + MultiField multifield(array::make_datatype(), vshape, var_names); + + const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); + nlev = multifield.array().shape(2); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 5); + EXPECT_EQ(nvar, 5); + + EXPECT_EQ(multifield.size(), 5); + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("clv")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("clv") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto clv = array::make_view(multifield.field("clv")); + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[3].name(), "clv"); + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma; + auto nproma_stride = 1; + + temp(1, 3) = 4; + clv(13, 14) = 16; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nproma); + + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), nproma_stride); + + EXPECT_EQ(multiarray(1, 0, 3), 4.); + EXPECT_EQ(multiarray(13, 3, 14), 16.); + + EXPECT_EQ(multiarray.size(), nblk * nvar * nproma); + } + + // access FieldSet through MultiField + auto fieldset = multifield->fieldset(); + auto field_v = array::make_view(fieldset.field("temperature")); + EXPECT_EQ(fieldset.size(), 5); + EXPECT(fieldset.has("temperature")); + EXPECT(fieldset.has("wind_u")); + EXPECT_EQ(field_v(1,3), 4); + } + + SECTION("Print configuration") { + Log::info() << "json = " << json() << std::endl; + } + + SECTION("test_MultiFieldArray_config") { + MultiField multifield{eckit::YAMLConfiguration{json()}}; + + const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 9); + EXPECT_EQ(nvar, 9); + + EXPECT_EQ(multifield.size(), 9); + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("pressure")); + EXPECT(multifield.has("density")); + EXPECT(multifield.has("clv_0")); + EXPECT(multifield.has("wind_u")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("pressure") << std::endl; + Log::info() << multifield.field("density") << std::endl; + Log::info() << multifield.field("clv_0") << std::endl; + Log::info() << multifield.field("wind_u") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto pres = array::make_view(multifield.field("pressure")); + auto dens = array::make_view(multifield.field("density")); + auto clv = array::make_view(multifield.field("clv_0")); // note rank 4 + auto wind_u = array::make_view(multifield.field("wind_u")); + + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[1].name(), "pressure"); + EXPECT_EQ(multifield[2].name(), "density"); + EXPECT_EQ(multifield[3].name(), "clv_0"); + EXPECT_EQ(multifield[8].name(), "wind_u"); + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma * nlev; + auto level_stride = nproma; + auto nproma_stride = 1; + + temp(1, 2, 3) = 4; + pres(5, 6, 7) = 8; + dens(9, 10, 11) = 12; + clv(13, 14, 15) = 16; + wind_u(17, 18, 3) = 19; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), level_stride); + EXPECT_EQ(temp.stride(2), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nlev * nproma); + + EXPECT_EQ(clv.stride(0), block_stride); + EXPECT_EQ(clv.stride(1), level_stride); + EXPECT_EQ(clv.stride(2), nproma_stride); + + EXPECT_EQ(clv.size(), nblk * nlev * nproma); + + + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), level_stride); + EXPECT_EQ(multiarray.stride(3), nproma_stride); + + EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); + EXPECT_EQ(multiarray(5, 1, 6, 7), 8.); + EXPECT_EQ(multiarray(9, 2, 10, 11), 12.); + EXPECT_EQ(multiarray(13, 3, 14, 15), 16.); + EXPECT_EQ(multiarray(17, 8, 18, 3), 19.); + + EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); + } + + // access FieldSet through MultiField + auto fieldset = multifield->fieldset(); + auto field_v = array::make_view(fieldset.field("temperature")); + EXPECT_EQ(fieldset.size(), 9); + EXPECT(fieldset.has("temperature")); + EXPECT(fieldset.has("wind_u")); + EXPECT_EQ(field_v(1,2,3), 4); + } + + SECTION("test registry") { + { + Field field = MultiField {eckit::YAMLConfiguration{json()}}.field("temperature"); + auto temp = array::make_view(field); + } + } +} + +//----------------------------------------------------------------------------- + } // namespace test } // namespace atlas From b944fba7aeb0dc61e1eade48dc5030ff2b6dbd9a Mon Sep 17 00:00:00 2001 From: Oliver Lomax Date: Tue, 8 Oct 2024 08:04:35 +0100 Subject: [PATCH 18/40] Added function to create `std::variant` for multiple array views. (#220) * Added array view variant class and tests. * Moved make_view_variant into array namespace. * Refactored ArrayViewVariant methods. * More refactoring. * Updated test. * Attempting to address gnu 7.3 compiler errors. * Typos in comments. * Added missing EXPECTs in test. * Refactored detial::VariantHelper template. * Merged in ArrayViewVariant refactor. * Refactored introspection helpers. * Refactor helper function signatures. Removed SFINAE test. * Cleaned up some garbage in test. * Removed reference qualifier on visitor template parameter. * Moved ValuesTypes and Ranks structs into array::detail namespace. * Tidied up naming consistency. * Renamed ValueType and Ranks structs. * Revert parameter names in ArrayViewVariant.h * Revert parameter names in ArrayViewVariant.h --- src/atlas/CMakeLists.txt | 2 + src/atlas/array.h | 1 + src/atlas/array/ArrayViewVariant.cc | 110 +++++++++++++++++ src/atlas/array/ArrayViewVariant.h | 103 ++++++++++++++++ src/tests/array/CMakeLists.txt | 5 + src/tests/array/test_array_view_variant.cc | 131 +++++++++++++++++++++ 6 files changed, 352 insertions(+) create mode 100644 src/atlas/array/ArrayViewVariant.cc create mode 100644 src/atlas/array/ArrayViewVariant.h create mode 100644 src/tests/array/test_array_view_variant.cc diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index ebecad3bb..60b93b9a1 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -759,6 +759,8 @@ array/Range.h array/Vector.h array/Vector.cc array/SVector.h +array/ArrayViewVariant.h +array/ArrayViewVariant.cc array/helpers/ArrayInitializer.h array/helpers/ArrayAssigner.h array/helpers/ArrayWriter.h diff --git a/src/atlas/array.h b/src/atlas/array.h index c2cf7f720..a7ac48d07 100644 --- a/src/atlas/array.h +++ b/src/atlas/array.h @@ -23,6 +23,7 @@ #include "atlas/array/ArraySpec.h" #include "atlas/array/ArrayStrides.h" #include "atlas/array/ArrayView.h" +#include "atlas/array/ArrayViewVariant.h" #include "atlas/array/DataType.h" #include "atlas/array/LocalView.h" #include "atlas/array/MakeView.h" diff --git a/src/atlas/array/ArrayViewVariant.cc b/src/atlas/array/ArrayViewVariant.cc new file mode 100644 index 000000000..f62efd2d8 --- /dev/null +++ b/src/atlas/array/ArrayViewVariant.cc @@ -0,0 +1,110 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "atlas/array/ArrayViewVariant.h" + +#include +#include + +#include "atlas/runtime/Exception.h" + +namespace atlas { +namespace array { + +using namespace detail; + +namespace { + +template +struct VariantTypeHelper { + using type = ArrayViewVariant; +}; + +template <> +struct VariantTypeHelper { + using type = ConstArrayViewVariant; +}; + +template +using VariantType = + typename VariantTypeHelper>::type; + +// Match array.rank() and array.datatype() to variant types. Return result of +// makeView on a successful pattern match. +template +VariantType executeMakeView(ArrayType& array, + const MakeView& makeView) { + using View = std::variant_alternative_t>; + using Value = typename View::non_const_value_type; + constexpr auto Rank = View::rank(); + + if (array.datatype() == DataType::kind() && array.rank() == Rank) { + return makeView(array, Value{}, std::integral_constant{}); + } + + if constexpr (TypeIndex < std::variant_size_v> - 1) { + return executeMakeView(array, makeView); + } else { + throw_Exception("ArrayView<" + array.datatype().str() + ", " + + std::to_string(array.rank()) + + "> is not an alternative in ArrayViewVariant.", + Here()); + } +} + +template +VariantType makeViewVariantImpl(ArrayType& array) { + const auto makeView = [](auto& array, auto value, auto rank) { + return make_view(array); + }; + return executeMakeView<>(array, makeView); +} + +template +VariantType makeHostViewVariantImpl(ArrayType& array) { + const auto makeView = [](auto& array, auto value, auto rank) { + return make_host_view(array); + }; + return executeMakeView<>(array, makeView); +} + +template +VariantType makeDeviceViewVariantImpl(ArrayType& array) { + const auto makeView = [](auto& array, auto value, auto rank) { + return make_device_view(array); + }; + return executeMakeView<>(array, makeView); +} + +} // namespace + +ArrayViewVariant make_view_variant(Array& array) { + return makeViewVariantImpl(array); +} + +ConstArrayViewVariant make_view_variant(const Array& array) { + return makeViewVariantImpl(array); +} + +ArrayViewVariant make_host_view_variant(Array& array) { + return makeHostViewVariantImpl(array); +} + +ConstArrayViewVariant make_host_view_variant(const Array& array) { + return makeHostViewVariantImpl(array); +} + +ArrayViewVariant make_device_view_variant(Array& array) { + return makeDeviceViewVariantImpl(array); +} + +ConstArrayViewVariant make_device_view_variant(const Array& array) { + return makeDeviceViewVariantImpl(array); +} + +} // namespace array +} // namespace atlas diff --git a/src/atlas/array/ArrayViewVariant.h b/src/atlas/array/ArrayViewVariant.h new file mode 100644 index 000000000..94dff8115 --- /dev/null +++ b/src/atlas/array/ArrayViewVariant.h @@ -0,0 +1,103 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include + +#include "atlas/array.h" + +namespace atlas { +namespace array { + +namespace detail { + +using namespace array; + +// Container struct for a list of types. +template +struct Types { + using add_const = Types...>; +}; + +// Container struct for a list of integers. +template +struct Ints {}; + +template +struct VariantHelper; + +// Recursively construct ArrayView std::variant from types Ts and ranks Is. +template +struct VariantHelper, Ints, ArrayViews...> { + using type = typename VariantHelper, Ints, ArrayViews..., + ArrayView...>::type; +}; + +// End recursion. +template +struct VariantHelper, Ints, ArrayViews...> { + using type = std::variant; +}; + +template +using Variant = typename VariantHelper::type; + +using VariantValueTypes = + detail::Types; + +using VariantRanks = detail::Ints<1, 2, 3, 4, 5, 6, 7, 8, 9>; + +} // namespace detail + +/// @brief Variant containing all supported non-const ArrayView alternatives. +using ArrayViewVariant = + detail::Variant; + +/// @brief Variant containing all supported const ArrayView alternatives. +using ConstArrayViewVariant = + detail::Variant; + +/// @brief Create an ArrayView and assign to an ArrayViewVariant. +ArrayViewVariant make_view_variant(Array& array); + +/// @brief Create a const ArrayView and assign to an ArrayViewVariant. +ConstArrayViewVariant make_view_variant(const Array& array); + +/// @brief Create a host ArrayView and assign to an ArrayViewVariant. +ArrayViewVariant make_host_view_variant(Array& array); + +/// @brief Create a const host ArrayView and assign to an ArrayViewVariant. +ConstArrayViewVariant make_host_view_variant(const Array& array); + +/// @brief Create a device ArrayView and assign to an ArrayViewVariant. +ArrayViewVariant make_device_view_variant(Array& array); + +/// @brief Create a const device ArrayView and assign to an ArrayViewVariant. +ConstArrayViewVariant make_device_view_variant(const Array& array); + +/// @brief Return true if View::rank() is any of Ranks... +template +constexpr bool is_rank(const View&) { + return ((std::decay_t::rank() == Ranks) || ...); +} +/// @brief Return true if View::value_type is any of ValuesTypes... +template +constexpr bool is_value_type(const View&) { + using ValueType = typename std::decay_t::value_type; + return ((std::is_same_v) || ...); +} + +/// @brief Return true if View::non_const_value_type is any of ValuesTypes... +template +constexpr bool is_non_const_value_type(const View&) { + using ValueType = typename std::decay_t::non_const_value_type; + return ((std::is_same_v) || ...); +} + +} // namespace array +} // namespace atlas diff --git a/src/tests/array/CMakeLists.txt b/src/tests/array/CMakeLists.txt index 3915caca7..880e46ba6 100644 --- a/src/tests/array/CMakeLists.txt +++ b/src/tests/array/CMakeLists.txt @@ -81,3 +81,8 @@ atlas_add_hic_test( ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_array_view_variant + SOURCES test_array_view_variant.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) diff --git a/src/tests/array/test_array_view_variant.cc b/src/tests/array/test_array_view_variant.cc new file mode 100644 index 000000000..f6eaeeacf --- /dev/null +++ b/src/tests/array/test_array_view_variant.cc @@ -0,0 +1,131 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include +#include + +#include "atlas/array.h" +#include "atlas/array/ArrayViewVariant.h" +#include "eckit/utils/Overloaded.h" +#include "tests/AtlasTestEnvironment.h" + +namespace atlas { +namespace test { + +using namespace array; + +CASE("test variant assignment") { + auto array1 = array::ArrayT(2); + auto array2 = array::ArrayT(2, 3); + auto array3 = array::ArrayT(2, 3, 4); + const auto& arrayRef = array1; + + array1.allocateDevice(); + array2.allocateDevice(); + array3.allocateDevice(); + + auto view1 = make_view_variant(array1); + auto view2 = make_view_variant(array2); + auto view3 = make_view_variant(array3); + auto view4 = make_view_variant(arrayRef); + + const auto hostView1 = make_host_view_variant(array1); + const auto hostView2 = make_host_view_variant(array2); + const auto hostView3 = make_host_view_variant(array3); + const auto hostView4 = make_host_view_variant(arrayRef); + + auto deviceView1 = make_device_view_variant(array1); + auto deviceView2 = make_device_view_variant(array2); + auto deviceView3 = make_device_view_variant(array3); + auto deviceView4 = make_device_view_variant(arrayRef); + + const auto visitVariants = [](auto& var1, auto& var2, auto var3, auto var4) { + std::visit( + [](auto view) { + EXPECT((is_rank<1>(view))); + EXPECT((is_value_type(view))); + EXPECT((is_non_const_value_type(view))); + }, + var1); + + std::visit( + [](auto view) { + EXPECT((is_rank<2>(view))); + EXPECT((is_value_type(view))); + EXPECT((is_non_const_value_type(view))); + }, + var2); + + std::visit( + [](auto view) { + EXPECT((is_rank<3>(view))); + EXPECT((is_value_type(view))); + EXPECT((is_non_const_value_type(view))); + }, + var3); + + std::visit( + [](auto view) { + EXPECT((is_rank<1>(view))); + EXPECT((is_value_type(view))); + EXPECT((is_non_const_value_type(view))); + }, + var4); + }; + + visitVariants(view1, view2, view3, view4); + visitVariants(hostView1, hostView2, hostView3, hostView4); + visitVariants(deviceView1, deviceView2, deviceView3, deviceView4); +} + +CASE("test std::visit") { + auto array1 = ArrayT(10); + make_view(array1).assign({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + auto array2 = ArrayT(5, 2); + make_view(array2).assign({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + const auto var1 = make_view_variant(array1); + const auto var2 = make_view_variant(array2); + auto rank1Tested = false; + auto rank2Tested = false; + + const auto visitor = [&](auto view) { + if constexpr (is_rank<1>(view)) { + EXPECT((is_value_type(view))); + auto testValue = int{0}; + for (auto i = size_t{0}; i < view.size(); ++i) { + const auto value = view(i); + EXPECT_EQ(value, static_cast(testValue++)); + } + rank1Tested = true; + } else if constexpr (is_rank<2>(view)) { + EXPECT((is_value_type(view))); + auto testValue = int{0}; + for (auto i = idx_t{0}; i < view.shape(0); ++i) { + for (auto j = idx_t{0}; j < view.shape(1); ++j) { + const auto value = view(i, j); + EXPECT_EQ(value, static_cast(testValue++)); + } + } + rank2Tested = true; + } else { + // Test should not reach here. + EXPECT(false); + } + }; + + std::visit(visitor, var1); + EXPECT(rank1Tested); + std::visit(visitor, var2); + EXPECT(rank2Tested); +} + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { return atlas::test::run(argc, argv); } From 84ec3e1c8c5089929566b30d38861838052f7572 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 8 Oct 2024 09:39:34 +0000 Subject: [PATCH 19/40] Fix CMake for ALIAS target transi when CMake version >= 3.18 --- cmake/features/ECTRANS.cmake | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmake/features/ECTRANS.cmake b/cmake/features/ECTRANS.cmake index 3d8ab3f08..20c893852 100644 --- a/cmake/features/ECTRANS.cmake +++ b/cmake/features/ECTRANS.cmake @@ -20,9 +20,13 @@ if( atlas_HAVE_ATLAS_FUNCTIONSPACE AND (ENABLE_ECTRANS OR NOT DEFINED ENABLE_ECT if( TARGET transi_dp ) set( transi_FOUND TRUE ) if( NOT TARGET transi ) - get_target_property( transi_dp_IMPORTED transi_dp IMPORTED ) - if( transi_dp_IMPORTED ) - set_target_properties( transi_dp PROPERTIES IMPORTED_GLOBAL TRUE) # required for aliasing imports + if( CMAKE_VERSION VERSION_LESS 3.18 ) + # Before CMake 3.18 it is not possible to alias a non-global imported target + # Make the import global. Warning, this may break further find_package + get_target_property( transi_dp_IMPORTED transi_dp IMPORTED ) + if( transi_dp_IMPORTED ) + set_target_properties( transi_dp PROPERTIES IMPORTED_GLOBAL TRUE) + endif() endif() add_library( transi ALIAS transi_dp ) endif() From df6290ead04f1c752862262978a9e74f78714faa Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:18:52 +0200 Subject: [PATCH 20/40] Bugfix for Qhull (#230) Co-authored-by: Willem Deconinck --- cmake/atlas-import.cmake.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/atlas-import.cmake.in b/cmake/atlas-import.cmake.in index 1c812e4de..778684a16 100644 --- a/cmake/atlas-import.cmake.in +++ b/cmake/atlas-import.cmake.in @@ -75,9 +75,9 @@ if( atlas_HAVE_ECTRANS AND atlas_REQUIRES_PRIVATE_DEPENDENCIES ) endif() endif() -## CGAL +## Qhull if( atlas_HAVE_TESSELATION AND atlas_REQUIRES_PRIVATE_DEPENDENCIES ) - find_dependency( CGAL HINTS @CGAL_DIR@ ) + find_dependency( Qhull HINTS @Qhull_DIR@ ) endif() ## Fortran From f1c0c1765012655aab0b16ade5299c8b4e9b1c04 Mon Sep 17 00:00:00 2001 From: Oliver Lomax Date: Thu, 10 Oct 2024 14:20:12 +0100 Subject: [PATCH 21/40] Tidied headers and error message. (#231) --- src/atlas/array/ArrayViewVariant.cc | 9 +++++---- src/atlas/array/ArrayViewVariant.h | 7 ++++--- src/atlas/runtime/Exception.h | 1 + 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/atlas/array/ArrayViewVariant.cc b/src/atlas/array/ArrayViewVariant.cc index f62efd2d8..cd0a2ad65 100644 --- a/src/atlas/array/ArrayViewVariant.cc +++ b/src/atlas/array/ArrayViewVariant.cc @@ -10,6 +10,8 @@ #include #include +#include "atlas/array/Array.h" +#include "atlas/array/MakeView.h" #include "atlas/runtime/Exception.h" namespace atlas { @@ -49,10 +51,9 @@ VariantType executeMakeView(ArrayType& array, if constexpr (TypeIndex < std::variant_size_v> - 1) { return executeMakeView(array, makeView); } else { - throw_Exception("ArrayView<" + array.datatype().str() + ", " + - std::to_string(array.rank()) + - "> is not an alternative in ArrayViewVariant.", - Here()); + ATLAS_THROW_EXCEPTION("Array with rank = " + std::to_string(array.rank()) + + " and datatype = " + array.datatype().str() + + " is not supported."); } } diff --git a/src/atlas/array/ArrayViewVariant.h b/src/atlas/array/ArrayViewVariant.h index 94dff8115..1193e9e59 100644 --- a/src/atlas/array/ArrayViewVariant.h +++ b/src/atlas/array/ArrayViewVariant.h @@ -7,17 +7,16 @@ #pragma once +#include #include -#include "atlas/array.h" +#include "atlas/array/ArrayView.h" namespace atlas { namespace array { namespace detail { -using namespace array; - // Container struct for a list of types. template struct Types { @@ -54,6 +53,8 @@ using VariantRanks = detail::Ints<1, 2, 3, 4, 5, 6, 7, 8, 9>; } // namespace detail +class Array; + /// @brief Variant containing all supported non-const ArrayView alternatives. using ArrayViewVariant = detail::Variant; diff --git a/src/atlas/runtime/Exception.h b/src/atlas/runtime/Exception.h index 5220c835a..194180b26 100644 --- a/src/atlas/runtime/Exception.h +++ b/src/atlas/runtime/Exception.h @@ -11,6 +11,7 @@ #pragma once #include +#include #include "eckit/log/CodeLocation.h" From 10cffa6e00d68d86651702e5faa64b91bdd3ca53 Mon Sep 17 00:00:00 2001 From: Oliver Lomax Date: Fri, 11 Oct 2024 15:26:09 +0100 Subject: [PATCH 22/40] Fixed ordering of fixup_halos and halo_exhange in StructuredColumns.cc (#223) --- src/atlas/functionspace/detail/StructuredColumns.cc | 8 ++++---- .../interpolation/test_interpolation_spherical_vector.cc | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/atlas/functionspace/detail/StructuredColumns.cc b/src/atlas/functionspace/detail/StructuredColumns.cc index 54b215feb..6c309b032 100644 --- a/src/atlas/functionspace/detail/StructuredColumns.cc +++ b/src/atlas/functionspace/detail/StructuredColumns.cc @@ -830,20 +830,20 @@ void dispatch_adjointHaloExchange(Field& field, const parallel::HaloExchange& ha const StructuredColumns& fs) { FixupHaloForVectors fixup_halos(fs); if (field.datatype() == array::DataType::kind()) { - halo_exchange.template execute_adjoint(field.array(), false); fixup_halos.template apply(field); + halo_exchange.template execute_adjoint(field.array(), false); } else if (field.datatype() == array::DataType::kind()) { - halo_exchange.template execute_adjoint(field.array(), false); fixup_halos.template apply(field); + halo_exchange.template execute_adjoint(field.array(), false); } else if (field.datatype() == array::DataType::kind()) { - halo_exchange.template execute_adjoint(field.array(), false); fixup_halos.template apply(field); + halo_exchange.template execute_adjoint(field.array(), false); } else if (field.datatype() == array::DataType::kind()) { - halo_exchange.template execute_adjoint(field.array(), false); fixup_halos.template apply(field); + halo_exchange.template execute_adjoint(field.array(), false); } else { throw_Exception("datatype not supported", Here()); diff --git a/src/tests/interpolation/test_interpolation_spherical_vector.cc b/src/tests/interpolation/test_interpolation_spherical_vector.cc index 16dbdabf9..fe48b47dd 100644 --- a/src/tests/interpolation/test_interpolation_spherical_vector.cc +++ b/src/tests/interpolation/test_interpolation_spherical_vector.cc @@ -290,7 +290,6 @@ void testInterpolation(const Config& config) { auto sourceAdjointView = array::make_view(sourceAdjoint); sourceAdjointView.assign(0.); - sourceAdjoint.set_dirty(false); interp.execute_adjoint(sourceAdjoint, targetAdjoint); // Check fields for nans or +/-inf From 96edef9dfbc025c86b88b9afe54f720052718d48 Mon Sep 17 00:00:00 2001 From: Oliver Lomax Date: Fri, 11 Oct 2024 20:43:27 +0100 Subject: [PATCH 23/40] Integrate `pack_vector_fields` into `SphericalVector` Interpolation method. (#224) --- .../method/sphericalvector/SphericalVector.cc | 32 ++++--- src/atlas/util/PackVectorFields.cc | 10 ++ .../test_interpolation_spherical_vector.cc | 91 +++++++++++++++++++ 3 files changed, 122 insertions(+), 11 deletions(-) diff --git a/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc b/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc index ab5f573d8..65e4e41d0 100644 --- a/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc +++ b/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc @@ -21,6 +21,7 @@ #include "atlas/runtime/Trace.h" #include "atlas/util/Constants.h" #include "atlas/util/Geometry.h" +#include "atlas/util/PackVectorFields.h" #include "eckit/config/LocalConfiguration.h" namespace atlas { @@ -95,10 +96,9 @@ void SphericalVector::do_setup(const FunctionSpace& source, const auto deltaAlpha = (alpha.first - alpha.second) * util::Constants::degreesToRadians(); - complexTriplets[dataIndex] = - ComplexTriplet{rowIndex, colIndex, - Complex{baseWeight * std::cos(deltaAlpha), - baseWeight * std::sin(deltaAlpha)}}; + complexTriplets[dataIndex] = ComplexTriplet{ + rowIndex, colIndex, + baseWeight * Complex{std::cos(deltaAlpha), std::sin(deltaAlpha)}}; realTriplets[dataIndex] = RealTriplet{rowIndex, colIndex, baseWeight}; } } @@ -120,9 +120,14 @@ void SphericalVector::do_execute(const FieldSet& sourceFieldSet, ATLAS_TRACE("atlas::interpolation::method::SphericalVector::do_execute()"); ATLAS_ASSERT(sourceFieldSet.size() == targetFieldSet.size()); - for (auto i = 0; i < sourceFieldSet.size(); ++i) { - do_execute(sourceFieldSet[i], targetFieldSet[i], metadata); + const auto packedSourceFieldSet = util::pack_vector_fields(sourceFieldSet); + auto packedTargetFieldSet = util::pack_vector_fields(targetFieldSet); + + for (auto i = 0; i < packedSourceFieldSet.size(); ++i) { + do_execute(packedSourceFieldSet[i], packedTargetFieldSet[i], metadata); } + + util::unpack_vector_fields(packedTargetFieldSet, targetFieldSet); } void SphericalVector::do_execute(const Field& sourceField, Field& targetField, @@ -130,7 +135,7 @@ void SphericalVector::do_execute(const Field& sourceField, Field& targetField, ATLAS_TRACE("atlas::interpolation::method::SphericalVector::do_execute()"); if (targetField.size() == 0) { - return; + return; } const auto fieldType = sourceField.metadata().getString("type", ""); @@ -156,9 +161,15 @@ void SphericalVector::do_execute_adjoint(FieldSet& sourceFieldSet, "atlas::interpolation::method::SphericalVector::do_execute_adjoint()"); ATLAS_ASSERT(sourceFieldSet.size() == targetFieldSet.size()); - for (auto i = 0; i < sourceFieldSet.size(); ++i) { - do_execute_adjoint(sourceFieldSet[i], targetFieldSet[i], metadata); + auto packedSourceFieldSet = util::pack_vector_fields(sourceFieldSet); + const auto packedTargetFieldSet = util::pack_vector_fields(targetFieldSet); + + for (auto i = 0; i < packedSourceFieldSet.size(); ++i) { + do_execute_adjoint(packedSourceFieldSet[i], packedTargetFieldSet[i], + metadata); } + + util::unpack_vector_fields(packedSourceFieldSet, sourceFieldSet); } void SphericalVector::do_execute_adjoint(Field& sourceField, @@ -168,7 +179,7 @@ void SphericalVector::do_execute_adjoint(Field& sourceField, "atlas::interpolation::method::SphericalVector::do_execute_adjoint()"); if (targetField.size() == 0) { - return; + return; } const auto fieldType = sourceField.metadata().getString("type", ""); @@ -192,7 +203,6 @@ template void SphericalVector::interpolate_vector_field(const Field& sourceField, Field& targetField, const MatMul& matMul) { - ATLAS_ASSERT_MSG(sourceField.variables() == 2 || sourceField.variables() == 3, "Vector field can only have 2 or 3 components."); diff --git a/src/atlas/util/PackVectorFields.cc b/src/atlas/util/PackVectorFields.cc index 640a1d46b..2734a1031 100644 --- a/src/atlas/util/PackVectorFields.cc +++ b/src/atlas/util/PackVectorFields.cc @@ -180,6 +180,15 @@ FieldSet pack_vector_fields(const FieldSet& fields, FieldSet packedFields) { componentFieldMetadataVector.push_back(componentFieldMetadata); vectorField.metadata().set("component_field_metadata", componentFieldMetadataVector); + + // If any component is dirty, the whole field is dirty. + if (vectorIndex == 0) { + vectorField.set_dirty(componentField.dirty()); + } else { + vectorField.set_dirty(vectorField.dirty() || componentField.dirty()); + } + + } return packedFields; } @@ -218,6 +227,7 @@ FieldSet unpack_vector_fields(const FieldSet& fields, FieldSet unpackedFields) { // Copy metadata. componentField.metadata() = componentFieldMetadata; + componentField.set_dirty(vectorField.dirty()); ++vectorIndex; } diff --git a/src/tests/interpolation/test_interpolation_spherical_vector.cc b/src/tests/interpolation/test_interpolation_spherical_vector.cc index fe48b47dd..398249d27 100644 --- a/src/tests/interpolation/test_interpolation_spherical_vector.cc +++ b/src/tests/interpolation/test_interpolation_spherical_vector.cc @@ -464,6 +464,97 @@ CASE("structured columns O96 vector interpolation (2d-field, 2-vector, hi-res)") testInterpolation((config)); } +CASE("separate vector field components") { + const auto sourceFunctionSpace = + FunctionSpaceFixtures::get("structured_columns"); + const auto targetFunctionSpace = + FunctionSpaceFixtures::get("cubedsphere_mesh"); + + auto sourceFieldSet = FieldSet{}; + auto targetFieldSet = FieldSet{}; + + const auto sourceLonLatView = + array::make_view(sourceFunctionSpace.lonlat()); + const auto targetLonLatView = + array::make_view(targetFunctionSpace.lonlat()); + + const auto createFieldView = [&](const FunctionSpace& functionSpace, + const std::string& name, + FieldSet& fieldSet) { + // Note: Vector field name can be anything that uniquely identifies field. + auto field = functionSpace.createField(option::name(name)); + field.metadata().set("vector_field_name", "wind"); + return array::make_view(fieldSet.add(field)); + }; + + auto uSourceView = createFieldView(sourceFunctionSpace, "u", sourceFieldSet); + auto vSourceView = createFieldView(sourceFunctionSpace, "v", sourceFieldSet); + const auto uTargetView = + createFieldView(targetFunctionSpace, "u", targetFieldSet); + const auto vTargetView = + createFieldView(targetFunctionSpace, "v", targetFieldSet); + + uSourceView.assign(0.); + vSourceView.assign(0.); + for (auto idx = idx_t{0}; idx < sourceFunctionSpace.size(); idx++) { + std::tie(uSourceView(idx), vSourceView(idx)) = + vortexHorizontal(sourceLonLatView(idx, 0), sourceLonLatView(idx, 1)); + } + + const auto interpScheme = + InterpSchemeFixtures::get("structured_linear_spherical"); + + const auto interp = + Interpolation(interpScheme, sourceFunctionSpace, targetFunctionSpace); + + interp.execute(sourceFieldSet, targetFieldSet); + targetFieldSet.haloExchange(); + + auto errorView = + createFieldView(targetFunctionSpace, "error", targetFieldSet); + + auto maxError = 0.; + for (auto idx = idx_t{0}; idx < targetFunctionSpace.size(); idx++) { + auto [uTrue, vTrue] = + vortexHorizontal(targetLonLatView(idx, 0), targetLonLatView(idx, 1)); + errorView(idx) = + std::hypot(uTrue - uTargetView(idx), vTrue - vTargetView(idx)); + maxError = std::max(maxError, errorView(idx)); + } + EXPECT_APPROX_EQ(maxError, 0., 0.00017); + + gmshOutput("vector_components_source.msh", sourceFieldSet); + gmshOutput("vector_components_target.msh", targetFieldSet); + + auto sourceAdjointFieldSet = FieldSet{}; + auto targetAdjointFieldSet = FieldSet{}; + + targetAdjointFieldSet.add(targetFieldSet["u"].clone()); + targetAdjointFieldSet.add(targetFieldSet["v"].clone()); + + targetAdjointFieldSet.adjointHaloExchange(); + + auto uSourceAdjointView = + createFieldView(sourceFunctionSpace, "u", sourceAdjointFieldSet); + auto vSourceAdjointView = + createFieldView(sourceFunctionSpace, "v", sourceAdjointFieldSet); + uSourceAdjointView.assign(0.); + vSourceAdjointView.assign(0.); + + // sourceAdjointFieldSet.set_dirty(false); + interp.execute_adjoint(sourceAdjointFieldSet, targetAdjointFieldSet); + + constexpr auto tinyNum = 1e-13; + const auto targetDotTarget = dotProduct(uTargetView, uTargetView) + + dotProduct(vTargetView, vTargetView); + const auto sourceDotSourceAdjoint = + dotProduct(uSourceView, uSourceAdjointView) + + dotProduct(vSourceView, vSourceAdjointView); + + const auto dotProdRatio = targetDotTarget / sourceDotSourceAdjoint; + EXPECT_APPROX_EQ(dotProdRatio, 1., tinyNum); +} + } // namespace test } // namespace atlas From 591a762dffe6627db2841512b21beebda7686d0d Mon Sep 17 00:00:00 2001 From: Oliver Lomax Date: Mon, 14 Oct 2024 15:10:29 +0100 Subject: [PATCH 24/40] Refactored (un)pack_vector_fields. (#226) --- src/atlas/util/PackVectorFields.cc | 59 ++++++++---------------------- 1 file changed, 16 insertions(+), 43 deletions(-) diff --git a/src/atlas/util/PackVectorFields.cc b/src/atlas/util/PackVectorFields.cc index 2734a1031..9fefb5b37 100644 --- a/src/atlas/util/PackVectorFields.cc +++ b/src/atlas/util/PackVectorFields.cc @@ -26,8 +26,6 @@ namespace util { namespace { using eckit::LocalConfiguration; - -using array::DataType; using array::helpers::arrayForEachDim; void addOrReplaceField(FieldSet& fieldSet, const Field& field) { @@ -84,51 +82,29 @@ void checkFieldCompatibility(const Field& componentField, template void copyFieldData(ComponentField& componentField, VectorField& vectorField, - const Functor& copier) { + const Functor& copier) { checkFieldCompatibility(componentField, vectorField); - const auto copyArrayData = [&](auto value, auto rank) { - // Resolve value-type and rank from arguments. - using Value = decltype(value); - constexpr auto Rank = decltype(rank)::value; - - // Iterate over fields. - auto vectorView = array::make_view(vectorField); - auto componentView = array::make_view(componentField); - constexpr auto Dims = std::make_integer_sequence{}; - arrayForEachDim(Dims, execution::par, std::tie(componentView, vectorView), - copier); - }; + auto componentViewVariant = array::make_view_variant(componentField); - const auto selectRank = [&](auto value) { - switch (vectorField.rank()) { - case 2: - return copyArrayData(value, std::integral_constant{}); - case 3: - return copyArrayData(value, std::integral_constant{}); - default: - ATLAS_THROW_EXCEPTION("Unsupported vector field rank: " + - std::to_string(vectorField.rank())); - } - }; + const auto componentVisitor = [&](auto componentView) { + if constexpr (array::is_rank<1, 2>(componentView)) { + using ComponentView = std::decay_t; + constexpr auto ComponentRank = ComponentView::rank(); + using Value = typename ComponentView::non_const_value_type; + + auto vectorView = array::make_view(vectorField); + constexpr auto Dims = std::make_integer_sequence{}; + arrayForEachDim(Dims, execution::par, std::tie(componentView, vectorView), + copier); - const auto selectType = [&]() { - switch (vectorField.datatype().kind()) { - case DataType::kind(): - return selectRank(double{}); - case DataType::kind(): - return selectRank(float{}); - case DataType::kind(): - return selectRank(long{}); - case DataType::kind(): - return selectRank(int{}); - default: - ATLAS_THROW_EXCEPTION("Unknown datatype: " + - std::to_string(vectorField.datatype().kind())); + } else { + ATLAS_THROW_EXCEPTION("Unsupported component field rank: " + + std::to_string(componentView.rank())); } }; - selectType(); + std::visit(componentVisitor, componentViewVariant); } } // namespace @@ -187,8 +163,6 @@ FieldSet pack_vector_fields(const FieldSet& fields, FieldSet packedFields) { } else { vectorField.set_dirty(vectorField.dirty() || componentField.dirty()); } - - } return packedFields; } @@ -208,7 +182,6 @@ FieldSet unpack_vector_fields(const FieldSet& fields, FieldSet unpackedFields) { auto vectorIndex = 0; for (const auto& componentFieldMetadata : componentFieldMetadataVector) { - // Get or create field. auto componentFieldName = std::string{}; componentFieldMetadata.get("name", componentFieldName); From 6a87b21585caa718a6188ba3158de7c61dfb8f6e Mon Sep 17 00:00:00 2001 From: Oliver Lomax Date: Mon, 14 Oct 2024 21:30:11 +0100 Subject: [PATCH 25/40] Refactored spherical vector interpolation method. (#227) --- .../method/sphericalvector/SphericalVector.cc | 84 ++++++++----------- 1 file changed, 34 insertions(+), 50 deletions(-) diff --git a/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc b/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc index 65e4e41d0..8bcd0bef1 100644 --- a/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc +++ b/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc @@ -5,6 +5,9 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ +#include +#include + #include "atlas/interpolation/method/sphericalvector/SphericalVector.h" #include "atlas/array/ArrayView.h" @@ -203,59 +206,40 @@ template void SphericalVector::interpolate_vector_field(const Field& sourceField, Field& targetField, const MatMul& matMul) { - ATLAS_ASSERT_MSG(sourceField.variables() == 2 || sourceField.variables() == 3, - "Vector field can only have 2 or 3 components."); - - if (sourceField.datatype().kind() == array::DataType::KIND_REAL64) { - interpolate_vector_field(sourceField, targetField, matMul); - return; - } - - if (sourceField.datatype().kind() == array::DataType::KIND_REAL32) { - interpolate_vector_field(sourceField, targetField, matMul); - return; - } + const auto sourceViewVariant = array::make_view_variant(sourceField); + + const auto sourceViewVisitor = [&](auto sourceView) { + if constexpr (array::is_rank<2, 3>(sourceView) && + array::is_non_const_value_type(sourceView)) { + using SourceView = std::decay_t; + using Value = typename SourceView::non_const_value_type; + constexpr auto Rank = SourceView::rank(); + auto targetView = array::make_view(targetField); + + switch (sourceField.variables()) { + case 2: + return matMul.apply(sourceView, targetView, twoVector); + case 3: + return matMul.apply(sourceView, targetView, threeVector); + default: + ATLAS_THROW_EXCEPTION("Error: no support for " + + std::to_string(sourceField.variables()) + + " variable vector fields.\n" + + " Number of variables must be 2 or 3."); + } + + } else { + ATLAS_THROW_EXCEPTION( + "Error: no support for rank = " + std::to_string(sourceField.rank()) + + " and value type = " + sourceField.datatype().str() + ".\n" + + "Vector field must have rank 2 or 3 with value type " + "float or double"); + } + }; - ATLAS_NOTIMPLEMENTED; + std::visit(sourceViewVisitor, sourceViewVariant); }; -template -void SphericalVector::interpolate_vector_field(const Field& sourceField, - Field& targetField, - const MatMul& matMul) { - if (sourceField.rank() == 2) { - interpolate_vector_field(sourceField, targetField, matMul); - return; - } - - if (sourceField.rank() == 3) { - interpolate_vector_field(sourceField, targetField, matMul); - return; - } - - ATLAS_NOTIMPLEMENTED; -} - -template -void SphericalVector::interpolate_vector_field(const Field& sourceField, - Field& targetField, - const MatMul& matMul) { - const auto sourceView = array::make_view(sourceField); - auto targetView = array::make_view(targetField); - - if (sourceField.variables() == 2) { - matMul.apply(sourceView, targetView, twoVector); - return; - } - - if (sourceField.variables() == 3) { - matMul.apply(sourceView, targetView, threeVector); - return; - } - - ATLAS_NOTIMPLEMENTED; -} - } // namespace method } // namespace interpolation } // namespace atlas From 33260ca02641919769f3df022911014dccb46c6d Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Mon, 14 Oct 2024 09:15:10 +0200 Subject: [PATCH 26/40] Add device_data Fortran access to Field Co-authored-by: Willem Deconinck --- src/atlas/array/Array.h | 4 ++++ src/atlas/array/ArraySpec.cc | 26 ++++++++++++++++++++- src/atlas/array/ArraySpec.h | 4 ++++ src/atlas/array/native/NativeMakeView.cc | 4 ++-- src/atlas/field/detail/FieldImpl.h | 3 +++ src/atlas/field/detail/FieldInterface.cc | 28 +++++++++++++++++++++++ src/atlas/field/detail/FieldInterface.h | 8 +++++++ src/atlas_f/field/atlas_Field_module.fypp | 23 +++++++++++++++++++ src/tests/field/fctest_field_gpu.F90 | 1 + 9 files changed, 98 insertions(+), 3 deletions(-) diff --git a/src/atlas/array/Array.h b/src/atlas/array/Array.h index 1e3958ee0..99bff9fac 100644 --- a/src/atlas/array/Array.h +++ b/src/atlas/array/Array.h @@ -85,12 +85,16 @@ class Array : public util::Object { const ArrayStrides& strides() const { return spec_.strides(); } + const ArrayStrides& device_strides() const { return spec_.device_strides(); } + const ArrayShape& shape() const { return spec_.shape(); } const std::vector& shapef() const { return spec_.shapef(); } const std::vector& stridesf() const { return spec_.stridesf(); } + const std::vector& device_stridesf() const { return spec_.device_stridesf(); } + bool contiguous() const { return spec_.contiguous(); } bool hasDefaultLayout() const { return spec_.hasDefaultLayout(); } diff --git a/src/atlas/array/ArraySpec.cc b/src/atlas/array/ArraySpec.cc index 999d4a982..4f27b8c4f 100644 --- a/src/atlas/array/ArraySpec.cc +++ b/src/atlas/array/ArraySpec.cc @@ -47,12 +47,18 @@ ArraySpec::ArraySpec(const ArrayShape& shape, const ArrayAlignment& alignment): shape_.resize(rank_); strides_.resize(rank_); layout_.resize(rank_); + device_strides_.resize(rank_); + device_strides_[rank_ - 1] = 1; for (int j = rank_ - 1; j >= 0; --j) { shape_[j] = shape[j]; strides_[j] = allocated_size_; layout_[j] = j; size_ *= size_t(shape_[j]); allocated_size_ *= size_t(aligned_shape[j]); + if( j < rank_ - 1) { + // Assume contiguous device data! + device_strides_[j] = strides_[j+1] * shape[j+1]; + } } ATLAS_ASSERT(allocated_size_ == compute_aligned_size(size_t(shape_[0]) * size_t(strides_[0]), size_t(alignment))); contiguous_ = (size_ == allocated_size_); @@ -81,11 +87,17 @@ ArraySpec::ArraySpec(const ArrayShape& shape, const ArrayStrides& strides, const shape_.resize(rank_); strides_.resize(rank_); layout_.resize(rank_); + device_strides_.resize(rank_); + device_strides_[rank_ - 1] = strides[rank_ - 1]; for (int j = rank_ - 1; j >= 0; --j) { shape_[j] = shape[j]; strides_[j] = strides[j]; layout_[j] = j; size_ *= size_t(shape_[j]); + if( j < rank_ - 1) { + // Assume contiguous device data! + device_strides_[j] = device_strides_[j+1] * shape[j+1]; + } } allocated_size_ = compute_aligned_size(size_t(shape_[0]) * size_t(strides_[0]), size_t(alignment)); contiguous_ = (size_ == allocated_size_); @@ -121,6 +133,8 @@ ArraySpec::ArraySpec(const ArrayShape& shape, const ArrayStrides& strides, const shape_.resize(rank_); strides_.resize(rank_); layout_.resize(rank_); + device_strides_.resize(rank_); + device_strides_[rank_ - 1] = strides[rank_ - 1]; default_layout_ = true; for (int j = rank_ - 1; j >= 0; --j) { shape_[j] = shape[j]; @@ -130,6 +144,10 @@ ArraySpec::ArraySpec(const ArrayShape& shape, const ArrayStrides& strides, const if (layout_[j] != idx_t(j)) { default_layout_ = false; } + if( j < rank_ - 1) { + // Assume contiguous device data! + device_strides_[j] = device_strides_[j+1] * shape[j+1]; + } } allocated_size_ = compute_aligned_size(size_t(shape_[layout_[0]]) * size_t(strides_[layout_[0]]), size_t(alignment)); contiguous_ = (size_ == allocated_size_); @@ -152,12 +170,18 @@ const std::vector& ArraySpec::stridesf() const { return stridesf_; } +const std::vector& ArraySpec::device_stridesf() const { + return device_stridesf_; +} + void ArraySpec::allocate_fortran_specs() { shapef_.resize(rank_); stridesf_.resize(rank_); + device_stridesf_.resize(rank_); for (idx_t j = 0; j < rank_; ++j) { shapef_[j] = shape_[rank_ - 1 - layout_[j]]; - stridesf_[j] = strides_[rank_ -1 - layout_[j]]; + stridesf_[j] = strides_[rank_ - 1 - layout_[j]]; + device_stridesf_[j] = device_strides_[rank_ - 1 - j]; } } diff --git a/src/atlas/array/ArraySpec.h b/src/atlas/array/ArraySpec.h index 55c9330c5..74b423689 100644 --- a/src/atlas/array/ArraySpec.h +++ b/src/atlas/array/ArraySpec.h @@ -33,10 +33,12 @@ class ArraySpec { DataType datatype_; ArrayShape shape_; ArrayStrides strides_; + ArrayStrides device_strides_; ArrayLayout layout_; ArrayAlignment alignment_; std::vector shapef_; std::vector stridesf_; + std::vector device_stridesf_; bool contiguous_; bool default_layout_; @@ -61,9 +63,11 @@ class ArraySpec { const ArrayShape& shape() const { return shape_; } const ArrayAlignment& alignment() const { return alignment_; } const ArrayStrides& strides() const { return strides_; } + const ArrayStrides& device_strides() const { return device_strides_; } const ArrayLayout& layout() const { return layout_; } const std::vector& shapef() const; const std::vector& stridesf() const; + const std::vector& device_stridesf() const; bool contiguous() const { return contiguous_; } bool hasDefaultLayout() const { return default_layout_; } diff --git a/src/atlas/array/native/NativeMakeView.cc b/src/atlas/array/native/NativeMakeView.cc index 5c24c0037..cdbcd27dc 100644 --- a/src/atlas/array/native/NativeMakeView.cc +++ b/src/atlas/array/native/NativeMakeView.cc @@ -51,7 +51,7 @@ template ArrayView make_device_view(Array& array) { #if ATLAS_HAVE_GPU ATLAS_ASSERT(array.deviceAllocated(),"make_device_view: Array not allocated on device"); - return ArrayView((array.device_data()), array.shape(), array.strides()); + return ArrayView((array.device_data()), array.shape(), array.device_strides()); #else return make_host_view(array); #endif @@ -61,7 +61,7 @@ template ArrayView make_device_view(const Array& array) { #if ATLAS_HAVE_GPU ATLAS_ASSERT(array.deviceAllocated(),"make_device_view: Array not allocated on device"); - return ArrayView(array.device_data(), array.shape(), array.strides()); + return ArrayView(array.device_data(), array.shape(), array.device_strides()); #else return make_host_view(array); #endif diff --git a/src/atlas/field/detail/FieldImpl.h b/src/atlas/field/detail/FieldImpl.h index 24af72d2c..d2ed04e37 100644 --- a/src/atlas/field/detail/FieldImpl.h +++ b/src/atlas/field/detail/FieldImpl.h @@ -121,6 +121,9 @@ class FieldImpl : public util::Object { /// @brief Strides of this field in Fortran style (reverse order of C style) const std::vector& stridesf() const { return array_->stridesf(); } + /// @brief Strides of this field on the device in Fortran style (reverse order of C style) + const std::vector& device_stridesf() const { return array_->device_stridesf(); } + /// @brief Shape of this field (reverse order of Fortran style) const array::ArrayShape& shape() const { return array_->shape(); } diff --git a/src/atlas/field/detail/FieldInterface.cc b/src/atlas/field/detail/FieldInterface.cc index 5ef12259e..2319f9974 100644 --- a/src/atlas/field/detail/FieldInterface.cc +++ b/src/atlas/field/detail/FieldInterface.cc @@ -38,6 +38,18 @@ void atlas__Field__data_specf(FieldImpl* This, Value*& data, int& rank, int*& sh rank = This->shapef().size(); } +template +void atlas__Field__device_data_specf(FieldImpl* This, Value*& data, int& rank, int*& shapef, int*& stridesf) { + ATLAS_ASSERT(This != nullptr, "Cannot access data of uninitialised atlas_Field"); + if (This->datatype() != array::make_datatype()) { + throw_Exception("Datatype mismatch for accessing field data"); + } + data = This->array().device_data(); + shapef = const_cast(This->shapef().data()); + stridesf = const_cast(This->device_stridesf().data()); + rank = This->shapef().size(); +} + template FieldImpl* atlas__Field__wrap_specf(const char* name, Value data[], int rank, int shapef[], int stridesf[]) { array::ArrayShape shape; @@ -189,6 +201,22 @@ void atlas__Field__data_double_specf(FieldImpl* This, double*& data, int& rank, atlas__Field__data_specf(This, data, rank, shapef, stridesf); } +void atlas__Field__device_data_int_specf(FieldImpl* This, int*& data, int& rank, int*& shapef, int*& stridesf) { + atlas__Field__device_data_specf(This, data, rank, shapef, stridesf); +} + +void atlas__Field__device_data_long_specf(FieldImpl* This, long*& data, int& rank, int*& shapef, int*& stridesf) { + atlas__Field__device_data_specf(This, data, rank, shapef, stridesf); +} + +void atlas__Field__device_data_float_specf(FieldImpl* This, float*& data, int& rank, int*& shapef, int*& stridesf) { + atlas__Field__device_data_specf(This, data, rank, shapef, stridesf); +} + +void atlas__Field__device_data_double_specf(FieldImpl* This, double*& data, int& rank, int*& shapef, int*& stridesf) { + atlas__Field__device_data_specf(This, data, rank, shapef, stridesf); +} + int atlas__Field__host_needs_update(const FieldImpl* This) { return This->hostNeedsUpdate(); } diff --git a/src/atlas/field/detail/FieldInterface.h b/src/atlas/field/detail/FieldInterface.h index a47ffe5f8..932f3dd78 100644 --- a/src/atlas/field/detail/FieldInterface.h +++ b/src/atlas/field/detail/FieldInterface.h @@ -52,6 +52,14 @@ void atlas__Field__data_float_specf(FieldImpl* This, float*& field_data, int& ra int*& field_stridesf); void atlas__Field__data_double_specf(FieldImpl* This, double*& field_data, int& rank, int*& field_shapef, int*& field_stridesf); +void atlas__Field__device_data_int_specf(FieldImpl* This, int*& field_data, int& rank, int*& field_shapef, + int*& field_stridesf); +void atlas__Field__device_data_long_specf(FieldImpl* This, long*& field_data, int& rank, int*& field_shapef, + int*& field_stridesf); +void atlas__Field__device_data_float_specf(FieldImpl* This, float*& field_data, int& rank, int*& field_shapef, + int*& field_stridesf); +void atlas__Field__device_data_double_specf(FieldImpl* This, double*& field_data, int& rank, int*& field_shapef, + int*& field_stridesf); util::Metadata* atlas__Field__metadata(FieldImpl* This); const functionspace::FunctionSpaceImpl* atlas__Field__functionspace(FieldImpl* This); void atlas__Field__rename(FieldImpl* This, const char* name); diff --git a/src/atlas_f/field/atlas_Field_module.fypp b/src/atlas_f/field/atlas_Field_module.fypp index acaadc3a3..671b8f873 100644 --- a/src/atlas_f/field/atlas_Field_module.fypp +++ b/src/atlas_f/field/atlas_Field_module.fypp @@ -75,6 +75,7 @@ contains #:for rank in ranks #:for dtype in dtypes procedure, private :: access_data_${dtype}$_r${rank}$ + procedure, private :: access_device_data_${dtype}$_r${rank}$ procedure, private :: access_data_${dtype}$_r${rank}$_shape procedure, private :: access_data_${dtype}$_r${rank}$_slice #:endfor @@ -87,6 +88,14 @@ contains & access_data_${dtype}$_r${rank}$_shape, & & access_data_${dtype}$_r${rank}$_slice, & #:endfor +#:endfor + & dummy + + generic, public :: device_data => & +#:for rank in ranks +#:for dtype in dtypes + & access_device_data_${dtype}$_r${rank}$, & +#:endfor #:endfor & dummy @@ -214,6 +223,20 @@ subroutine access_data_${dtype}$_r${rank}$(this, field) call atlas__Field__data_${ctype}$_specf(this%CPTR_PGIBUG_A, field_cptr, rank, shape_cptr, strides_cptr) call array_c_to_f(field_cptr,rank,shape_cptr,strides_cptr, field) end subroutine + +subroutine access_device_data_${dtype}$_r${rank}$(this, field) + use atlas_field_c_binding + use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_long, c_float, c_double + class(atlas_Field), intent(in) :: this + ${ftype}$, pointer, intent(inout) :: field(${dim[rank]}$) + type(c_ptr) :: field_cptr + type(c_ptr) :: shape_cptr + type(c_ptr) :: strides_cptr + integer(c_int) :: rank + call atlas__Field__device_data_${ctype}$_specf(this%CPTR_PGIBUG_A, field_cptr, rank, shape_cptr, strides_cptr) + call array_c_to_f(field_cptr,rank,shape_cptr,strides_cptr, field) +end subroutine + subroutine access_data_${dtype}$_r${rank}$_slice(this, slice, iblk) use atlas_field_c_binding use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_long, c_float, c_double diff --git a/src/tests/field/fctest_field_gpu.F90 b/src/tests/field/fctest_field_gpu.F90 index 1d9101200..b2d87bc60 100644 --- a/src/tests/field/fctest_field_gpu.F90 +++ b/src/tests/field/fctest_field_gpu.F90 @@ -8,6 +8,7 @@ ! This File contains Unit Tests for testing the ! C++ / Fortran Interfaces to the State Datastructure ! @author Willem Deconinck +! @author Slavko Brdar #include "fckit/fctest.h" From cc888671f7453c9637eff4f74f4bd1adae5a0cb6 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Mon, 14 Oct 2024 09:21:43 +0200 Subject: [PATCH 27/40] Update test_field_acc to test acc deviceptr Co-authored-by: Willem Deconinck --- src/tests/field/test_field_acc.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tests/field/test_field_acc.cc b/src/tests/field/test_field_acc.cc index 8dad81fbd..968a71d0f 100644 --- a/src/tests/field/test_field_acc.cc +++ b/src/tests/field/test_field_acc.cc @@ -72,10 +72,19 @@ CASE("test_field_acc") { { cpu_ptr[view.index(3,2)] = 3.; } + field.updateHost(); + EXPECT_EQ( view(3,2), 3. ); + + auto dview = array::make_device_view(field); + double* dptr = dview.data(); +#pragma acc parallel deviceptr(dptr) + { + dptr[dview.index(3,2)] = 4.; + } field.updateHost(); + EXPECT_EQ( view(3,2), 4. ); - EXPECT_EQ( view(3,2), 3. ); #endif } From 828735219d8bf584a8ea23f178442b8ad9433e4c Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Mon, 14 Oct 2024 09:49:16 +0200 Subject: [PATCH 28/40] Update fctest_field_gpu to test acc deviceptr Co-authored-by: Willem Deconinck --- src/tests/field/fctest_field_gpu.F90 | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/tests/field/fctest_field_gpu.F90 b/src/tests/field/fctest_field_gpu.F90 index b2d87bc60..f7f75279f 100644 --- a/src/tests/field/fctest_field_gpu.F90 +++ b/src/tests/field/fctest_field_gpu.F90 @@ -36,6 +36,26 @@ subroutine module_acc_routine(view) end subroutine module_acc_routine +subroutine kernel_host_ptr(view) + implicit none + real(4), intent(inout) :: view(:,:) + !$acc data present(view) + !$acc kernels present(view) + view(2,1) = 4. + !$acc end kernels + !$acc end data +end subroutine kernel_host_ptr + +subroutine kernel_device_ptr(dview) + implicit none + real(4), intent(inout) :: dview(:,:) + !$acc data deviceptr(dview) + !$acc kernels + dview(2,1) = 5. + !$acc end kernels + !$acc end data +end subroutine kernel_device_ptr + end module ! ----------------------------------------------------------------------------- @@ -108,5 +128,29 @@ end subroutine external_acc_routine ! ----------------------------------------------------------------------------- +TEST( test_device_data ) +implicit none +type(atlas_Field) :: field +real(4), pointer :: view(:,:) +real(4), pointer :: dview(:,:) + +field = atlas_Field(kind=atlas_real(4),shape=[5,3]) + +call field%data(view) +view(:,:) = 0 + +call field%allocate_device() +call field%update_device() +call field%device_data(dview) +call kernel_device_ptr(dview) +call field%update_host() +FCTEST_CHECK_EQUAL( view(2,1), 5. ) + +call field%final() +END_TEST + +! ----------------------------------------------------------------------------- + + END_TESTSUITE From 5476d1f154b32389376dc3ddd0720c5267506ddd Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Mon, 14 Oct 2024 09:58:23 +0200 Subject: [PATCH 29/40] Add fieldset device operators Co-authored-by: Willem Deconinck --- src/atlas_f/field/atlas_FieldSet_module.fypp | 357 +++++++++++++++++++ 1 file changed, 357 insertions(+) diff --git a/src/atlas_f/field/atlas_FieldSet_module.fypp b/src/atlas_f/field/atlas_FieldSet_module.fypp index 36a2a38da..377470e99 100644 --- a/src/atlas_f/field/atlas_FieldSet_module.fypp +++ b/src/atlas_f/field/atlas_FieldSet_module.fypp @@ -81,6 +81,32 @@ contains procedure, private :: dummy + procedure, public :: set_host_needs_update_idx + procedure, public :: set_host_needs_update_value + procedure, public :: set_host_needs_update_name + generic :: set_host_needs_update => set_host_needs_update_idx, set_host_needs_update_value, & + set_host_needs_update_name + procedure, public :: set_device_needs_update_idx + procedure, public :: set_device_needs_update_value + procedure, public :: set_device_needs_update_name + generic :: set_device_needs_update => set_device_needs_update_idx, set_device_needs_update_value, & + set_device_needs_update_name + procedure, public :: sync_host_device_idx + procedure, public :: sync_host_device_name + generic :: sync_host_device => sync_host_device_idx, sync_host_device_name + procedure, public :: allocate_device_idx + procedure, public :: allocate_device_name + generic :: allocate_device => allocate_device_idx, allocate_device_name + procedure, public :: update_device_idx + procedure, public :: update_device_name + generic :: update_device => update_device_idx, update_device_name + procedure, public :: update_host_idx + procedure, public :: update_host_name + generic :: update_host => update_host_idx, update_host_name + procedure, public :: deallocate_device_idx + procedure, public :: deallocate_device_name + generic :: deallocate_device => deallocate_device_idx, deallocate_device_name + #if FCKIT_FINAL_NOT_INHERITING final :: atlas_FieldSet__final_auto #endif @@ -304,6 +330,337 @@ end subroutine !------------------------------------------------------------------------------- +subroutine set_host_needs_update_value(this, value) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + logical, intent(in), optional :: value + type(atlas_Field) :: field + integer(c_int) :: i, value_int + value_int = 1 + if (present(value)) then + if (.not. value) then + value_int = 0 + end if + end if + do i = 1, this%size() + field = this%field(i) + call atlas__Field__set_host_needs_update(field%CPTR_PGIBUG_A, value_int) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine set_host_needs_update_idx(this, field_indices, value) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + integer, intent(in) :: field_indices(:) + logical, intent(in), optional :: value + type(atlas_Field) :: field + integer(c_int) :: i, value_int + value_int = 1 + if (present(value)) then + if (.not. value) then + value_int = 0 + end if + end if + do i = 1, size(field_indices) + field = this%field(field_indices(i)) + call atlas__Field__set_host_needs_update(field%CPTR_PGIBUG_A, value_int) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine set_host_needs_update_name(this, field_names, value) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + character(*), intent(in) :: field_names(:) + logical, intent(in), optional :: value + type(atlas_Field) :: field + integer(c_int) :: i, value_int + value_int = 1 + if (present(value)) then + if (.not. value) then + value_int = 0 + end if + end if + do i = 1, size(field_names) + field = this%field(field_names(i)) + call atlas__Field__set_host_needs_update(field%CPTR_PGIBUG_A, value_int) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine set_device_needs_update_value(this, value) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + logical, intent(in), optional :: value + type(atlas_Field) :: field + integer(c_int) :: i, value_int + value_int = 1 + if (present(value)) then + if (.not. value) then + value_int = 0 + end if + end if + do i = 1, this%size() + field = this%field(i) + call atlas__Field__set_device_needs_update(field%CPTR_PGIBUG_A, value_int) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine set_device_needs_update_idx(this, field_indices, value) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + integer, intent(in) :: field_indices(:) + logical, intent(in), optional :: value + type(atlas_Field) :: field + integer(c_int) :: i, value_int + value_int = 1 + if (present(value)) then + if (.not. value) then + value_int = 0 + end if + end if + do i = 1, size(field_indices) + field = this%field(field_indices(i)) + call atlas__Field__set_device_needs_update(field%CPTR_PGIBUG_A, value_int) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine set_device_needs_update_name(this, field_names, value) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + character(*), intent(in) :: field_names(:) + logical, intent(in), optional :: value + type(atlas_Field) :: field + integer(c_int) :: i, value_int + value_int = 1 + if (present(value)) then + if (.not. value) then + value_int = 0 + end if + end if + do i = 1, size(field_names) + field = this%field(field_names(i)) + call atlas__Field__set_device_needs_update(field%CPTR_PGIBUG_A, value_int) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine sync_host_device_idx(this, field_indices) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + integer, intent(in), optional :: field_indices(:) + type(atlas_Field) :: field + integer(c_int) :: i + if (present(field_indices)) then + do i = 1, size(field_indices) + field = this%field(field_indices(i)) + call atlas__Field__sync_host_device(field%CPTR_PGIBUG_A) + end do + else + do i = 1, this%size() + field = this%field(i) + call atlas__Field__sync_host_device(field%CPTR_PGIBUG_A) + end do + end if +end subroutine + +!------------------------------------------------------------------------------- + +subroutine sync_host_device_name(this, field_names) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + character(*), intent(in) :: field_names(:) + type(atlas_Field) :: field + integer(c_int) :: i + do i = 1, size(field_names) + field = this%field(field_names(i)) + call atlas__Field__sync_host_device(field%CPTR_PGIBUG_A) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine allocate_device_idx(this, field_indices) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + integer, intent(in), optional :: field_indices(:) + type(atlas_Field) :: field + integer(c_int) :: i + if (present(field_indices)) then + do i = 1, size(field_indices) + field = this%field(field_indices(i)) + call atlas__Field__allocate_device(field%CPTR_PGIBUG_A) + end do + else + do i = 1, this%size() + field = this%field(i) + call atlas__Field__allocate_device(field%CPTR_PGIBUG_A) + end do + end if +end subroutine + +!------------------------------------------------------------------------------- + +subroutine allocate_device_name(this, field_names) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + character(*), intent(in) :: field_names(:) + type(atlas_Field) :: field + integer(c_int) :: i + do i = 1, size(field_names) + field = this%field(field_names(i)) + call atlas__Field__allocate_device(field%CPTR_PGIBUG_A) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine update_device_idx(this, field_indices) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + integer, intent(in), optional :: field_indices(:) + type(atlas_Field) :: field + integer(c_int) :: i + if (present(field_indices)) then + do i = 1, size(field_indices) + field = this%field(field_indices(i)) + call atlas__Field__update_device(field%CPTR_PGIBUG_A) + end do + else + do i = 1, this%size() + field = this%field(i) + call atlas__Field__update_device(field%CPTR_PGIBUG_A) + end do + end if +end subroutine + +!------------------------------------------------------------------------------- + +subroutine update_device_name(this, field_names) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + character(*), intent(in) :: field_names(:) + type(atlas_Field) :: field + integer(c_int) :: i + do i = 1, size(field_names) + field = this%field(field_names(i)) + call atlas__Field__update_device(field%CPTR_PGIBUG_A) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine update_host_idx(this, field_indices) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + integer, intent(in), optional :: field_indices(:) + type(atlas_Field) :: field + integer(c_int) :: i + if (present(field_indices)) then + do i = 1, size(field_indices) + field = this%field(field_indices(i)) + call atlas__Field__update_host(field%CPTR_PGIBUG_A) + end do + else + do i = 1, this%size() + field = this%field(i) + call atlas__Field__update_host(field%CPTR_PGIBUG_A) + end do + end if +end subroutine + +!------------------------------------------------------------------------------- + +subroutine update_host_name(this, field_names) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + character(*), intent(in) :: field_names(:) + type(atlas_Field) :: field + integer(c_int) :: i + do i = 1, size(field_names) + field = this%field(field_names(i)) + call atlas__Field__update_host(field%CPTR_PGIBUG_A) + end do +end subroutine + +!------------------------------------------------------------------------------- + +subroutine deallocate_device_idx(this, field_indices) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + integer, optional :: field_indices(:) + type(atlas_Field) :: field + integer(c_int) :: i + if (present(field_indices)) then + do i = 1, size(field_indices) + field = this%field(field_indices(i)) + call atlas__Field__deallocate_device(field%CPTR_PGIBUG_A) + end do + else + do i = 1, this%size() + field = this%field(i) + call atlas__Field__deallocate_device(field%CPTR_PGIBUG_A) + end do + end if +end subroutine + +!------------------------------------------------------------------------------- + +subroutine deallocate_device_name(this, field_names) + use, intrinsic :: iso_c_binding, only : c_int + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + class(atlas_FieldSet), intent(inout) :: this + character(*), intent(in) :: field_names(:) + type(atlas_Field) :: field + integer(c_int) :: i + do i = 1, size(field_names) + field = this%field(field_names(i)) + call atlas__Field__deallocate_device(field%CPTR_PGIBUG_A) + end do +end subroutine + +!------------------------------------------------------------------------------- + subroutine set_dirty(this,value) use, intrinsic :: iso_c_binding, only : c_int use atlas_fieldset_c_binding From e830813cbcb2386a06be779aa5dc51519768201e Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Mon, 14 Oct 2024 10:01:47 +0200 Subject: [PATCH 30/40] Add test for fieldset device operators Co-authored-by: Willem Deconinck --- src/tests/field/CMakeLists.txt | 13 ++ src/tests/field/fctest_fieldset_gpu.F90 | 256 ++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 src/tests/field/fctest_fieldset_gpu.F90 diff --git a/src/tests/field/CMakeLists.txt b/src/tests/field/CMakeLists.txt index dbf51ee37..c740187f8 100644 --- a/src/tests/field/CMakeLists.txt +++ b/src/tests/field/CMakeLists.txt @@ -92,6 +92,19 @@ if( HAVE_FCTEST ) set_tests_properties ( atlas_fctest_field_device PROPERTIES LABELS "gpu;acc") endif() + add_fctest( TARGET atlas_fctest_fieldset_device + CONDITION atlas_HAVE_ACC + LINKER_LANGUAGE Fortran + SOURCES fctest_fieldset_gpu.F90 + LIBS atlas_f OpenACC::OpenACC_Fortran + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 + ) + + if( TARGET atlas_fctest_fieldset_device ) + set_tests_properties ( atlas_fctest_fieldset_device PROPERTIES LABELS "gpu;acc") + endif() + + add_fctest( TARGET atlas_fctest_field_wrap_device CONDITION atlas_HAVE_ACC LINKER_LANGUAGE Fortran diff --git a/src/tests/field/fctest_fieldset_gpu.F90 b/src/tests/field/fctest_fieldset_gpu.F90 new file mode 100644 index 000000000..0b75d8f13 --- /dev/null +++ b/src/tests/field/fctest_fieldset_gpu.F90 @@ -0,0 +1,256 @@ +! (C) Copyright 2013 ECMWF. +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +! This File contains Unit Tests for testing the +! C++ / Fortran Interfaces to the State Datastructure +! @author Willem Deconinck +! @author Slavko Brdar + +#include "fckit/fctest.h" + + + +! ----------------------------------------------------------------------------- + +module fcta_FieldSet_gpu_fxt + use atlas_module + use, intrinsic :: iso_c_binding, only: c_ptr + implicit none + +contains + + subroutine kernel_host_ptr(view) + implicit none + real(4), intent(inout) :: view(:,:) + !$acc data present(view) + !$acc kernels present(view) + view(2,1) = 4. + !$acc end kernels + !$acc end data + end subroutine kernel_host_ptr + + subroutine kernel_device_ptr(dview) + implicit none + real(4) :: dview(:,:) + !$acc data deviceptr(dview) + !$acc kernels + dview(2,1) = 5. + !$acc end kernels + !$acc end data + end subroutine kernel_device_ptr + +! ----------------------------------------------------------------------------- + +end module + +! ----------------------------------------------------------------------------- + +! ----------------------------------------------------------------------------- + +TESTSUITE_WITH_FIXTURE(fcta_FieldSet_gpu,fcta_FieldSet_gpu_fxt) + +! ----------------------------------------------------------------------------- + +TESTSUITE_INIT + call atlas_initialize() +END_TESTSUITE_INIT + +! ----------------------------------------------------------------------------- + +TESTSUITE_FINALIZE + call atlas_finalise() +END_TESTSUITE_FINALIZE + +! ----------------------------------------------------------------------------- + +TEST( test_fieldset_cummulative_gpu_calls_all_fields ) +implicit none +type(atlas_FieldSet) :: fset +type(atlas_Field) :: field +real(4), pointer :: fview(:,:) +real(4), pointer :: fview_dev(:,:) + +print *, "test_fieldset_cummulative_gpu_calls_all_fields" +fset = atlas_FieldSet() +call fset%add(atlas_Field(name="f1", kind=atlas_real(4), shape=[1,2])) +call fset%add(atlas_Field(name="f2", kind=atlas_real(4), shape=[2,2])) +call fset%add(atlas_Field(name="f3", kind=atlas_real(4), shape=[2,1])) + +print *, "... by idx" +field = fset%field(2) +call field%data(fview) +fview(:,:) = 1. +fview(2,1) = 2. + +call fset%set_host_needs_update(.false.) +call fset%allocate_device() +FCTEST_CHECK_EQUAL(field%device_allocated(), .true.) +call fset%update_device() + +!$acc kernels present(fview) +fview(2,1) = 3. +!$acc end kernels + +FCTEST_CHECK_EQUAL( fview(2,1), 2. ) +call fset%update_host() +FCTEST_CHECK_EQUAL( fview(2,1), 3. ) + +call kernel_host_ptr(fview) +FCTEST_CHECK_EQUAL( fview(2,1), 3. ) +call fset%update_host() +FCTEST_CHECK_EQUAL( fview(2,1), 4. ) + +call field%device_data(fview_dev) +call kernel_device_ptr(fview_dev) +FCTEST_CHECK_EQUAL( fview(2,1), 4. ) +call fset%update_host() +FCTEST_CHECK_EQUAL( fview(2,1), 5. ) + +call fset%deallocate_device() + +print *, "... by name" +field = fset%field("f3") +call field%data(fview) +fview(:,:) = 3. +fview(2,1) = 4. + +call fset%allocate_device() +call fset%update_device() + +!$acc kernels present(fview) +fview(2,1) = 7. +!$acc end kernels + +FCTEST_CHECK_EQUAL( fview(2,1), 4. ) +call fset%update_host() +FCTEST_CHECK_EQUAL( fview(2,1), 7. ) + +call fset%final() +END_TEST + +! ----------------------------------------------------------------------------- + +TEST( test_fieldset_cummulative_gpu_calls_subset_field) +implicit none +type(atlas_FieldSet) :: fset +type(atlas_Field) :: field +real(4), pointer :: fview(:,:) +print *, "fieldset_cummulative_gpu_calls subset fields" + +fset = atlas_FieldSet() +call fset%add(atlas_Field(name="f1", kind=atlas_real(4), shape=[1,2])) +call fset%add(atlas_Field(name="f2", kind=atlas_real(4), shape=[2,2])) +call fset%add(atlas_Field(name="f3", kind=atlas_real(4), shape=[2,1])) + +print *, "... by idx" +field = fset%field(2) +call field%data(fview) +fview(:,:) = 1. +fview(2,1) = 2. + +call fset%set_host_needs_update((/2, 3/), .false.) +call fset%allocate_device((/ 2, 3 /)) ! device allocate only for field 2 and 3 +call fset%sync_host_device((/ 2, 3 /)) ! device-memcpy for field 2 and 3 + +!$acc kernels present(fview) +fview(2,1) = 5. +!$acc end kernels + +call fset%set_host_needs_update((/ 2 /)) + +FCTEST_CHECK_EQUAL( fview(2,1), 2. ) +call fset%sync_host_device() +FCTEST_CHECK_EQUAL( fview(2,1), 5. ) +call fset%deallocate_device() + +print *, "... by name" +field = fset%field("f3") +call field%data(fview) +fview(:,:) = 3. +fview(2,1) = 4. + +call fset%allocate_device((/ "f2", "f3" /)) +call fset%sync_host_device((/ "f2", "f3" /)) + +!$acc kernels present(fview) +fview(2,1) = 7. +!$acc end kernels + +call fset%set_host_needs_update((/ "f3" /)) + +FCTEST_CHECK_EQUAL( fview(2,1), 4. ) +call fset%sync_host_device() +FCTEST_CHECK_EQUAL( fview(2,1), 7. ) + +call fset%final() +END_TEST + +! ----------------------------------------------------------------------------- + +TEST( test_fieldset_cummulative_gpu_calls_with_sync_host_device) +implicit none +type(atlas_FieldSet) :: fset +type(atlas_Field) :: field +real(4), pointer :: fview(:,:) + +print *, "test_fieldset_cummulative_gpu_calls_with_sync_host_device" +fset = atlas_FieldSet() +call fset%add(atlas_Field(name="f1", kind=atlas_real(4), shape=[1,2])) +call fset%add(atlas_Field(name="f2", kind=atlas_real(4), shape=[2,2])) +call fset%add(atlas_Field(name="f3", kind=atlas_real(4), shape=[2,1])) + +print *, "... by idx" +field = fset%field(2) +call field%data(fview) +fview(:,:) = 1. +fview(2,1) = 2. + +call fset%set_host_needs_update(.false.) +call fset%allocate_device((/ 2, 3 /)) ! device allocate only for field 2 and 3 +call fset%set_device_needs_update((/ 2 /)) +call fset%sync_host_device((/ 2 /)) ! device-memcpy for field 2 and 3 + +!$acc kernels present(fview) +fview(2,1) = 5. +!$acc end kernels + +call fset%set_host_needs_update((/ 2 /)) + +FCTEST_CHECK_EQUAL( fview(2,1), 2. ) +call fset%sync_host_device() +FCTEST_CHECK_EQUAL( fview(2,1), 5. ) + +call fset%deallocate_device() + +print *, "... by name" +field = fset%field("f2") +call field%data(fview) +fview(:,:) = 3. +fview(2,1) = 4. + +call fset%set_host_needs_update(.false.) +call fset%allocate_device((/ "f2", "f3" /)) ! device allocate only for field 2 and 3 +call fset%set_device_needs_update((/ "f2" /)) ! set fields for sync_host_device +call fset%sync_host_device((/ "f2" /)) ! device-memcpy for field 'f2' + +!$acc kernels present(fview) +fview(2,1) = 7. +!$acc end kernels + +call fset%set_host_needs_update((/ "f2" /)) + +FCTEST_CHECK_EQUAL( fview(2,1), 4. ) +call fset%sync_host_device() +FCTEST_CHECK_EQUAL( fview(2,1), 7. ) + +call fset%final() +END_TEST + +! ----------------------------------------------------------------------------- + +END_TESTSUITE + From 3816e838eb347154befdc644d6d93e0859e2dcce Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Mon, 14 Oct 2024 15:36:51 +0000 Subject: [PATCH 31/40] add device data on FieldSet --- src/atlas_f/field/atlas_FieldSet_module.fypp | 53 ++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/atlas_f/field/atlas_FieldSet_module.fypp b/src/atlas_f/field/atlas_FieldSet_module.fypp index 377470e99..b895404bb 100644 --- a/src/atlas_f/field/atlas_FieldSet_module.fypp +++ b/src/atlas_f/field/atlas_FieldSet_module.fypp @@ -65,6 +65,8 @@ contains procedure, private :: access_data_${dtype}$_r${rank}$_by_idx procedure, private :: access_data_${dtype}$_r${rank}$_slice_by_name procedure, private :: access_data_${dtype}$_r${rank}$_slice_by_idx + procedure, private :: access_device_data_${dtype}$_r${rank}$_by_name + procedure, private :: access_device_data_${dtype}$_r${rank}$_by_idx #:endfor #:endfor @@ -76,6 +78,15 @@ contains & access_data_${dtype}$_r${rank}$_slice_by_name, & & access_data_${dtype}$_r${rank}$_slice_by_idx, & #:endfor +#:endfor + & dummy + + generic, public :: device_data => & +#:for rank in ranks +#:for dtype in dtypes + & access_device_data_${dtype}$_r${rank}$_by_name, & + & access_device_data_${dtype}$_r${rank}$_by_idx, & +#:endfor #:endfor & dummy @@ -199,10 +210,52 @@ subroutine access_data_${dtype}$_r${rank}$_slice_by_idx(this, idx, slice, iblk) slice => field(iblk) #:endif end subroutine + +!------------------------------------------------------------------------------- + +subroutine access_device_data_${dtype}$_r${rank}$_by_name(this, name, field) + use fckit_c_interop_module, only: c_str + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_long, c_float, c_double + class(atlas_FieldSet), intent(in) :: this + character(len=*), intent(in) :: name + ${ftype}$, pointer, intent(inout) :: field(${dim[rank]}$) + type(c_ptr) :: field_cptr + type(c_ptr) :: shape_cptr + type(c_ptr) :: strides_cptr + integer(c_int) :: rank + type(atlas_Field) :: field_sel + field_sel = this%field(name) + call atlas__Field__device_data_${ctype}$_specf(field_sel%CPTR_PGIBUG_A, field_cptr, rank, shape_cptr, strides_cptr) + call array_c_to_f(field_cptr, rank, shape_cptr, strides_cptr, field) +end subroutine + +!------------------------------------------------------------------------------- + +subroutine access_device_data_${dtype}$_r${rank}$_by_idx(this, idx, field) + use fckit_c_interop_module, only: c_str + use atlas_field_c_binding + use atlas_Field_module, only: atlas_Field + use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_long, c_float, c_double + class(atlas_FieldSet), intent(in) :: this + integer, intent(in) :: idx + ${ftype}$, pointer, intent(inout) :: field(${dim[rank]}$) + type(c_ptr) :: field_cptr + type(c_ptr) :: shape_cptr + type(c_ptr) :: strides_cptr + integer(c_int) :: rank + type(atlas_Field) :: field_loc + field_loc = this%field(idx) + call atlas__Field__device_data_${ctype}$_specf(field_loc%CPTR_PGIBUG_A, field_cptr, rank, shape_cptr, strides_cptr) + call array_c_to_f(field_cptr, rank, shape_cptr, strides_cptr, field) +end subroutine + !------------------------------------------------------------------------------- #:endfor #:endfor + subroutine dummy(this) use atlas_fieldset_c_binding class(atlas_FieldSet), intent(in) :: this From 450908564815b6166dea592034af906bbc08e0fe Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 28 Oct 2024 12:22:05 +0100 Subject: [PATCH 32/40] Fix possible device-memory leak in WrappedDataStore (fixes #234) --- src/atlas/array/native/NativeDataStore.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/atlas/array/native/NativeDataStore.h b/src/atlas/array/native/NativeDataStore.h index 60d35dd50..ed89476d6 100644 --- a/src/atlas/array/native/NativeDataStore.h +++ b/src/atlas/array/native/NativeDataStore.h @@ -121,7 +121,7 @@ class DataStore : public ArrayDataStore { } } - ~DataStore() { + virtual ~DataStore() { deallocateDevice(); deallocateHost(); } @@ -318,6 +318,10 @@ class WrappedDataStore : public ArrayDataStore { } } + virtual ~WrappedDataStore() { + deallocateDevice(); + } + void updateDevice() const override { if (ATLAS_HAVE_GPU && devices()) { if (not device_allocated_) { From fc85c7ef559e1de8edb4e03cca764a1b46245a75 Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:05:07 +0100 Subject: [PATCH 33/40] Move example-fortran inside atlas_HAVE_DOCS (#235) --- doc/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 05c754c4b..302647e53 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -6,12 +6,12 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +if(atlas_HAVE_DOCS) + if(atlas_HAVE_FORTRAN) add_subdirectory(example-fortran) endif() -if(atlas_HAVE_DOCS) - add_subdirectory(user-guide) list( APPEND DOC_CODE_TARGETS From 3de9361405a16d5ed96d416e7456e70fa6fc897b Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 15 Nov 2024 14:13:01 +0000 Subject: [PATCH 34/40] Protect atlas_test_array_view_variant to eckit v 1.27.0 --- src/tests/array/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tests/array/CMakeLists.txt b/src/tests/array/CMakeLists.txt index 880e46ba6..bc44cbf68 100644 --- a/src/tests/array/CMakeLists.txt +++ b/src/tests/array/CMakeLists.txt @@ -81,8 +81,11 @@ atlas_add_hic_test( ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +if(eckit_VERSION VERSION_GREATER_EQUAL 1.27.0) +# The test uses a header file from eckit "Overloaded.h" which only is available in v 1.27.0 ecbuild_add_test( TARGET atlas_test_array_view_variant SOURCES test_array_view_variant.cc LIBS atlas ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +endif() From e4f6b8736b99b5f7389094e3f0246c01c8a72d0b Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 13 Nov 2024 21:56:58 +0000 Subject: [PATCH 35/40] Fix application of limiter for StructuredInterpolation2D --- .../method/structured/StructuredInterpolation2D.h | 3 ++- .../method/structured/StructuredInterpolation2D.tcc | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.h b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.h index 77598fefc..35fdf99e1 100644 --- a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.h +++ b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.h @@ -75,8 +75,9 @@ class StructuredInterpolation2D : public Method { FunctionSpace source_; FunctionSpace target_; - bool matrix_free_; bool verbose_; + bool limiter_; + bool matrix_free_; double convert_units_; idx_t out_npts_; diff --git a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc index c9376d79e..de71fb24c 100644 --- a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc +++ b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc @@ -234,10 +234,15 @@ double StructuredInterpolation2D::convert_units_multiplier( const Field& template StructuredInterpolation2D::StructuredInterpolation2D( const Method::Config& config ) : Method( config ), - matrix_free_{false} , - verbose_{false} { - config.get( "matrix_free", matrix_free_ ); + verbose_{false}, + limiter_{false}, + matrix_free_{false} { config.get( "verbose", verbose_ ); + config.get( "limiter", limiter_ ); + config.get( "matrix_free", matrix_free_ ); + if (limiter_ && not matrix_free_) { + ATLAS_THROW_EXCEPTION("Cannot apply configuration 'limiter=true' and 'matrix_free=false' together"); + } } @@ -328,7 +333,7 @@ template void StructuredInterpolation2D::setup( const FunctionSpace& source ) { using namespace structured2d; - kernel_.reset( new Kernel( source ) ); + kernel_.reset( new Kernel( source, util::Config( "limiter", limiter_ ) ) ); if ( functionspace::StructuredColumns( source ).halo() < 1 ) { throw_Exception( "The source functionspace must have (halo >= 1) for pole treatment" ); From 91c64308b738ae39141d9641c94a658ac10bb6ea Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 15 Nov 2024 15:43:45 +0000 Subject: [PATCH 36/40] Implement StructureInterpolation2D for 3-dimensional fields with extra variables) --- .../structured/StructuredInterpolation2D.tcc | 15 ++++-- .../kernels/CubicHorizontalKernel.h | 33 ++++++++++++ .../kernels/CubicHorizontalLimiter.h | 33 ++++++++++++ .../kernels/LinearHorizontalKernel.h | 26 ++++++++++ .../kernels/QuasiCubicHorizontalKernel.h | 51 +++++++++++++++++++ 5 files changed, 155 insertions(+), 3 deletions(-) diff --git a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc index de71fb24c..2f2fa75a8 100644 --- a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc +++ b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc @@ -472,15 +472,24 @@ void StructuredInterpolation2D::do_execute( const FieldSet& src_fields, if ( datatype.kind() == array::DataType::KIND_REAL64 && rank == 1 ) { execute_impl( *kernel_, src_fields, tgt_fields ); } - if ( datatype.kind() == array::DataType::KIND_REAL32 && rank == 1 ) { + else if ( datatype.kind() == array::DataType::KIND_REAL32 && rank == 1 ) { execute_impl( *kernel_, src_fields, tgt_fields ); } - if ( datatype.kind() == array::DataType::KIND_REAL64 && rank == 2 ) { + else if ( datatype.kind() == array::DataType::KIND_REAL64 && rank == 2 ) { execute_impl( *kernel_, src_fields, tgt_fields ); } - if ( datatype.kind() == array::DataType::KIND_REAL32 && rank == 2 ) { + else if ( datatype.kind() == array::DataType::KIND_REAL32 && rank == 2 ) { execute_impl( *kernel_, src_fields, tgt_fields ); } + else if ( datatype.kind() == array::DataType::KIND_REAL64 && rank == 3 ) { + execute_impl( *kernel_, src_fields, tgt_fields ); + } + else if ( datatype.kind() == array::DataType::KIND_REAL32 && rank == 3 ) { + execute_impl( *kernel_, src_fields, tgt_fields ); + } + else { + ATLAS_NOTIMPLEMENTED; + } tgt_fields.set_dirty(); } diff --git a/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h b/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h index 2d5fa016b..d4828e833 100644 --- a/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h @@ -234,6 +234,39 @@ class CubicHorizontalKernel { } } + template + typename std::enable_if<(Rank == 3), void>::type interpolate(const stencil_t& stencil, const weights_t& weights, + const array::ArrayView& input, + array::ArrayView& output, idx_t r) const { + std::array, stencil_width()> index; + const auto& weights_j = weights.weights_j; + const idx_t Nk = output.shape(1); + const idx_t Nl = output.shape(2); + + for (idx_t k = 0; k < Nk; ++k) { + for (idx_t l = 0; l < Nl; ++l) { + output(r, k, l) = 0.; + } + } + for (idx_t j = 0; j < stencil_width(); ++j) { + const auto& weights_i = weights.weights_i[j]; + for (idx_t i = 0; i < stencil_width(); ++i) { + idx_t n = src_.index(stencil.i(i, j), stencil.j(j)); + Value w = static_cast(weights_i[i] * weights_j[j]); + for (idx_t k = 0; k < Nk; ++k) { + for (idx_t l = 0; l < Nl; ++l) { + output(r, k, l) += w * input(n, k, l); + } + } + index[j][i] = n; + } + } + + if (limiter_) { + Limiter::limit(index, input, output, r); + } + } + template typename array_t::value_type operator()(double x, double y, const array_t& input) const { Stencil stencil; diff --git a/src/atlas/interpolation/method/structured/kernels/CubicHorizontalLimiter.h b/src/atlas/interpolation/method/structured/kernels/CubicHorizontalLimiter.h index 04fecd246..75371985b 100644 --- a/src/atlas/interpolation/method/structured/kernels/CubicHorizontalLimiter.h +++ b/src/atlas/interpolation/method/structured/kernels/CubicHorizontalLimiter.h @@ -108,6 +108,39 @@ class CubicHorizontalLimiter { } } } + + template + static typename std::enable_if<(Rank == 3), void>::type limit(const std::array, 4>& index, + const array::ArrayView& input, + array::ArrayView& output, idx_t r) { + // Limit output to max/min of values in stencil marked by '*' + // x x x x + // x *-----* x + // / P | + // x *------ * x + // x x x x + for (idx_t k = 0; k < output.shape(1); ++k) { + for (idx_t l = 0; l < output.shape(2); ++l) { + Value maxval = std::numeric_limits::lowest(); + Value minval = std::numeric_limits::max(); + for (idx_t j = 1; j < 3; ++j) { + for (idx_t i = 1; i < 3; ++i) { + idx_t n = index[j][i]; + Value val = input(n, k, l); + maxval = std::max(maxval, val); + minval = std::min(minval, val); + } + } + if (output(r, k, l) < minval) { + output(r, k, l) = minval; + } + else if (output(r, k, l) > maxval) { + output(r, k, l) = maxval; + } + } + } + } + }; } // namespace method diff --git a/src/atlas/interpolation/method/structured/kernels/LinearHorizontalKernel.h b/src/atlas/interpolation/method/structured/kernels/LinearHorizontalKernel.h index d0815c7c2..c4298c5ae 100644 --- a/src/atlas/interpolation/method/structured/kernels/LinearHorizontalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/LinearHorizontalKernel.h @@ -186,6 +186,32 @@ class LinearHorizontalKernel { } } + template + typename std::enable_if<(Rank == 3), void>::type interpolate(const stencil_t& stencil, const weights_t& weights, + const array::ArrayView& input, + array::ArrayView& output, idx_t r) const { + const auto& weights_j = weights.weights_j; + const idx_t Nk = output.shape(1); + const idx_t Nl = output.shape(2); + for (idx_t k = 0; k < Nk; ++k) { + for (idx_t l = 0; l < Nl; ++l) { + output(r, k, l) = 0.; + } + } + for (idx_t j = 0; j < stencil_width(); ++j) { + const auto& weights_i = weights.weights_i[j]; + for (idx_t i = 0; i < stencil_width(); ++i) { + idx_t n = src_.index(stencil.i(i, j), stencil.j(j)); + Value w = static_cast(weights_i[i] * weights_j[j]); + for (idx_t k = 0; k < Nk; ++k) { + for (idx_t l = 0; l < Nl; ++l) { + output(r, k, l) += w * input(n, k, l); + } + } + } + } + } + template typename array_t::value_type operator()(double x, double y, const array_t& input) const { Stencil stencil; diff --git a/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h b/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h index 463cb9765..54777e7b9 100644 --- a/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h @@ -285,6 +285,57 @@ class QuasiCubicHorizontalKernel { } } + + template + typename std::enable_if<(Rank == 3), void>::type interpolate(const stencil_t& stencil, const weights_t& weights, + const array::ArrayView& input, + array::ArrayView& output, idx_t r) const { + std::array, stencil_width()> index; + const auto& weights_j = weights.weights_j; + const idx_t Nk = output.shape(1); + const idx_t Nl = output.shape(2); + for (idx_t k = 0; k < Nk; ++k) { + for (idx_t l = 0; l < Nl; ++l) { + output(r, k, l) = 0.; + } + } + + // LINEAR for outer rows ( j = {0,3} ) + for (idx_t j = 0; j < 4; j += 3) { + const auto& weights_i = weights.weights_i[j]; + for (idx_t i = 1; i < 3; ++i) { // i = {1,2} + idx_t n = src_.index(stencil.i(i, j), stencil.j(j)); + Value w = weights_i[i] * weights_j[j]; + for (idx_t k = 0; k < Nk; ++k) { + for (idx_t l = 0; l < Nl; ++l) { + output(r, k, l) += w * input(n, k, l); + } + } + index[j][i] = n; + } + } + // CUBIC for inner rows ( j = {1,2} ) + for (idx_t j = 1; j < 3; ++j) { + const auto& weights_i = weights.weights_i[j]; + for (idx_t i = 0; i < stencil_width(); ++i) { + idx_t n = src_.index(stencil.i(i, j), stencil.j(j)); + Value w = weights_i[i] * weights_j[j]; + for (idx_t k = 0; k < Nk; ++k) { + for (idx_t l = 0; l < Nl; ++l) { + output(r, k, l) += w * input(n, k, l); + } + } + index[j][i] = n; + } + } + + if (limiter_) { + Limiter::limit(index, input, output, r); + } + } + + + template typename array_t::value_type operator()(double x, double y, const array_t& input) const { Stencil stencil; From 4a59c34ba5d642d3abf6a611777df96eb426e937 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 18 Nov 2024 13:49:13 +0100 Subject: [PATCH 37/40] Add test for various use cases of StructuredInterpolation2D with limiter --- src/tests/interpolation/CMakeLists.txt | 6 + ...test_interpolation_structured2D_limiter.cc | 354 ++++++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 src/tests/interpolation/test_interpolation_structured2D_limiter.cc diff --git a/src/tests/interpolation/CMakeLists.txt b/src/tests/interpolation/CMakeLists.txt index a5ac0e86a..a8300526f 100644 --- a/src/tests/interpolation/CMakeLists.txt +++ b/src/tests/interpolation/CMakeLists.txt @@ -75,6 +75,12 @@ ecbuild_add_executable( TARGET atlas_test_interpolation_structured2D NOINSTALL ) +ecbuild_add_executable( TARGET atlas_test_interpolation_structured_limiter + SOURCES test_interpolation_structured2D_limiter.cc + LIBS atlas + NOINSTALL +) + ecbuild_add_test( TARGET atlas_test_interpolation_structured2D_regional SOURCES test_interpolation_structured2D_regional.cc LIBS atlas diff --git a/src/tests/interpolation/test_interpolation_structured2D_limiter.cc b/src/tests/interpolation/test_interpolation_structured2D_limiter.cc new file mode 100644 index 000000000..755317fdd --- /dev/null +++ b/src/tests/interpolation/test_interpolation_structured2D_limiter.cc @@ -0,0 +1,354 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include +#include +#include +#include + +#include "atlas/library.h" +#include "atlas/runtime/Log.h" +#include "atlas/functionspace.h" +#include "atlas/interpolation.h" +#include "atlas/output/Gmsh.h" + +#include "tests/AtlasTestEnvironment.h" + +using atlas::functionspace::NodeColumns; +using atlas::functionspace::StructuredColumns; +using atlas::functionspace::PointCloud; +using atlas::util::Config; + +namespace atlas { +namespace test { + +//----------------------------------------------------------------------------- + + +std::vector read_array(const std::string& filename) { + std::string files_dir="/Users/willem/work/debug-ifs-mgrids-remap/files/H15p3_cubicatlasinterp/"; + std::ifstream file (files_dir+filename); + assert(file.is_open()); + std::size_t size; + file >> size; + std::vector array(size); + for(std::size_t j=0; j> array[j]; + } + return array; +} +std::vector to_points(const std::vector& lon, const std::vector& lat) { + std::vector points(lon.size()); + for( std::size_t j=0; j ignore_overshoot{6271, 6272, 7196, 7212}; +std::set ignore_undershoot{43138,43319}; + +template +std::pair view_max_loc(View v) { + double m = -std::numeric_limits::max(); + atlas::idx_t loc = 0; + for(atlas::idx_t j=0; j m) { + if (ignore_overshoot.find(j) == ignore_overshoot.end()) { + m = v[j]; + loc = j; + } + } + } + return {m,loc}; +} + +std::pair view_max_loc(atlas::array::ArrayView v) { + auto all = atlas::array::Range::all(); + return view_max_loc( v.slice(all,0) ); +} + +std::pair view_max_loc(atlas::array::ArrayView v) { + auto all = atlas::array::Range::all(); + return view_max_loc( v.slice(all,0,0) ); +} + +std::pair max_loc(const Field& field) { + if( field.rank() == 1 ) { + return view_max_loc(atlas::array::make_view(field)); + } + else if( field.rank() == 2 ) { + return view_max_loc(atlas::array::make_view(field)); + } + else if( field.rank() == 3 ) { + return view_max_loc(atlas::array::make_view(field)); + } + else { + ATLAS_NOTIMPLEMENTED; + } +} + +template +std::pair view_min_loc(View v) { + double m = std::numeric_limits::max(); + atlas::idx_t loc = 0; + for(atlas::idx_t j=0; j view_min_loc(atlas::array::ArrayView v) { + auto all = atlas::array::Range::all(); + return view_min_loc( v.slice(all,0) ); +} + +std::pair view_min_loc(atlas::array::ArrayView v) { + auto all = atlas::array::Range::all(); + return view_min_loc( v.slice(all,0,0) ); +} + + +std::pair min_loc(const Field& field) { + if( field.rank() == 1 ) { + return view_min_loc(atlas::array::make_view(field)); + } + else if( field.rank() == 2 ) { + return view_min_loc(atlas::array::make_view(field)); + } + else if( field.rank() == 3 ) { + return view_min_loc(atlas::array::make_view(field)); + } + else { + ATLAS_NOTIMPLEMENTED; + } +} + + +std::ostream& operator<<(std::ostream& out, const std::pair& mloc) { + out << std::setw(13) << std::fixed << std::setprecision(10) << mloc.first << " at index " << mloc.second; + return out; +} + +std::vector overshooting_dg_points() { + return std::vector{ + {180., 45.2048}, + {0.,-88.9603}, // undershoot + {360.,-88.9603} // undershoot + }; +} + +std::vector read_dg_points() { + static auto dg_lon = read_array("lambda_dg_nod.txt"); + static auto dg_lat = read_array("theta_dg_nod.txt"); + return to_points(dg_lon,dg_lat); +}; + +void read_tracer(atlas::Field& field_ifs) { + // the tracer read from file should match closely the analytical function below 'init_tracer' + static auto ifs_trc = read_array("trc_ifs_gp.txt"); + if( field_ifs.rank() == 1 ) { + auto view_ifs = array::make_view(field_ifs); + for( std::size_t j=0; j(ifs_trc.size(), field_ifs.shape(0)); ++j) { + view_ifs(j) = ifs_trc[j]; + } + } + else if( field_ifs.rank() == 2 ) { + auto view_ifs = array::make_view(field_ifs); + for( std::size_t j=0; j(ifs_trc.size(), field_ifs.shape(0)); ++j) { + view_ifs(j,0) = ifs_trc[j]; + } + } + else if( field_ifs.rank() == 3 ) { + auto view_ifs = array::make_view(field_ifs); + for( std::size_t j=0; j(ifs_trc.size(), field_ifs.shape(0)); ++j) { + view_ifs(j,0,0) = ifs_trc[j]; + } + } + else { + ATLAS_NOTIMPLEMENTED; + } + field_ifs.set_dirty(); +} + +void init_tracer(atlas::Field& field_ifs) { + auto func = [](double lon, double lat) -> double { + constexpr double lon_ctr = M_PI; + constexpr double lat_ctr = M_PI/4.; + constexpr double sigma_lon = 2.*M_PI/5.; + constexpr double sigma_lat = 2.*M_PI/5.; + constexpr double to_rad = M_PI/180.; + lon *= to_rad; + lat *= to_rad; + auto sqr = [](double x){ return x*x;}; + return std::exp( - sqr( (lon-lon_ctr) / (std::sqrt(2.)*sigma_lon)) ) + * std::exp( - sqr( (lat-lat_ctr) / (std::sqrt(2.)*sigma_lat)) ); + }; + + + auto ifs_lonlat = atlas::array::make_view(field_ifs.functionspace().lonlat()); + if( field_ifs.rank() == 1 ) { + auto view_ifs = array::make_view(field_ifs); + for( std::size_t j=0; j(field_ifs); + for( std::size_t j=0; j(field_ifs); + for( std::size_t j=0; j(field).assign(99999999999.); + } + else if( field.rank() == 2 ) { + array::make_view(field).assign(99999999999.); + } + else if( field.rank() == 3 ) { + array::make_view(field).assign(99999999999.); + } + else { + ATLAS_NOTIMPLEMENTED; + } +} + + + +// ---------------------------------------------------------------------- + +void do_test(bool limiter, int rank) { + + // std::vector dg_points = read_dg_points(); + // This reads coordinates from files in ascii with format "nb_points value[0] value[1] ... value[nb_points-1]" + std::vector dg_points = overshooting_dg_points(); + + atlas::functionspace::PointCloud dg_fs(dg_points); + atlas::functionspace::StructuredColumns ifs_fs(atlas::Grid("O80"), atlas::option::halo(3)); + atlas::Field field_ifs; + atlas::Field field_dg; + + if( rank == 1 ) { + field_ifs = ifs_fs.createField(); + field_dg = dg_fs.createField(); + } + else if( rank == 2 ) { + int nlev = 60; + field_ifs = ifs_fs.createField(atlas::option::levels(nlev)); + field_dg = dg_fs.createField(atlas::option::levels(nlev)); + } + else if( rank == 3 ) { + int nlev = 60; + int ntrc = 1; + field_ifs = ifs_fs.createField(atlas::option::levels(nlev)|atlas::option::variables(ntrc)); + field_dg = dg_fs.createField(atlas::option::levels(nlev)|atlas::option::variables(ntrc)); + } + else { + ATLAS_NOTIMPLEMENTED; + } + init_crazy(field_dg); + init_tracer(field_ifs); + // read_tracer(field_ifs); + + atlas::util::Config config; + config.set("type", "structured-quasicubic2D"); + config.set("limiter", limiter); + config.set("matrix_free", true); + atlas::Interpolation interpolator(config, ifs_fs, dg_fs); + interpolator.execute(field_ifs, field_dg); + + auto max_ifs = max_loc(field_ifs); + auto min_ifs = min_loc(field_ifs); + auto max_dg = max_loc(field_dg); + auto min_dg = min_loc(field_dg); + + int precision = 10; + std::cout << "max_ifs: " << max_ifs << std::endl; + std::cout << "max_dg: " << max_dg << std::endl; + std::cout << "max_diff: " << std::setw(13) << std::fixed << std::setprecision(precision) << max_ifs.first - max_dg.first << std::endl; + std::cout << std::endl; + std::cout << "min_ifs: " << min_ifs << std::endl; + std::cout << "min_dg: " << min_dg << std::endl; + std::cout << "min_diff: " << std::setw(13) << std::fixed << std::setprecision(precision) << min_dg.first - min_ifs.first << std::endl; + + if( limiter && max_dg.first > max_ifs.first ) { + std::cout << std::endl; + std::cout << "FAILURE: overshoot in interpolation detected " << std::endl; + auto lonlat = atlas::array::make_view(dg_fs.lonlat()); + std::cout << " coordinate ["<(dg_fs.lonlat()); + std::cout << " coordinate ["<= min_ifs.first); + } +} + +CASE("structured-quasicubic2D without limiter, rank 1") { + bool limiter = false; + int rank = 1; + do_test(limiter, rank); +} +CASE("structured-quasicubic2D with limiter, rank 1") { + bool limiter = true; + int rank = 1; + do_test(limiter, rank); +} +CASE("structured-quasicubic2D without limiter, rank 2") { + bool limiter = false; + int rank = 2; + do_test(limiter, rank); +} +CASE("structured-quasicubic2D with limiter, rank 2") { + bool limiter = true; + int rank = 2; + do_test(limiter, rank); +} +CASE("structured-quasicubic2D without limiter, rank 3") { + bool limiter = false; + int rank = 3; + do_test(limiter, rank); +} +CASE("structured-quasicubic2D with limiter, rank 3") { + bool limiter = true; + int rank = 3; + do_test(limiter, rank); +} + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} From e351a31a928a455c8c507395927458b509b35161 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 18 Nov 2024 14:48:44 +0000 Subject: [PATCH 38/40] Add hicLaunchHostFunc for async host functions --- hic/src/hic/hic_dummy/hic_dummy_runtime.h | 1 + hic/src/hic/hic_runtime.h | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/hic/src/hic/hic_dummy/hic_dummy_runtime.h b/hic/src/hic/hic_dummy/hic_dummy_runtime.h index 1560b790c..265b7c76d 100644 --- a/hic/src/hic/hic_dummy/hic_dummy_runtime.h +++ b/hic/src/hic/hic_dummy/hic_dummy_runtime.h @@ -29,6 +29,7 @@ namespace { using dummyError_t = int; using dummyEvent_t = void*; +using dummyHostFn_t = void*; using dummyStream_t = void*; const char* dummyGetErrorString(dummyError_t) { diff --git a/hic/src/hic/hic_runtime.h b/hic/src/hic/hic_runtime.h index 25af23cb0..6ad9646dc 100644 --- a/hic/src/hic/hic_runtime.h +++ b/hic/src/hic/hic_runtime.h @@ -110,6 +110,16 @@ HIC_NAMESPACE_BEGIN //------------------------------------------------ +HIC_TYPE(HostFn_t) +HIC_TYPE(Error_t) +HIC_TYPE(Event_t) +HIC_TYPE(Stream_t) +#if !HIC_BACKEND_HIP +HIC_TYPE(PointerAttributes) +#elif HIP_VERSION_MAJOR >= 6 +using HIC_SYMBOL(PointerAttributes) = HIC_BACKEND_SYMBOL(PointerAttribute_t); +#endif + HIC_FUNCTION(DeviceSynchronize) HIC_FUNCTION(Free) HIC_FUNCTION(FreeAsync) @@ -117,6 +127,7 @@ HIC_FUNCTION(GetDeviceCount) HIC_FUNCTION(GetErrorString) HIC_FUNCTION(GetLastError) HIC_FUNCTION(HostGetDevicePointer) +HIC_FUNCTION(LaunchHostFunc) HIC_FUNCTION(HostRegister) HIC_FUNCTION(HostUnregister) HIC_FUNCTION(Malloc) @@ -137,15 +148,6 @@ HIC_FUNCTION(StreamCreate) HIC_FUNCTION(StreamDestroy) HIC_FUNCTION(StreamSynchronize) -HIC_TYPE(Error_t) -HIC_TYPE(Event_t) -HIC_TYPE(Stream_t) -#if !HIC_BACKEND_HIP -HIC_TYPE(PointerAttributes) -#elif HIP_VERSION_MAJOR >= 6 -using HIC_SYMBOL(PointerAttributes) = HIC_BACKEND_SYMBOL(PointerAttribute_t); -#endif - HIC_VALUE(CpuDeviceId) HIC_VALUE(HostRegisterMapped) #if !HIC_BACKEND_HIP From 6484a0a40c031c677df938fcf3362ddcdf24f523 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 13 Nov 2024 16:00:04 +0000 Subject: [PATCH 39/40] Version 0.40.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4ef2eb086..9b0025a78 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.39.0 +0.40.0 From d43194087cc6f758d53f6cd09dceb32f89c5bd74 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 18 Nov 2024 21:30:32 +0100 Subject: [PATCH 40/40] Update changelog --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6417ccd00..c69cf15f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ## [Unreleased] +## [0.40.0] - 2024-11-18 + +### Added +- Add MultiField to pack field allocations by @sbrdar in https://github.com/ecmwf/atlas/pull/229 +- Add function to create `std::variant` for multiple array views. by @odlomax in https://github.com/ecmwf/atlas/pull/220 +- Add Fortran access to device data through Field and FieldSet by @sbrdar in https://github.com/ecmwf/atlas/pull/232 +- Add Fortran API for structured stencil computations by @wdeconinck in https://github.com/ecmwf/atlas/pull/228 + +### Changed +- Support OpenACC with Cray compiler + use OpenACC_Fortran_FLAGS by @wdeconinck in https://github.com/ecmwf/atlas/pull/225 +- Refactor `SphericalVector` interpolation method to use `array::ArrayViewVariant`. by @odlomax in https://github.com/ecmwf/atlas/pull/227 +- Integrate `pack_vector_fields` into `SphericalVector` Interpolation method. by @odlomax in https://github.com/ecmwf/atlas/pull/224 +- Refactor `util::pack_vector_fields` to use `array::ArrayViewVariant` by @odlomax in https://github.com/ecmwf/atlas/pull/226 + +### Fixed +- Fix application of limiter for StructuredInterpolation2D by @wdeconinck in https://github.com/ecmwf/atlas/pull/236 +- Fix use of ATLAS_ENABLE_CUDA ON/OFF +- Bugfix for Qhull by @benjaminmenetrier in https://github.com/ecmwf/atlas/pull/230 +- Fixed ordering of `fixup_halos` and `halo_exhange.execute_adjoint` in `StructuredColumns.cc` by @odlomax in https://github.com/ecmwf/atlas/pull/223 + + ## [0.39.0] - 2024-09-18 ### Added @@ -564,6 +585,7 @@ Fix StructuredInterpolation2D with retry for failed stencils ## 0.13.0 - 2018-02-16 [Unreleased]: https://github.com/ecmwf/atlas/compare/master...develop +[0.40.0]: https://github.com/ecmwf/atlas/compare/0.39.0...0.40.0 [0.39.0]: https://github.com/ecmwf/atlas/compare/0.38.1...0.39.0 [0.38.1]: https://github.com/ecmwf/atlas/compare/0.38.0...0.38.1 [0.38.0]: https://github.com/ecmwf/atlas/compare/0.37.0...0.38.0