diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fbb41bf93..b7037788f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,11 +121,11 @@ jobs: - name: macos # Xcode compiler requires empty environment variables, so we pass null (~) here - os: macos-12 - compiler: clang-14 + os: macos-14 + compiler: clang-15 compiler_cc: ~ compiler_cxx: ~ - compiler_fc: gfortran-11 + compiler_fc: gfortran-13 caching: true coverage: false cmake_options: -DMPI_SLOTS=4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28bd7a5c4..791bee0c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,3 +73,20 @@ jobs: repository: private-downstream-ci event_type: downstream-ci-hpc payload: '{"atlas": "ecmwf/atlas@${{ github.event.pull_request.head.sha || github.sha }}"}' + + + notify: + runs-on: ubuntu-latest + needs: + - downstream-ci + - private-downstream-ci + - downstream-ci-hpc + - private-downstream-ci-hpc + if: ${{ always() && !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + steps: + - name: Trigger Teams notification + uses: ecmwf-actions/notify-teams@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} + needs_context: ${{ toJSON(needs) }} + diff --git a/.github/workflows/notify-new-issue.yml b/.github/workflows/notify-new-issue.yml new file mode 100644 index 000000000..384a35706 --- /dev/null +++ b/.github/workflows/notify-new-issue.yml @@ -0,0 +1,15 @@ +name: Notify new issue + +on: + issues: + types: + - "opened" + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Notify new issue + uses: ecmwf-actions/notify-teams-issue@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} diff --git a/.github/workflows/notify-new-pr.yml b/.github/workflows/notify-new-pr.yml new file mode 100644 index 000000000..6ed1a93ee --- /dev/null +++ b/.github/workflows/notify-new-pr.yml @@ -0,0 +1,16 @@ +name: Notify new PR + +# Needs the worklow to be located in the branche the PR is merged to +on: + pull_request_target: + types: + - "opened" + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Notify new PR + uses: ecmwf-actions/notify-teams-pr@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f7a0e8ece..6417ccd00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,43 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ## [Unreleased] +## [0.39.0] - 2024-09-18 + +### Added +- Add HIC abstraction layer for HIP and CUDA by @wdeconinck in https://github.com/ecmwf/atlas/pull/219 +- Support for HIP via HIC +- Add regional interpolation by @benjaminmenetrier in https://github.com/ecmwf/atlas/pull/215 +- Pack vector components into higher-rank vector fields. by @odlomax in https://github.com/ecmwf/atlas/pull/218 + +### Fixed +- Fix missing header in FieldImpl.h by @tehrengruber in https://github.com/ecmwf/atlas/pull/214 +- Bug fixes to vector interpolation with StructuredColumns and spherical vector interpolation by @MarekWlasak in https://github.com/ecmwf/atlas/pull/222 + + +## [0.38.1] - 2024-07-15 + +### Fixed +- Compilation and running with NAG compiler +- Wrap out-of-bounds source cells into the domain (#211) + +## [0.38.0] - 2024-06-20 + +### Added +- Make non_linear interpolation independent of a chosen value type by @wdeconinck in https://github.com/ecmwf/atlas/pull/176 +- Procedure to carry out a regridding from high to low resolution (binning) by @mo-lormi in https://github.com/ecmwf/atlas/pull/191 +- Add Fortran interface for node-to-edge connectivity building by @benjaminmenetrier in https://github.com/ecmwf/atlas/pull/209 +- CUDA/OpenACC capable fields with Native storage backend @sbrdar and @wdeconinck in https://github.com/ecmwf/atlas/pull/182 + +### Changed +- Make non_linear interpolation independent of a chosen value type by @wdeconinck in https://github.com/ecmwf/atlas/pull/176 + +### Fixed +- Remove float in Triag2D intersection algorithm by @fmahebert in https://github.com/ecmwf/atlas/pull/203 +- Allow zero-sized interpolation target functionspace by @odlomax in https://github.com/ecmwf/atlas/pull/206 +- Made sure cubed-sphere interpolation method always sets metadata. by @odlomax in https://github.com/ecmwf/atlas/pull/208 +- Avoid silent errors accessing Fieldset fields by ambiguous names by @wdeconinck in https://github.com/ecmwf/atlas/pull/210 +- Fixes opposite pole coordinates by @benjaminmenetrier in https://github.com/ecmwf/atlas/pull/202 + ## [0.37.0] - 2024-04-09 ### Added - Add SphericalVector interpolation method using parallel transport (#163) @@ -527,6 +564,9 @@ Fix StructuredInterpolation2D with retry for failed stencils ## 0.13.0 - 2018-02-16 [Unreleased]: https://github.com/ecmwf/atlas/compare/master...develop +[0.39.0]: https://github.com/ecmwf/atlas/compare/0.38.1...0.39.0 +[0.38.1]: https://github.com/ecmwf/atlas/compare/0.38.0...0.38.1 +[0.38.0]: https://github.com/ecmwf/atlas/compare/0.37.0...0.38.0 [0.37.0]: https://github.com/ecmwf/atlas/compare/0.36.0...0.37.0 [0.36.0]: https://github.com/ecmwf/atlas/compare/0.35.1...0.36.0 [0.35.1]: https://github.com/ecmwf/atlas/compare/0.35.0...0.35.1 diff --git a/CMakeLists.txt b/CMakeLists.txt index df9963eaa..0b59e607f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,16 @@ ecbuild_add_option( FEATURE ECKIT_DEVELOP DESCRIPTION "Used to enable new features or API depending on eckit develop branch, not yet in a tagged release" DEFAULT OFF ) +if (DEFINED ATLAS_ENABLE_CUDA AND NOT DEFINED HIC_ENABLE_CUDA ) + set( HIC_ENABLE_CUDA ${ATLAS_ENABLE_CUDA} ) +endif() +if (DEFINED ATLAS_ENABLE_HIP AND NOT DEFINED HIC_ENABLE_HIP ) + set( HIC_ENABLE_HIP ${ATLAS_ENABLE_HIP} ) +endif() + +add_subdirectory( hic ) +find_package(hic) + include( features/BOUNDSCHECKING ) include( features/FORTRAN ) diff --git a/VERSION b/VERSION index 0f1a7dfc7..4ef2eb086 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.37.0 +0.39.0 diff --git a/cmake/atlas-import.cmake.in b/cmake/atlas-import.cmake.in index f60f44ad6..61a3f4642 100644 --- a/cmake/atlas-import.cmake.in +++ b/cmake/atlas-import.cmake.in @@ -23,6 +23,7 @@ if( atlas_HAVE_FORTRAN ) endif() find_dependency( atlas_io HINTS ${CMAKE_CURRENT_LIST_DIR}/../atlas_io @atlas_io_DIR@ @atlas_io_BINARY_DIR@ ) +find_dependency( hic HINTS ${CMAKE_CURRENT_LIST_DIR}/../hic @hic_DIR@ @hic_BINARY_DIR@ ) ## Eigen3 set( Eigen3_HINT @Eigen3_DIR@ ) diff --git a/cmake/atlas_compile_flags.cmake b/cmake/atlas_compile_flags.cmake index 8b334f149..8261a407d 100644 --- a/cmake/atlas_compile_flags.cmake +++ b/cmake/atlas_compile_flags.cmake @@ -11,8 +11,9 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) if( CMAKE_CXX_COMPILER_ID MATCHES Cray ) - - ecbuild_add_cxx_flags("-hnomessage=3140" NAME atlas_cxx_disable_warnings ) # colon separated numbers + if( NOT CMAKE_CXX_COMPILER_ID MATCHES CrayClang ) + ecbuild_add_cxx_flags("-hnomessage=3140" NAME atlas_cxx_disable_warnings ) # colon separated numbers + endif() ecbuild_add_fortran_flags("-hnomessage=3140" NAME atlas_fortran_disable_warnings ) # colon separated numbers # CC-3140 crayc++: WARNING File = atlas/functionspace/NodeColumns.cc, Line = 1, Column = 1 diff --git a/cmake/atlas_host_device.cmake b/cmake/atlas_host_device.cmake index 3a404a8e4..d4e9a5d67 100644 --- a/cmake/atlas_host_device.cmake +++ b/cmake/atlas_host_device.cmake @@ -6,7 +6,7 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. -function( create_cuda_wrapper variable ) +function( create_hic_wrapper variable ) set( options "" ) set( single_value_args SOURCE ) set( multi_value_args "" ) @@ -17,18 +17,24 @@ function( create_cuda_wrapper variable ) get_filename_component(name ${_PAR_SOURCE} NAME) get_filename_component(abspath ${_PAR_SOURCE} ABSOLUTE) + if( HAVE_CUDA ) + set( extension "cu" ) + elseif( HAVE_HIP ) + set( extension "hip" ) + endif() + if( directory ) - set(cuda_wrapper ${CMAKE_CURRENT_BINARY_DIR}/${directory}/${base}.cu) + set(hic_wrapper ${CMAKE_CURRENT_BINARY_DIR}/${directory}/${base}.${extension}) else() - set(cuda_wrapper ${CMAKE_CURRENT_BINARY_DIR}/${base}.cu) + set(hic_wrapper ${CMAKE_CURRENT_BINARY_DIR}/${base}.${extension}) endif() - set(${variable} ${cuda_wrapper} PARENT_SCOPE) + set(${variable} ${hic_wrapper} PARENT_SCOPE) set(content " #include \"atlas/${directory}/${name}\" ") - if( ${abspath} IS_NEWER_THAN ${cuda_wrapper} ) - file(WRITE ${cuda_wrapper} "${content}") + if( ${abspath} IS_NEWER_THAN ${hic_wrapper} ) + file(WRITE ${hic_wrapper} "${content}") endif() endfunction() @@ -40,12 +46,12 @@ function( atlas_host_device srclist ) set( multi_value_args SOURCES ) cmake_parse_arguments( _PAR "${options}" "${single_value_args}" "${multi_value_args}" ${ARGN} ) - if( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA ) - set( use_cuda_srclist ${_PAR_SOURCES} ) + if( HAVE_GPU ) + set( use_hic_srclist ${_PAR_SOURCES} ) - foreach( src ${use_cuda_srclist} ) - create_cuda_wrapper( cuda_wrapper SOURCE ${src} ) - list( APPEND ${srclist} ${cuda_wrapper} ) + foreach( src ${use_hic_srclist} ) + create_hic_wrapper( hic_wrapper SOURCE ${src} ) + list( APPEND ${srclist} ${hic_wrapper} ) ecbuild_list_exclude_pattern( LIST ${srclist} REGEX ${src} ) endforeach() diff --git a/cmake/features/ACC.cmake b/cmake/features/ACC.cmake index 1464d9bc0..a1537142e 100644 --- a/cmake/features/ACC.cmake +++ b/cmake/features/ACC.cmake @@ -1,32 +1,27 @@ ### OpenACC -if( atlas_HAVE_ATLAS_FIELD ) +if( atlas_HAVE_ATLAS_FIELD AND HAVE_GPU ) -set( ATLAS_ACC_CAPABLE FALSE ) -if( HAVE_CUDA ) - if( CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC" ) - set( ATLAS_ACC_CAPABLE TRUE ) + if( DEFINED ATLAS_ENABLE_ACC ) + set( ENABLE_ACC ${ATLAS_ENABLE_ACC} ) endif() -endif() - -ecbuild_add_option( FEATURE ACC - DESCRIPTION "OpenACC capable data structures" - CONDITION ATLAS_ACC_CAPABLE ) - -if( atlas_HAVE_ACC ) - if( CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC" ) - #set( ACC_Fortran_FLAGS -acc -ta=tesla,nordc ) - set( ACC_Fortran_FLAGS "-acc=gpu;-gpu=gvmode,lineinfo,fastmath,rdc" ) - set( ACC_C_FLAGS ${ACC_Fortran_FLAGS} ) - find_program( ACC_C_COMPILER NAMES pgcc HINTS ${PGI_DIR} ${NVPHC_DIR} ENV PGI_DIR NVHPC_DIR PATH_SUFFIXES bin ) - if( NOT ACC_C_COMPILER ) - ecbuild_error( "Could not find OpenACC capable C compiler" ) + if( ENABLE_ACC ) + if( NOT HAVE_FORTRAN ) + enable_language(Fortran) endif() + find_package( OpenACC COMPONENTS Fortran CXX ) + endif() + + ecbuild_add_option( FEATURE ACC + DESCRIPTION "OpenACC capable data structures" + CONDITION OpenACC_Fortran_FOUND ) + if( HAVE_ACC ) + set( ACC_LINK_OPTIONS ${OpenACC_Fortran_FLAGS} ) endif() -endif() else() + set( HAVE_ACC 0 ) set( atlas_HAVE_ACC 0 ) -endif() +endif() diff --git a/cmake/features/CUDA.cmake b/cmake/features/CUDA.cmake index 1bd57058a..80d5723d8 100644 --- a/cmake/features/CUDA.cmake +++ b/cmake/features/CUDA.cmake @@ -1,15 +1,37 @@ -ecbuild_add_option( FEATURE CUDA - DESCRIPTION "Enable CUDA support" - DEFAULT OFF - ) +# ecbuild_add_option( FEATURE CUDA +# DESCRIPTION "Enable CUDA support" +# DEFAULT OFF +# ) +# ecbuild_add_option( FEATURE HIP +# DESCRIPTION "Enable CUDA support" +# DEFAULT OFF +# ) -if( HAVE_CUDA ) - - enable_language( CUDA ) - ecbuild_info( "CUDA language enabled" ) - - find_package( CUDAToolkit REQUIRED ) +set( atlas_HAVE_CUDA 0 ) +set( atlas_HAVE_HIP 0 ) +set( atlas_HAVE_GPU 0 ) +if( hic_HAVE_CUDA ) + enable_language( CUDA ) + ecbuild_info( "CUDA language enabled" ) + find_package(CUDAToolkit REQUIRED) + set( atlas_HAVE_CUDA 1 ) + set( atlas_HAVE_GPU 1 ) +elseif( hic_HAVE_HIP ) + enable_language( HIP ) + ecbuild_info( "HIP language enabled" ) + find_package(hip CONFIG REQUIRED) + set( atlas_HAVE_HIP 1 ) + set( atlas_HAVE_GPU 1 ) endif() +set( HAVE_CUDA ${atlas_HAVE_CUDA} ) +set( HAVE_HIP ${atlas_HAVE_HIP} ) +set( HAVE_GPU ${atlas_HAVE_GPU} ) + +if( HAVE_GPU ) + ecbuild_info("GPU support enabled") +else() + ecbuild_info("GPU support not enabled") +endif() \ No newline at end of file diff --git a/cmake/features/ECTRANS.cmake b/cmake/features/ECTRANS.cmake index 3d8ab3f08..20c893852 100644 --- a/cmake/features/ECTRANS.cmake +++ b/cmake/features/ECTRANS.cmake @@ -20,9 +20,13 @@ if( atlas_HAVE_ATLAS_FUNCTIONSPACE AND (ENABLE_ECTRANS OR NOT DEFINED ENABLE_ECT if( TARGET transi_dp ) set( transi_FOUND TRUE ) if( NOT TARGET transi ) - get_target_property( transi_dp_IMPORTED transi_dp IMPORTED ) - if( transi_dp_IMPORTED ) - set_target_properties( transi_dp PROPERTIES IMPORTED_GLOBAL TRUE) # required for aliasing imports + if( CMAKE_VERSION VERSION_LESS 3.18 ) + # Before CMake 3.18 it is not possible to alias a non-global imported target + # Make the import global. Warning, this may break further find_package + get_target_property( transi_dp_IMPORTED transi_dp IMPORTED ) + if( transi_dp_IMPORTED ) + set_target_properties( transi_dp PROPERTIES IMPORTED_GLOBAL TRUE) + endif() endif() add_library( transi ALIAS transi_dp ) endif() diff --git a/cmake/project_summary.cmake b/cmake/project_summary.cmake index eb4face8b..b5bfa6e3c 100644 --- a/cmake/project_summary.cmake +++ b/cmake/project_summary.cmake @@ -42,9 +42,7 @@ endif() if( atlas_HAVE_ACC ) ecbuild_info( "ACC" ) - ecbuild_info( " ACC_C_COMPILER : [${ACC_C_COMPILER}]" ) - ecbuild_info( " ACC_C_FLAGS : [${ACC_C_FLAGS}]" ) - ecbuild_info( " ACC_Fortran_FLAGS : [${ACC_Fortran_FLAGS}]" ) + ecbuild_info( " OpenACC_Fortran_FLAGS : [${OpenACC_Fortran_FLAGS}]" ) endif() @@ -60,10 +58,10 @@ if( atlas_HAVE_GRIDTOOLS_STORAGE ) else() - if( NOT atlas_HAVE_CUDA ) + if( NOT atlas_HAVE_GPU ) ecbuild_info( "Array storage backend: Native [HOST]" ) else() - ecbuild_info( "Array storage backend: Native [CUDA]" ) + ecbuild_info( "Array storage backend: Native [GPU]" ) endif() endif() diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 6c7fa7d3a..05c754c4b 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -6,6 +6,10 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +if(atlas_HAVE_FORTRAN) + add_subdirectory(example-fortran) +endif() + if(atlas_HAVE_DOCS) add_subdirectory(user-guide) diff --git a/doc/example-fortran/CMakeLists.txt b/doc/example-fortran/CMakeLists.txt new file mode 100644 index 000000000..564a289f0 --- /dev/null +++ b/doc/example-fortran/CMakeLists.txt @@ -0,0 +1,10 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +add_subdirectory(structured-grid-stencils) + diff --git a/doc/example-fortran/structured-grid-stencils/CMakeLists.txt b/doc/example-fortran/structured-grid-stencils/CMakeLists.txt new file mode 100644 index 000000000..c3a221662 --- /dev/null +++ b/doc/example-fortran/structured-grid-stencils/CMakeLists.txt @@ -0,0 +1,12 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +ecbuild_add_executable( TARGET example-fortran-structured-grid-stencils + OUTPUT_NAME program NOINSTALL + SOURCES program.F90 + LIBS atlas_f ) diff --git a/doc/example-fortran/structured-grid-stencils/program.F90 b/doc/example-fortran/structured-grid-stencils/program.F90 new file mode 100644 index 000000000..6517ad3a4 --- /dev/null +++ b/doc/example-fortran/structured-grid-stencils/program.F90 @@ -0,0 +1,210 @@ +! (C) Copyright 2023 ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +! ------------------------------------------------------------------------------------------------ +! Example program on how to access and use StructuredColumns functionspace +! Also includes computation of stencils around a given coordinate +! ------------------------------------------------------------------------------------------------ + +subroutine run + +use, intrinsic :: iso_fortran_env, only : real64, real32 +use atlas_module, only : & + & atlas_ReducedGaussianGrid ,& + & atlas_functionspace_StructuredColumns ,& + & atlas_Field, & + & ATLAS_KIND_IDX, & + & atlas_StructuredGrid_ComputeNorth ,& + & atlas_StructuredGrid_ComputeWest ,& + & atlas_StructuredGrid_ComputeStencil ,& + & atlas_StructuredGrid_Stencil, & + & atlas_real + +use fckit_mpi_module, only : fckit_mpi + +! ------------------------------------------------ +implicit none + +character(len=:), allocatable :: gridname +type(atlas_ReducedGaussianGrid) :: grid +type(atlas_functionspace_StructuredColumns) :: functionspace +real(real64) :: zlon, zlat +integer(ATLAS_KIND_IDX) :: jlon,jlat, jgp +real(real64), pointer :: xy(:,:) +integer :: log_rank + +! ------------------------------------------------ + +log_rank = 1 ! First partition has rank=0 !!! +if (fckit_mpi%size() == 1) then + log_rank = 0 +endif + +gridname = "O320" + +! Create grid and write some properties +!grid = atlas_StructuredGrid(gridname) +grid = atlas_ReducedGaussianGrid([20,24,28,32,36,40,44,48,52,56,60,64,& + 64,60,56,52,48,44,40,36,32,28,24,20]) + +if (fckit_mpi%rank() == log_rank) then + write(0,*) "grid%name() : ", grid%name() + write(0,*) "grid%size() : ", grid%size() + write(0,*) "grid%ny() : ", grid%ny() + write(0,*) "grid%nxmax() : ", grid%nxmax() +endif + +! Create functionspace of grid, distributed across MPI tasks with default partitioner, and with halo of 2 wide +! The default partitioner is "equal_regions" +functionspace = atlas_functionspace_StructuredColumns(grid, halo=2) + +! Latitude bounds without and with halos of this partition +if (fckit_mpi%rank() == log_rank) then + write(0,*) "functionspace%size_owned() : ", functionspace%size_owned() + write(0,*) "functionspace%size() : ", functionspace%size() + write(0,'(A,I0,A,I0)') " halo points between index ", functionspace%size_owned()+1 , " and " , functionspace%size() + write(0,*) "functionspace%j_begin(), functionspace%j_end() : ", & + & functionspace%j_begin(), functionspace%j_end() + write(0,*) "functionspace%j_begin_halo(), functionspace%j_end_halo() : ", & + & functionspace%j_begin_halo(), functionspace%j_end_halo() +endif + +! Longitude bounds without and with halos of this partition for the first latitude +jlat = functionspace%j_begin() +if (fckit_mpi%rank() == log_rank) then + write(0,*) "functionspace%i_begin(jlat), functionspace%i_end(jlat) : ", functionspace%i_begin(jlat), & + & functionspace%i_end(jlat) + write(0,*) "functionspace%i_begin_halo(jlat), functionspace%i_end_halo(jlat) : ", functionspace%i_begin_halo(jlat),& + & functionspace%i_end_halo(jlat) +endif + +! Access i,j indices of grid points +block + type(atlas_Field) :: field_index_i, field_index_j + integer(ATLAS_KIND_IDX), pointer :: index_i(:), index_j(:) + field_index_i = functionspace%index_i() + field_index_j = functionspace%index_j() + call field_index_i%data(index_i) + call field_index_j%data(index_j) + if (fckit_mpi%rank() == log_rank) then + write(0,*) "i,j of first partition point :", index_i(1), index_j(1) + endif + call field_index_i%final() + call field_index_j%final() +end block + +! Access to xy coordinates +block + type(atlas_Field) :: field_xy + field_xy = functionspace%xy() + call field_xy%data(xy) + call field_xy%final() +end block + +! Creating a horizontal field and perform halo exchange +block + type(atlas_Field) :: field + real(real32), pointer :: view(:) + field = functionspace%create_field(name="myfield", kind=atlas_real(real32)) + call field%data(view) + if (fckit_mpi%rank() == log_rank) then + write(0,*) "shape( horizontal field )", shape(view) + endif + view(1:functionspace%size_owned()) = 1. + view(functionspace%size_owned()+1:functionspace%size()) = 0 + call field%set_dirty() ! This marks that halos are in "dirty" state + call field%halo_exchange() ! Only exchanges halos when halos are marked as "dirty" + if (fckit_mpi%rank() == log_rank) then + write(0,*) "halo exhange success : ", all(view == 1.) + endif + call field%final() +end block + +! Creating a horizontal/vertical field +block + type(atlas_Field) :: field + real(real32), pointer :: view(:,:) + field = functionspace%create_field(name="myfield", kind=atlas_real(real32), levels=10) + call field%data(view) + if (fckit_mpi%rank() == log_rank) then + write(0,*) "shape( horizontal/vertical field )", shape(view) + endif + call field%final() +end block + +! Set a coordinate somewhere south-east of the first grid point, but still within this partition +jgp = 1 +zlon = xy(1,jgp) + 0.1 +zlat = xy(2,jgp) - 0.1 + +! Compute nearest points to the north-west of a coordinate +block + type(atlas_StructuredGrid_ComputeNorth) :: compute_north + type(atlas_StructuredGrid_ComputeWest) :: compute_west + compute_north = atlas_StructuredGrid_ComputeNorth(grid, halo=2) + compute_west = atlas_StructuredGrid_ComputeWest(grid, halo=2) + + jlat = compute_north%execute(zlat) + jlon = compute_west%execute(zlon, jlat) + jgp = functionspace%index(jlon,jlat) ! gridpoint index in 1-dimensional array + + if (fckit_mpi%rank() == log_rank) then + write(0,'(A,F6.2,A,I0)') "compute_north%execute(y=",zlat,") : ", jlat + write(0,'(A,F6.2,A,I0,A,I0)') "compute_west%execute(x=",zlon,", j=", jlat,") : ", jlon + write(0,'(A,I0,A,F6.2,A,F6.2,A)') "gridpoint north-west: jpg=",jgp," (lon,lat)=(", xy(1,jgp), ",", xy(2,jgp), ")" + endif + call compute_west%final() + call compute_north%final() +end block + +! Stencil computer of 4x4 stencil around a coordinate +block + type(atlas_StructuredGrid_ComputeStencil) :: compute_stencil + type(atlas_StructuredGrid_Stencil) :: stencil + call compute_stencil%setup(grid, stencil_width=4) + call compute_stencil%execute(zlon,zlat,stencil) + if (fckit_mpi%rank() == log_rank) then + write(0,'(A,A,dt)') 'stencil:', new_line('a'), stencil + endif + if (fckit_mpi%rank() == log_rank) then + write(0,'(A,A,dt)') 'stencil gridpoints:' + do jlat=1,stencil%width + do jlon=1,stencil%width + write(0,'(I8)',advance='no') functionspace%index(stencil%i(jlon,jlat),stencil%j(jlat)) + enddo + write(0,'(A)') "" + enddo + + write(0,'(A,A,dt)') 'stencil coordinates:' + do jlat=1,stencil%width + jgp = functionspace%index(stencil%i(1,jlat),stencil%j(jlat)) + write(0,'(A, F6.2,A)', advance='no') " lat : ", xy(2,jgp), " --- lon : " + do jlon=1,stencil%width + jgp = functionspace%index(stencil%i(jlon,jlat),stencil%j(jlat)) + write(0,'(F8.2)',advance='no') xy(1,jgp) + enddo + write(0,'(A)') "" + enddo + + endif + call compute_stencil%final() +end block + +call functionspace%final() +call grid%final() +end subroutine + +! ------------------------------------------------------------------------------------------------ + +program main +use atlas_module, only : atlas_library +implicit none +call atlas_library%initialize() +call run() +call atlas_library%finalize() +end program \ No newline at end of file diff --git a/doc/example-grids/classic_gaussian_2.yml b/doc/example-grids/classic_gaussian_2.yml index a9c6b4165..c4cf93018 100644 --- a/doc/example-grids/classic_gaussian_2.yml +++ b/doc/example-grids/classic_gaussian_2.yml @@ -10,7 +10,7 @@ projection : check : size : 1688 - lonlat(first) : [3,44.8796] - lonlat(last) : [-172.453,-54.9736] - uid : 64d609c6ee4b036b209047aef97a10eb + lonlat(first) : [3,49.1204] + lonlat(last) : [179.6485,-38.8936] + uid : 02ef7ed866af557e04882342a90fddfe bounding_box(n,w,s,e) : [90,0,-90,360] diff --git a/doc/example-grids/custom_structured_5.yml b/doc/example-grids/custom_structured_5.yml index 6dc221daa..e7960017a 100644 --- a/doc/example-grids/custom_structured_5.yml +++ b/doc/example-grids/custom_structured_5.yml @@ -10,5 +10,5 @@ check : size : 144 lonlat(first) : [3,40] lonlat(last) : [-177,-40] - uid : d55211f27c4a8c6ce94ea67f0c706e22 + uid : 23f0a23085bbea10277b457f482927c0 bounding_box(n,w,s,e) : [90,0,-90,360] diff --git a/doc/example-grids/octahedral_gaussian_2.yml b/doc/example-grids/octahedral_gaussian_2.yml index 0b8354d23..04379e5f7 100644 --- a/doc/example-grids/octahedral_gaussian_2.yml +++ b/doc/example-grids/octahedral_gaussian_2.yml @@ -7,7 +7,7 @@ projection : { type: "rotated_schmidt", stretching_factor : 4.0, north_pole : [ check : size : 1600 - lonlat(first) : [3,45.9397] - lonlat(last) : [-165.776,-62.6128] - uid : 20308244515b7c78e22709ebaa842246 + lonlat(first) : [3,48.0603] + lonlat(last) : [177.0166,-30.8] + uid : 1297f9ab088b831da3139eb5f44e20a9 bounding_box(n,w,s,e) : [90,0,-90,360] diff --git a/doc/example-grids/octahedral_gaussian_4.yml b/doc/example-grids/octahedral_gaussian_4.yml index 5ea110afe..9b860b59b 100644 --- a/doc/example-grids/octahedral_gaussian_4.yml +++ b/doc/example-grids/octahedral_gaussian_4.yml @@ -6,7 +6,7 @@ projection : { type: "rotated_schmidt", stretching_factor: 4.0, north_pole: [3. check : size : 1600 - lonlat(first) : [3,45.9397] - lonlat(last) : [-165.776,-62.6128] - uid : 20308244515b7c78e22709ebaa842246 + lonlat(first) : [3,48.0603] + lonlat(last) : [177.0166,-30.8] + uid : 1297f9ab088b831da3139eb5f44e20a9 bounding_box(n,w,s,e) : [90,0,-90,360] diff --git a/doc/example-grids/regional_rotated_mercator_1.yml b/doc/example-grids/regional_rotated_mercator_1.yml index 8ea84ecfc..6ccb19ca3 100644 --- a/doc/example-grids/regional_rotated_mercator_1.yml +++ b/doc/example-grids/regional_rotated_mercator_1.yml @@ -14,5 +14,5 @@ check : size : 2000 lonlat(first) : [-10.319,40.2109] lonlat(last) : [24.4206,57.2263] - uid : 9d4b8c38db08c6f3fe3221c8770d8d57 + uid : 88d856331d46d30703b38e3ed2e8b2de bounding_box(n,w,s,e) : [58.734,-16.4206,40.2109,24.4206] diff --git a/doc/example-grids/regular_gaussian_2.yml b/doc/example-grids/regular_gaussian_2.yml index b7cf9d110..81508365f 100644 --- a/doc/example-grids/regular_gaussian_2.yml +++ b/doc/example-grids/regular_gaussian_2.yml @@ -10,7 +10,7 @@ projection : check : size : 128 - lonlat(first) : [3,38.8589] - lonlat(last) : [-135.011,-72.4668] - uid : d179fd0b56811242a1a0a8e83dade6a1 + lonlat(first) : [3,55.1411] + lonlat(last) : [170.8438,-16.8513] + uid : 2efc83bfae617ec2fe1a7cebe523cf41 bounding_box(n,w,s,e) : [90,0,-90,360] diff --git a/hic/AUTHORS b/hic/AUTHORS new file mode 100644 index 000000000..e4c77d93c --- /dev/null +++ b/hic/AUTHORS @@ -0,0 +1,8 @@ +Authors +======= + +- Willem Deconinck + +Thanks for contributions from +============================= + diff --git a/hic/CHANGELOG.md b/hic/CHANGELOG.md new file mode 100644 index 000000000..e69de29bb diff --git a/hic/CMakeLists.txt b/hic/CMakeLists.txt new file mode 100644 index 000000000..904f7a077 --- /dev/null +++ b/hic/CMakeLists.txt @@ -0,0 +1,56 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +############################################################################################ +# hic + +cmake_minimum_required( VERSION 3.18 FATAL_ERROR ) + +find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild ${CMAKE_CURRENT_SOURCE_DIR}/../../ecbuild ) + +################################################################################ +# Initialise project + +if( NOT DEFINED atlas_VERSION ) + set( atlas_VERSION 0.0.0 ) +endif() + +project( hic VERSION ${atlas_VERSION} LANGUAGES CXX ) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +################################################################################ +# Features that can be enabled / disabled with -DENABLE_ + +ecbuild_add_option( FEATURE CUDA DESCRIPTION "Use CUDA as backend to HIC" DEFAULT OFF ) +ecbuild_add_option( FEATURE HIP DESCRIPTION "Use HIP as backend to HIC" DEFAULT OFF ) + +if( HAVE_CUDA ) + find_package(CUDAToolkit REQUIRED) +elseif( HAVE_HIP ) + find_package(hip CONFIG REQUIRED) +endif() + +add_subdirectory( src ) +add_subdirectory( tests ) + +################################################################################ +# Export and summarize + +ecbuild_add_resources( + TARGET hic-others + SOURCES_PACK + README.md + CHANGELOG.md + LICENSE +) + +ecbuild_install_project( NAME hic ) +ecbuild_print_summary() + diff --git a/hic/LICENSE b/hic/LICENSE new file mode 100644 index 000000000..af6ca0287 --- /dev/null +++ b/hic/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 1996-2018 ECMWF + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/hic/README.md b/hic/README.md new file mode 100644 index 000000000..1b756fb40 --- /dev/null +++ b/hic/README.md @@ -0,0 +1,5 @@ +hic +=== + +HIC: An abstraction to HIP and CUDA + diff --git a/hic/cmake/hic-import.cmake.in b/hic/cmake/hic-import.cmake.in new file mode 100644 index 000000000..23ac59885 --- /dev/null +++ b/hic/cmake/hic-import.cmake.in @@ -0,0 +1,13 @@ + +include( CMakeFindDependencyMacro ) + +set( hic_HAVE_CUDA @hic_HAVE_CUDA@ ) +set( hic_HAVE_HIP @hic_HAVE_HIP@ ) + +if( hic_HAVE_CUDA ) + find_dependency( CUDAToolkit ) +endif() +if( hic_HAVE_HIP ) + find_dependency( hip CONFIG ) +endif() + diff --git a/hic/src/CMakeLists.txt b/hic/src/CMakeLists.txt new file mode 100644 index 000000000..5048de357 --- /dev/null +++ b/hic/src/CMakeLists.txt @@ -0,0 +1,39 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +set(HIC_BACKEND_CUDA 0) +set(HIC_BACKEND_HIP 0) +set(HIC_BACKEND_DUMMY 0) +if( HAVE_CUDA ) + set(HIC_BACKEND_CUDA 1) +elseif( HAVE_HIP ) + set(HIC_BACKEND_HIP 1) +else() + set(HIC_BACKEND_DUMMY 1) +endif() + +configure_file(${PROJECT_SOURCE_DIR}/src/hic/hic_config.h.in ${PROJECT_BINARY_DIR}/src/hic/hic_config.h @ONLY ) + +ecbuild_add_library( TARGET hic + TYPE INTERFACE + PUBLIC_INCLUDES + $ + $ + $ +) + +install( FILES ${PROJECT_BINARY_DIR}/src/hic/hic_config.h DESTINATION include/hic ) +install( FILES hic/hic.h DESTINATION include/hic ) +install( FILES hic/hic_runtime.h DESTINATION include/hic ) +install( FILES hic/hic_dummy/hic_dummy_runtime.h DESTINATION include/hic/hic_dummy ) + +if( HAVE_CUDA ) + target_link_libraries( hic INTERFACE CUDA::cudart ) +elseif( HAVE_HIP ) + target_link_libraries( hic INTERFACE hip::host ) +endif() diff --git a/hic/src/hic/hic.h b/hic/src/hic/hic.h new file mode 100644 index 000000000..66a2530bf --- /dev/null +++ b/hic/src/hic/hic.h @@ -0,0 +1,38 @@ +/* + * (C) Copyright 2024- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ +#pragma once + +#include "hic/hic_config.h" +#include "hic/hic_runtime.h" + +#include +#include + +#if HIC_BACKEND_DUMMY +#define HIC_CALL(val) +#else +#define HIC_CALL(val) hic_assert((val), #val, __FILE__, __LINE__) +#endif + +inline void hic_assert(hicError_t err, const char* const func, const char* const file, const int line) { + // Ignore errors when HIP/CUDA runtime is unloaded or deinitialized. + // This happens when calling HIP/CUDA after main has ended, e.g. in teardown of static variables calling `hicFree` + // --> ignore hicErrorDeinitialized (a.k.a. cudaErrorCudartUnloading / hipErrorDeinitialized) + if (err != hicSuccess && err != hicErrorDeinitialized) { + std::ostringstream msg; + msg << "HIC Runtime Error [code="< +#include + +#define DUMMY_SHOULD_NOT_BE_CALLED(SYMBOL) dummyShouldNotBeCalled( #SYMBOL ) +#define DUMMY_FUNCTION(SYMBOL) \ + template inline \ + dummyError_t dummy##SYMBOL(Args&&... args) { \ + DUMMY_SHOULD_NOT_BE_CALLED( hic##SYMBOL ); \ + return dummyError_t{0}; \ + } +#define DUMMY_VALUE(SYMBOL) \ + constexpr int dummy##SYMBOL = 0; + +namespace { + +[[noreturn]] void dummyShouldNotBeCalled(const char* symbol) { + throw std::runtime_error(std::string(symbol)+" is using the dummy backend and should not be called"); +} + +using dummyError_t = int; +using dummyEvent_t = void*; +using dummyStream_t = void*; + +const char* dummyGetErrorString(dummyError_t) { + DUMMY_SHOULD_NOT_BE_CALLED( hicGetErrorString ); +} + +dummyError_t dummyGetLastError( void ) { + DUMMY_SHOULD_NOT_BE_CALLED( hicGetLastError ); +} + +dummyError_t dummyPeekAtLastError( void ) { + DUMMY_SHOULD_NOT_BE_CALLED( hicPeekAtLastError ); +} + +struct dummyPointerAttributes { + int type{0}; + int device{-2}; + void* hostPointer{nullptr}; + void* devicePointer{nullptr}; +}; + +DUMMY_FUNCTION(DeviceSynchronize) +DUMMY_FUNCTION(Free) +DUMMY_FUNCTION(FreeAsync) +DUMMY_FUNCTION(GetDeviceCount); +DUMMY_FUNCTION(GetErrorString) +DUMMY_FUNCTION(GetLastError) +DUMMY_FUNCTION(HostGetDevicePointer) +DUMMY_FUNCTION(HostRegister) +DUMMY_FUNCTION(HostUnregister) +DUMMY_FUNCTION(Malloc) +DUMMY_FUNCTION(MallocAsync) +DUMMY_FUNCTION(MallocManaged) +DUMMY_FUNCTION(Memcpy) +DUMMY_FUNCTION(Memcpy2D) +DUMMY_FUNCTION(MemcpyAsync) +DUMMY_FUNCTION(Memcpy2DAsync) +DUMMY_FUNCTION(MemPrefetchAsync) +DUMMY_FUNCTION(StreamCreate) +DUMMY_FUNCTION(StreamDestroy) +DUMMY_FUNCTION(StreamSynchronize) +DUMMY_FUNCTION(PointerGetAttributes) + +DUMMY_VALUE(CpuDeviceId) +DUMMY_VALUE(HostRegisterMapped) +DUMMY_VALUE(MemoryTypeDevice) +DUMMY_VALUE(MemoryTypeHost) +DUMMY_VALUE(MemoryTypeUnregistered) +DUMMY_VALUE(MemoryTypeManaged) +DUMMY_VALUE(MemcpyDeviceToHost) +DUMMY_VALUE(MemcpyHostToDevice) +DUMMY_VALUE(Success) + +} + +#undef DUMMY_FUNCTION +#undef DUMMY_VALUE +#undef DUMMY_SHOULD_NOT_BE_CALLED \ No newline at end of file diff --git a/hic/src/hic/hic_runtime.h b/hic/src/hic/hic_runtime.h new file mode 100644 index 000000000..25af23cb0 --- /dev/null +++ b/hic/src/hic/hic_runtime.h @@ -0,0 +1,180 @@ +/* + * (C) Copyright 2024- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ +#pragma once + +#if !defined(HIC_NAMESPACE) + #define HIC_NAMESPACE + #define HIC_NAMESPACE_BEGIN + #define HIC_NAMESPACE_END +#else + #define HIC_NAMESPACE_BEGIN namespace HIC_NAMESPACE { + #define HIC_NAMESPACE_END } +#endif + +#if HIC_BACKEND_CUDA + #define HIC_BACKEND cuda + #include +#elif HIC_BACKEND_HIP + #define HIC_BACKEND hip + #if defined(DEPRECATED) + #define DEFINED_OUTERSCOPE DEPRECATED + #undef DEPRECATED + #endif + #include + #if defined(DEPRECATED) + #undef DEPRECATED + #endif + #if defined(DEFINED_OUTERSCOPE) + #define DEPRECATED DEFINED_OUTERSCOPE + #endif + +#if HIP_VERSION_MAJOR < 6 + enum hicMemoryType { + hicMemoryTypeUnregistered = 0 , + hicMemoryTypeHost = 1 , + hicMemoryTypeDevice = 2 , + hicMemoryTypeManaged = 3 + }; + struct hicPointerAttributes { + hicMemoryType type; + int device; + void* devicePointer; + void* hostPointer; + }; + + [[nodiscard]] inline hipError_t hicPointerGetAttributes(hicPointerAttributes* attributes, const void* ptr) { + hipPointerAttribute_t attr_; + auto err = hipPointerGetAttributes(&attr_, ptr); + if( err != hipSuccess ) { + attributes->device = 0; + attributes->devicePointer = nullptr; + attributes->hostPointer = nullptr; + return err; + } + const auto& type = attr_.memoryType; + if(attr_.isManaged) { + attributes->type = hicMemoryTypeManaged; + } + else if(type == hipMemoryTypeHost) { + attributes->type = hicMemoryTypeHost; + } + else if(type == hipMemoryTypeDevice) { + attributes->type = hicMemoryTypeDevice; + } + else if(type == hipMemoryTypeUnified) { + attributes->type = hicMemoryTypeManaged; + } + else { + attributes->type = hicMemoryTypeUnregistered; + } + attributes->device = attr_.device; + attributes->devicePointer = attr_.devicePointer; + attributes->hostPointer = attr_.hostPointer; + return err; + }; +#endif + +#elif HIC_BACKEND_DUMMY + #define HIC_BACKEND dummy + #include "hic/hic_dummy/hic_dummy_runtime.h" +#else + #error Unsupported hic backend. Please define HIC_BACKEND_CUDA or HIC_BACKEND_HIP or HIC_BACKEND_DUMMY +#endif + +#define HIC_PREFIX hic +#define HIC_CONCAT_(A, B) A ## B +#define HIC_CONCAT(A, B) HIC_CONCAT_(A, B) +#define HIC_SYMBOL(API) HIC_CONCAT(HIC_PREFIX, API) +#define HIC_BACKEND_SYMBOL(API) HIC_CONCAT(HIC_BACKEND, API) + +#define HIC_TYPE(TYPE) \ + using HIC_SYMBOL(TYPE) = HIC_BACKEND_SYMBOL(TYPE); + +#define HIC_FUNCTION(FUNCTION) \ + template inline \ + auto HIC_SYMBOL(FUNCTION)(Args&&... args) -> decltype(HIC_BACKEND_SYMBOL(FUNCTION)(std::forward(args)...)) { \ + return HIC_BACKEND_SYMBOL(FUNCTION)(std::forward(args)...); \ + } + +#define HIC_VALUE(VALUE) \ + constexpr decltype(HIC_BACKEND_SYMBOL(VALUE)) HIC_SYMBOL(VALUE) = HIC_BACKEND_SYMBOL(VALUE); + +//------------------------------------------------ +HIC_NAMESPACE_BEGIN +//------------------------------------------------ + +HIC_FUNCTION(DeviceSynchronize) +HIC_FUNCTION(Free) +HIC_FUNCTION(FreeAsync) +HIC_FUNCTION(GetDeviceCount) +HIC_FUNCTION(GetErrorString) +HIC_FUNCTION(GetLastError) +HIC_FUNCTION(HostGetDevicePointer) +HIC_FUNCTION(HostRegister) +HIC_FUNCTION(HostUnregister) +HIC_FUNCTION(Malloc) +HIC_FUNCTION(MallocAsync) +HIC_FUNCTION(MallocManaged) +HIC_FUNCTION(Memcpy) +HIC_FUNCTION(Memcpy2D) +HIC_FUNCTION(MemcpyAsync) +HIC_FUNCTION(Memcpy2DAsync) +HIC_FUNCTION(MemPrefetchAsync) +HIC_FUNCTION(PeekAtLastError) +#if !HIC_BACKEND_HIP +HIC_FUNCTION(PointerGetAttributes) +#elif HIP_VERSION_MAJOR >= 6 +HIC_FUNCTION(PointerGetAttributes) +#endif +HIC_FUNCTION(StreamCreate) +HIC_FUNCTION(StreamDestroy) +HIC_FUNCTION(StreamSynchronize) + +HIC_TYPE(Error_t) +HIC_TYPE(Event_t) +HIC_TYPE(Stream_t) +#if !HIC_BACKEND_HIP +HIC_TYPE(PointerAttributes) +#elif HIP_VERSION_MAJOR >= 6 +using HIC_SYMBOL(PointerAttributes) = HIC_BACKEND_SYMBOL(PointerAttribute_t); +#endif + +HIC_VALUE(CpuDeviceId) +HIC_VALUE(HostRegisterMapped) +#if !HIC_BACKEND_HIP +HIC_VALUE(MemoryTypeDevice) +HIC_VALUE(MemoryTypeHost) +HIC_VALUE(MemoryTypeUnregistered) +HIC_VALUE(MemoryTypeManaged) +#endif +HIC_VALUE(MemcpyDeviceToHost) +HIC_VALUE(MemcpyHostToDevice) +HIC_VALUE(Success) + +#if HIC_BACKEND_CUDA + constexpr hicError_t hicErrorDeinitialized = cudaErrorCudartUnloading; +#elif HIC_BACKEND_HIP + constexpr hicError_t hicErrorDeinitialized = hipErrorDeinitialized; +#else + constexpr hicError_t hicErrorDeinitialized = 4; +#endif + +//------------------------------------------------ +HIC_NAMESPACE_END +//------------------------------------------------ + +#undef HIC_FUNCTION +#undef HIC_TYPE +#undef HIC_VALUE +#undef HIC_CONCAT +#undef HIC_CONCAT_ +#undef HIC_PREFIX +#undef HIC_SYMBOL +#undef HIC_BACKEND_SYMBOL diff --git a/hic/tests/CMakeLists.txt b/hic/tests/CMakeLists.txt new file mode 100644 index 000000000..ecd23347d --- /dev/null +++ b/hic/tests/CMakeLists.txt @@ -0,0 +1,22 @@ +# (C) Copyright 2024 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +ecbuild_add_test( TARGET hic_test_dummy + SOURCES test_hic_dummy.cc + LIBS hic + DEFINITIONS HIC_BACKEND_DUMMY=1 +) + +if( HAVE_HIP OR HAVE_CUDA ) + ecbuild_add_test( TARGET hic_test_hic + SOURCES test_hic.cc + LIBS hic + ) +endif() + +add_subdirectory(test_find_hic) diff --git a/hic/tests/test_find_hic/CMakeLists.txt b/hic/tests/test_find_hic/CMakeLists.txt new file mode 100644 index 000000000..e47613247 --- /dev/null +++ b/hic/tests/test_find_hic/CMakeLists.txt @@ -0,0 +1,31 @@ + +# This test builds a package that requires fypp processing +# It uses the overriding of compile flags, like IFS is using. +# +# Test created to avoid regression after fixing issue FCKIT-19, +# where compile flags were not propagated to fypp-generated files. + +if( HAVE_TESTS ) + + configure_file( test.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test.sh @ONLY ) + + unset( _test_args ) + if( CMAKE_TOOLCHAIN_FILE ) + if( NOT IS_ABSOLUTE ${CMAKE_TOOLCHAIN_FILE}) + set( CMAKE_TOOLCHAIN_FILE "${CMAKE_BINARY_DIR}/${CMAKE_TOOLCHAIN_FILE}" ) + endif() + list( APPEND _test_args "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" ) + endif() + foreach( lang CXX ) + if( CMAKE_${lang}_COMPILER ) + list( APPEND _test_args "-DCMAKE_${lang}_COMPILER=${CMAKE_${lang}_COMPILER}" ) + endif() + if( CMAKE_${lang}_FLAGS ) + list( APPEND _test_args "-DCMAKE_${lang}_FLAGS=${CMAKE_${lang}_FLAGS}" ) + endif() + endforeach() + + add_test( NAME hic_test_find_hic + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test.sh ${_test_args} ) + +endif() diff --git a/hic/tests/test_find_hic/find_hic/CMakeLists.txt b/hic/tests/test_find_hic/find_hic/CMakeLists.txt new file mode 100644 index 000000000..4900f462c --- /dev/null +++ b/hic/tests/test_find_hic/find_hic/CMakeLists.txt @@ -0,0 +1,6 @@ + +cmake_minimum_required( VERSION 3.18 FATAL_ERROR ) + +project( find_hic LANGUAGES CXX ) + +find_package( hic REQUIRED ) diff --git a/hic/tests/test_find_hic/test.sh.in b/hic/tests/test_find_hic/test.sh.in new file mode 100755 index 000000000..81b37eda1 --- /dev/null +++ b/hic/tests/test_find_hic/test.sh.in @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Description: +# Build downstream example projects +# each individually with make/install +# +# Usage: +# test-individual.sh [CMAKE_ARGUMENTS] + +SOURCE=@PROJECT_SOURCE_DIR@/tests/test_find_hic/find_hic +BUILD=@CMAKE_CURRENT_BINARY_DIR@/build + +# Error handling +function test_failed { + EXIT_CODE=$? + { set +ex; } 2>/dev/null + if [ $EXIT_CODE -ne 0 ]; then + echo "+++++++++++++++++" + echo "Test failed" + echo "+++++++++++++++++" + fi + exit $EXIT_CODE +} +trap test_failed EXIT +set -e -o pipefail +set -x + +# Start with clean build +rm -rf $BUILD + +export hic_ROOT=@PROJECT_BINARY_DIR@ +export ecbuild_DIR=@ecbuild_DIR@ + +# Build +mkdir -p $BUILD && cd $BUILD +cmake $SOURCE \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + "$@" + +# make VERBOSE=1 + +# ctest -VV + +{ set +ex; } 2>/dev/null +echo "+++++++++++++++++" +echo "Test passed" +echo "+++++++++++++++++" diff --git a/hic/tests/test_hic.cc b/hic/tests/test_hic.cc new file mode 100644 index 000000000..7355ec807 --- /dev/null +++ b/hic/tests/test_hic.cc @@ -0,0 +1,66 @@ +/* + * (C) Copyright 2024- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ +#include +#include +#include +#include + +#include "hic/hic.h" + +// ----------------------------------------------------------------------------- + +int test_hicMalloc() { + std::cout << "--- " << __FUNCTION__ << std::endl; + void* p; + HIC_CALL( hicMalloc(&p, 8) ); + HIC_CALL( hicFree(p) ); + std::cout << "--- " << __FUNCTION__ << " SUCCEEDED " << std::endl; + return 0; // success +} + +// ----------------------------------------------------------------------------- + +int test_hicMallocManaged() { + std::cout << "--- " << __FUNCTION__ << std::endl; + void* p; + HIC_CALL( hicMallocManaged(&p, 8) ); + HIC_CALL( hicFree(p) ); + std::cout << "--- " << __FUNCTION__ << " SUCCEEDED " << std::endl; + return 0; // success +} + +// ----------------------------------------------------------------------------- + +std::vector> tests = { + test_hicMalloc, + test_hicMallocManaged, +}; + +int main(int argc, char* argv[]) { + int num_devices = 0; + hicGetDeviceCount(&num_devices); + if( num_devices == 0 ) { + std::ignore = hicGetLastError(); + std::cout << "TEST IGNORED, hicGetDeviceCount -> 0" << std::endl; + return 0; + } + std::cout << "hicGetDeviceCount -> " << num_devices << std::endl; + int error = 0; + for( auto& test: tests) { + try { + error += test(); + } + catch( std::exception& e ) { + error += 1; + std::cout << e.what() << std::endl; + } + } + return error; +} diff --git a/hic/tests/test_hic_dummy.cc b/hic/tests/test_hic_dummy.cc new file mode 100644 index 000000000..cb78b8186 --- /dev/null +++ b/hic/tests/test_hic_dummy.cc @@ -0,0 +1,43 @@ +/* + * (C) Copyright 2024- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ +#include +#include +#include +#include + +#include "hic/hic.h" + +int test_throw() { + std::cout << "--- " << __FUNCTION__ << std::endl; + // For the dummy backend we expect to have an error thrown when used + void* p; + try { + std::ignore = hicMalloc(&p, 8); + std::ignore = hicFree(p); + } catch(std::runtime_error& e) { + std::string expected_message = "hicMalloc is using the dummy backend and should not be called"; + if (e.what() == expected_message) { + std::cout << "--- " << __FUNCTION__ << " SUCCEEDED " << std::endl; + return 0; // success + } + } + std::cout << "--- " << __FUNCTION__ << " SUCCEEDED " << std::endl; + return 1; // fail +} + +std::vector> tests = { test_throw }; + +int main(int argc, char* argv[]) { + int error = 0; + for( auto& test: tests) { + error += test(); + } + return error; +} diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 757f40fd2..60b93b9a1 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -60,6 +60,8 @@ library/detail/BlackMagic.h library/detail/Debug.h +parallel/acc/acc.cc +parallel/acc/acc.h parallel/mpi/mpi.cc parallel/mpi/mpi.h parallel/omp/omp.cc @@ -173,6 +175,8 @@ grid/detail/grid/RegionalVariableResolution.h grid/detail/grid/RegionalVariableResolution.cc grid/detail/grid/Healpix.h grid/detail/grid/Healpix.cc +grid/detail/grid/StencilComputerInterface.h +grid/detail/grid/StencilComputerInterface.cc grid/detail/tiles/Tiles.cc grid/detail/tiles/Tiles.h @@ -376,8 +380,6 @@ mesh/detail/AccumulateFacets.cc util/Unique.h util/Unique.cc -mesh/actions/ExtendNodesGlobal.h -mesh/actions/ExtendNodesGlobal.cc mesh/actions/BuildDualMesh.h mesh/actions/BuildCellCentres.cc mesh/actions/BuildCellCentres.h @@ -401,6 +403,10 @@ mesh/actions/BuildStatistics.cc mesh/actions/BuildStatistics.h mesh/actions/BuildXYZField.cc mesh/actions/BuildXYZField.h +mesh/actions/ExtendNodesGlobal.h +mesh/actions/ExtendNodesGlobal.cc +mesh/actions/GetCubedSphereNodalArea.cc +mesh/actions/GetCubedSphereNodalArea.h mesh/actions/WriteLoadBalanceReport.cc mesh/actions/BuildTorusXYZField.h mesh/actions/BuildTorusXYZField.cc @@ -468,12 +474,24 @@ field/FieldSet.cc field/FieldSet.h field/MissingValue.cc field/MissingValue.h +field/MultiField.cc +field/MultiField.h +field/MultiFieldCreator.cc +field/MultiFieldCreator.h +field/MultiFieldCreatorIFS.cc +field/MultiFieldCreatorIFS.h +field/MultiFieldCreatorArray.cc +field/MultiFieldCreatorArray.h field/State.cc field/State.h field/detail/FieldImpl.cc field/detail/FieldImpl.h field/detail/FieldInterface.cc field/detail/FieldInterface.h +field/detail/MultiFieldImpl.cc +field/detail/MultiFieldImpl.h +field/detail/MultiFieldInterface.cc +field/detail/MultiFieldInterface.h field/detail/MissingValue.cc field/detail/MissingValue.h ) @@ -618,6 +636,8 @@ interpolation/method/PointIndex3.cc interpolation/method/PointIndex3.h interpolation/method/PointIndex2.cc interpolation/method/PointIndex2.h +interpolation/method/binning/Binning.cc +interpolation/method/binning/Binning.h interpolation/method/cubedsphere/CubedSphereBilinear.cc interpolation/method/cubedsphere/CubedSphereBilinear.h interpolation/method/knn/GridBox.cc @@ -651,6 +671,8 @@ interpolation/method/structured/QuasiCubic2D.cc interpolation/method/structured/QuasiCubic2D.h interpolation/method/structured/QuasiCubic3D.cc interpolation/method/structured/QuasiCubic3D.h +interpolation/method/structured/RegionalLinear2D.cc +interpolation/method/structured/RegionalLinear2D.h interpolation/method/structured/StructuredInterpolation2D.h interpolation/method/structured/StructuredInterpolation2D.tcc interpolation/method/structured/StructuredInterpolation3D.h @@ -737,6 +759,8 @@ array/Range.h array/Vector.h array/Vector.cc array/SVector.h +array/ArrayViewVariant.h +array/ArrayViewVariant.cc array/helpers/ArrayInitializer.h array/helpers/ArrayAssigner.h array/helpers/ArrayWriter.h @@ -820,6 +844,8 @@ util/PolygonXY.cc util/PolygonXY.h util/Metadata.cc util/Metadata.h +util/PackVectorFields.cc +util/PackVectorFields.h util/Point.cc util/Point.h util/Polygon.cc @@ -919,14 +945,15 @@ list( APPEND source_list ) -if( atlas_HAVE_CUDA ) +if( atlas_HAVE_GPU ) include( atlas_host_device ) list( APPEND source_list - parallel/HaloExchangeCUDA.h - parallel/HaloExchangeCUDA.cu + parallel/HaloExchangeGPU.h + parallel/HaloExchangeGPU.hic ) atlas_host_device( source_list SOURCES + parallel/HaloExchangeGPU.hic mesh/Connectivity.cc ) @@ -940,7 +967,7 @@ if( atlas_HAVE_CUDA ) atlas_host_device( source_list SOURCES array/native/NativeArrayView.cc - array/native/NativeDataStore.h + array/native/NativeIndexView.cc ) endif() endif() @@ -973,10 +1000,10 @@ ecbuild_add_library( TARGET atlas eckit_mpi eckit_option atlas_io + hic $<${atlas_HAVE_EIGEN}:Eigen3::Eigen> $<${atlas_HAVE_OMP_CXX}:OpenMP::OpenMP_CXX> $<${atlas_HAVE_GRIDTOOLS_STORAGE}:GridTools::gridtools> - $<${atlas_HAVE_CUDA}:CUDA::cudart> PRIVATE_INCLUDES ${CGAL_INCLUDE_DIRS} @@ -990,4 +1017,12 @@ ecbuild_add_library( TARGET atlas ) +if( HAVE_ACC AND CMAKE_Fortran_COMPILER_ID MATCHES NVHPC ) + target_link_options( atlas INTERFACE + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> ) +endif() + target_compile_features( atlas PUBLIC cxx_std_17 ) diff --git a/src/atlas/array.h b/src/atlas/array.h index c2cf7f720..a7ac48d07 100644 --- a/src/atlas/array.h +++ b/src/atlas/array.h @@ -23,6 +23,7 @@ #include "atlas/array/ArraySpec.h" #include "atlas/array/ArrayStrides.h" #include "atlas/array/ArrayView.h" +#include "atlas/array/ArrayViewVariant.h" #include "atlas/array/DataType.h" #include "atlas/array/LocalView.h" #include "atlas/array/MakeView.h" diff --git a/src/atlas/array/ArrayViewVariant.cc b/src/atlas/array/ArrayViewVariant.cc new file mode 100644 index 000000000..f62efd2d8 --- /dev/null +++ b/src/atlas/array/ArrayViewVariant.cc @@ -0,0 +1,110 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "atlas/array/ArrayViewVariant.h" + +#include +#include + +#include "atlas/runtime/Exception.h" + +namespace atlas { +namespace array { + +using namespace detail; + +namespace { + +template +struct VariantTypeHelper { + using type = ArrayViewVariant; +}; + +template <> +struct VariantTypeHelper { + using type = ConstArrayViewVariant; +}; + +template +using VariantType = + typename VariantTypeHelper>::type; + +// Match array.rank() and array.datatype() to variant types. Return result of +// makeView on a successful pattern match. +template +VariantType executeMakeView(ArrayType& array, + const MakeView& makeView) { + using View = std::variant_alternative_t>; + using Value = typename View::non_const_value_type; + constexpr auto Rank = View::rank(); + + if (array.datatype() == DataType::kind() && array.rank() == Rank) { + return makeView(array, Value{}, std::integral_constant{}); + } + + if constexpr (TypeIndex < std::variant_size_v> - 1) { + return executeMakeView(array, makeView); + } else { + throw_Exception("ArrayView<" + array.datatype().str() + ", " + + std::to_string(array.rank()) + + "> is not an alternative in ArrayViewVariant.", + Here()); + } +} + +template +VariantType makeViewVariantImpl(ArrayType& array) { + const auto makeView = [](auto& array, auto value, auto rank) { + return make_view(array); + }; + return executeMakeView<>(array, makeView); +} + +template +VariantType makeHostViewVariantImpl(ArrayType& array) { + const auto makeView = [](auto& array, auto value, auto rank) { + return make_host_view(array); + }; + return executeMakeView<>(array, makeView); +} + +template +VariantType makeDeviceViewVariantImpl(ArrayType& array) { + const auto makeView = [](auto& array, auto value, auto rank) { + return make_device_view(array); + }; + return executeMakeView<>(array, makeView); +} + +} // namespace + +ArrayViewVariant make_view_variant(Array& array) { + return makeViewVariantImpl(array); +} + +ConstArrayViewVariant make_view_variant(const Array& array) { + return makeViewVariantImpl(array); +} + +ArrayViewVariant make_host_view_variant(Array& array) { + return makeHostViewVariantImpl(array); +} + +ConstArrayViewVariant make_host_view_variant(const Array& array) { + return makeHostViewVariantImpl(array); +} + +ArrayViewVariant make_device_view_variant(Array& array) { + return makeDeviceViewVariantImpl(array); +} + +ConstArrayViewVariant make_device_view_variant(const Array& array) { + return makeDeviceViewVariantImpl(array); +} + +} // namespace array +} // namespace atlas diff --git a/src/atlas/array/ArrayViewVariant.h b/src/atlas/array/ArrayViewVariant.h new file mode 100644 index 000000000..94dff8115 --- /dev/null +++ b/src/atlas/array/ArrayViewVariant.h @@ -0,0 +1,103 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include + +#include "atlas/array.h" + +namespace atlas { +namespace array { + +namespace detail { + +using namespace array; + +// Container struct for a list of types. +template +struct Types { + using add_const = Types...>; +}; + +// Container struct for a list of integers. +template +struct Ints {}; + +template +struct VariantHelper; + +// Recursively construct ArrayView std::variant from types Ts and ranks Is. +template +struct VariantHelper, Ints, ArrayViews...> { + using type = typename VariantHelper, Ints, ArrayViews..., + ArrayView...>::type; +}; + +// End recursion. +template +struct VariantHelper, Ints, ArrayViews...> { + using type = std::variant; +}; + +template +using Variant = typename VariantHelper::type; + +using VariantValueTypes = + detail::Types; + +using VariantRanks = detail::Ints<1, 2, 3, 4, 5, 6, 7, 8, 9>; + +} // namespace detail + +/// @brief Variant containing all supported non-const ArrayView alternatives. +using ArrayViewVariant = + detail::Variant; + +/// @brief Variant containing all supported const ArrayView alternatives. +using ConstArrayViewVariant = + detail::Variant; + +/// @brief Create an ArrayView and assign to an ArrayViewVariant. +ArrayViewVariant make_view_variant(Array& array); + +/// @brief Create a const ArrayView and assign to an ArrayViewVariant. +ConstArrayViewVariant make_view_variant(const Array& array); + +/// @brief Create a host ArrayView and assign to an ArrayViewVariant. +ArrayViewVariant make_host_view_variant(Array& array); + +/// @brief Create a const host ArrayView and assign to an ArrayViewVariant. +ConstArrayViewVariant make_host_view_variant(const Array& array); + +/// @brief Create a device ArrayView and assign to an ArrayViewVariant. +ArrayViewVariant make_device_view_variant(Array& array); + +/// @brief Create a const device ArrayView and assign to an ArrayViewVariant. +ConstArrayViewVariant make_device_view_variant(const Array& array); + +/// @brief Return true if View::rank() is any of Ranks... +template +constexpr bool is_rank(const View&) { + return ((std::decay_t::rank() == Ranks) || ...); +} +/// @brief Return true if View::value_type is any of ValuesTypes... +template +constexpr bool is_value_type(const View&) { + using ValueType = typename std::decay_t::value_type; + return ((std::is_same_v) || ...); +} + +/// @brief Return true if View::non_const_value_type is any of ValuesTypes... +template +constexpr bool is_non_const_value_type(const View&) { + using ValueType = typename std::decay_t::non_const_value_type; + return ((std::is_same_v) || ...); +} + +} // namespace array +} // namespace atlas diff --git a/src/atlas/array/SVector.h b/src/atlas/array/SVector.h index af6c81853..b275dc09d 100644 --- a/src/atlas/array/SVector.h +++ b/src/atlas/array/SVector.h @@ -16,7 +16,7 @@ #include "atlas/library/config.h" #include "atlas/util/Allocate.h" -#ifndef __CUDA_ARCH__ +#if ATLAS_HOST_COMPILE #include "atlas/runtime/Exception.h" #endif @@ -34,7 +34,7 @@ class SVector { ATLAS_HOST_DEVICE SVector(const T* data, const idx_t size): data_(data), size_(size), externally_allocated_(true) {} -#ifdef __CUDACC__ +#if ATLAS_HIC_COMPILER // Note that this does not copy!!! It is mainly intended to be passed to a CUDA kernel which requires value semantics for this class ATLAS_HOST_DEVICE SVector(SVector const& other): data_(other.data_), size_(other.size_), externally_allocated_(true) {} @@ -70,7 +70,7 @@ class SVector { ATLAS_HOST_DEVICE void clear() { if (data_ && !externally_allocated_) { -#ifndef __CUDA_ARCH__ +#if ATLAS_HOST_COMPILE deallocate(data_, size_); #endif } @@ -103,23 +103,23 @@ class SVector { ATLAS_HOST_DEVICE T& operator()(const idx_t idx) { - assert(data_ && idx < size_); + //assert(data_ && idx < size_); return data_[idx]; } ATLAS_HOST_DEVICE T const& operator()(const idx_t idx) const { - assert(data_ && idx < size_); + //assert(data_ && idx < size_); return data_[idx]; } ATLAS_HOST_DEVICE T& operator[](const idx_t idx) { - assert(data_ && idx < size_); + //assert(data_ && idx < size_); return data_[idx]; } ATLAS_HOST_DEVICE T const& operator[](const idx_t idx) const { - assert(data_ && idx < size_); + //assert(data_ && idx < size_); return data_[idx]; } @@ -140,7 +140,7 @@ class SVector { } void resize(idx_t N) { -#ifndef __CUDA_ARCH__ +#if ATLAS_HOST_COMPILE ATLAS_ASSERT(not externally_allocated_, "Cannot resize externally allocated (or wrapped) data"); #endif resize_impl(N); @@ -161,7 +161,7 @@ class SVector { for (idx_t c = 0; c < size; ++c) { ptr[c].~T(); } - util::delete_managedmem(ptr); + util::delete_managedmem(ptr, size); } } diff --git a/src/atlas/array/Vector.h b/src/atlas/array/Vector.h index aeadd9bfc..ecef21769 100644 --- a/src/atlas/array/Vector.h +++ b/src/atlas/array/Vector.h @@ -16,9 +16,7 @@ #include "atlas/library/config.h" #include "atlas/runtime/Exception.h" -#if ATLAS_HAVE_CUDA -#include -#endif +#include "hic/hic.h" namespace atlas { namespace array { @@ -59,10 +57,9 @@ class Vector { size_ = N; } - void updateDevice() { - if (!data_gpu_) { -#if ATLAS_HAVE_CUDA - ::cudaMalloc((void**)(&data_gpu_), sizeof(T*) * size_); + void allocateDevice() { + if constexpr (ATLAS_HAVE_GPU) { + HIC_CALL(hicMalloc((void**)(&data_gpu_), sizeof(T*) * size_)); T* buff = new T[size_]; @@ -70,32 +67,36 @@ class Vector { data_[i]->updateDevice(); buff[i] = data_[i]->gpu_object_ptr(); } - ::cudaMemcpy(data_gpu_, buff, sizeof(T*) * size_, cudaMemcpyHostToDevice); - delete buff; -#else + HIC_CALL(hicMemcpy(data_gpu_, buff, sizeof(T*) * size_, hicMemcpyHostToDevice)); + delete[] buff; + } + else { data_gpu_ = data_; -#endif - size_gpu_ = size_; + } + size_gpu_ = size_; + } + + void updateDevice() { + if (!data_gpu_) { + allocateDevice(); } else { ATLAS_ASSERT(size_gpu_ == size_); -#if ATLAS_HAVE_CUDA - for (idx_t i = 0; i < size(); ++i) { - data_[i]->updateDevice(); - assert(data_gpu_[i] == data_[i]->gpu_object_ptr()); + if constexpr (ATLAS_HAVE_GPU) { + for (idx_t i = 0; i < size(); ++i) { + data_[i]->updateDevice(); + assert(data_gpu_[i] == data_[i]->gpu_object_ptr()); + } } -#endif } } void updateHost() { ATLAS_ASSERT(data_gpu_ != nullptr); - -#if ATLAS_HAVE_CUDA - - for (idx_t i = 0; i < size(); ++i) { - data_[i]->updateHost(); + if constexpr (ATLAS_HAVE_GPU) { + for (idx_t i = 0; i < size(); ++i) { + data_[i]->updateHost(); + } } -#endif } T* gpu_object_ptr() { return data_gpu_; } diff --git a/src/atlas/array/gridtools/GridToolsArray.cc b/src/atlas/array/gridtools/GridToolsArray.cc index ac660742a..572c62385 100644 --- a/src/atlas/array/gridtools/GridToolsArray.cc +++ b/src/atlas/array/gridtools/GridToolsArray.cc @@ -25,7 +25,7 @@ #include "atlas/runtime/Log.h" #if ATLAS_HAVE_ACC -#include "atlas_acc_support/atlas_acc_map_data.h" +#include "atlas_acc_support/atlas_acc.h" #endif //------------------------------------------------------------------------------ diff --git a/src/atlas/array/gridtools/GridToolsArrayView.cu b/src/atlas/array/gridtools/GridToolsArrayView.cu deleted file mode 100644 index fe607a93b..000000000 --- a/src/atlas/array/gridtools/GridToolsArrayView.cu +++ /dev/null @@ -1,11 +0,0 @@ -/* - * (C) Copyright 2013 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - -#include "GridToolsArrayView.cc" diff --git a/src/atlas/array/gridtools/GridToolsDataStore.h b/src/atlas/array/gridtools/GridToolsDataStore.h index d2017f14e..058f726e5 100644 --- a/src/atlas/array/gridtools/GridToolsDataStore.h +++ b/src/atlas/array/gridtools/GridToolsDataStore.h @@ -10,11 +10,12 @@ #pragma once +#include "atlas/library/config.h" #include "atlas/array/ArrayDataStore.h" #include "atlas/array/gridtools/GridToolsTraits.h" #if ATLAS_HAVE_ACC -#include "atlas_acc_support/atlas_acc_map_data.h" +#include "atlas_acc_support/atlas_acc.h" #endif //------------------------------------------------------------------------------ @@ -54,7 +55,7 @@ struct GridToolsDataStore : ArrayDataStore { accUnmap(); } - bool deviceAllocated() const override { return ATLAS_HAVE_CUDA; } + bool deviceAllocated() const override { return ATLAS_HAVE_GPU; } bool hostNeedsUpdate() const override { return data_store_->host_needs_update(); } @@ -112,7 +113,7 @@ struct GridToolsDataStore : ArrayDataStore { return ::gridtools::make_host_view<::gridtools::access_mode::read_only>(*data_store_).data(); } void* device_data() const { -#if ATLAS_HAVE_CUDA +#if ATLAS_HAVE_GPU return ::gridtools::make_device_view<::gridtools::access_mode::read_only>(*data_store_).data(); #else return ::gridtools::make_host_view<::gridtools::access_mode::read_only>(*data_store_).data(); diff --git a/src/atlas/array/gridtools/GridToolsIndexView.cu b/src/atlas/array/gridtools/GridToolsIndexView.cu deleted file mode 100644 index a4a8427b2..000000000 --- a/src/atlas/array/gridtools/GridToolsIndexView.cu +++ /dev/null @@ -1,11 +0,0 @@ -/* - * (C) Copyright 2013 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - -#include "GridToolsIndexView.cc" diff --git a/src/atlas/array/native/NativeArrayView.h b/src/atlas/array/native/NativeArrayView.h index 5c85f115f..e9e90a47e 100644 --- a/src/atlas/array/native/NativeArrayView.h +++ b/src/atlas/array/native/NativeArrayView.h @@ -352,7 +352,9 @@ class ArrayView { ATLAS_HOST_DEVICE void check_bounds(Ints... idx) const { static_assert(sizeof...(idx) == Rank, "Expected number of indices is different from rank of array"); +#if ATLAS_HOST_COMPILE return check_bounds_part<0>(idx...); +#endif } #else template @@ -366,11 +368,13 @@ class ArrayView { ATLAS_HOST_DEVICE void check_bounds_force(Ints... idx) const { static_assert(sizeof...(idx) == Rank, "Expected number of indices is different from rank of array"); +#if ATLAS_HOST_COMPILE return check_bounds_part<0>(idx...); +#endif } template - ATLAS_HOST_DEVICE + ATLAS_HOST void check_bounds_part(Int idx, Ints... next_idx) const { if (idx_t(idx) >= shape_[Dim]) { throw_OutOfRange("ArrayView", array_dim(), idx, shape_[Dim]); @@ -379,7 +383,7 @@ class ArrayView { } template - ATLAS_HOST_DEVICE + ATLAS_HOST void check_bounds_part(Int last_idx) const { if (idx_t(last_idx) >= shape_[Dim]) { throw_OutOfRange("ArrayView", array_dim(), last_idx, shape_[Dim]); diff --git a/src/atlas/array/native/NativeDataStore.h b/src/atlas/array/native/NativeDataStore.h index d0ce0baef..60d35dd50 100644 --- a/src/atlas/array/native/NativeDataStore.h +++ b/src/atlas/array/native/NativeDataStore.h @@ -16,20 +16,18 @@ #include // std::numeric_limits::signaling_NaN #include -#if ATLAS_HAVE_CUDA -#include -#endif - #include "atlas/array/ArrayDataStore.h" #include "atlas/library/Library.h" #include "atlas/library/config.h" +#include "atlas/parallel/acc/acc.h" #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "eckit/log/Bytes.h" -#if ATLAS_HAVE_ACC -#include "atlas_acc_support/atlas_acc_map_data.h" -#endif +#include "hic/hic.h" + + +#define ATLAS_ACC_DEBUG 0 //------------------------------------------------------------------------------ @@ -96,17 +94,31 @@ template void initialise(Value[], size_t) {} #endif +static int devices() { + static int devices_ = [](){ + int n = 0; + auto err = hicGetDeviceCount(&n); + if (err != hicSuccess) { + n = 0; + static_cast(hicGetLastError()); + } + return n; + }(); + return devices_; +} + template class DataStore : public ArrayDataStore { public: DataStore(size_t size): size_(size) { allocateHost(); initialise(host_data_, size_); -#if ATLAS_HAVE_CUDA - device_updated_ = false; -#else - device_data_ = host_data_; -#endif + if (ATLAS_HAVE_GPU && devices()) { + device_updated_ = false; + } + else { + device_data_ = host_data_; + } } ~DataStore() { @@ -115,28 +127,28 @@ class DataStore : public ArrayDataStore { } void updateDevice() const override { -#if ATLAS_HAVE_CUDA - if (not device_allocated_) { - allocateDevice(); - } - cudaError_t err = cudaMemcpy(device_data_, host_data_, size_*sizeof(Value), cudaMemcpyHostToDevice); - if (err != cudaSuccess) { - throw_AssertionFailed("Failed to updateDevice: "+std::string(cudaGetErrorString(err)), Here()); + if (ATLAS_HAVE_GPU && devices()) { + if (not device_allocated_) { + allocateDevice(); + } + hicError_t err = hicMemcpy(device_data_, host_data_, size_*sizeof(Value), hicMemcpyHostToDevice); + if (err != hicSuccess) { + throw_AssertionFailed("Failed to updateDevice: "+std::string(hicGetErrorString(err)), Here()); + } + device_updated_ = true; } - device_updated_ = true; -#endif } void updateHost() const override { -#if ATLAS_HAVE_CUDA - if (device_allocated_) { - cudaError_t err = cudaMemcpy(host_data_, device_data_, size_*sizeof(Value), cudaMemcpyDeviceToHost); - if (err != cudaSuccess) { - throw_AssertionFailed("Failed to updateHost: "+std::string(cudaGetErrorString(err)), Here()); + if constexpr (ATLAS_HAVE_GPU) { + if (device_allocated_) { + hicError_t err = hicMemcpy(host_data_, device_data_, size_*sizeof(Value), hicMemcpyDeviceToHost); + if (err != hicSuccess) { + throw_AssertionFailed("Failed to updateHost: "+std::string(hicGetErrorString(err)), Here()); + } + host_updated_ = true; } - host_updated_ = true; } -#endif } bool valid() const override { return true; } @@ -162,33 +174,33 @@ class DataStore : public ArrayDataStore { bool deviceAllocated() const override { return device_allocated_; } void allocateDevice() const override { -#if ATLAS_HAVE_CUDA - if (device_allocated_) { - return; - } - if (size_) { - cudaError_t err = cudaMalloc((void**)&device_data_, sizeof(Value)*size_); - if (err != cudaSuccess) { - throw_AssertionFailed("Failed to allocate GPU memory: " + std::string(cudaGetErrorString(err)), Here()); + if (ATLAS_HAVE_GPU && devices()) { + if (device_allocated_) { + return; + } + if (size_) { + hicError_t err = hicMalloc((void**)&device_data_, sizeof(Value)*size_); + if (err != hicSuccess) { + throw_AssertionFailed("Failed to allocate GPU memory: " + std::string(hicGetErrorString(err)), Here()); + } + device_allocated_ = true; + accMap(); } - device_allocated_ = true; - accMap(); } -#endif } void deallocateDevice() const override { -#if ATLAS_HAVE_CUDA - if (device_allocated_) { - accUnmap(); - cudaError_t err = cudaFree(device_data_); - if (err != cudaSuccess) { - throw_AssertionFailed("Failed to deallocate GPU memory: " + std::string(cudaGetErrorString(err)), Here()); + if constexpr (ATLAS_HAVE_GPU) { + if (device_allocated_) { + accUnmap(); + hicError_t err = hicFree(device_data_); + if (err != hicSuccess) { + throw_AssertionFailed("Failed to deallocate GPU memory: " + std::string(hicGetErrorString(err)), Here()); + } + device_data_ = nullptr; + device_allocated_ = false; } - device_data_ = nullptr; - device_allocated_ = false; } -#endif } bool hostNeedsUpdate() const override { return (not host_updated_); } @@ -210,13 +222,17 @@ class DataStore : public ArrayDataStore { void* voidDeviceData() override { return static_cast(device_data_); } void accMap() const override { -#if ATLAS_HAVE_ACC - if (not acc_mapped_) { + if (not acc_mapped_ && acc::devices()) { ATLAS_ASSERT(deviceAllocated(),"Could not accMap as device data is not allocated"); - atlas_acc_map_data((void*)host_data_, (void*)device_data_, size_ * sizeof(Value)); + ATLAS_ASSERT(!atlas::acc::is_present(host_data_, size_ * sizeof(Value))); + if constexpr(ATLAS_ACC_DEBUG) { + std::cout << " + acc_map_data(hostptr:"< class WrappedDataStore : public ArrayDataStore { public: WrappedDataStore(Value* host_data, size_t size): host_data_(host_data), size_(size) { -#if ATLAS_HAVE_CUDA - device_updated_ = false; -#else - device_data_ = host_data_; -#endif + if (ATLAS_HAVE_GPU && devices()) { + device_updated_ = false; + } + else { + device_data_ = host_data_; + } } void updateDevice() const override { -#if ATLAS_HAVE_CUDA - if (not device_allocated_) { - allocateDevice(); - } - cudaError_t err = cudaMemcpy(device_data_, host_data_, size_*sizeof(Value), cudaMemcpyHostToDevice); - if (err != cudaSuccess) { - throw_AssertionFailed("Failed to updateDevice: "+std::string(cudaGetErrorString(err)), Here()); + if (ATLAS_HAVE_GPU && devices()) { + if (not device_allocated_) { + allocateDevice(); + } + hicError_t err = hicMemcpy(device_data_, host_data_, size_*sizeof(Value), hicMemcpyHostToDevice); + if (err != hicSuccess) { + throw_AssertionFailed("Failed to updateDevice: "+std::string(hicGetErrorString(err)), Here()); + } + device_updated_ = true; } - device_updated_ = true; -#endif } void updateHost() const override { -#if ATLAS_HAVE_CUDA - if (device_allocated_) { - cudaError_t err = cudaMemcpy(host_data_, device_data_, size_*sizeof(Value), cudaMemcpyDeviceToHost); - if (err != cudaSuccess) { - throw_AssertionFailed("Failed to updateHost: "+std::string(cudaGetErrorString(err)), Here()); + if constexpr (ATLAS_HAVE_GPU) { + if (device_allocated_) { + hicError_t err = hicMemcpy(host_data_, device_data_, size_*sizeof(Value), hicMemcpyDeviceToHost); + if (err != hicSuccess) { + throw_AssertionFailed("Failed to updateHost: "+std::string(hicGetErrorString(err)), Here()); + } + host_updated_ = true; } - host_updated_ = true; } -#endif } bool valid() const override { return true; } @@ -346,33 +366,33 @@ class WrappedDataStore : public ArrayDataStore { bool deviceAllocated() const override { return device_allocated_; } void allocateDevice() const override { -#if ATLAS_HAVE_CUDA - if (device_allocated_) { - return; - } - if (size_) { - cudaError_t err = cudaMalloc((void**)&device_data_, sizeof(Value)*size_); - if (err != cudaSuccess) { - throw_AssertionFailed("Failed to allocate GPU memory: " + std::string(cudaGetErrorString(err)), Here()); + if (ATLAS_HAVE_GPU && devices()) { + if (device_allocated_) { + return; + } + if (size_) { + hicError_t err = hicMalloc((void**)&device_data_, sizeof(Value)*size_); + if (err != hicSuccess) { + throw_AssertionFailed("Failed to allocate GPU memory: " + std::string(hicGetErrorString(err)), Here()); + } + device_allocated_ = true; + accMap(); } - device_allocated_ = true; - accMap(); } -#endif } void deallocateDevice() const override { -#if ATLAS_HAVE_CUDA - if (device_allocated_) { - accUnmap(); - cudaError_t err = cudaFree(device_data_); - if (err != cudaSuccess) { - throw_AssertionFailed("Failed to deallocate GPU memory: " + std::string(cudaGetErrorString(err)), Here()); + if constexpr (ATLAS_HAVE_GPU) { + if (device_allocated_) { + accUnmap(); + hicError_t err = hicFree(device_data_); + if (err != hicSuccess) { + throw_AssertionFailed("Failed to deallocate GPU memory: " + std::string(hicGetErrorString(err)), Here()); + } + device_data_ = nullptr; + device_allocated_ = false; } - device_data_ = nullptr; - device_allocated_ = false; } -#endif } bool hostNeedsUpdate() const override { return (not host_updated_); } @@ -394,13 +414,17 @@ class WrappedDataStore : public ArrayDataStore { void* voidDeviceData() override { return static_cast(device_data_); } void accMap() const override { -#if ATLAS_HAVE_ACC - if (not acc_mapped_) { + if (not acc_mapped_ && acc::devices()) { ATLAS_ASSERT(deviceAllocated(),"Could not accMap as device data is not allocated"); - atlas_acc_map_data((void*)host_data_, (void*)device_data_, size_ * sizeof(Value)); + ATLAS_ASSERT(!atlas::acc::is_present(host_data_, size_ * sizeof(Value))); + if constexpr(ATLAS_ACC_DEBUG) { + std::cout << " + acc_map_data(hostptr:"< make_host_view(const Array& array) { template ArrayView make_device_view(Array& array) { -#if ATLAS_HAVE_CUDA +#if ATLAS_HAVE_GPU ATLAS_ASSERT(array.deviceAllocated(),"make_device_view: Array not allocated on device"); return ArrayView((array.device_data()), array.shape(), array.strides()); #else @@ -59,7 +59,7 @@ ArrayView make_device_view(Array& array) { template ArrayView make_device_view(const Array& array) { -#if ATLAS_HAVE_CUDA +#if ATLAS_HAVE_GPU ATLAS_ASSERT(array.deviceAllocated(),"make_device_view: Array not allocated on device"); return ArrayView(array.device_data(), array.shape(), array.strides()); #else diff --git a/src/atlas/field/FieldSet.cc b/src/atlas/field/FieldSet.cc index b1c4a461e..d9ee713f9 100644 --- a/src/atlas/field/FieldSet.cc +++ b/src/atlas/field/FieldSet.cc @@ -22,11 +22,11 @@ namespace field { //------------------------------------------------------------------------------------------------------ void FieldSetImpl::FieldObserver::onFieldRename(FieldImpl& field) { - std::string name = field.name(); - for (auto& kv: fieldset_.index_) { - const auto old_name = kv.first; - const auto idx = kv.second; - if (&field == fieldset_.fields_[idx].get()) { + + for (idx_t idx=0; idxattachObserver(field_observer_); return field; } +void FieldSetImpl::add(const FieldSet& fieldset) { + for( auto& field : fieldset) { + add(field); + } +} + bool FieldSetImpl::has(const std::string& name) const { return index_.count(name); } @@ -76,9 +115,16 @@ bool FieldSetImpl::has(const std::string& name) const { Field& FieldSetImpl::field(const std::string& name) const { if (!has(name)) { - const std::string msg("FieldSet" + (name_.length() ? " \"" + name_ + "\"" : "") + ": cannot find field \"" + - name + "\""); - throw_Exception(msg, Here()); + std::stringstream msg; + msg << "FieldSet" << (name_.length() ? " \"" + name_ + "\"" : "") << ": cannot find field \"" << name << "\""; + throw_Exception(msg.str(), Here()); + } + if (duplicates_.at(name) > 1) { + std::stringstream msg; + msg << "FieldSet" << (name_.length() ? " \"" + name_ + "\"" : "") << ": cannot get field with ambiguous name \n" << name << "\". " + << duplicates_.at(name) << " fields are registered with same name. Access field by index or iterator instead."; + throw_Exception(msg.str(), Here()); + } return const_cast(fields_[index_.at(name)]); } @@ -101,14 +147,8 @@ void FieldSetImpl::set_dirty(bool value) const { } } -std::vector FieldSetImpl::field_names() const { - std::vector ret; - - for (const_iterator field = cbegin(); field != cend(); ++field) { - ret.push_back(field->name()); - } - - return ret; +const std::vector& FieldSetImpl::field_names() const { + return field_names_; } //----------------------------------------------------------------------------- @@ -170,6 +210,12 @@ void atlas__FieldSet__add_field(FieldSetImpl* This, FieldImpl* field) { This->add(field); } +void atlas__FieldSet__add_fieldset(FieldSetImpl* This, FieldSetImpl* fieldset) { + ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldSet"); + ATLAS_ASSERT(fieldset != nullptr, "Reason: Use of uninitialised atlas_FieldSet"); + This->add(fieldset); +} + int atlas__FieldSet__has_field(const FieldSetImpl* This, char* name) { ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldSet"); return This->has(std::string(name)); diff --git a/src/atlas/field/FieldSet.h b/src/atlas/field/FieldSet.h index 4c7a480de..7e110b121 100644 --- a/src/atlas/field/FieldSet.h +++ b/src/atlas/field/FieldSet.h @@ -114,9 +114,10 @@ class FieldSetImpl : public util::Object { return fields_[i]; } - std::vector field_names() const; + const std::vector& field_names() const; Field add(const Field&); + void add(const FieldSet&); bool has(const std::string& name) const; @@ -141,6 +142,8 @@ class FieldSetImpl : public util::Object { std::string name_; ///< internal name util::Metadata metadata_; ///< metadata associated with the FieldSet std::map index_; ///< name-to-index map, to refer fields by name + std::vector field_names_; ///< field names + std::map duplicates_; ///< name-to-duplicates map, to refer fields by name friend class FieldObserver; FieldObserver field_observer_; @@ -151,6 +154,7 @@ extern "C" { FieldSetImpl* atlas__FieldSet__new(char* name); void atlas__FieldSet__delete(FieldSetImpl* This); void atlas__FieldSet__add_field(FieldSetImpl* This, FieldImpl* field); +void atlas__FieldSet__add_fieldset(FieldSetImpl* This, FieldSetImpl* fieldset); int atlas__FieldSet__has_field(const FieldSetImpl* This, char* name); const char* atlas__FieldSet__name(FieldSetImpl* This); idx_t atlas__FieldSet__size(const FieldSetImpl* This); @@ -234,9 +238,10 @@ class FieldSet : DOXYGEN_HIDE(public util::ObjectHandle) { return get()->field(i); } - std::vector field_names() const { return get()->field_names(); } + const std::vector& field_names() const { return get()->field_names(); } Field add(const Field& field) { return get()->add(field); } + void add(const FieldSet& fieldset) { return get()->add(fieldset); } bool has(const std::string& name) const { return get()->has(name); } diff --git a/src/atlas/field/MultiField.cc b/src/atlas/field/MultiField.cc new file mode 100644 index 000000000..e646f17af --- /dev/null +++ b/src/atlas/field/MultiField.cc @@ -0,0 +1,74 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "atlas/field/MultiField.h" + +#include +#include +#include +#include +#include +#include + +#include "atlas/field/MultiFieldCreator.h" +#include "atlas/field/detail/MultiFieldImpl.h" +#include "atlas/runtime/Exception.h" + +namespace atlas { +namespace field { + +//----------------------------------------------------------------------------- + +MultiField::MultiField(const eckit::Configuration& config) { + std::string type; + if (!config.get("type", type)) { + ATLAS_THROW_EXCEPTION("Could not find \"type\" in configuration"); + } + std::unique_ptr creator(MultiFieldCreatorFactory::build(type, config)); + reset(creator->create(config)); +} + +MultiField::MultiField(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) { + std::unique_ptr creator(MultiFieldCreatorFactory::build("MultiFieldCreatorArray")); + reset(creator->create(datatype, shape, var_names)); +} + +const Field& MultiField::field(const std::string& name) const { return get()->field(name); } +Field& MultiField::field(const std::string& name) { return get()->field(name); } +bool MultiField::has(const std::string& name) const { return get()->has(name); } +std::vector MultiField::field_names() const { return get()->field_names(); } + +const Field& MultiField::field(const idx_t idx) const { return get()->field(idx); } +Field& MultiField::field(const idx_t idx) { return get()->field(idx); } +idx_t MultiField::size() const { return get()->size(); } + +const Field& MultiField::operator[](const idx_t idx) const { return get()->field(idx); } +Field& MultiField::operator[](const idx_t idx) { return get()->field(idx); } + +const Field& MultiField::operator[](const std::string& name) const { return get()->field(name); } +Field& MultiField::operator[](const std::string& name) { return get()->field(name); } + +const util::Metadata& MultiField::metadata() const { return get()->metadata(); } +util::Metadata& MultiField::metadata() { return get()->metadata(); } + +MultiField::operator const array::Array&() const { return get()->array(); } +MultiField::operator array::Array&() { return get()->array(); } + +MultiField::operator const FieldSet&() const { return get()->fieldset_; } +MultiField::operator FieldSet&() { return get()->fieldset_; } + +const array::Array& MultiField::array() const { return get()->array(); } +array::Array& MultiField::array() { return get()->array(); } + +//----------------------------------------------------------------------------- + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiField.h b/src/atlas/field/MultiField.h new file mode 100644 index 000000000..2cc3c6903 --- /dev/null +++ b/src/atlas/field/MultiField.h @@ -0,0 +1,132 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Willem Deconinck +/// @date June 2015 + +#pragma once + +#include +#include + +#include "atlas/array/Array.h" +#include "atlas/field/Field.h" +#include "atlas/field/FieldSet.h" +#include "atlas/util/Config.h" +#include "atlas/util/Factory.h" +#include "atlas/util/Metadata.h" +#include "atlas/util/Object.h" +#include "atlas/util/ObjectHandle.h" + +namespace eckit { +class Parametrisation; +} + +namespace atlas { +namespace field { +class MultiFieldImpl; +} +} + +namespace atlas { +namespace field { + +/** + * \brief MultiField class that owns a collection of fields that are co-allocated + * + * Fields can only be described by parametrisation during the construction. + * Once setup, no additional fields can be added. + * + * Fields have to all be of same memory layout and data type + */ + +class MultiField : public util::ObjectHandle { +public: // methods + //-- Constructors + using Handle::Handle; + + MultiField(const eckit::Configuration&); + MultiField(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names); + + //-- Accessors + + const Field& field(const std::string& name) const; + Field& field(const std::string& name); + bool has(const std::string& name) const; + std::vector field_names() const; + + const Field& field(const idx_t idx) const; + Field& field(const idx_t idx); + idx_t size() const; + + const Field& operator[](const idx_t idx) const; + Field& operator[](const idx_t idx); + + const Field& operator[](const std::string& name) const; + Field& operator[](const std::string& name); + + const util::Metadata& metadata() const; + util::Metadata& metadata(); + + // -- Modifiers + + /// @brief Implicit conversion to Array + operator const array::Array&() const; + operator array::Array&(); + + operator const FieldSet&() const; + operator FieldSet&(); + + /// @brief Access contained Array + const array::Array& array() const; + array::Array& array(); + +private: + template + void create(const std::vector shape, const std::vector var_names); +}; + +/** + * \brief MultiFieldArrayRegistry + */ + +class MultiFieldArrayRegistry : public field::FieldObserver { +private: + MultiFieldArrayRegistry() {} + +public: + static MultiFieldArrayRegistry& instance() { + static MultiFieldArrayRegistry inst; + return inst; + } + void onFieldDestruction(FieldImpl& field) override { + std::lock_guard guard(lock_); + map_.erase(&field); + } + + ~MultiFieldArrayRegistry() override = default; + + void add(Field& field, std::shared_ptr array) { + std::lock_guard guard(lock_); + map_.emplace(field.get(), array); + field->attachObserver(*this); + } + +public: + std::mutex lock_; + std::map> map_; + +}; + +// ------------------------------------------------------------------------------------ + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreator.cc b/src/atlas/field/MultiFieldCreator.cc new file mode 100644 index 000000000..72908f997 --- /dev/null +++ b/src/atlas/field/MultiFieldCreator.cc @@ -0,0 +1,58 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +// file deepcode ignore CppMemoryLeak: static pointers for global registry are OK and will be cleaned up at end + +#include "atlas/field/MultiFieldCreator.h" + +#include +#include + +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/Mutex.h" + +#include "atlas/field/MultiFieldCreatorIFS.h" +#include "atlas/field/MultiFieldCreatorArray.h" +#include "atlas/grid/Grid.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + + +namespace atlas { +namespace field { + + +namespace { + +void force_link() { + static struct Link { + Link() { + MultiFieldCreatorBuilder(); + MultiFieldCreatorBuilder(); + } + } link; +} + +} // namespace + +// ------------------------------------------------------------------ + +MultiFieldCreator::MultiFieldCreator() = default; + +MultiFieldCreator::~MultiFieldCreator() = default; + +MultiFieldCreator* MultiFieldCreatorFactory::build(const std::string& builder, const eckit::Configuration& config) { + force_link(); + auto factory = get(builder); + return factory->make(config); +} + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreator.h b/src/atlas/field/MultiFieldCreator.h new file mode 100644 index 000000000..84a0cf48e --- /dev/null +++ b/src/atlas/field/MultiFieldCreator.h @@ -0,0 +1,89 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#ifndef atlas_field_MultiFieldCreator_h +#define atlas_field_MultiFieldCreator_h + +#include + +#include "atlas/util/Object.h" + +#include "atlas/field/MultiField.h" + +namespace eckit { +class Configuration; +} + +//------------------------------------------------------------------------------------------------------ + +namespace atlas { +namespace field { + +//------------------------------------------------------------------------------------------------------ + +/*! + * \brief Base class for creating new multifields based on Configuration + * + * \details + * Example to create field[100][3] of default type double: + * \code{.cpp} + * FieldImpl* field = Field::create( + * Config + * ("creator","ArraySpec") // ArraySpec MultiFieldCreator + * ("shape",array::make_shape(100,3)) // Rank 2 field with indexing [100][3] + * ); + * \endcode + */ +class MultiFieldCreator : public util::Object { +public: + MultiFieldCreator(); + MultiFieldCreator(const eckit::Configuration& config); + + virtual ~MultiFieldCreator(); + + virtual MultiFieldImpl* create(const eckit::Configuration& config = util::Config()) const = 0; + virtual MultiFieldImpl* create(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) const = 0; +}; + +//------------------------------------------------------------------------------------------------------ + +class MultiFieldCreatorFactory : public util::Factory { +public: + static std::string className() { return "MultiFieldCreatorFactory"; } + + /*! + * \brief build MultiFieldCreator with options specified in parametrisation + * \return mesh generator + */ + static MultiFieldCreator* build(const std::string&, const eckit::Configuration& = util::Config()); + + using Factory::Factory; + +private: + virtual MultiFieldCreator* make() = 0; + virtual MultiFieldCreator* make(const eckit::Configuration&) = 0; +}; + +template +class MultiFieldCreatorBuilder : public MultiFieldCreatorFactory { + virtual MultiFieldCreator* make() { return new T(); } + virtual MultiFieldCreator* make(const eckit::Configuration& config) { return new T(config); } + +public: + using MultiFieldCreatorFactory::MultiFieldCreatorFactory; +}; + +//------------------------------------------------------------------------------------------------------ + +} // namespace field +} // namespace atlas + +#endif diff --git a/src/atlas/field/MultiFieldCreatorArray.cc b/src/atlas/field/MultiFieldCreatorArray.cc new file mode 100644 index 000000000..5ecc5c2aa --- /dev/null +++ b/src/atlas/field/MultiFieldCreatorArray.cc @@ -0,0 +1,150 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Slavko Brdar +/// @author Willem Deconinck +/// @date September 2024 + +#include +#include + +#include "atlas/array/ArrayDataStore.h" +#include "atlas/array/DataType.h" +#include "atlas/array/Range.h" +#include "atlas/field/MultiFieldCreatorArray.h" +#include "atlas/field/detail/MultiFieldImpl.h" +#include "atlas/grid/Grid.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + +namespace atlas { +namespace field { + +MultiFieldCreatorArray::MultiFieldCreatorArray() {} + +MultiFieldCreatorArray::MultiFieldCreatorArray(const eckit::Configuration&) {} + +MultiFieldCreatorArray::~MultiFieldCreatorArray() = default; + +MultiFieldImpl* MultiFieldCreatorArray::create(const eckit::Configuration& config) const { + array::DataType datatype = array::DataType::create(); + std::string datatype_str; + if (config.get("datatype", datatype_str)) { + datatype = array::DataType(datatype_str); + } + else { + array::DataType::kind_t kind(array::DataType::kind()); + config.get("kind", kind); + if (!array::DataType::kind_valid(kind)) { + std::stringstream msg; + msg << "Could not create field. kind parameter unrecognized"; + throw_Exception(msg.str()); + } + datatype = array::DataType(kind); + } + std::vector shape; + config.get("shape", shape); + const auto fields = config.getSubConfigurations("fields"); + int nflds = 0; + for (size_t i = 0; i < fields.size(); ++i) { + long nvar = 1; + fields[i].get("nvar", nvar); + nflds += nvar; + } + std::vector var_names; + var_names.resize(nflds); + for (size_t i = 0, cnt = 0; i < fields.size(); ++i) { + std::string name; + fields[i].get("name", name); + long nvar = 1; + fields[i].get("nvar", nvar); + if (nvar > 1) { + for (int ivar = 0; ivar < nvar; ivar++) { + std::stringstream ss; + ss << name << "_" << ivar; + var_names[cnt++] = ss.str(); + } + + } + else { + var_names[cnt++] = name; + } + } + return create(datatype, shape, var_names); +} + +MultiFieldImpl* MultiFieldCreatorArray::create(const array::DataType datatype, const std::vector& shape, const std::vector& var_names) const { + const int dim = shape.size(); + const int nvar = var_names.size(); + ATLAS_ASSERT(nvar > 0 && dim > 2, "MultiField must have at least one field name."); + + int varidx = -1; + for (int i = 0; i < dim; i++) { + if (shape[i] == -1) { + varidx = i; + break; + } + } + + array::ArrayShape multiarray_shape = shape; + multiarray_shape[varidx] = nvar; + + MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; + auto& multiarray = multifield->array(); + + array::ArrayShape field_shape; + field_shape.resize(dim - 1); + array::ArrayStrides field_strides; + field_strides.resize(dim - 1); + for (int i = 0, j = 0; i < dim; i++) { + if (i != varidx) { + field_shape[j] = multiarray.shape()[i]; + field_strides[j] = multiarray.strides()[i]; + ++j; + } + } + array::ArraySpec field_array_spec(field_shape, field_strides); + + for (size_t ifield = 0; ifield < nvar; ++ifield) { + idx_t start_index = multiarray.strides()[varidx] * ifield; + Field field; + if (datatype.kind() == array::DataType::KIND_REAL64) { + double* slice_begin = multiarray.data() + start_index; + field = Field(var_names[ifield], slice_begin, field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + float* slice_begin = multiarray.data() + start_index; + field = Field(var_names[ifield], slice_begin, field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_INT32) { + int* slice_begin = multiarray.data() + start_index; + field = Field(var_names[ifield], slice_begin, field_array_spec); + } + else if (datatype.kind() == array::DataType::KIND_INT64) { + long* slice_begin = multiarray.data() + start_index; + field = Field(var_names[ifield], slice_begin, field_array_spec); + } + else { + ATLAS_NOTIMPLEMENTED; + } + multifield->add(field); + } + Log::debug() << "Creating multifield of " << datatype.str() << " type" << std::endl; + return multifield; +} + +// ------------------------------------------------------------------ + +namespace { +static MultiFieldCreatorBuilder __MultiFieldCreatorArray("MultiFieldCreatorArray"); +} + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreatorArray.h b/src/atlas/field/MultiFieldCreatorArray.h new file mode 100644 index 000000000..c66f2c144 --- /dev/null +++ b/src/atlas/field/MultiFieldCreatorArray.h @@ -0,0 +1,58 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Slavko Brdar +/// @date September 2024 + +#pragma once + +#include "atlas/field/MultiFieldCreator.h" + +namespace eckit { +class Configuration; +} +namespace atlas { +namespace field { +class MultiFieldImpl; +} +} // namespace atlas + +namespace atlas { +namespace field { + +// ------------------------------------------------------------------ + +/*! + * \brief MultiField creator using datatype, shape, variable names as arguments + * \details + * shape argument contains -1 at the position which gets filled with variable names + * Example use: + * \code{.cpp} + * MultiFieldImpl* multifield = MultiField::create( + * datatype, + * shape, + * var_names + * ); + * \endcode + */ +class MultiFieldCreatorArray : public MultiFieldCreator { +public: + MultiFieldCreatorArray(); + MultiFieldCreatorArray(const eckit::Configuration& config); + ~MultiFieldCreatorArray() override; + MultiFieldImpl* create(const eckit::Configuration& config = util::Config()) const override; + MultiFieldImpl* create(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) const override; +}; + +// ------------------------------------------------------------------ + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreatorIFS.cc b/src/atlas/field/MultiFieldCreatorIFS.cc new file mode 100644 index 000000000..c90681694 --- /dev/null +++ b/src/atlas/field/MultiFieldCreatorIFS.cc @@ -0,0 +1,230 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Willem Deconinck +/// @author Slavko Brdar +/// @date June 2024 + +#include +#include + +#include "eckit/config/Parametrisation.h" + +#include "atlas/array/ArrayDataStore.h" +#include "atlas/array/DataType.h" +#include "atlas/field/MultiFieldCreatorIFS.h" +#include "atlas/field/detail/MultiFieldImpl.h" +#include "atlas/grid/Grid.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + +namespace atlas { +namespace field { + +MultiFieldCreatorIFS::MultiFieldCreatorIFS() {} + +MultiFieldCreatorIFS::MultiFieldCreatorIFS(const eckit::Configuration& config) {} + +MultiFieldCreatorIFS::~MultiFieldCreatorIFS() = default; + +MultiFieldImpl* MultiFieldCreatorIFS::create(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) const { + ATLAS_NOTIMPLEMENTED; + return nullptr; +} + +MultiFieldImpl* MultiFieldCreatorIFS::create(const eckit::Configuration& config) const { + long nproma; + config.get("nproma", nproma); + long nlev; + config.get("nlev", nlev); + long nblk = 0; + if (config.has("nblk")) { + config.get("nblk", nblk); + } + else if (config.has("ngptot")) { + long ngptot; + config.get("ngptot", ngptot); + nblk = std::ceil(static_cast(ngptot) / static_cast(nproma)); + } + else { + ATLAS_THROW_EXCEPTION("Configuration not found: ngptot or nblk"); + } + array::DataType datatype = array::DataType::create(); + std::string datatype_str; + if (config.get("datatype", datatype_str)) { + datatype = array::DataType(datatype_str); + } + else { + array::DataType::kind_t kind(array::DataType::kind()); + config.get("kind", kind); + if (!array::DataType::kind_valid(kind)) { + std::stringstream msg; + msg << "Could not create field. kind parameter unrecognized"; + throw_Exception(msg.str()); + } + datatype = array::DataType(kind); + } + + auto fields = config.getSubConfigurations("fields"); + long nfld = 0; + for (const auto& field_params : fields) { + long nvar = 1; + field_params.get("nvar", nvar); + nfld += nvar; + } + array::ArrayShape multiarray_shape = ( (nlev > 0 and nfld > 0) ? array::make_shape(nblk, nfld, nlev, nproma) : + ( (nlev > 0) ? array::make_shape(nblk, nlev, nproma) : ( (nfld > 0) ? array::make_shape(nblk, nfld, nproma) : + array::make_shape(nblk, nproma) ) ) ); + + MultiFieldImpl* multifield = new MultiFieldImpl{array::ArraySpec{datatype, multiarray_shape}}; + auto& multiarray = multifield->array(); + + size_t multiarray_field_idx = 0; + for (size_t i = 0; i < fields.size(); ++i) { + std::string name; + fields[i].get("name", name); + Field field; + size_t field_vars = 1; + if (fields[i].get("nvar", field_vars)) { + array::ArrayShape field_shape = + ( (nlev > 0 and field_vars > 0) ? + array::make_shape(multiarray.shape(0), field_vars, multiarray.shape(2), multiarray.shape(3)) : + ( nlev > 0 ? + array::make_shape(multiarray.shape(0), multiarray.shape(1), multiarray.shape(2)) : + ( field_vars > 0 ? + array::make_shape(multiarray.shape(0), field_vars, multiarray.shape(2)) : + array::make_shape(multiarray.shape(0), multiarray.shape(1)) ) ) ); + array::ArrayShape multiarray_shape = + ( (nlev > 0 and field_vars > 0) ? array::make_shape(nblk, field_vars, nlev, nproma) : + ( (nlev > 0) ? array::make_shape(nblk, nlev, nproma) : ( (field_vars > 0) ? + array::make_shape(nblk, field_vars, nproma) : array::make_shape(nblk, nproma) ) ) ); + auto field_strides = multiarray.strides(); + auto field_array_spec = array::ArraySpec(field_shape, field_strides); + + constexpr auto all = array::Range::all(); + const auto range = array::Range(multiarray_field_idx, multiarray_field_idx + field_vars); + if (datatype.kind() == array::DataType::KIND_REAL64) { + if (nlev > 0 and field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, range, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (nlev > 0) { + auto slice = array::make_view(multiarray).slice(all, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, range, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + auto slice = array::make_view(multiarray).slice(all, all); + field = Field(name, slice.data(), field_array_spec); + } + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + if (nlev > 0 and field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, range, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (nlev > 0) { + auto slice = array::make_view(multiarray).slice(all, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, range, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + auto slice = array::make_view(multiarray).slice(all, all); + field = Field(name, slice.data(), field_array_spec); + } + } + else { + ATLAS_NOTIMPLEMENTED; + } + field.set_variables(field_vars); + } + else { + array::ArraySpec field_array_spec; + if (nlev > 0) { + auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2), multiarray.shape(3)); + auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2), multiarray.stride(3)); + field_array_spec = array::ArraySpec(field_shape, field_strides); + } + else if (field_vars > 0) { + auto field_shape = array::make_shape(multiarray.shape(0), multiarray.shape(2)); + auto field_strides = array::make_strides(multiarray.stride(0), multiarray.stride(2)); + field_array_spec = array::ArraySpec(field_shape, field_strides); + } + + constexpr auto all = array::Range::all(); + if (datatype.kind() == array::DataType::KIND_REAL64) { + if (nlev > 0 and field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (nlev > 0) { + auto slice = array::make_view(multiarray).slice(all, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + auto slice = array::make_view(multiarray).slice(all, all); + field = Field(name, slice.data(), field_array_spec); + } + } + else if (datatype.kind() == array::DataType::KIND_REAL32) { + if (nlev > 0 and field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (nlev > 0) { + auto slice = array::make_view(multiarray).slice(all, all, all); + field = Field(name, slice.data(), field_array_spec); + } + else if (field_vars > 0) { + auto slice = array::make_view(multiarray).slice(all, multiarray_field_idx, all); + field = Field(name, slice.data(), field_array_spec); + } + else { + auto slice = array::make_view(multiarray).slice(all, all); + field = Field(name, slice.data(), field_array_spec); + } + } + else { + ATLAS_NOTIMPLEMENTED; + } + } + field.set_levels(nlev); + //field.set_blocks(nblk); + + multifield->add(field); + + multiarray_field_idx += field_vars; + } + std::string name; + config.get("name", name); + Log::debug() << "Creating IFS " << datatype.str() << " multifield: " << name << "[nblk=" << nblk << "][nvar=" << nfld + << "][nlev=" << nlev << "][nproma=" << nproma << "]\n"; + return multifield; +} + +// ------------------------------------------------------------------ + +namespace { +static MultiFieldCreatorBuilder __MultiFieldCreatorIFS("MultiFieldCreatorIFS"); +} + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/MultiFieldCreatorIFS.h b/src/atlas/field/MultiFieldCreatorIFS.h new file mode 100644 index 000000000..dddd7cd04 --- /dev/null +++ b/src/atlas/field/MultiFieldCreatorIFS.h @@ -0,0 +1,63 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Willem Deconinck +/// @date June 2015 + +#pragma once + +#include "atlas/field/MultiFieldCreator.h" + +namespace eckit { +class Configuration; +} +namespace atlas { +namespace field { +class MultiFieldImpl; +} +} // namespace atlas + +namespace atlas { +namespace field { + +// ------------------------------------------------------------------ + +/*! + * \brief MultiField creator using IFS parametrisation + * \details + * Ideally this class should belong to IFS. + * The only reference to IFS in Atlas::MultiField should be here. + * Example use: + * \code{.cpp} + * MultiFieldImpl* multifield = MultiField::create( + * Config + * ("creator","MultiFieldIFS") // MultiFieldIFS FieldCreator + * ("ngptot",ngptot) // Total number of grid points + * ("nproma",nproma) // Grouping of grid points for vectorlength + * ("nlev",nlev) // Number of levels + * ("nvar",nvar) // Number of variables + * ("kind",8) // Real kind in bytes + * ); + * \endcode + */ +class MultiFieldCreatorIFS : public MultiFieldCreator { +public: + MultiFieldCreatorIFS(); + MultiFieldCreatorIFS(const eckit::Configuration& config); + ~MultiFieldCreatorIFS() override; + MultiFieldImpl* create(const eckit::Configuration& config = util::Config()) const override; + MultiFieldImpl* create(const array::DataType datatype, const std::vector& shape, + const std::vector& var_names) const override; +}; + +// ------------------------------------------------------------------ + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/detail/FieldImpl.cc b/src/atlas/field/detail/FieldImpl.cc index 78d929f23..3d655926d 100644 --- a/src/atlas/field/detail/FieldImpl.cc +++ b/src/atlas/field/detail/FieldImpl.cc @@ -94,6 +94,9 @@ FieldImpl::FieldImpl(const std::string& name, array::Array* array) } FieldImpl::~FieldImpl() { + for (FieldObserver* observer : field_observers_) { + observer->onFieldDestruction(*this); + } array_->detach(); if (array_->owners() == 0) { for (auto& f : callback_on_destruction_) { diff --git a/src/atlas/field/detail/FieldImpl.h b/src/atlas/field/detail/FieldImpl.h index 066f2cba7..24af72d2c 100644 --- a/src/atlas/field/detail/FieldImpl.h +++ b/src/atlas/field/detail/FieldImpl.h @@ -16,6 +16,7 @@ #include #include #include +#include #include "atlas/util/Object.h" @@ -256,7 +257,8 @@ class FieldObserver { } } - virtual void onFieldRename(FieldImpl&) = 0; + virtual void onFieldRename(FieldImpl&) {} + virtual void onFieldDestruction(FieldImpl&) {} }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/atlas/field/detail/MultiFieldImpl.cc b/src/atlas/field/detail/MultiFieldImpl.cc new file mode 100644 index 000000000..c3a6abb73 --- /dev/null +++ b/src/atlas/field/detail/MultiFieldImpl.cc @@ -0,0 +1,32 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +// file deepcode ignore CppMemoryLeak: static pointers for global registry are OK and will be cleaned up at end + +#include "atlas/field/MultiField.h" +#include "atlas/field/MultiFieldCreator.h" + +#include +#include +#include + +#include "atlas/field/detail/MultiFieldImpl.h" + +namespace atlas { +namespace field { + +void MultiFieldImpl::add(Field& field) { + ATLAS_ASSERT(not fieldset_.has(field.name()), "Field with name \"" + field.name() + "\" already exists!"); + fieldset_.add(field); + MultiFieldArrayRegistry::instance().add(field, array_); +} + +} +} diff --git a/src/atlas/field/detail/MultiFieldImpl.h b/src/atlas/field/detail/MultiFieldImpl.h new file mode 100644 index 000000000..4a1445e1d --- /dev/null +++ b/src/atlas/field/detail/MultiFieldImpl.h @@ -0,0 +1,96 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @author Willem Deconinck +/// @date June 2015 + +#pragma once + +#include +#include + +#include "atlas/array/Array.h" +#include "atlas/field/Field.h" +#include "atlas/field/FieldSet.h" +#include "atlas/util/Config.h" +#include "atlas/util/Metadata.h" +#include "atlas/util/Object.h" + +namespace atlas { +namespace field { + +class MultiFieldImpl : public util::Object { +public: // methods + //-- Constructors + + MultiFieldImpl() {} + + MultiFieldImpl(const array::ArraySpec& spec) { + array::ArraySpec s(spec); + array_.reset(array::Array::create(std::move(s))); + } + + virtual ~MultiFieldImpl() {} + + + //-- Accessors + + const Field& field(const std::string& name) const { return fieldset_.field(name); } + Field& field(const std::string& name) { return fieldset_.field(name); } + bool has(const std::string& name) const { return fieldset_.has(name); } + std::vector field_names() const { return fieldset_.field_names(); } + + const Field& field(const idx_t idx) const { return fieldset_[idx]; } + Field& field(const idx_t idx) { return fieldset_[idx]; } + idx_t size() const { return fieldset_.size(); } + + const Field& operator[](const idx_t idx) const { return fieldset_[idx]; } + Field& operator[](const idx_t idx) { return fieldset_[idx]; } + + const Field& operator[](const std::string& name) const { return fieldset_.field(name); } + Field& operator[](const std::string& name) { return fieldset_.field(name); } + + const util::Metadata& metadata() const { return metadata_; } + util::Metadata& metadata() { return metadata_; } + + // -- Modifiers + + /// @brief Implicit conversion to Array + operator const array::Array&() const { return array(); } + operator array::Array&() { return array(); } + + operator const FieldSet&() const { return fieldset_; } + + operator FieldSet&() { return fieldset_; } + + /// @brief Access contained Array + const array::Array& array() const { + ATLAS_ASSERT(array_); + return *array_; + } + array::Array& array() { + ATLAS_ASSERT(array_); + return *array_; + } + + /// @brief Access contained FieldSet + const FieldSet& fieldset() const { return fieldset_; } + FieldSet& fieldset() { return fieldset_; } + + void add(Field& field); + +public: // temporary public for prototyping + FieldSet fieldset_; + std::shared_ptr array_; + util::Metadata metadata_; +}; + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/detail/MultiFieldInterface.cc b/src/atlas/field/detail/MultiFieldInterface.cc new file mode 100644 index 000000000..805f5e162 --- /dev/null +++ b/src/atlas/field/detail/MultiFieldInterface.cc @@ -0,0 +1,74 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include +#include + +#include "atlas/library/config.h" +#include "atlas/field/MultiField.h" +#include "atlas/field/detail/MultiFieldImpl.h" +#include "atlas/field/detail/MultiFieldInterface.h" +#include "atlas/runtime/Exception.h" + +namespace atlas { +namespace field { + +extern "C" { +MultiFieldImpl* atlas__MultiField__create(eckit::Configuration* config) { + ATLAS_ASSERT(config != nullptr); + auto multifield = new MultiField(*config); + ATLAS_ASSERT(multifield); + return multifield->get(); +} + +MultiFieldImpl* atlas__MultiField__create_shape(int kind, int rank, int shapef[], const char* var_names, + size_t length, size_t size) { + array::ArrayShape shape; + shape.resize(rank); + array::ArrayStrides strides; + for (idx_t j = 0, jf = rank - 1; j < rank; ++j) { + shape[j] = shapef[jf--]; + } + + std::vector var_names_str; + for (size_t jj = 0; jj < size; ++jj) { + char str[length + 1]; + ATLAS_ASSERT(snprintf(str, sizeof(str), "%s", var_names + jj * length ) >= 0); + std::string sstr(str); + sstr.erase(std::find_if(sstr.rbegin(), sstr.rend(), [](unsigned char ch) { + return !std::isspace(ch); + }).base(), sstr.end()); + var_names_str.push_back(sstr); + } + + auto multifield = new MultiField(array::DataType{kind}, shape, var_names_str); + ATLAS_ASSERT(multifield); + return multifield->get(); +} + +void atlas__MultiField__delete(MultiFieldImpl* This) { + delete This; +} + +int atlas__MultiField__size(MultiFieldImpl* This) { + return This->size(); +} + +FieldSetImpl* atlas__MultiField__fieldset(MultiFieldImpl* This) { + return This->fieldset().get(); +} + +} + +// ------------------------------------------------------------------ + +} // namespace field +} // namespace atlas diff --git a/src/atlas/field/detail/MultiFieldInterface.h b/src/atlas/field/detail/MultiFieldInterface.h new file mode 100644 index 000000000..c63d4f755 --- /dev/null +++ b/src/atlas/field/detail/MultiFieldInterface.h @@ -0,0 +1,43 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +/// @file MultiFieldInterface.h +/// @author Willem Deconinck +/// @date Sep 2014 + +#pragma once + +#include "atlas/field/FieldSet.h" +#include "atlas/field/MultiField.h" + +namespace atlas { +namespace functionspace { +} +} // namespace atlas + +namespace atlas { +namespace field { + +//---------------------------------------------------------------------------------------------------------------------- + +// C wrapper interfaces to C++ routines +extern "C" { +MultiFieldImpl* atlas__MultiField__create(eckit::Configuration* config); +MultiFieldImpl* atlas__MultiField__create_shape(int kind, int rank, int shapef[], const char* var_names, + size_t length, size_t size); +void atlas__MultiField__delete(MultiFieldImpl* This); +int atlas__MultiField__size(MultiFieldImpl* This); +FieldSetImpl* atlas__MultiField__fieldset(MultiFieldImpl* This); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace field +} // namespace atlas diff --git a/src/atlas/functionspace/Spectral.h b/src/atlas/functionspace/Spectral.h index dbb32b0fd..7e5fe7c43 100644 --- a/src/atlas/functionspace/Spectral.h +++ b/src/atlas/functionspace/Spectral.h @@ -47,9 +47,10 @@ class Spectral : public functionspace::FunctionSpaceImpl { n = total wavenumber const auto zonal_wavenumbers = Spectral::zonal_wavenumbers(); + const int truncation = Spectral::truncation(); idx_t jc=0; for( int jm=0; jmblock_size(jblk); } +idx_t atlas__fs__BStructuredColumns__nproma(const detail::BlockStructuredColumns* This) { + return This->nproma(); +} idx_t atlas__fs__BStructuredColumns__nblks(const detail::BlockStructuredColumns* This) { return This->nblks(); } diff --git a/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.h b/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.h index 7428afdf1..5a660132c 100644 --- a/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.h +++ b/src/atlas/functionspace/detail/BlockStructuredColumnsInterface.h @@ -102,6 +102,7 @@ idx_t atlas__fs__BStructuredColumns__i_end_halo(const detail::BlockStructuredCol idx_t atlas__fs__BStructuredColumns__levels(const detail::BlockStructuredColumns* This); idx_t atlas__fs__BStructuredColumns__block_begin(const detail::BlockStructuredColumns* This, idx_t jblk); idx_t atlas__fs__BStructuredColumns__block_size(const detail::BlockStructuredColumns* This, idx_t jblk); +idx_t atlas__fs__BStructuredColumns__nproma(const detail::BlockStructuredColumns* This); idx_t atlas__fs__BStructuredColumns__nblks(const detail::BlockStructuredColumns* This); field::FieldImpl* atlas__fs__BStructuredColumns__xy(const detail::BlockStructuredColumns* This); diff --git a/src/atlas/functionspace/detail/StructuredColumns.cc b/src/atlas/functionspace/detail/StructuredColumns.cc index 653ea10ab..54b215feb 100644 --- a/src/atlas/functionspace/detail/StructuredColumns.cc +++ b/src/atlas/functionspace/detail/StructuredColumns.cc @@ -540,7 +540,7 @@ const util::PartitionPolygon& StructuredColumns::polygon(idx_t halo) const { } if (not polygons_[halo]) { if (halo > this->halo()) { - throw_Exception("StructuredColumsn does not contain a halo of size " + std::to_string(halo) + ".", Here()); + throw_Exception("StructuredColumns does not contain a halo of size " + std::to_string(halo) + ".", Here()); } polygons_[halo].reset(new grid::StructuredPartitionPolygon(*this, halo)); } @@ -726,7 +726,7 @@ struct FixupHaloForVectors { template void apply(Field& field) { std::string type = field.metadata().getString("type", "scalar"); - if (type == "vector ") { + if (type == "vector") { ATLAS_NOTIMPLEMENTED; } } @@ -770,12 +770,16 @@ struct FixupHaloForVectors<3> { template void apply(Field& field) { std::string type = field.metadata().getString("type", "scalar"); + + // if levels not setup in functionspace use levels from field + idx_t k_end = (fs.k_begin() == 0) && (fs.k_end() == 0) ? field.levels() : fs.k_end(); + if (type == "vector") { auto array = array::make_view(field); for (idx_t j = fs.j_begin_halo(); j < 0; ++j) { for (idx_t i = fs.i_begin_halo(j); i < fs.i_end_halo(j); ++i) { idx_t n = fs.index(i, j); - for (idx_t k = fs.k_begin(); k < fs.k_end(); ++k) { + for (idx_t k = fs.k_begin(); k < k_end; ++k) { array(n, k, XX) = -array(n, k, XX); array(n, k, YY) = -array(n, k, YY); } @@ -784,7 +788,7 @@ struct FixupHaloForVectors<3> { for (idx_t j = fs.grid().ny(); j < fs.j_end_halo(); ++j) { for (idx_t i = fs.i_begin_halo(j); i < fs.i_end_halo(j); ++i) { idx_t n = fs.index(i, j); - for (idx_t k = fs.k_begin(); k < fs.k_end(); ++k) { + for (idx_t k = fs.k_begin(); k < k_end; ++k) { array(n, k, XX) = -array(n, k, XX); array(n, k, YY) = -array(n, k, YY); } diff --git a/src/atlas/functionspace/detail/StructuredColumns_setup.cc b/src/atlas/functionspace/detail/StructuredColumns_setup.cc index 4bdc0a4e9..094b47849 100644 --- a/src/atlas/functionspace/detail/StructuredColumns_setup.cc +++ b/src/atlas/functionspace/detail/StructuredColumns_setup.cc @@ -588,13 +588,20 @@ void StructuredColumns::setup(const grid::Distribution& distribution, const ecki atlas_omp_parallel_for(idx_t n = 0; n < gridpoints.size(); ++n) { const GridPoint& gp = gridpoints[n]; - if (gp.j >= 0 && gp.j < grid_->ny()) { - xy(gp.r, XX) = grid_->x(gp.i, gp.j); - xy(gp.r, YY) = grid_->y(gp.j); - } - else { - xy(gp.r, XX) = compute_x(gp.i, gp.j); - xy(gp.r, YY) = compute_y(gp.j); + if (regional) { + std::array lonlatVec; + grid_->lonlat(gp.i, gp.j, lonlatVec.data()); + xy(gp.r, XX) = lonlatVec[0]; + xy(gp.r, YY) = lonlatVec[1]; + } else { + if (gp.j >= 0 && gp.j < grid_->ny()) { + xy(gp.r, XX) = grid_->x(gp.i, gp.j); + xy(gp.r, YY) = grid_->y(gp.j); + } + else { + xy(gp.r, XX) = compute_x(gp.i, gp.j); + xy(gp.r, YY) = compute_y(gp.j); + } } bool in_domain(false); diff --git a/src/atlas/grid/detail/grid/CubedSphere.h b/src/atlas/grid/detail/grid/CubedSphere.h index 0f1df4803..7abd9b4f9 100644 --- a/src/atlas/grid/detail/grid/CubedSphere.h +++ b/src/atlas/grid/detail/grid/CubedSphere.h @@ -241,7 +241,7 @@ class CubedSphere : public Grid { virtual std::string name() const override; virtual std::string type() const override; - // Return number of faces on cube + // Return N_, where (N_ * N_) is the number of cells on a tile inline idx_t N() const { return N_; } // Access to the tile class @@ -436,7 +436,7 @@ class CubedSphere : public Grid { void xyt2xy(const double xyt[], double xy[]) const; protected: - // Number of faces on tile + // (N_ * N_) = number of cells on a tile idx_t N_; // Number of tiles diff --git a/src/atlas/grid/detail/grid/Grid.cc b/src/atlas/grid/detail/grid/Grid.cc index e9750b3d7..e2ec13d69 100644 --- a/src/atlas/grid/detail/grid/Grid.cc +++ b/src/atlas/grid/detail/grid/Grid.cc @@ -163,7 +163,7 @@ Grid::Spec* atlas__grid__Grid__spec(Grid* This) { } void atlas__grid__Grid__uid(const Grid* This, char*& uid, int& size) { - ATLAS_ASSERT(This != nullptr, "Cannot access uninitialised atlas_Projection"); + ATLAS_ASSERT(This != nullptr, "Cannot access uninitialised atlas_Grid"); eckit::MD5 md5; This->hash(md5); std::string s = This->uid(); @@ -172,6 +172,14 @@ void atlas__grid__Grid__uid(const Grid* This, char*& uid, int& size) { std::strncpy(uid, s.c_str(), size + 1); } +void atlas__grid__Grid__name(const Grid* This, char*& name, int& size) { + ATLAS_ASSERT(This != nullptr, "Cannot access uninitialised atlas_Grid"); + std::string s = This->name(); + size = static_cast(s.size()); + name = new char[size + 1]; + std::strncpy(name, s.c_str(), size + 1); +} + Grid::Domain::Implementation* atlas__grid__Grid__lonlat_bounding_box(const Grid* This) { Grid::Domain::Implementation* lonlatboundingbox; { diff --git a/src/atlas/grid/detail/grid/Grid.h b/src/atlas/grid/detail/grid/Grid.h index 35fe9eeed..db652cf82 100644 --- a/src/atlas/grid/detail/grid/Grid.h +++ b/src/atlas/grid/detail/grid/Grid.h @@ -183,6 +183,7 @@ class GridObserver { //---------------------------------------------------------------------------------------------------------------------- extern "C" { +void atlas__grid__Grid__name(const Grid* This, char*& uid, int& size); idx_t atlas__grid__Grid__size(Grid* This); Grid::Spec* atlas__grid__Grid__spec(Grid* This); void atlas__grid__Grid__uid(const Grid* This, char*& uid, int& size); diff --git a/src/atlas/grid/detail/grid/StencilComputerInterface.cc b/src/atlas/grid/detail/grid/StencilComputerInterface.cc new file mode 100644 index 000000000..4e1df4765 --- /dev/null +++ b/src/atlas/grid/detail/grid/StencilComputerInterface.cc @@ -0,0 +1,48 @@ +/* + * (C) Copyright 2023 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "StencilComputerInterface.h" + +extern "C" { + atlas::grid::ComputeNorth* atlas__grid__ComputeNorth__new(const atlas::StructuredGrid::Implementation* grid, int halo) { + return new atlas::grid::ComputeNorth{atlas::StructuredGrid{grid}, halo}; + } + + void atlas__grid__ComputeNorth__delete(atlas::grid::ComputeNorth* This) { + delete This; + } + + atlas::idx_t atlas__grid__ComputeNorth__execute_real32(const atlas::grid::ComputeNorth* This, float y) { + return This->operator()(y); + } + + atlas::idx_t atlas__grid__ComputeNorth__execute_real64(const atlas::grid::ComputeNorth* This, double y) { + return This->operator()(y); + } + + + atlas::grid::ComputeWest* atlas__grid__ComputeWest__new(const atlas::StructuredGrid::Implementation* grid, int halo) { + return new atlas::grid::ComputeWest{atlas::StructuredGrid{grid}, halo}; + } + + void atlas__grid__ComputeWest__delete(atlas::grid::ComputeWest* This) { + delete This; + } + + atlas::idx_t atlas__grid__ComputeWest__execute_real32(const atlas::grid::ComputeWest* This, float x, atlas::idx_t j) { + return This->operator()(x, j); + } + + atlas::idx_t atlas__grid__ComputeWest__execute_real64(const atlas::grid::ComputeWest* This, double x, atlas::idx_t j) { + return This->operator()(x, j); + } + + +} diff --git a/src/atlas/grid/detail/grid/StencilComputerInterface.h b/src/atlas/grid/detail/grid/StencilComputerInterface.h new file mode 100644 index 000000000..87e4172d2 --- /dev/null +++ b/src/atlas/grid/detail/grid/StencilComputerInterface.h @@ -0,0 +1,27 @@ +/* + * (C) Copyright 2023 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include "atlas/grid/StencilComputer.h" +#include "atlas/grid/StructuredGrid.h" + +extern "C" { + atlas::grid::ComputeNorth* atlas__grid__ComputeNorth__new(const atlas::StructuredGrid::Implementation* grid, int halo); + void atlas__grid__ComputeNorth__delete(atlas::grid::ComputeNorth* This); + atlas::idx_t atlas__grid__ComputeNorth__execute_real32(const atlas::grid::ComputeNorth* This, float y); + atlas::idx_t atlas__grid__ComputeNorth__execute_real64(const atlas::grid::ComputeNorth* This, double y); + + atlas::grid::ComputeWest* atlas__grid__ComputeWest__new(const atlas::StructuredGrid::Implementation* grid, int halo); + void atlas__grid__ComputeWest__delete(atlas::grid::ComputeWest* This); + atlas::idx_t atlas__grid__ComputeWest__execute_real32(const atlas::grid::ComputeWest* This, float x, atlas::idx_t j); + atlas::idx_t atlas__grid__ComputeWest__execute_real64(const atlas::grid::ComputeWest* This, double x, atlas::idx_t j); + +} diff --git a/src/atlas/interpolation/element/Triag2D.cc b/src/atlas/interpolation/element/Triag2D.cc index 9210762e6..285f399e2 100644 --- a/src/atlas/interpolation/element/Triag2D.cc +++ b/src/atlas/interpolation/element/Triag2D.cc @@ -42,7 +42,7 @@ method::Intersect Triag2D::intersects(const PointXY& r, double edgeEpsilon, doub Vector2D pvec{rvec - v00}; // solve u e1 + v e2 = pvec for u and v - float invDet = 1. / (e1.x() * e2.y() - e2.x() * e1.y()); + double invDet = 1. / (e1.x() * e2.y() - e2.x() * e1.y()); isect.u = (pvec.x() * e2.y() - e2.x() * pvec.y()) * invDet; isect.v = (e1.x() * pvec.y() - pvec.x() * e1.y()) * invDet; diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index fb7f90221..3a2349a1d 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -301,7 +301,7 @@ void Method::setup(const FunctionSpace& source, const FunctionSpace& target) { ATLAS_TRACE("atlas::interpolation::method::Method::setup(FunctionSpace, FunctionSpace)"); this->do_setup(source, target); - if (adjoint_) { + if (adjoint_ && target.size() > 0) { Matrix tmp(*matrix_); // if interpolation is matrix free then matrix->nonZeros() will be zero. @@ -346,14 +346,14 @@ Method::Metadata Method::execute(const Field& source, Field& target) const { } Method::Metadata Method::execute_adjoint(FieldSet& source, const FieldSet& target) const { - ATLAS_TRACE("atlas::interpolation::method::Method::execute(FieldSet, FieldSet)"); + ATLAS_TRACE("atlas::interpolation::method::Method::execute_adjoint(FieldSet, FieldSet)"); Metadata metadata; this->do_execute_adjoint(source, target, metadata); return metadata; } Method::Metadata Method::execute_adjoint(Field& source, const Field& target) const { - ATLAS_TRACE("atlas::interpolation::method::Method::execute(Field, Field)"); + ATLAS_TRACE("atlas::interpolation::method::Method::execute_adjoint(Field, Field)"); Metadata metadata; this->do_execute_adjoint(source, target, metadata); return metadata; @@ -438,8 +438,8 @@ void Method::do_execute_adjoint(Field& src, const Field& tgt, Metadata&) const { throw_NotImplemented("Adjoint Interpolation does not work for fields that have missing data. ", Here()); } - if (matrix_transpose_.empty()) { - throw_AssertionFailed("Need to set 'adjoint coefficients' to true in config for adjoint interpolation to work"); + if (!adjoint_) { + throw_AssertionFailed("Need to set 'adjoint' to true in config for adjoint interpolation to work"); } if (src.datatype().kind() == array::DataType::KIND_REAL64) { diff --git a/src/atlas/interpolation/method/Method.h b/src/atlas/interpolation/method/Method.h index 5e5abbfde..7e52d5bf3 100644 --- a/src/atlas/interpolation/method/Method.h +++ b/src/atlas/interpolation/method/Method.h @@ -160,10 +160,10 @@ class Method : public util::Object { interpolation::MatrixCache matrix_cache_; NonLinear nonLinear_; std::string linalg_backend_; - bool adjoint_{false}; Matrix matrix_transpose_; protected: + bool adjoint_{false}; bool allow_halo_exchange_{true}; std::vector missing_; }; diff --git a/src/atlas/interpolation/method/MethodFactory.cc b/src/atlas/interpolation/method/MethodFactory.cc index 4baa44c37..f7a25c72f 100644 --- a/src/atlas/interpolation/method/MethodFactory.cc +++ b/src/atlas/interpolation/method/MethodFactory.cc @@ -11,6 +11,7 @@ #include "MethodFactory.h" // for static linking +#include "binning/Binning.h" #include "cubedsphere/CubedSphereBilinear.h" #include "knn/GridBoxAverage.h" #include "knn/GridBoxMaximum.h" @@ -23,6 +24,7 @@ #include "structured/Linear3D.h" #include "structured/QuasiCubic2D.h" #include "structured/QuasiCubic3D.h" +#include "structured/RegionalLinear2D.h" #include "unstructured/FiniteElement.h" #include "unstructured/UnstructuredBilinearLonLat.h" @@ -45,10 +47,12 @@ void force_link() { MethodBuilder(); MethodBuilder(); MethodBuilder(); + MethodBuilder(); MethodBuilder(); MethodBuilder(); MethodBuilder(); MethodBuilder(); + MethodBuilder(); } } link; } diff --git a/src/atlas/interpolation/method/binning/Binning.cc b/src/atlas/interpolation/method/binning/Binning.cc new file mode 100644 index 000000000..f5a6b0743 --- /dev/null +++ b/src/atlas/interpolation/method/binning/Binning.cc @@ -0,0 +1,187 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + + +#include "atlas/functionspace/NodeColumns.h" +#include "atlas/functionspace/StructuredColumns.h" +#include "atlas/grid.h" +#include "atlas/interpolation/Interpolation.h" +#include "atlas/interpolation/method/binning/Binning.h" +#include "atlas/interpolation/method/MethodFactory.h" +#include "atlas/mesh.h" +#include "atlas/mesh/actions/GetCubedSphereNodalArea.h" +#include "atlas/runtime/Trace.h" + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/linalg/SparseMatrix.h" +#include "eckit/linalg/Triplet.h" +#include "eckit/mpi/Comm.h" + + +namespace atlas { +namespace interpolation { +namespace method { + + +namespace { + +MethodBuilder __builder("binning"); + +} + + +Binning::Binning(const Config& config) : Method(config) { + const auto* conf = dynamic_cast(&config); + ATLAS_ASSERT(conf, "config must be derived from eckit::LocalConfiguration"); + interpAncillaryScheme_ = conf->getSubConfiguration("scheme"); + // enabling or disabling the adjoint operation + adjoint_ = conf->getBool("adjoint", false); + // enabling or disabling the halo exchange + allow_halo_exchange_ = conf->getBool("halo_exchange", true); +} + + +void Binning::do_setup(const Grid& source, + const Grid& target, + const Cache&) { + ATLAS_NOTIMPLEMENTED; +} + + +void Binning::do_setup(const FunctionSpace& source, + const FunctionSpace& target) { + ATLAS_TRACE("atlas::interpolation::method::Binning::do_setup()"); + + using Index = eckit::linalg::Index; + using Triplet = eckit::linalg::Triplet; + using SMatrix = eckit::linalg::SparseMatrix; + + source_ = source; + target_ = target; + + if (target_.size() == 0) { + return; + } + + // note that the 'source' grid for the low-to-high regridding (interpolation) + // is the 'target' grid for high-to-low regridding (binning) and + // the 'target' grid for the low-to-high regridding (interpolation) is the + // 'source' grid for the for high-to-low regridding (binning) + const auto& fs_source_interp = target_; + const auto& fs_target_interp = source_; + + const auto interp = Interpolation( + interpAncillaryScheme_, fs_source_interp, fs_target_interp); + auto smx_interp_cache = interpolation::MatrixCache(interp); + + auto smx_interp = smx_interp_cache.matrix(); + + auto smx_interp_tr = smx_interp.transpose(); + + const auto rows_tamx = smx_interp_tr.rows(); + const auto cols_tamx = smx_interp_tr.cols(); + + const double* ptr_tamx_data = smx_interp_tr.data(); + const Index* ptr_tamx_idxs_col = smx_interp_tr.inner(); + const Index* ptr_tamx_o = smx_interp_tr.outer(); + + // diagonal of 'area weights matrix', W + auto ds_aweights = getAreaWeights(source_); + + auto smx_binning_els = std::vector{}; + size_t idx_row_next = 0; + + for (size_t idx_row = 0; idx_row < rows_tamx; ++idx_row) { + idx_row_next = (idx_row+1); + // start of the indexes associated with the row 'i' + size_t lbound = ptr_tamx_o[idx_row]; + // start of the indexes associated with the row 'i+1' + size_t ubound = ptr_tamx_o[idx_row_next]; + + if (lbound == ubound) { + continue; + } + + double sum_row = 0; + for (size_t i = lbound; i < ubound; ++i) { + sum_row += (ptr_tamx_data[i] * ds_aweights.at(ptr_tamx_idxs_col[i])); + } + + // normalization factor + double nfactor = 1/sum_row; + + for (size_t i = lbound; i < ubound; ++i) { + // evaluating the non-zero elements of the binning matrix + smx_binning_els.emplace_back( + idx_row, ptr_tamx_idxs_col[i], + (nfactor * (ptr_tamx_data[i] * ds_aweights.at(ptr_tamx_idxs_col[i])))); + } + } + + // 'binning matrix' (sparse matrix), B = N A^T W + SMatrix smx_binning{rows_tamx, cols_tamx, smx_binning_els}; + setMatrix(smx_binning); +} + + +void Binning::print(std::ostream&) const { + ATLAS_NOTIMPLEMENTED; +} + + +std::vector Binning::getAreaWeights(const FunctionSpace& fspace) const { + // diagonal of 'area weights matrix', W + std::vector ds_aweights; + + bool is_cubed_sphere {false}; + if (auto csfs = functionspace::NodeColumns(fspace)) { + if (CubedSphereGrid(csfs.mesh().grid())) { + is_cubed_sphere = true; + } + } + + if (is_cubed_sphere) { + + const auto csfs = functionspace::NodeColumns(fspace); + auto csmesh = csfs.mesh(); + + // areas of the cells (geographic coord. system) + auto gcell_areas = mesh::actions::GetCubedSphereNodalArea()(csmesh); + auto gcell_areas_view = array::make_view(gcell_areas); + + auto is_ghost = array::make_view(csfs.ghost()); + + double total_area {0.}; + for (idx_t i = 0; i < csfs.size(); i++) { + if (!is_ghost[i]) { + total_area += gcell_areas_view(i); + } + } + eckit::mpi::comm().allReduceInPlace(total_area, eckit::mpi::Operation::SUM); + + double aweight_temp {0.}; + for (idx_t i = 0; i < csfs.size(); i++) { + if (!is_ghost[i]) { + aweight_temp = gcell_areas_view(i)/total_area; + ds_aweights.emplace_back(aweight_temp); + } + } + + } else { + + // area weights (default) + ds_aweights.assign(fspace.size(), 1.); + + } + + return ds_aweights; +} + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/binning/Binning.h b/src/atlas/interpolation/method/binning/Binning.h new file mode 100644 index 000000000..110975d01 --- /dev/null +++ b/src/atlas/interpolation/method/binning/Binning.h @@ -0,0 +1,74 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include + +#include "atlas/functionspace/FunctionSpace.h" +#include "atlas/interpolation/Cache.h" +#include "atlas/interpolation/method/Method.h" +#include "atlas/grid/Grid.h" + +#include "eckit/config/Configuration.h" + + +namespace atlas { +namespace interpolation { +namespace method { + + +class Binning : public Method { + public: + /// @brief Binning procedure to carry out a regridding from high + /// to low resolution + /// + /// @details This method is part of the family of the Interpolation operations; + /// it relies on the evaluation of the transpose of an interpolation matrix. + /// The rigridding from high to low resolution is carried out by using + /// a 'binning matrix': + /// binning matrix: B = N A^T W + /// area weights matrix: W + /// interpolation matrix: A + /// normalization factor matrix: N + /// + /// Setup, configuration variables: + /// : method used to evaluate the 'B' matrix; + /// value: 'binning' + /// : method used to evaluate the 'A' matrix; + /// value: 'cubedsphere-bilinear', 'structured-bilinear', ... + /// : flag to control the halo exchange procedure + /// value: 'true', 'false' + /// : flag to control the adjoint operation + /// value: 'true', 'false' + /// + Binning(const Config& config); + ~Binning() override {} + + void print(std::ostream&) const override; + const FunctionSpace& source() const override { return source_; } + const FunctionSpace& target() const override { return target_; } + + private: + using Method::do_setup; + void do_setup(const FunctionSpace& source, + const FunctionSpace& target) override; + void do_setup(const Grid& source, const Grid& target, const Cache&) override; + + std::vector getAreaWeights(const FunctionSpace& source) const; + + eckit::LocalConfiguration interpAncillaryScheme_{}; + + FunctionSpace source_{}; + FunctionSpace target_{}; +}; + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/cubedsphere/CellFinder.cc b/src/atlas/interpolation/method/cubedsphere/CellFinder.cc index 9912d16f1..4a77231a0 100644 --- a/src/atlas/interpolation/method/cubedsphere/CellFinder.cc +++ b/src/atlas/interpolation/method/cubedsphere/CellFinder.cc @@ -37,7 +37,7 @@ CellFinder::CellFinder(const Mesh& mesh, const util::Config& config): mesh_{mesh auto halo = config.getInt("halo", 0); for (idx_t i = 0; i < mesh_.cells().size(); ++i) { if (haloView(i) <= halo) { - points.emplace_back(PointLonLat(lonlatView(i, LON), lonlatView(i, LAT))); + points.emplace_back(lonlatView(i, LON), lonlatView(i, LAT)); payloads.emplace_back(i); } } diff --git a/src/atlas/interpolation/method/cubedsphere/CubedSphereBilinear.cc b/src/atlas/interpolation/method/cubedsphere/CubedSphereBilinear.cc index 0dd6527d6..366df8cfe 100644 --- a/src/atlas/interpolation/method/cubedsphere/CubedSphereBilinear.cc +++ b/src/atlas/interpolation/method/cubedsphere/CubedSphereBilinear.cc @@ -33,6 +33,10 @@ void CubedSphereBilinear::do_setup(const FunctionSpace& source, const FunctionSp ATLAS_ASSERT(ncSource); ATLAS_ASSERT(target_); + // Enable or disable halo exchange. + this->allow_halo_exchange_ = halo_exchange_; + + // return early if no output points on this partition reserve is called on // the triplets but also during the sparseMatrix constructor. This won't // work for empty matrices @@ -46,9 +50,6 @@ void CubedSphereBilinear::do_setup(const FunctionSpace& source, const FunctionSp const auto N = CubedSphereGrid(ncSource.mesh().grid()).N(); const auto tolerance = 2. * std::numeric_limits::epsilon() * N; - // Enable or disable halo exchange. - this->allow_halo_exchange_ = halo_exchange_; - // Loop over target at calculate interpolation weights. auto weights = std::vector{}; const auto ghostView = array::make_view(target_.ghost()); diff --git a/src/atlas/interpolation/method/sphericalvector/ComplexMatrixMultiply.h b/src/atlas/interpolation/method/sphericalvector/ComplexMatrixMultiply.h index 34cfdacb2..8d15d035a 100644 --- a/src/atlas/interpolation/method/sphericalvector/ComplexMatrixMultiply.h +++ b/src/atlas/interpolation/method/sphericalvector/ComplexMatrixMultiply.h @@ -70,7 +70,7 @@ class ComplexMatrixMultiply { // tinyNum ~= 2.3e-13 for double. constexpr auto tinyNum = 1024 * std::numeric_limits::epsilon(); const auto complexMagnitude = std::abs(complexRowIter.value()); - const auto realValue = realRowIter.value(); + const auto realValue = std::abs(realRowIter.value()); const auto error = std::abs(complexMagnitude - realValue); const auto printError = [&]() { diff --git a/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc b/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc index 7e29f41c1..ab5f573d8 100644 --- a/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc +++ b/src/atlas/interpolation/method/sphericalvector/SphericalVector.cc @@ -72,7 +72,7 @@ void SphericalVector::do_setup(const FunctionSpace& source, // whereas eckit does not. auto complexTriplets = ComplexTriplets(nNonZeros); auto realTriplets = RealTriplets(nNonZeros); - + const auto sourceLonLatsView = array::make_view(source_.lonlat()); const auto targetLonLatsView = array::make_view(target_.lonlat()); const auto unitSphere = geometry::UnitSphere{}; @@ -128,6 +128,11 @@ void SphericalVector::do_execute(const FieldSet& sourceFieldSet, void SphericalVector::do_execute(const Field& sourceField, Field& targetField, Metadata&) const { ATLAS_TRACE("atlas::interpolation::method::SphericalVector::do_execute()"); + + if (targetField.size() == 0) { + return; + } + const auto fieldType = sourceField.metadata().getString("type", ""); if (fieldType != "vector") { auto metadata = Metadata(); @@ -162,6 +167,10 @@ void SphericalVector::do_execute_adjoint(Field& sourceField, ATLAS_TRACE( "atlas::interpolation::method::SphericalVector::do_execute_adjoint()"); + if (targetField.size() == 0) { + return; + } + const auto fieldType = sourceField.metadata().getString("type", ""); if (fieldType != "vector") { auto metadata = Metadata(); @@ -183,9 +192,6 @@ template void SphericalVector::interpolate_vector_field(const Field& sourceField, Field& targetField, const MatMul& matMul) { - if (targetField.size() == 0) { - return; - } ATLAS_ASSERT_MSG(sourceField.variables() == 2 || sourceField.variables() == 3, "Vector field can only have 2 or 3 components."); diff --git a/src/atlas/interpolation/method/sphericalvector/SphericalVector.h b/src/atlas/interpolation/method/sphericalvector/SphericalVector.h index c3514b0d1..a8ee1573b 100644 --- a/src/atlas/interpolation/method/sphericalvector/SphericalVector.h +++ b/src/atlas/interpolation/method/sphericalvector/SphericalVector.h @@ -72,7 +72,6 @@ class SphericalVector : public Method { void do_setup(const Grid& source, const Grid& target, const Cache&) override; eckit::LocalConfiguration interpolationScheme_{}; - bool adjoint_{}; FunctionSpace source_{}; FunctionSpace target_{}; diff --git a/src/atlas/interpolation/method/structured/RegionalLinear2D.cc b/src/atlas/interpolation/method/structured/RegionalLinear2D.cc new file mode 100644 index 000000000..e8588b617 --- /dev/null +++ b/src/atlas/interpolation/method/structured/RegionalLinear2D.cc @@ -0,0 +1,572 @@ +/* + * (C) Copyright 2024 Meteorologisk Institutt + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "atlas/interpolation/method/structured/RegionalLinear2D.h" + +#include + +#include "atlas/array.h" +#include "atlas/interpolation/method/MethodFactory.h" +#include "atlas/util/KDTree.h" +#include "atlas/util/Point.h" + +namespace atlas { +namespace interpolation { +namespace method { + +namespace { +MethodBuilder __builder("regional-linear-2d"); +} + +void RegionalLinear2D::print(std::ostream&) const { ATLAS_NOTIMPLEMENTED; } + +void RegionalLinear2D::do_setup(const Grid& source, const Grid& target, + const Cache&) { + ATLAS_NOTIMPLEMENTED; +} + +void RegionalLinear2D::do_setup(const FunctionSpace& source, + const FunctionSpace& target) { + ATLAS_TRACE("interpolation::method::RegionalLinear2D::do_setup"); + source_ = source; + target_ = target; + + if (target_.size() == 0) { + return; + } + ASSERT(source_.type() == "StructuredColumns"); + + // Get grid parameters + const functionspace::StructuredColumns sourceFs(source_); + const RegularGrid sourceGrid(sourceFs.grid()); + const Projection & sourceProj = sourceGrid.projection(); + const size_t sourceNx = sourceGrid.nx(); + const size_t sourceNy = sourceGrid.ny(); + const double sourceDx = sourceGrid.dx(); + const double sourceDy = std::abs(sourceGrid.y(1)-sourceGrid.y(0)); + const bool reversedY = sourceGrid.y(1) < sourceGrid.y(0); + + // Check grid regularity in y direction + for (size_t sourceJ = 0; sourceJ < sourceNy-1; ++sourceJ) { + if (reversedY) { + ASSERT(std::abs(sourceGrid.y(sourceJ)-sourceGrid.y(sourceJ+1)-sourceDy) < 1.0e-12*sourceDy); + } else { + ASSERT(std::abs(sourceGrid.y(sourceJ+1)-sourceGrid.y(sourceJ)-sourceDy) < 1.0e-12*sourceDy); + } + } + + // Source grid indices + const Field sourceFieldIndexI = sourceFs.index_i(); + const Field sourceFieldIndexJ = sourceFs.index_j(); + const auto sourceIndexIView = array::make_view(sourceFieldIndexI); + const auto sourceIndexJView = array::make_view(sourceFieldIndexJ); + sourceSize_ = sourceFs.size(); + + // Destination grid size + targetSize_ = target_.size(); + + // Ghost points + const auto sourceGhostView = array::make_view(sourceFs.ghost()); + const auto targetGhostView = array::make_view(target_.ghost()); + + // Define reduced grid horizontal distribution + std::vector mpiTask(sourceNx*sourceNy, 0); + for (size_t sourceJnode = 0; sourceJnode < sourceSize_; ++sourceJnode) { + if (sourceGhostView(sourceJnode) == 0) { + mpiTask[(sourceIndexIView(sourceJnode)-1)*sourceNy+sourceIndexJView(sourceJnode)-1] = comm_.rank(); + } + } + comm_.allReduceInPlace(mpiTask.begin(), mpiTask.end(), eckit::mpi::sum()); + + // Define local tree on destination grid + std::vector targetPoints; + std::vector targetIndices; + targetPoints.reserve(targetSize_); + targetIndices.reserve(targetSize_); + const auto targetLonLatView = array::make_view(target_.lonlat()); + for (size_t targetJnode = 0; targetJnode < targetSize_; ++targetJnode) { + PointLonLat p({targetLonLatView(targetJnode, 0), targetLonLatView(targetJnode, 1)}); + sourceProj.lonlat2xy(p); + targetPoints.emplace_back(p[0], p[1], 0.0); + targetIndices.emplace_back(targetJnode); + } + util::IndexKDTree targetTree; + if (targetSize_ > 0) { + targetTree.build(targetPoints, targetIndices); + } + const double radius = std::sqrt(sourceDx*sourceDx+sourceDy*sourceDy); + + // Delta for colocation + const double eps = 1.0e-8; + + // RecvCounts and received points list + targetRecvCounts_.resize(comm_.size()); + std::fill(targetRecvCounts_.begin(), targetRecvCounts_.end(), 0); + std::vector targetRecvPointsList; + for (size_t sourceJ = 0; sourceJ < sourceNy; ++sourceJ) { + double yMin, yMax; + if (reversedY) { + yMin = sourceJ < sourceNy-1 ? sourceGrid.y(sourceJ+1)-eps : -std::numeric_limits::max(); + yMax = sourceJ > 0 ? sourceGrid.y(sourceJ-1)+eps : std::numeric_limits::max(); + } else { + yMin = sourceJ > 0 ? sourceGrid.y(sourceJ-1)-eps : -std::numeric_limits::max(); + yMax = sourceJ < sourceNy-1 ? sourceGrid.y(sourceJ+1)+eps : std::numeric_limits::max(); + } + for (size_t sourceI = 0; sourceI < sourceNx; ++sourceI) { + const double xMin = sourceI > 0 ? sourceGrid.x(sourceI-1)-eps : -std::numeric_limits::max(); + const double xMax = sourceI < sourceNx-1 ? sourceGrid.x(sourceI+1)+eps : + std::numeric_limits::max(); + + bool pointsNeeded = false; + if (targetSize_ > 0) { + const Point3 p(sourceGrid.x(sourceI), sourceGrid.y(sourceJ), 0.0); + const auto list = targetTree.closestPointsWithinRadius(p, radius); + for (const auto & item : list) { + const PointXYZ targetPoint = item.point(); + const size_t targetJnode = item.payload(); + if (targetGhostView(targetJnode) == 0) { + const bool inX = (xMin <= targetPoint[0] && targetPoint[0] <= xMax); + const bool inY = (yMin <= targetPoint[1] && targetPoint[1] <= yMax); + if (inX && inY) { + pointsNeeded = true; + break; + } + } + } + } + if (pointsNeeded) { + ++targetRecvCounts_[mpiTask[sourceI*sourceNy+sourceJ]]; + targetRecvPointsList.push_back(sourceI*sourceNy+sourceJ); + } + } + } + + // Buffer size + targetRecvSize_ = targetRecvPointsList.size(); + + if (targetRecvSize_ > 0) { + // RecvDispls + targetRecvDispls_.push_back(0); + for (size_t jt = 0; jt < comm_.size()-1; ++jt) { + targetRecvDispls_.push_back(targetRecvDispls_[jt]+targetRecvCounts_[jt]); + } + + // Allgather RecvCounts + eckit::mpi::Buffer targetRecvCountsBuffer(comm_.size()); + comm_.allGatherv(targetRecvCounts_.begin(), targetRecvCounts_.end(), targetRecvCountsBuffer); + std::vector targetRecvCountsGlb_ = std::move(targetRecvCountsBuffer.buffer); + + // SendCounts + for (size_t jt = 0; jt < comm_.size(); ++jt) { + sourceSendCounts_.push_back(targetRecvCountsGlb_[jt*comm_.size()+comm_.rank()]); + } + + // Buffer size + sourceSendSize_ = 0; + for (const auto & n : sourceSendCounts_) { + sourceSendSize_ += n; + } + + // SendDispls + sourceSendDispls_.push_back(0); + for (size_t jt = 0; jt < comm_.size()-1; ++jt) { + sourceSendDispls_.push_back(sourceSendDispls_[jt]+sourceSendCounts_[jt]); + } + + // Ordered received points list + std::vector targetRecvOffset(comm_.size(), 0); + std::vector targetRecvPointsListOrdered(targetRecvSize_); + for (size_t jr = 0; jr < targetRecvSize_; ++jr) { + const size_t sourceI = targetRecvPointsList[jr]/sourceNy; + const size_t sourceJ = targetRecvPointsList[jr]-sourceI*sourceNy; + size_t jt = mpiTask[sourceI*sourceNy+sourceJ]; + size_t jro = targetRecvDispls_[jt]+targetRecvOffset[jt]; + targetRecvPointsListOrdered[jro] = targetRecvPointsList[jr]; + ++targetRecvOffset[jt]; + } + std::vector sourceSentPointsList(sourceSendSize_); + comm_.allToAllv(targetRecvPointsListOrdered.data(), targetRecvCounts_.data(), targetRecvDispls_.data(), + sourceSentPointsList.data(), sourceSendCounts_.data(), sourceSendDispls_.data()); + + // Sort indices + std::vector gij; + for (size_t sourceJnode = 0; sourceJnode < sourceSize_; ++sourceJnode) { + if (sourceGhostView(sourceJnode) == 0) { + gij.push_back((sourceIndexIView(sourceJnode)-1)*sourceNy+sourceIndexJView(sourceJnode)-1); + } else { + gij.push_back(-1); + } + } + std::vector gidx(sourceSize_); + std::iota(gidx.begin(), gidx.end(), 0); + std::stable_sort(gidx.begin(), gidx.end(), [&gij](size_t i1, size_t i2) + {return gij[i1] < gij[i2];}); + std::vector ridx(sourceSendSize_); + std::iota(ridx.begin(), ridx.end(), 0); + std::stable_sort(ridx.begin(), ridx.end(), [&sourceSentPointsList](size_t i1, size_t i2) + {return sourceSentPointsList[i1] < sourceSentPointsList[i2];}); + + // Mapping for sent points + sourceSendMapping_.resize(sourceSendSize_); + size_t sourceJnode = 0; + for (size_t js = 0; js < sourceSendSize_; ++js) { + while (gij[gidx[sourceJnode]] < sourceSentPointsList[ridx[js]]) { + ++sourceJnode; + ASSERT(sourceJnode < sourceSize_); + } + sourceSendMapping_[ridx[js]] = gidx[sourceJnode]; + } + + // Sort indices + std::vector idx(targetRecvPointsListOrdered.size()); + std::iota(idx.begin(), idx.end(), 0); + std::stable_sort(idx.begin(), idx.end(), [&targetRecvPointsListOrdered](size_t i1, size_t i2) + {return targetRecvPointsListOrdered[i1] < targetRecvPointsListOrdered[i2];}); + + // Compute horizontal interpolation + stencil_.resize(targetSize_); + weights_.resize(targetSize_); + stencilSize_.resize(targetSize_); + for (size_t targetJnode = 0; targetJnode < targetSize_; ++targetJnode) { + // Interpolation element default values + if (targetGhostView(targetJnode) == 0) { + // Destination grid indices + const double targetX = targetPoints[targetJnode][0]; + bool colocatedX = false; + int indexI = -1; + for (size_t sourceI = 0; sourceI < sourceNx-1; ++sourceI) { + if (std::abs(targetX-sourceGrid.x(sourceI)) < eps) { + indexI = sourceI; + colocatedX = true; + } + if (sourceGrid.x(sourceI)+eps < targetX && targetX < sourceGrid.x(sourceI+1)-eps) { + indexI = sourceI; + colocatedX = false; + } + } + if (std::abs(targetX-sourceGrid.x(sourceNx-1)) < eps) { + indexI = sourceNx-1; + colocatedX = true; + } + const double targetY = targetPoints[targetJnode][1]; + bool colocatedY = false; + int indexJ = -1; + for (size_t sourceJ = 0; sourceJ < sourceNy-1; ++sourceJ) { + if (std::abs(targetY-sourceGrid.y(sourceJ)) < eps) { + indexJ = sourceJ; + colocatedY = true; + } + if (reversedY) { + if (sourceGrid.y(sourceJ+1)+eps < targetY && targetY < sourceGrid.y(sourceJ)-eps) { + indexJ = sourceJ; + colocatedY = false; + } + } else { + if (sourceGrid.y(sourceJ)+eps < targetY && targetY < sourceGrid.y(sourceJ+1)-eps) { + indexJ = sourceJ; + colocatedY = false; + } + } + } + if (std::abs(targetY-sourceGrid.y(sourceNy-1)) < eps) { + indexJ = sourceNy-1; + colocatedY = true; + } + + if (indexI == -1 || indexJ == -1) { + // Point outside of the domain, using nearest neighbor + if (indexI > -1) { + if (!colocatedX && + (std::abs(targetX-sourceGrid.x(indexI+1)) < std::abs(targetX-sourceGrid.x(indexI)))) { + indexI += 1; + } + } else { + if (std::abs(targetX-sourceGrid.x(0)) < std::abs(targetX-sourceGrid.x(sourceNx-1))) { + indexI = 0; + } else { + indexI = sourceNx-1; + } + } + if (indexJ > -1) { + if (!colocatedY && + (std::abs(targetY-sourceGrid.y(indexJ+1)) < std::abs(targetY-sourceGrid.y(indexJ)))) { + indexJ += 1; + } + } else { + if (std::abs(targetY-sourceGrid.y(0)) < std::abs(targetY-sourceGrid.y(sourceNy-1))) { + indexJ = 0; + } else { + indexJ = sourceNy-1; + } + Log::info() << "WARNING: point outside of the domain" << std::endl; + } + + // Colocated point (actually nearest neighbor) + colocatedX = true; + colocatedY = true; + } + + // Bilinear interpolation factor + const double alphaX = 1.0-(sourceGrid.x(indexI)+sourceDx-targetX)/sourceDx; + const double alphaY = reversedY ? (sourceGrid.y(indexJ)-targetY)/sourceDy + : 1.0-(sourceGrid.y(indexJ)+sourceDy-targetY)/sourceDy; + + // Points to find + std::vector toFind = {true, !colocatedX, !colocatedY, !colocatedX && !colocatedY}; + std::vector valueToFind = {indexI*sourceNy+indexJ, (indexI+1)*sourceNy+indexJ, + indexI*sourceNy+(indexJ+1), (indexI+1)*sourceNy+(indexJ+1)}; + std::array foundIndex; + foundIndex.fill(-1); + + // Binary search for each point + for (size_t jj = 0; jj < 4; ++jj) { + if (toFind[jj]) { + size_t low = 0; + size_t high = targetRecvPointsListOrdered.size()-1; + while (low <= high) { + size_t mid = low+(high-low)/2; + if (valueToFind[jj] == static_cast(targetRecvPointsListOrdered[idx[mid]])) { + foundIndex[jj] = idx[mid]; + break; + } + if (valueToFind[jj] > static_cast(targetRecvPointsListOrdered[idx[mid]])) { + low = mid+1; + } + if (valueToFind[jj] < static_cast(targetRecvPointsListOrdered[idx[mid]])) { + high = mid-1; + } + } + ASSERT(foundIndex[jj] > -1); + ASSERT(static_cast(targetRecvPointsListOrdered[foundIndex[jj]]) == + valueToFind[jj]); + } + } + + // Create interpolation operations + if (colocatedX && colocatedY) { + // Colocated point + stencil_[targetJnode][0] = foundIndex[0]; + weights_[targetJnode][0] = 1.0; + stencilSize_[targetJnode] = 1; + } else if (colocatedY) { + // Linear interpolation along x + stencil_[targetJnode][0] = foundIndex[0]; + weights_[targetJnode][0] = 1.0-alphaX; + stencil_[targetJnode][1] = foundIndex[1]; + weights_[targetJnode][1] = alphaX; + stencilSize_[targetJnode] = 2; + } else if (colocatedX) { + // Linear interpolation along y + stencil_[targetJnode][0] = foundIndex[0]; + weights_[targetJnode][0] = 1.0-alphaY; + stencil_[targetJnode][1] = foundIndex[2]; + weights_[targetJnode][1] = alphaY; + stencilSize_[targetJnode] = 2; + } else { + // Bilinear interpolation + stencil_[targetJnode][0] = foundIndex[0]; + weights_[targetJnode][0] = (1.0-alphaX)*(1.0-alphaY); + stencil_[targetJnode][1] = foundIndex[1]; + weights_[targetJnode][1] = alphaX*(1.0-alphaY); + stencil_[targetJnode][2] = foundIndex[2]; + weights_[targetJnode][2] = (1.0-alphaX)*alphaY; + stencil_[targetJnode][3] = foundIndex[3]; + weights_[targetJnode][3] = alphaX*alphaY; + stencilSize_[targetJnode] = 4; + } + } else { + // Ghost point + stencilSize_[targetJnode] = 0; + } + } + } +} + +void RegionalLinear2D::do_execute(const FieldSet& sourceFieldSet, + FieldSet& targetFieldSet, + Metadata& metadata) const { + ATLAS_TRACE("atlas::interpolation::method::RegionalLinear2D::do_execute()"); + ATLAS_ASSERT(sourceFieldSet.size() == targetFieldSet.size()); + + for (auto i = 0; i < sourceFieldSet.size(); ++i) { + do_execute(sourceFieldSet[i], targetFieldSet[i], metadata); + } +} + +void RegionalLinear2D::do_execute(const Field& sourceField, Field& targetField, + Metadata&) const { + ATLAS_TRACE("atlas::interpolation::method::RegionalLinear2D::do_execute()"); + + if (targetField.size() == 0) { + return; + } + + // Check number of levels + ASSERT(sourceField.levels() == targetField.levels()); + const size_t nz = sourceField.levels() > 0 ? sourceField.levels() : 1; + const size_t ndim = sourceField.levels() > 0 ? 2 : 1; + + // Scale counts and displs for all levels + std::vector sourceSendCounts3D(comm_.size()); + std::vector sourceSendDispls3D(comm_.size()); + std::vector targetRecvCounts3D(comm_.size()); + std::vector targetRecvDispls3D(comm_.size()); + for (size_t jt = 0; jt < comm_.size(); ++jt) { + sourceSendCounts3D[jt] = sourceSendCounts_[jt]*nz; + sourceSendDispls3D[jt] = sourceSendDispls_[jt]*nz; + targetRecvCounts3D[jt] = targetRecvCounts_[jt]*nz; + targetRecvDispls3D[jt] = targetRecvDispls_[jt]*nz; + } + + // Halo exchange + haloExchange(sourceField); + + // Serialize + std::vector sourceSendVec(sourceSendSize_*nz); + if (ndim == 1) { + const auto sourceView = array::make_view(sourceField); + for (size_t js = 0; js < sourceSendSize_; ++js) { + size_t sourceJnode = sourceSendMapping_[js]; + sourceSendVec[js] = sourceView(sourceJnode); + } + } else if (ndim == 2) { + const auto sourceView = array::make_view(sourceField); + for (size_t js = 0; js < sourceSendSize_; ++js) { + for (size_t k = 0; k < nz; ++k) { + size_t sourceJnode = sourceSendMapping_[js]; + sourceSendVec[js*nz+k] = sourceView(sourceJnode, k); + } + } + } + + // Communication + std::vector targetRecvVec(targetRecvSize_*nz); + comm_.allToAllv(sourceSendVec.data(), sourceSendCounts3D.data(), sourceSendDispls3D.data(), + targetRecvVec.data(), targetRecvCounts3D.data(), targetRecvDispls3D.data()); + + // Interpolation + if (ndim == 1) { + auto targetView = array::make_view(targetField); + targetView.assign(0.0); + for (size_t targetJnode = 0; targetJnode < targetSize_; ++targetJnode) { + for (size_t jj = 0; jj < stencilSize_[targetJnode]; ++jj) { + targetView(targetJnode) += weights_[targetJnode][jj] + *targetRecvVec[stencil_[targetJnode][jj]]; + } + } + } else if (ndim == 2) { + auto targetView = array::make_view(targetField); + targetView.assign(0.0); + for (size_t targetJnode = 0; targetJnode < targetSize_; ++targetJnode) { + for (size_t jj = 0; jj < stencilSize_[targetJnode]; ++jj) { + for (size_t k = 0; k < nz; ++k) { + targetView(targetJnode, k) += weights_[targetJnode][jj] + *targetRecvVec[stencil_[targetJnode][jj]*nz+k]; + } + } + } + } + + // Set target field dirty + targetField.set_dirty(); +} + +void RegionalLinear2D::do_execute_adjoint(FieldSet& sourceFieldSet, + const FieldSet& targetFieldSet, + Metadata& metadata) const { + ATLAS_TRACE( + "atlas::interpolation::method::RegionalLinear2D::do_execute_adjoint()"); + ATLAS_ASSERT(sourceFieldSet.size() == targetFieldSet.size()); + + for (auto i = 0; i < sourceFieldSet.size(); ++i) { + do_execute_adjoint(sourceFieldSet[i], targetFieldSet[i], metadata); + } +} + +void RegionalLinear2D::do_execute_adjoint(Field& sourceField, + const Field& targetField, + Metadata& metadata) const { + ATLAS_TRACE( + "atlas::interpolation::method::RegionalLinear2D::do_execute_adjoint()"); + + if (targetField.size() == 0) { + return; + } + + // Check number of levels + ASSERT(sourceField.levels() == targetField.levels()); + const size_t nz = sourceField.levels() > 0 ? sourceField.levels() : 1; + const size_t ndim = sourceField.levels() > 0 ? 2 : 1; + + // Scale counts and displs for all levels + std::vector sourceSendCounts3D(comm_.size()); + std::vector sourceSendDispls3D(comm_.size()); + std::vector targetRecvCounts3D(comm_.size()); + std::vector targetRecvDispls3D(comm_.size()); + for (size_t jt = 0; jt < comm_.size(); ++jt) { + sourceSendCounts3D[jt] = sourceSendCounts_[jt]*nz; + sourceSendDispls3D[jt] = sourceSendDispls_[jt]*nz; + targetRecvCounts3D[jt] = targetRecvCounts_[jt]*nz; + targetRecvDispls3D[jt] = targetRecvDispls_[jt]*nz; + } + + // Copy destination field + Field targetTmpField = targetField.clone(); + + // Interpolation adjoint + std::vector targetRecvVec(targetRecvSize_*nz, 0.0); + if (ndim == 1) { + const auto targetView = array::make_view(targetTmpField); + for (size_t targetJnode = 0; targetJnode < targetSize_; ++targetJnode) { + for (size_t jj = 0; jj < stencilSize_[targetJnode]; ++jj) { + targetRecvVec[stencil_[targetJnode][jj]] += weights_[targetJnode][jj] + *targetView(targetJnode); + } + } + } else if (ndim == 2) { + const auto targetView = array::make_view(targetTmpField); + for (size_t targetJnode = 0; targetJnode < targetSize_; ++targetJnode) { + for (size_t jj = 0; jj < stencilSize_[targetJnode]; ++jj) { + for (size_t k = 0; k < nz; ++k) { + targetRecvVec[stencil_[targetJnode][jj]*nz+k] += weights_[targetJnode][jj] + *targetView(targetJnode, k); + } + } + } + } + + // Communication + std::vector sourceSendVec(sourceSendSize_*nz); + comm_.allToAllv(targetRecvVec.data(), targetRecvCounts3D.data(), targetRecvDispls3D.data(), + sourceSendVec.data(), sourceSendCounts3D.data(), sourceSendDispls3D.data()); + + // Deserialize + if (ndim == 1) { + auto sourceView = array::make_view(sourceField); + sourceView.assign(0.0); + for (size_t js = 0; js < sourceSendSize_; ++js) { + size_t sourceJnode = sourceSendMapping_[js]; + sourceView(sourceJnode) += sourceSendVec[js]; + } + } else if (ndim == 2) { + auto sourceView = array::make_view(sourceField); + sourceView.assign(0.0); + for (size_t js = 0; js < sourceSendSize_; ++js) { + size_t sourceJnode = sourceSendMapping_[js]; + for (size_t k = 0; k < nz; ++k) { + sourceView(sourceJnode, k) += sourceSendVec[js*nz+k]; + } + } + } + + // Adjoint halo exchange + adjointHaloExchange(sourceField); +} + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/structured/RegionalLinear2D.h b/src/atlas/interpolation/method/structured/RegionalLinear2D.h new file mode 100644 index 000000000..462f88662 --- /dev/null +++ b/src/atlas/interpolation/method/structured/RegionalLinear2D.h @@ -0,0 +1,77 @@ +/* + * (C) Copyright 2024 Meteorologisk Institutt + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include "atlas/interpolation/method/Method.h" + +#include + +#include "atlas/field.h" +#include "atlas/functionspace.h" +#include "atlas/grid/Grid.h" + +#include "eckit/config/Configuration.h" +#include "eckit/mpi/Comm.h" + +namespace atlas { +namespace interpolation { +namespace method { + + +class RegionalLinear2D : public Method { + public: + /// @brief Regional linear interpolation + /// + /// @details + /// + RegionalLinear2D(const Config& config) : Method(config), comm_(eckit::mpi::comm()) {} + ~RegionalLinear2D() override {} + + void print(std::ostream&) const override; + const FunctionSpace& source() const override { return source_; } + const FunctionSpace& target() const override { return target_; } + + void do_execute(const FieldSet& sourceFieldSet, FieldSet& targetFieldSet, + Metadata& metadata) const override; + void do_execute(const Field& sourceField, Field& targetField, + Metadata& metadata) const override; + + void do_execute_adjoint(FieldSet& sourceFieldSet, + const FieldSet& targetFieldSet, + Metadata& metadata) const override; + void do_execute_adjoint(Field& sourceField, const Field& targetField, + Metadata& metadata) const override; + + private: + using Method::do_setup; + void do_setup(const FunctionSpace& source, const FunctionSpace& target) override; + void do_setup(const Grid& source, const Grid& target, const Cache&) override; + + FunctionSpace source_{}; + FunctionSpace target_{}; + + const eckit::mpi::Comm & comm_; + size_t sourceSize_; + std::vector mpiTask_; + size_t targetSize_; + size_t sourceSendSize_; + size_t targetRecvSize_; + std::vector sourceSendCounts_; + std::vector sourceSendDispls_; + std::vector targetRecvCounts_; + std::vector targetRecvDispls_; + std::vector sourceSendMapping_; + std::vector> stencil_; + std::vector> weights_; + std::vector stencilSize_; +}; + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc index b40436f97..c9376d79e 100644 --- a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc +++ b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc @@ -421,7 +421,7 @@ void StructuredInterpolation2D::setup( const FunctionSpace& source ) { } // fill sparse matrix - if( failed_points.empty() ) { + if( failed_points.empty() && out_npts_) { idx_t inp_npts = source.size(); Matrix A( out_npts_, inp_npts, triplets ); setMatrix(A); diff --git a/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h b/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h index 3dd79099b..6d4db4dba 100644 --- a/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h +++ b/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h @@ -110,9 +110,10 @@ class ConservativeSphericalPolygonInterpolation : public Method { SRCTGT_INTERSECTPLG_DIFF, // index, 1/(unit_sphere.area) ( \sum_{scell} scell.area - \sum{tcell} tcell.area ) REMAP_CONS, // index, error in mass conservation REMAP_L2, // index, error accuracy for given analytical function - REMAP_LINF // index, like REMAP_L2 but in L_infinity norm + REMAP_LINF, // index, like REMAP_L2 but in L_infinity norm + ERRORS_ENUM_SIZE }; - std::array errors; + std::array errors; double tgt_area_sum; double src_area_sum; diff --git a/src/atlas/interpolation/method/unstructured/UnstructuredBilinearLonLat.cc b/src/atlas/interpolation/method/unstructured/UnstructuredBilinearLonLat.cc index 62276db31..840c851a8 100644 --- a/src/atlas/interpolation/method/unstructured/UnstructuredBilinearLonLat.cc +++ b/src/atlas/interpolation/method/unstructured/UnstructuredBilinearLonLat.cc @@ -419,15 +419,22 @@ Method::Triplets UnstructuredBilinearLonLat::projectPointToElements(size_t ip, c } // shift cells on the east-west periodic boundary from the east to the west // so that the quad surrounds a point with output longitude in [0,360) - if (lons[0] > lons[1] || lons[3] > lons[2]) { - lons[0] -= 360; - lons[3] -= 360; + double minlon = std::numeric_limits::max(); + for ( int i = 0; i < 4; i++ ) { + minlon = std::min( minlon, lons[i] ); + } + for ( int i = 0; i < 4; i++ ) { + if ( (lons[i] - minlon) > 180 ) { + lons[i] -= 360; + } } element::Quad2D quad( PointLonLat{lons[0], (*ilonlat_)(idx[0], LAT)}, PointLonLat{lons[1], (*ilonlat_)(idx[1], LAT)}, PointLonLat{lons[2], (*ilonlat_)(idx[2], LAT)}, PointLonLat{lons[3], (*ilonlat_)(idx[3], LAT)}); + ATLAS_ASSERT( quad.validate() ); + if (itc == elems.begin()) { inv_dist_weight_quad(quad, o_loc, inv_dist_w); } diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index 6ed6b227a..99bebfe96 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -485,8 +485,8 @@ void Library::Information::print(std::ostream& out) const { feature_MPI = true; #endif std::string array_data_store = "Native-host"; -#if ATLAS_HAVE_CUDA - array_data_store = "Native-CUDA"; +#if ATLAS_HAVE_GPU + array_data_store = "Native-GPU"; #endif #if ATLAS_HAVE_GRIDTOOLS_STORAGE array_data_store = "Gridtools-host"; diff --git a/src/atlas/library/config.h b/src/atlas/library/config.h index d684c3716..82c2b2e7e 100644 --- a/src/atlas/library/config.h +++ b/src/atlas/library/config.h @@ -16,6 +16,7 @@ #include "atlas/atlas_ecbuild_config.h" #include "atlas/library/defines.h" +#include "hic/hic_config.h" #ifndef DOXYGEN_SHOULD_SKIP_THIS #define DOXYGEN_HIDE(X) X @@ -24,6 +25,21 @@ #define ATLAS_HAVE_TRACE 1 #define ATLAS_HAVE_TRACE_BARRIERS 1 +#define ATLAS_HIC_COMPILER HIC_COMPILER +#define ATLAS_HOST_COMPILE HIC_HOST_COMPILE +#define ATLAS_DEVICE_COMPILE HIC_DEVICE_COMPILE +#define ATLAS_HOST_DEVICE HIC_HOST_DEVICE +#define ATLAS_DEVICE HIC_DEVICE +#define ATLAS_HOST HIC_HOST +#define ATLAS_GLOBAL HIC_GLOBAL + +#if HIC_BACKEND_CUDA || HIC_BACKEND_HIP +#define ATLAS_HAVE_GPU 1 +#else +#define ATLAS_HAVE_GPU 0 +#endif + + namespace atlas { /// @typedef gidx_t diff --git a/src/atlas/library/defines.h.in b/src/atlas/library/defines.h.in index cf329bfd5..de1672dcf 100644 --- a/src/atlas/library/defines.h.in +++ b/src/atlas/library/defines.h.in @@ -17,7 +17,7 @@ #define ATLAS_HAVE_OMP @atlas_HAVE_OMP_CXX@ #define ATLAS_OMP_TASK_SUPPORTED @ATLAS_OMP_TASK_SUPPORTED@ #define ATLAS_OMP_TASK_UNTIED_SUPPORTED @ATLAS_OMP_TASK_UNTIED_SUPPORTED@ -#define ATLAS_HAVE_CUDA @atlas_HAVE_CUDA@ +#define ATLAS_HAVE_GPU @atlas_HAVE_GPU@ #define ATLAS_HAVE_ACC @atlas_HAVE_ACC@ #define ATLAS_HAVE_QHULL @atlas_HAVE_QHULL@ #define ATLAS_HAVE_CGAL @atlas_HAVE_CGAL@ @@ -45,16 +45,6 @@ #define ATLAS_ECKIT_DEVELOP @ATLAS_ECKIT_DEVELOP@ #define ATLAS_HAVE_FUNCTIONSPACE @atlas_HAVE_ATLAS_FUNCTIONSPACE@ -#ifdef __CUDACC__ -#define ATLAS_HOST_DEVICE __host__ __device__ -#define ATLAS_DEVICE __device__ -#define ATLAS_HOST __host__ -#else -#define ATLAS_HOST_DEVICE -#define ATLAS_DEVICE -#define ATLAS_HOST -#endif - #define ATLAS_BITS_LOCAL @ATLAS_BITS_LOCAL@ #if defined( __GNUC__ ) || defined( __clang__ ) diff --git a/src/atlas/mesh/Connectivity.h b/src/atlas/mesh/Connectivity.h index a77eafdfe..1631384d7 100644 --- a/src/atlas/mesh/Connectivity.h +++ b/src/atlas/mesh/Connectivity.h @@ -183,7 +183,7 @@ class IrregularConnectivityImpl { /// No resizing can be performed as data is not owned. IrregularConnectivityImpl(idx_t values[], idx_t rows, idx_t displs[], idx_t counts[]); -#ifdef __CUDACC__ +#if ATLAS_HIC_COMPILER /// @brief Copy ctr (only to be used when calling a cuda kernel) // This ctr has to be defined in the header, since __CUDACC__ will identify // whether @@ -529,7 +529,7 @@ class BlockConnectivityImpl { /// No resizing can be performed as data is not owned. BlockConnectivityImpl(idx_t rows, idx_t cols, idx_t values[]); -#ifdef __CUDACC__ +#if ATLAS_HIC_COMPILER /// @brief Copy ctr (only to be used when calling a cuda kernel) // This ctr has to be defined in the header, since __CUDACC__ will identify // whether it is compiled it for a GPU kernel @@ -645,6 +645,7 @@ using Connectivity = IrregularConnectivity; // ----------------------------------------------------------------------------------------------------- +ATLAS_HOST_DEVICE inline idx_t IrregularConnectivityImpl::operator()(idx_t row_idx, idx_t col_idx) const { assert(counts_[row_idx] > (col_idx)); return values_[displs_[row_idx] + col_idx] FROM_FORTRAN; @@ -662,36 +663,43 @@ inline void IrregularConnectivityImpl::set(idx_t row_idx, idx_t col_idx, const i values_[displs_[row_idx] + col_idx] = value TO_FORTRAN; } +ATLAS_HOST_DEVICE inline IrregularConnectivityImpl::Row IrregularConnectivityImpl::row(idx_t row_idx) const { return IrregularConnectivityImpl::Row(const_cast(values_.data()) + displs_(row_idx), counts_(row_idx)); } // ----------------------------------------------------------------------------------------------------- +ATLAS_HOST_DEVICE inline idx_t MultiBlockConnectivityImpl::operator()(idx_t row_idx, idx_t col_idx) const { return IrregularConnectivityImpl::operator()(row_idx, col_idx); } +ATLAS_HOST_DEVICE inline idx_t MultiBlockConnectivityImpl::operator()(idx_t block_idx, idx_t block_row_idx, idx_t block_col_idx) const { return block(block_idx)(block_row_idx, block_col_idx); } // ----------------------------------------------------------------------------------------------------- +ATLAS_HOST_DEVICE inline idx_t BlockConnectivityImpl::operator()(idx_t row_idx, idx_t col_idx) const { return values_[index(row_idx, col_idx)] FROM_FORTRAN; } +ATLAS_HOST_DEVICE inline void BlockConnectivityImpl::set(idx_t row_idx, const idx_t column_values[]) { for (idx_t n = 0; n < cols_; ++n) { values_[index(row_idx, n)] = column_values[n] TO_FORTRAN; } } +ATLAS_HOST_DEVICE inline void BlockConnectivityImpl::set(idx_t row_idx, idx_t col_idx, const idx_t value) { values_[index(row_idx, col_idx)] = value TO_FORTRAN; } +ATLAS_HOST_DEVICE inline idx_t BlockConnectivityImpl::index(idx_t i, idx_t j) const { return i * cols_ + j; } diff --git a/src/atlas/mesh/actions/BuildEdges.cc b/src/atlas/mesh/actions/BuildEdges.cc index 08a568a17..4ab0f453a 100644 --- a/src/atlas/mesh/actions/BuildEdges.cc +++ b/src/atlas/mesh/actions/BuildEdges.cc @@ -93,7 +93,7 @@ void build_element_to_edge_connectivity(Mesh& mesh) { UniqueLonLat compute_uid(mesh); for (idx_t jedge = 0; jedge < nb_edges; ++jedge) { - edge_sort.emplace_back(Sort(compute_uid(edge_node_connectivity.row(jedge)), jedge)); + edge_sort.emplace_back(compute_uid(edge_node_connectivity.row(jedge)), jedge); } std::sort(edge_sort.data(), edge_sort.data() + nb_edges); diff --git a/src/atlas/mesh/actions/BuildParallelFields.cc b/src/atlas/mesh/actions/BuildParallelFields.cc index 17f617d18..bd403a29b 100644 --- a/src/atlas/mesh/actions/BuildParallelFields.cc +++ b/src/atlas/mesh/actions/BuildParallelFields.cc @@ -984,7 +984,7 @@ Field& build_edges_global_idx(Mesh& mesh) { std::vector edge_sort; edge_sort.reserve(glb_nb_edges); for (idx_t jedge = 0; jedge < glb_edge_id.shape(0); ++jedge) { - edge_sort.emplace_back(Node(glb_edge_id(jedge), jedge)); + edge_sort.emplace_back(glb_edge_id(jedge), jedge); } std::sort(edge_sort.begin(), edge_sort.end()); diff --git a/src/atlas/mesh/actions/GetCubedSphereNodalArea.cc b/src/atlas/mesh/actions/GetCubedSphereNodalArea.cc new file mode 100644 index 000000000..17ad5f410 --- /dev/null +++ b/src/atlas/mesh/actions/GetCubedSphereNodalArea.cc @@ -0,0 +1,59 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + + +#include "atlas/array/MakeView.h" +#include "atlas/grid/CubedSphereGrid.h" +#include "atlas/mesh/actions/GetCubedSphereNodalArea.h" +#include "atlas/util/NormaliseLongitude.h" +#include "atlas/util/Point.h" + + +namespace atlas { +namespace mesh { +namespace actions { + + +Field& GetCubedSphereNodalArea::operator()(Mesh& mesh) { + if (mesh.nodes().has_field("grid_cell_areas")) { + return mesh.nodes().field("grid_cell_areas"); + } + else { + constexpr auto deg2rad = M_PI / 180.; + const auto& proj = mesh.projection(); + auto lonlat = array::make_view(mesh.nodes().lonlat()); + + auto gcell_area_field = Field("grid_cell_areas", + make_datatype(), + array::make_shape(mesh.nodes().size())); + auto gcell_area_fview = array::make_view(gcell_area_field); + + double gcell_area_cs = [&] { + ATLAS_ASSERT(CubedSphereGrid(mesh.grid())); + // (grid_res * grid_res) = no. of cells on a tile + auto grid_res = CubedSphereGrid(mesh.grid()).N(); + // area of a grid cell (cubed-sphere coord. system) + return M_PI/(2*grid_res) * M_PI/(2*grid_res); + }(); + + for (size_t i = 0; i < gcell_area_fview.size(); ++i) { + PointLonLat loc = PointLonLat(lonlat(i, atlas::LON), lonlat(i, atlas::LAT)); + double cos_lat = std::cos(deg2rad * loc.lat()); + double grid_jac_det = 1/proj.jacobian(loc).determinant(); + // area of a grid cell (geographic coord. system) + gcell_area_fview(i) = grid_jac_det * gcell_area_cs * cos_lat; + } + + mesh.nodes().add(gcell_area_field); + + return mesh.nodes().field("grid_cell_areas"); + } +} + +} // namespace actions +} // namespace mesh +} // namespace atlas diff --git a/src/atlas/mesh/actions/GetCubedSphereNodalArea.h b/src/atlas/mesh/actions/GetCubedSphereNodalArea.h new file mode 100644 index 000000000..d571f729d --- /dev/null +++ b/src/atlas/mesh/actions/GetCubedSphereNodalArea.h @@ -0,0 +1,32 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include "atlas/field/Field.h" +#include "atlas/mesh/Mesh.h" +#include "atlas/mesh/Nodes.h" + + +namespace atlas { + +class Mesh; +class Field; + +namespace mesh { +namespace actions { + + +/// Provide the area around nodes of cubed sphere mesh +class GetCubedSphereNodalArea { +public: + Field& operator()(Mesh&); +}; + +} // namespace actions +} // namespace mesh +} // namespace atlas diff --git a/src/atlas/parallel/HaloExchange.h b/src/atlas/parallel/HaloExchange.h index 0d6b0e229..2c4931007 100644 --- a/src/atlas/parallel/HaloExchange.h +++ b/src/atlas/parallel/HaloExchange.h @@ -33,8 +33,8 @@ #include "atlas/runtime/Exception.h" #include "atlas/util/Object.h" -#if ATLAS_HAVE_CUDA -#include "atlas/parallel/HaloExchangeCUDA.h" +#if ATLAS_HAVE_GPU +#include "atlas/parallel/HaloExchangeGPU.h" #endif namespace atlas { @@ -90,7 +90,7 @@ class HaloExchange : public util::Object { DATA_TYPE* allocate_buffer(const int buffer_size, const bool on_device) const; template - void deallocate_buffer(DATA_TYPE* buffer, const bool on_device) const; + void deallocate_buffer(DATA_TYPE* buffer, const int buffer_size, const bool on_device) const; template void pack_send_buffer(const array::ArrayView& hfield, @@ -174,6 +174,14 @@ void HaloExchange::execute(array::Array& field, bool on_device) const { DATA_TYPE* inner_buffer = allocate_buffer(inner_size, on_device); DATA_TYPE* halo_buffer = allocate_buffer(halo_size, on_device); +#if ATLAS_HAVE_GPU + if (on_device) { + ATLAS_ASSERT( is_device_accessible(inner_buffer) ); + ATLAS_ASSERT( is_device_accessible(halo_buffer) ); + ATLAS_ASSERT( is_device_accessible(field_dv.data()) ); + } +#endif + counts_displs_setup(var_size, inner_counts_init, halo_counts_init, inner_counts, halo_counts, inner_displs, halo_displs); @@ -190,8 +198,8 @@ void HaloExchange::execute(array::Array& field, bool on_device) const { wait_for_send(inner_counts_init, inner_req); - deallocate_buffer(inner_buffer, on_device); - deallocate_buffer(halo_buffer, on_device); + deallocate_buffer(inner_buffer, inner_size, on_device); + deallocate_buffer(halo_buffer, halo_size, on_device); } template @@ -241,8 +249,8 @@ void HaloExchange::execute_adjoint(array::Array& field, bool on_device) const { zero_halos(field_hv, field_dv, halo_buffer, halo_size, on_device); - deallocate_buffer(halo_buffer, on_device); - deallocate_buffer(inner_buffer, on_device); + deallocate_buffer(halo_buffer, halo_size, on_device); + deallocate_buffer(inner_buffer, inner_size, on_device); } template @@ -261,12 +269,12 @@ DATA_TYPE* HaloExchange::allocate_buffer(const int buffer_size, const bool on_de template -void HaloExchange::deallocate_buffer(DATA_TYPE* buffer, const bool on_device) const { +void HaloExchange::deallocate_buffer(DATA_TYPE* buffer, const int buffer_size, const bool on_device) const { if (on_device) { - util::delete_devicemem(buffer); + util::delete_devicemem(buffer, buffer_size); } else { - util::delete_hostmem(buffer); + util::delete_hostmem(buffer, buffer_size); } } @@ -294,7 +302,7 @@ void HaloExchange::ireceive(int tag, std::vector& recv_displs, std::vector< for (size_t jproc = 0; jproc < static_cast(nproc); ++jproc) { if (recv_counts[jproc] > 0) { recv_req[jproc] = - comm().iReceive(&recv_buffer[recv_displs[jproc]], recv_counts[jproc], jproc, tag); + comm().iReceive(recv_buffer+recv_displs[jproc], recv_counts[jproc], jproc, tag); } } } @@ -309,7 +317,7 @@ void HaloExchange::isend_and_wait_for_receive(int tag, std::vector& recv_co ATLAS_TRACE_MPI(ISEND) { for (size_t jproc = 0; jproc < static_cast(nproc); ++jproc) { if (send_counts[jproc] > 0) { - send_req[jproc] = comm().iSend(&send_buffer[send_displs[jproc]], send_counts[jproc], jproc, tag); + send_req[jproc] = comm().iSend(send_buffer+send_displs[jproc], send_counts[jproc], jproc, tag); } } } @@ -380,7 +388,7 @@ void HaloExchange::zero_halos(ATLAS_MAYBE_UNUSED const array::ArrayView& dfield, DATA_TYPE* recv_buffer, int recv_size, ATLAS_MAYBE_UNUSED const bool on_device) const { ATLAS_TRACE(); -#if ATLAS_HAVE_CUDA +#if ATLAS_HAVE_GPU if (on_device) { ATLAS_NOTIMPLEMENTED; } @@ -394,9 +402,9 @@ void HaloExchange::pack_send_buffer(ATLAS_MAYBE_UNUSED const array::ArrayView& dfield, DATA_TYPE* send_buffer, int send_size, ATLAS_MAYBE_UNUSED const bool on_device) const { ATLAS_TRACE(); -#if ATLAS_HAVE_CUDA +#if ATLAS_HAVE_GPU if (on_device) { - halo_packer_cuda::pack(sendcnt_, sendmap_, hfield, dfield, send_buffer, + halo_packer_hic::pack(sendcnt_, sendmap_, hfield, dfield, send_buffer, send_size); } else @@ -410,9 +418,9 @@ void HaloExchange::unpack_recv_buffer(const DATA_TYPE* recv_buffer, int recv_siz array::ArrayView& dfield, ATLAS_MAYBE_UNUSED const bool on_device) const { ATLAS_TRACE(); -#if ATLAS_HAVE_CUDA +#if ATLAS_HAVE_GPU if (on_device) { - halo_packer_cuda::unpack(recvcnt_, recvmap_, recv_buffer, recv_size, hfield, + halo_packer_hic::unpack(recvcnt_, recvmap_, recv_buffer, recv_size, hfield, dfield); } else @@ -425,9 +433,9 @@ void HaloExchange::pack_recv_adjoint_buffer(ATLAS_MAYBE_UNUSED const array::Arra const array::ArrayView& dfield, DATA_TYPE* recv_buffer, int recv_size, ATLAS_MAYBE_UNUSED const bool on_device) const { ATLAS_TRACE(); -#if ATLAS_HAVE_CUDA +#if ATLAS_HAVE_GPU if (on_device) { - halo_packer_cuda::pack(recvcnt_, recvmap_, hfield, dfield, recv_buffer, + halo_packer_hic::pack(recvcnt_, recvmap_, hfield, dfield, recv_buffer, recv_size); } else @@ -441,9 +449,9 @@ void HaloExchange::unpack_send_adjoint_buffer(const DATA_TYPE* send_buffer, int array::ArrayView& dfield, ATLAS_MAYBE_UNUSED const bool on_device) const { ATLAS_TRACE(); -#if ATLAS_HAVE_CUDA +#if ATLAS_HAVE_GPU if (on_device) { - halo_packer_cuda::unpack(sendcnt_, sendmap_, send_buffer, send_size, hfield, + halo_packer_hic::unpack(sendcnt_, sendmap_, send_buffer, send_size, hfield, dfield); } else diff --git a/src/atlas/parallel/HaloExchangeCUDA.h b/src/atlas/parallel/HaloExchangeGPU.h similarity index 94% rename from src/atlas/parallel/HaloExchangeCUDA.h rename to src/atlas/parallel/HaloExchangeGPU.h index ada9641e6..f3b9a46c5 100644 --- a/src/atlas/parallel/HaloExchangeCUDA.h +++ b/src/atlas/parallel/HaloExchangeGPU.h @@ -18,7 +18,7 @@ namespace atlas { namespace parallel { template -struct halo_packer_cuda { +struct halo_packer_hic { static void pack(const int sendcnt, array::SVector const& sendmap, const array::ArrayView& hfield, const array::ArrayView& dfield, DATA_TYPE* send_buffer, int send_buffer_size); @@ -28,5 +28,8 @@ struct halo_packer_cuda { array::ArrayView& dfield); }; +bool is_device_accessible(const void* data); + + } // namespace parallel } // namespace atlas diff --git a/src/atlas/parallel/HaloExchangeCUDA.cu b/src/atlas/parallel/HaloExchangeGPU.hic similarity index 80% rename from src/atlas/parallel/HaloExchangeCUDA.cu rename to src/atlas/parallel/HaloExchangeGPU.hic index a9242515a..c96893d1c 100644 --- a/src/atlas/parallel/HaloExchangeCUDA.cu +++ b/src/atlas/parallel/HaloExchangeGPU.hic @@ -9,14 +9,26 @@ */ -#include "atlas/parallel/HaloExchangeCUDA.h" +#include "atlas/parallel/HaloExchangeGPU.h" #include "atlas/parallel/HaloExchangeImpl.h" #include "atlas/array/SVector.h" #include "atlas/runtime/Exception.h" +#include "hic/hic.h" + namespace atlas { namespace parallel { +bool is_device_accessible(const void* ptr) { + hicPointerAttributes attr; + hicError_t code = hicPointerGetAttributes(&attr, ptr); + if( code != hicSuccess ) { + static_cast(hicGetLastError()); + return false; + } + return ptr == attr.devicePointer; +} + template struct get_buffer_index{ template @@ -120,7 +132,7 @@ struct get_first_non_parallel_dim }; template -struct get_n_cuda_blocks +struct get_n_hic_blocks { template static unsigned int apply(const array::ArrayView& hfield, const unsigned int block_size_y) { @@ -130,7 +142,7 @@ struct get_n_cuda_blocks }; template<> -struct get_n_cuda_blocks<0, 1> { +struct get_n_hic_blocks<0, 1> { template static unsigned int apply(const array::ArrayView& hfield, const unsigned int block_size_y) { return 1; @@ -138,70 +150,71 @@ struct get_n_cuda_blocks<0, 1> { }; template -void halo_packer_cuda::pack( const int sendcnt, array::SVector const & sendmap, +void halo_packer_hic::pack( const int sendcnt, array::SVector const & sendmap, const array::ArrayView& hfield, const array::ArrayView& dfield, DATA_TYPE* send_buffer, int send_buffer_size ) { const unsigned int block_size_x = 32; const unsigned int block_size_y = (RANK==1) ? 1 : 4; - unsigned int nblocks_y = get_n_cuda_blocks::apply(hfield, block_size_y); + const unsigned int nblocks_x = (sendcnt+block_size_x-1)/block_size_x; + const unsigned int nblocks_y = get_n_hic_blocks::apply(hfield, block_size_y); dim3 threads(block_size_x, block_size_y); - dim3 blocks((sendcnt+block_size_x-1)/block_size_x, nblocks_y); - cudaDeviceSynchronize(); - cudaError_t err = cudaGetLastError(); - if (err != cudaSuccess) { - std::string msg = std::string("Error synchronizing device")+ cudaGetErrorString(err); + dim3 blocks(nblocks_x, nblocks_y); + HIC_CALL(hicDeviceSynchronize()); + hicError_t err = hicGetLastError(); + if (err != hicSuccess) { + std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err); throw_Exception(msg); } pack_kernel<<>>(sendcnt, sendmap.data(), sendmap.size(), dfield, send_buffer, send_buffer_size); - err = cudaGetLastError(); - if (err != cudaSuccess) + err = hicGetLastError(); + if (err != hicSuccess) throw_Exception("Error launching GPU packing kernel"); - cudaDeviceSynchronize(); - err = cudaGetLastError(); - if (err != cudaSuccess) { - std::string msg = std::string("Error synchronizing device")+ cudaGetErrorString(err); + HIC_CALL(hicDeviceSynchronize()); + err = hicGetLastError(); + if (err != hicSuccess) { + std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err); throw_Exception(msg); } } template -void halo_packer_cuda::unpack(const int recvcnt, array::SVector const & recvmap, +void halo_packer_hic::unpack(const int recvcnt, array::SVector const & recvmap, const DATA_TYPE* recv_buffer , int recv_buffer_size, array::ArrayView &hfield, array::ArrayView &dfield) { const unsigned int block_size_x = 32; const unsigned int block_size_y = (RANK==1) ? 1 : 4; - unsigned int nblocks_y = get_n_cuda_blocks::apply(hfield, block_size_y); + unsigned int nblocks_y = get_n_hic_blocks::apply(hfield, block_size_y); dim3 threads(block_size_x, block_size_y); dim3 blocks((recvcnt+block_size_x-1)/block_size_x, nblocks_y); - cudaDeviceSynchronize(); - cudaError_t err = cudaGetLastError(); - if (err != cudaSuccess) { - std::string msg = std::string("Error synchronizing device")+ cudaGetErrorString(err); + HIC_CALL(hicDeviceSynchronize()); + hicError_t err = hicGetLastError(); + if (err != hicSuccess) { + std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err); throw_Exception(msg); } unpack_kernel<<>>(recvcnt, recvmap.data(), recvmap.size(), recv_buffer, recv_buffer_size, dfield); - err = cudaGetLastError(); - if (err != cudaSuccess) { - std::string msg = std::string("Error launching GPU packing kernel")+ cudaGetErrorString(err); + err = hicGetLastError(); + if (err != hicSuccess) { + std::string msg = std::string("Error launching GPU packing kernel")+ hicGetErrorString(err); throw_Exception(msg); } - cudaDeviceSynchronize(); - err = cudaGetLastError(); - if (err != cudaSuccess) { - std::string msg = std::string("Error synchronizing device")+ cudaGetErrorString(err); + HIC_CALL(hicDeviceSynchronize()); + err = hicGetLastError(); + if (err != hicSuccess) { + std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err); throw_Exception(msg); } } @@ -220,11 +233,11 @@ void halo_packer_cuda::unpack(const int recvcnt, a #define EXPLICIT_TEMPLATE_INSTANTIATION( ParallelDim, RANK ) \ - template class halo_packer_cuda; \ - template class halo_packer_cuda; \ - template class halo_packer_cuda; \ - template class halo_packer_cuda; \ - template class halo_packer_cuda; + template class halo_packer_hic; \ + template class halo_packer_hic; \ + template class halo_packer_hic; \ + template class halo_packer_hic; \ + template class halo_packer_hic; #define EXPLICIT_TEMPLATE_INSTANTIATION_RANK(RANK) \ ATLAS_REPEAT_MACRO(RANK, EXPLICIT_TEMPLATE_INSTANTIATION, RANK) diff --git a/src/atlas/parallel/acc/acc.cc b/src/atlas/parallel/acc/acc.cc new file mode 100644 index 000000000..c73bdea2c --- /dev/null +++ b/src/atlas/parallel/acc/acc.cc @@ -0,0 +1,81 @@ +/* + * (C) Copyright 2024- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "acc.h" + +#include "atlas/library/defines.h" + +#if ATLAS_HAVE_ACC +#include "hic/hic.h" +#include "atlas_acc_support/atlas_acc.h" +static int hic_devices() { + static int devices_ = [](){ + int n = 0; + auto err = hicGetDeviceCount(&n); + if (err != hicSuccess) { + n = 0; + static_cast(hicGetLastError()); + } + return n; + }(); + return devices_; +} +#endif + +namespace atlas::acc { + +int devices() { +#if ATLAS_HAVE_ACC + static int num_devices = [](){ + if (hic_devices() == 0) { + return 0; + } + auto devicetype = atlas_acc_get_device_type(); + int _num_devices = atlas_acc_get_num_devices(); + if (_num_devices == 1 && devicetype == atlas_acc_device_host) { + --_num_devices; + } + return _num_devices; + }(); + return num_devices; +#else + return 0; +#endif +} + +void map(void* host_data, void* device_data, std::size_t bytes) { +#if ATLAS_HAVE_ACC + atlas_acc_map_data(host_data, device_data, bytes); +#endif +} +void unmap(void* host_data) { +#if ATLAS_HAVE_ACC + atlas_acc_unmap_data(host_data); +#endif +} + +bool is_present(void* host_data, std::size_t bytes) { +#if ATLAS_HAVE_ACC + return atlas_acc_is_present(host_data, bytes); +#else + return false; +#endif +} + +void* deviceptr(void* host_data) { +#if ATLAS_HAVE_ACC + return atlas_acc_deviceptr(host_data); +#else + return nullptr; +#endif +} + +} + diff --git a/src/atlas_acc_support/atlas_acc_map_data.h b/src/atlas/parallel/acc/acc.h similarity index 58% rename from src/atlas_acc_support/atlas_acc_map_data.h rename to src/atlas/parallel/acc/acc.h index 4fa1208c2..3a10f1e25 100644 --- a/src/atlas_acc_support/atlas_acc_map_data.h +++ b/src/atlas/parallel/acc/acc.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2013 ECMWF. + * (C) Copyright 2024- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -9,14 +9,15 @@ */ #pragma once +#include -#ifdef __cplusplus -extern "C" { -#endif +namespace atlas::acc { -void atlas_acc_map_data(void* cpu_ptr, void* gpu_ptr, unsigned long size); -void atlas_acc_unmap_data(void* cpu_ptr); +int devices(); +void map(void* host_data, void* device_data, std::size_t bytes); +void unmap(void* host_data); +bool is_present(void* host_data, std::size_t bytes); +void* deviceptr(void* host_data); -#ifdef __cplusplus } -#endif + diff --git a/src/atlas/runtime/trace/Timings.cc b/src/atlas/runtime/trace/Timings.cc index a477969d6..cbc847213 100644 --- a/src/atlas/runtime/trace/Timings.cc +++ b/src/atlas/runtime/trace/Timings.cc @@ -274,8 +274,8 @@ void TimingsRegistry::report(std::ostream& out, const eckit::Configuration& conf } } - size_t max_title_length(0); - size_t max_location_length(0); + size_t max_title_length(2); + size_t max_location_length(9); size_t max_nest(0); long max_count(0); double max_seconds(0); diff --git a/src/atlas/util/Allocate.cc b/src/atlas/util/Allocate.cc index b93446abb..58f2648d6 100644 --- a/src/atlas/util/Allocate.cc +++ b/src/atlas/util/Allocate.cc @@ -16,9 +16,7 @@ #include "atlas/library/config.h" #include "atlas/runtime/Exception.h" -#if ATLAS_HAVE_CUDA -#include -#endif +#include "hic/hic.h" namespace atlas { namespace util { @@ -27,50 +25,66 @@ namespace util { namespace detail { //------------------------------------------------------------------------------ -void allocate_cudamanaged(void** ptr, size_t size) { -#if ATLAS_HAVE_CUDA - cudaError_t err = cudaMallocManaged(ptr, size); - if (err != cudaSuccess) - throw_AssertionFailed("failed to allocate GPU memory", Here()); -#else - *ptr = malloc(size); -#endif -} - -void deallocate_cudamanaged(void* ptr) { -#if ATLAS_HAVE_CUDA - cudaError_t err = cudaDeviceSynchronize(); - if (err != cudaSuccess) - throw_AssertionFailed("failed to synchronize memory", Here()); - - err = cudaFree(ptr); - // The following throws an invalid device memory - if (err != cudaSuccess) - throw_AssertionFailed("failed to free GPU memory", Here()); -#else - free(ptr); -#endif +static int devices() { + static int devices_ = [](){ + int n = 0; + auto err = hicGetDeviceCount(&n); + if (err != hicSuccess) { + n = 0; + static_cast(hicGetLastError()); + } + return n; + }(); + return devices_; +} + +void allocate_managed(void** ptr, size_t bytes) { + if constexpr (not ATLAS_HAVE_GPU) { + return allocate_host(ptr, bytes); + } + if (devices() == 0) { + return allocate_host(ptr, bytes); + } + HIC_CALL(hicMallocManaged(ptr, bytes)); +} + +void deallocate_managed(void* ptr, size_t bytes) { + if constexpr (not ATLAS_HAVE_GPU) { + return deallocate_host(ptr, bytes); + } + if (devices() == 0) { + return deallocate_host(ptr, bytes); + } + HIC_CALL(hicDeviceSynchronize()); + HIC_CALL(hicFree(ptr)); } -void allocate_cuda(void** ptr, size_t size) { -#if ATLAS_HAVE_CUDA - cudaError_t err = cudaMalloc(ptr, size); - if (err != cudaSuccess) - throw_AssertionFailed("failed to allocate GPU memory", Here()); -#else - *ptr = malloc(size); -#endif +void allocate_device(void** ptr, size_t bytes) { + if constexpr (not ATLAS_HAVE_GPU) { + return allocate_host(ptr, bytes); + } + if (devices() == 0) { + return allocate_host(ptr, bytes); + } + HIC_CALL(hicMalloc(ptr, bytes)); } -void deallocate_cuda(void* ptr) { - deallocate_cudamanaged(ptr); +void deallocate_device(void* ptr, size_t bytes) { + if constexpr (not ATLAS_HAVE_GPU) { + return deallocate_host(ptr, bytes); + } + if (devices() == 0) { + return deallocate_host(ptr, bytes); + } + HIC_CALL(hicDeviceSynchronize()); + HIC_CALL(hicFree(ptr)); } -void allocate_host(void** ptr, size_t size) { - *ptr = malloc(size); +void allocate_host(void** ptr, size_t bytes) { + *ptr = malloc(bytes); } -void deallocate_host(void* ptr) { +void deallocate_host(void* ptr, size_t /*bytes*/) { free(ptr); } @@ -91,8 +105,17 @@ void atlas__allocate_managedmem_int(int*& a, size_t N) { void atlas__allocate_managedmem_long(long*& a, size_t N) { allocate_managedmem(a, N); } -void atlas__deallocate_managedmem(void*& a) { - delete_managedmem(a); +void atlas__deallocate_managedmem_double(double*& a, size_t N) { + delete_managedmem(a, N); +} +void atlas__deallocate_managedmem_float(float*& a, size_t N) { + delete_managedmem(a, N); +} +void atlas__deallocate_managedmem_int(int*& a, size_t N) { + delete_managedmem(a, N); +} +void atlas__deallocate_managedmem_long(long*& a, size_t N) { + delete_managedmem(a, N); } } diff --git a/src/atlas/util/Allocate.h b/src/atlas/util/Allocate.h index cac37e40a..b8fac7179 100644 --- a/src/atlas/util/Allocate.h +++ b/src/atlas/util/Allocate.h @@ -18,28 +18,28 @@ namespace util { //------------------------------------------------------------------------------ namespace detail { -void allocate_cudamanaged(void** ptr, size_t size); -void deallocate_cudamanaged(void* ptr); +void allocate_managed(void** ptr, size_t bytes); +void deallocate_managed(void* ptr, size_t bytes); -void allocate_cuda(void** ptr, size_t size); -void deallocate_cuda(void* ptr); +void allocate_device(void** ptr, size_t bytes); +void deallocate_device(void* ptr, size_t bytes); -void allocate_host(void** ptr, size_t size); -void deallocate_host(void* ptr); +void allocate_host(void** ptr, size_t bytes); +void deallocate_host(void* ptr, size_t bytes); } // namespace detail template void allocate_managedmem(T*& data, size_t N) { if (N != 0) { - detail::allocate_cudamanaged(reinterpret_cast(&data), N * sizeof(T)); + detail::allocate_managed(reinterpret_cast(&data), N * sizeof(T)); } } template -void delete_managedmem(T*& data) { +void delete_managedmem(T*& data, size_t N) { if (data) { - detail::deallocate_cudamanaged(data); + detail::deallocate_managed(data, N * sizeof(T)); data = nullptr; } } @@ -47,14 +47,14 @@ void delete_managedmem(T*& data) { template void allocate_devicemem(T*& data, size_t N) { if (N != 0) { - detail::allocate_cuda(reinterpret_cast(&data), N * sizeof(T)); + detail::allocate_device(reinterpret_cast(&data), N * sizeof(T)); } } template -void delete_devicemem(T*& data) { +void delete_devicemem(T*& data, size_t N) { if (data) { - detail::deallocate_cuda(data); + detail::deallocate_device(data, N * sizeof(T)); data = nullptr; } } @@ -67,9 +67,9 @@ void allocate_hostmem(T*& data, size_t N) { } template -void delete_hostmem(T*& data) { +void delete_hostmem(T*& data, size_t N) { if (data) { - detail::deallocate_host(data); + detail::deallocate_host(data, N * sizeof(T)); data = nullptr; } } @@ -82,7 +82,10 @@ void atlas__allocate_managedmem_double(double*& a, size_t N); void atlas__allocate_managedmem_float(float*& a, size_t N); void atlas__allocate_managedmem_int(int*& a, size_t N); void atlas__allocate_managedmem_long(long*& a, size_t N); -void atlas__deallocate_managedmem(void*& a); +void atlas__deallocate_managedmem_double(double*& a, size_t N); +void atlas__deallocate_managedmem_float(float*& a, size_t N); +void atlas__deallocate_managedmem_int(int*& a, size_t N); +void atlas__deallocate_managedmem_long(long*& a, size_t N); } //------------------------------------------------------------------------------ diff --git a/src/atlas/util/GPUClonable.h b/src/atlas/util/GPUClonable.h index fcee74129..704a94429 100644 --- a/src/atlas/util/GPUClonable.h +++ b/src/atlas/util/GPUClonable.h @@ -12,9 +12,7 @@ #include "atlas/library/config.h" -#if ATLAS_HAVE_CUDA -#include -#endif +#include "hic/hic.h" namespace atlas { namespace util { @@ -22,30 +20,28 @@ namespace util { template struct GPUClonable { GPUClonable(Base* base_ptr): base_ptr_(base_ptr), gpu_object_ptr_(nullptr) { -#if ATLAS_HAVE_CUDA - cudaMalloc(&gpu_object_ptr_, sizeof(Base)); -#endif + if constexpr (ATLAS_HAVE_GPU) { + HIC_CALL(hicMalloc(&gpu_object_ptr_, sizeof(Base))); + } } ~GPUClonable() { - if (gpu_object_ptr_) { -#if ATLAS_HAVE_CUDA - cudaFree(gpu_object_ptr_); -#endif + if (gpu_object_ptr_ != nullptr) { + HIC_CALL(hicFree(gpu_object_ptr_)); } } Base* gpu_object_ptr() { return static_cast(gpu_object_ptr_); } void updateDevice() { -#if ATLAS_HAVE_CUDA - cudaMemcpy(gpu_object_ptr_, base_ptr_, sizeof(Base), cudaMemcpyHostToDevice); -#endif + if constexpr (ATLAS_HAVE_GPU) { + HIC_CALL(hicMemcpy(gpu_object_ptr_, base_ptr_, sizeof(Base), hicMemcpyHostToDevice)); + } } void updateHost() { -#if ATLAS_HAVE_CUDA - cudaMemcpy(base_ptr_, gpu_object_ptr_, sizeof(Base), cudaMemcpyDeviceToHost); -#endif + if constexpr (ATLAS_HAVE_GPU) { + HIC_CALL(hicMemcpy(base_ptr_, gpu_object_ptr_, sizeof(Base), hicMemcpyDeviceToHost)); + } } Base* base_ptr_; diff --git a/src/atlas/util/PackVectorFields.cc b/src/atlas/util/PackVectorFields.cc new file mode 100644 index 000000000..640a1d46b --- /dev/null +++ b/src/atlas/util/PackVectorFields.cc @@ -0,0 +1,229 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "atlas/util/PackVectorFields.h" + +#include +#include +#include +#include + +#include "atlas/array.h" +#include "atlas/array/helpers/ArrayForEach.h" +#include "atlas/functionspace.h" +#include "atlas/option.h" +#include "atlas/runtime/Exception.h" +#include "atlas/util/Config.h" +#include "eckit/config/LocalConfiguration.h" + +namespace atlas { +namespace util { + +namespace { + +using eckit::LocalConfiguration; + +using array::DataType; +using array::helpers::arrayForEachDim; + +void addOrReplaceField(FieldSet& fieldSet, const Field& field) { + const auto fieldName = field.name(); + if (fieldSet.has(fieldName)) { + fieldSet[fieldName] = field; + } else { + fieldSet.add(field); + } +} + +Field& getOrCreateField(FieldSet& fieldSet, const FunctionSpace& functionSpace, + const Config& config) { + const auto fieldName = config.getString("name"); + if (!fieldSet.has(fieldName)) { + fieldSet.add(functionSpace.createField(config)); + } + return fieldSet[fieldName]; +} + +void checkFieldCompatibility(const Field& componentField, + const Field& vectorField) { + ATLAS_ASSERT(componentField.functionspace().size() == + vectorField.functionspace().size()); + ATLAS_ASSERT(componentField.levels() == vectorField.levels()); + ATLAS_ASSERT(componentField.variables() == 0); + ATLAS_ASSERT(vectorField.variables() > 0); + + const auto checkStandardShape = [](const Field& field) { + // Check for "standard" Atlas field shape. + auto dim = 0; + const auto rank = field.rank(); + const auto shape = field.shape(); + if (field.functionspace().size() != shape[dim++]) { + return false; + } + if (const auto levels = field.levels(); + levels && (dim >= rank || levels != shape[dim++])) { + return false; + } + if (const auto variables = field.variables(); + variables && (dim >= rank || variables != shape[dim++])) { + return false; + } + if (dim != rank) { + return false; + } + return true; + }; + + ATLAS_ASSERT(checkStandardShape(componentField)); + ATLAS_ASSERT(checkStandardShape(vectorField)); +} + +template +void copyFieldData(ComponentField& componentField, VectorField& vectorField, + const Functor& copier) { + checkFieldCompatibility(componentField, vectorField); + + const auto copyArrayData = [&](auto value, auto rank) { + // Resolve value-type and rank from arguments. + using Value = decltype(value); + constexpr auto Rank = decltype(rank)::value; + + // Iterate over fields. + auto vectorView = array::make_view(vectorField); + auto componentView = array::make_view(componentField); + constexpr auto Dims = std::make_integer_sequence{}; + arrayForEachDim(Dims, execution::par, std::tie(componentView, vectorView), + copier); + }; + + const auto selectRank = [&](auto value) { + switch (vectorField.rank()) { + case 2: + return copyArrayData(value, std::integral_constant{}); + case 3: + return copyArrayData(value, std::integral_constant{}); + default: + ATLAS_THROW_EXCEPTION("Unsupported vector field rank: " + + std::to_string(vectorField.rank())); + } + }; + + const auto selectType = [&]() { + switch (vectorField.datatype().kind()) { + case DataType::kind(): + return selectRank(double{}); + case DataType::kind(): + return selectRank(float{}); + case DataType::kind(): + return selectRank(long{}); + case DataType::kind(): + return selectRank(int{}); + default: + ATLAS_THROW_EXCEPTION("Unknown datatype: " + + std::to_string(vectorField.datatype().kind())); + } + }; + + selectType(); +} + +} // namespace + +FieldSet pack_vector_fields(const FieldSet& fields, FieldSet packedFields) { + // Get the number of variables for each vector field. + auto vectorSizeMap = std::map{}; + for (const auto& field : fields) { + auto vectorFieldName = std::string{}; + if (field.metadata().get("vector_field_name", vectorFieldName)) { + ++vectorSizeMap[vectorFieldName]; + } + } + auto vectorIndexMap = std::map{}; + + // Pack vector fields. + for (const auto& field : fields) { + auto vectorFieldName = std::string{}; + if (!field.metadata().get("vector_field_name", vectorFieldName)) { + // Not a vector component field. + addOrReplaceField(packedFields, field); + continue; + } + + // Field is vector field component. + const auto& componentField = field; + + // Get or create vector field. + const auto vectorFieldConfig = + option::name(vectorFieldName) | + option::levels(componentField.levels()) | + option::vector(vectorSizeMap[vectorFieldName]) | + option::datatype(componentField.datatype()); + auto& vectorField = getOrCreateField( + packedFields, componentField.functionspace(), vectorFieldConfig); + + // Copy field data. + const auto vectorIndex = vectorIndexMap[vectorFieldName]++; + const auto copier = [&](auto&& componentElem, auto&& vectorElem) { + vectorElem(vectorIndex) = componentElem; + }; + copyFieldData(componentField, vectorField, copier); + + // Copy metadata. + const auto componentFieldMetadata = componentField.metadata(); + auto componentFieldMetadataVector = std::vector{}; + vectorField.metadata().get("component_field_metadata", + componentFieldMetadataVector); + componentFieldMetadataVector.push_back(componentFieldMetadata); + vectorField.metadata().set("component_field_metadata", + componentFieldMetadataVector); + } + return packedFields; +} + +FieldSet unpack_vector_fields(const FieldSet& fields, FieldSet unpackedFields) { + for (const auto& field : fields) { + auto componentFieldMetadataVector = std::vector{}; + if (!field.metadata().get("component_field_metadata", + componentFieldMetadataVector)) { + // Not a vector field. + addOrReplaceField(unpackedFields, field); + continue; + } + + // Field is vector. + const auto& vectorField = field; + + auto vectorIndex = 0; + for (const auto& componentFieldMetadata : componentFieldMetadataVector) { + + // Get or create field. + auto componentFieldName = std::string{}; + componentFieldMetadata.get("name", componentFieldName); + const auto componentFieldConfig = + option::name(componentFieldName) | + option::levels(vectorField.levels()) | + option::datatype(vectorField.datatype()); + auto& componentField = getOrCreateField( + unpackedFields, vectorField.functionspace(), componentFieldConfig); + + // Copy field data. + const auto copier = [&](auto&& componentElem, auto&& vectorElem) { + componentElem = vectorElem(vectorIndex); + }; + copyFieldData(componentField, vectorField, copier); + + // Copy metadata. + componentField.metadata() = componentFieldMetadata; + + ++vectorIndex; + } + } + return unpackedFields; +} + +} // namespace util +} // namespace atlas diff --git a/src/atlas/util/PackVectorFields.h b/src/atlas/util/PackVectorFields.h new file mode 100644 index 000000000..fc77da6c5 --- /dev/null +++ b/src/atlas/util/PackVectorFields.h @@ -0,0 +1,40 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include "atlas/field.h" + +namespace eckit { +class LocalConfiguration; +} + +namespace atlas { +namespace util { + +/// @brief Packs vector field components into vector fields +/// +/// @details Iterates through @param fields and creates vector fields from any +/// component field with the "vector name" string metadata. These, as +/// well as any present scalar fields, are added to the return-value +/// field set. +/// Note, a mutable @param packedFields field set can be supplied if +/// one needs to guarantee the order of the packed fields +FieldSet pack_vector_fields(const FieldSet& fields, + FieldSet packedFields = FieldSet{}); + +/// @brief Unpacks vector field into vector field components. +/// +/// @details Undoes "pack" operation when a set of packed fields are supplied +/// as @param fields. A mutable @param unpackedFields field set can be +/// supplied if one needs to guarantee the order of the unpacked +/// fields. +FieldSet unpack_vector_fields(const FieldSet& fields, + FieldSet unpackedFields = FieldSet{}); + +} // namespace util +} // namespace atlas diff --git a/src/atlas/util/Rotation.cc b/src/atlas/util/Rotation.cc index d83d5cca4..a7baccfaf 100644 --- a/src/atlas/util/Rotation.cc +++ b/src/atlas/util/Rotation.cc @@ -126,9 +126,9 @@ void Rotation::precompute() { Rotation::Rotation(const PointLonLat& south_pole, double rotation_angle) { spole_ = south_pole; - npole_ = PointLonLat(spole_.lon() - 180., spole_.lat() + 180.); - if (npole_.lat() > 90) { - npole_.lon() += 180.; + npole_ = PointLonLat(spole_.lon() - 180., -spole_.lat()); + if (npole_.lon() < 0.) { + npole_.lon() += 360.; } angle_ = wrap_angle(rotation_angle); @@ -144,16 +144,16 @@ Rotation::Rotation(const eckit::Parametrisation& p) { std::vector pole(2); if (p.get("north_pole", pole)) { npole_ = PointLonLat(pole.data()); - spole_ = PointLonLat(npole_.lon() + 180., npole_.lat() - 180.); - if (spole_.lat() < -90) { - spole_.lon() -= 180.; + spole_ = PointLonLat(npole_.lon() - 180., -npole_.lat()); + if (spole_.lon() < 0.) { + spole_.lon() += 360.; } } else if (p.get("south_pole", pole)) { spole_ = PointLonLat(pole.data()); - npole_ = PointLonLat(spole_.lon() - 180., spole_.lat() + 180.); - if (npole_.lat() > 90) { - npole_.lon() += 180.; + npole_ = PointLonLat(spole_.lon() - 180., -spole_.lat()); + if (npole_.lon() < 0.) { + npole_.lon() += 360.; } } diff --git a/src/atlas_acc_support/CMakeLists.txt b/src/atlas_acc_support/CMakeLists.txt index ffe144a37..7bf818b1d 100644 --- a/src/atlas_acc_support/CMakeLists.txt +++ b/src/atlas_acc_support/CMakeLists.txt @@ -8,31 +8,15 @@ if( atlas_HAVE_ACC ) - if( NOT (CMAKE_C_COMPILER_ID MATCHES ${CMAKE_Fortran_COMPILER_ID}) ) - add_custom_command( - OUTPUT ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so ${CMAKE_CURRENT_BINARY_DIR}/atlas_acc_map_data.c.o - COMMAND ${ACC_C_COMPILER} ${ACC_C_FLAGS} ${ACC_C_INCLUDE} -fPIC -o ${CMAKE_CURRENT_BINARY_DIR}/atlas_acc_map_data.c.o - -c ${CMAKE_CURRENT_SOURCE_DIR}/atlas_acc_map_data.c - COMMAND mkdir -p ${CMAKE_BINARY_DIR}/lib - COMMAND ${ACC_C_COMPILER} ${ACC_C_FLAGS} -shared -o ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so - ${CMAKE_CURRENT_BINARY_DIR}/atlas_acc_map_data.c.o - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/atlas_acc_map_data.c - COMMENT "Building atlas_acc_support with ${ACC_C_COMPILER}" - ) - add_custom_target( build-atlas_acc_support ALL DEPENDS ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so ) - add_library( atlas_acc_support SHARED IMPORTED GLOBAL ) - set_property( TARGET atlas_acc_support PROPERTY IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so ) - set_property( TARGET atlas_acc_support PROPERTY IMPORTED_NO_SONAME TRUE ) - set_property( TARGET atlas_acc_support PROPERTY IMPORTED_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR} ) - add_dependencies( atlas_acc_support build-atlas_acc_support ) - install( FILES ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so DESTINATION ${INSTALL_LIB_DIR}/ ) - + if( CMAKE_CXX_COMPILER_ID MATCHES NVHPC ) + if( NOT TARGET OpenACC::OpenACC_CXX ) + ecbuild_error("ERROR: OpenACC::OpenACC_CXX TARGET not found") + endif() + ecbuild_add_library( TARGET atlas_acc_support SOURCES atlas_acc.cc ) + target_link_libraries( atlas_acc_support PRIVATE OpenACC::OpenACC_CXX ) else() - - ecbuild_add_library( TARGET atlas_acc_support SOURCES atlas_acc_map_data.c ) - target_compile_options( atlas_acc_support PRIVATE ${ACC_C_FLAGS} ) - target_link_libraries( atlas_acc_support PRIVATE ${ACC_C_FLAGS} ) - + ecbuild_add_library( TARGET atlas_acc_support SOURCES atlas_acc.F90 ) + target_link_libraries( atlas_acc_support PRIVATE OpenACC::OpenACC_Fortran ) endif() endif() diff --git a/src/atlas_acc_support/atlas_acc.F90 b/src/atlas_acc_support/atlas_acc.F90 new file mode 100644 index 000000000..bb73b5237 --- /dev/null +++ b/src/atlas_acc_support/atlas_acc.F90 @@ -0,0 +1,70 @@ +module atlas_acc +use openacc +implicit none +private + +public :: atlas_acc_get_num_devices +public :: atlas_acc_map_data +public :: atlas_acc_unmap_data +public :: atlas_acc_is_present +public :: atlas_acc_get_device_type +public :: atlas_acc_deviceptr + +contains + +function atlas_acc_get_num_devices() bind(C,name="atlas_acc_get_num_devices") result(num_devices) + use, intrinsic :: iso_c_binding, only : c_int + integer(c_int) :: num_devices + integer(acc_device_kind) :: devicetype + + devicetype = acc_get_device_type() + num_devices = acc_get_num_devices(devicetype) +end function + +subroutine atlas_acc_map_data(data_arg, data_dev, bytes) bind(C,name="atlas_acc_map_data") + use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t + type(*), dimension(*) :: data_arg + type(c_ptr), value :: data_dev + integer(c_size_t), value :: bytes + call acc_map_data(data_arg, data_dev, bytes) +end subroutine + +subroutine atlas_acc_unmap_data(data_arg) bind(C,name="atlas_acc_unmap_data") + use, intrinsic :: iso_c_binding, only : c_ptr + type(*), dimension(*) :: data_arg + call acc_unmap_data(data_arg) +end subroutine + +function atlas_acc_is_present(data_arg, bytes) bind(C,name="atlas_acc_is_present") result(is_present) + use, intrinsic :: iso_c_binding, only : c_size_t, c_ptr, c_char, c_int + integer(c_int) :: is_present + logical :: lpresent + type(c_ptr), value :: data_arg + integer(c_size_t), value :: bytes + character(kind=c_char), pointer :: data_f(:) + call c_f_pointer(data_arg, data_f,[bytes]) + lpresent = acc_is_present(data_f) + is_present = 0 + if (lpresent) is_present = 1 +end function + +function atlas_acc_deviceptr(data_arg) bind(C,name="atlas_acc_deviceptr") result(deviceptr) + use, intrinsic :: iso_c_binding, only : c_ptr + type(*), dimension(*) :: data_arg + type(c_ptr):: deviceptr + deviceptr = acc_deviceptr(data_arg) +end function + +function atlas_acc_get_device_type() bind(C,name="atlas_acc_get_device_type") result(devicetype) + use, intrinsic :: iso_c_binding, only : c_int + integer(c_int) :: devicetype + integer(acc_device_kind) :: acc_devicetype + acc_devicetype = acc_get_device_type() + if (acc_devicetype == acc_device_host .or. acc_devicetype == acc_device_none) then + devicetype = 0 + else + devicetype = 1 + endif +end function + +end module diff --git a/src/atlas_acc_support/atlas_acc.cc b/src/atlas_acc_support/atlas_acc.cc new file mode 100644 index 000000000..8cc58981a --- /dev/null +++ b/src/atlas_acc_support/atlas_acc.cc @@ -0,0 +1,130 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +// This file needs to be compiled with an OpenACC capable C compiler that is +// compatible with the Fortran compiler + +#ifndef _OPENACC +#error atlas_acc_map_data must be compiled with OpenACC capable compiler +#endif + +#include +#include + +#include "atlas_acc.h" + +extern "C" { + +void atlas_acc_map_data(void* cpu_ptr, void* gpu_ptr, unsigned long bytes) { + acc_map_data(cpu_ptr, gpu_ptr, bytes); +} + + +void atlas_acc_unmap_data(void* cpu_ptr) { + acc_unmap_data(cpu_ptr); +} + +int atlas_acc_is_present(void* cpu_ptr, unsigned long bytes) { + return acc_is_present(cpu_ptr, bytes); +} + +void* atlas_acc_deviceptr(void* cpu_ptr) { + return acc_deviceptr(cpu_ptr); +} + +atlas_acc_device_t atlas_acc_get_device_type() { + acc_device_t device_type = acc_get_device_type(); + if (device_type == acc_device_host || device_type == acc_device_none) { + return atlas_acc_device_host; + } + return atlas_acc_device_not_host; +} + +#define BUFFER_PRINTF(buffer, ...) snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer), __VA_ARGS__); + +const char* atlas_acc_version_str() { + static char buffer[16]; + if( strlen(buffer) != 0 ) { + return buffer; + } + + BUFFER_PRINTF(buffer, "%i", _OPENACC); + switch( _OPENACC ) { + case 201111: BUFFER_PRINTF(buffer, " (1.0)"); break; + case 201306: BUFFER_PRINTF(buffer, " (2.0)"); break; + case 201308: BUFFER_PRINTF(buffer, " (2.0)"); break; + case 201510: BUFFER_PRINTF(buffer, " (2.5)"); break; + case 201711: BUFFER_PRINTF(buffer, " (2.6)"); break; + default: break; + } + return buffer; +} +const char* atlas_acc_info_str() { + static char buffer[1024]; + if( strlen(buffer) != 0 ) { + return buffer; + } + // Platform information + acc_device_t devicetype = acc_get_device_type(); + int num_devices = acc_get_num_devices(devicetype); + BUFFER_PRINTF(buffer, " OpenACC version: %s\n", atlas_acc_version_str()); + if (devicetype == acc_device_host ) { + BUFFER_PRINTF(buffer, " No OpenACC GPU devices available\n"); + return buffer; + } + BUFFER_PRINTF(buffer, " Number of OpenACC devices: %i\n", num_devices); + int device_num = acc_get_device_num(devicetype); + BUFFER_PRINTF(buffer, " OpenACC Device number: %i\n", device_num); + BUFFER_PRINTF(buffer, " OpenACC acc_device_t (enum value): %d", devicetype); + switch( devicetype ) { + case acc_device_host: BUFFER_PRINTF(buffer, " (acc_device_host)"); break; + case acc_device_none: BUFFER_PRINTF(buffer, " (acc_device_none)"); break; + case acc_device_not_host: BUFFER_PRINTF(buffer, " (acc_device_not_host)"); break; + case acc_device_nvidia: BUFFER_PRINTF(buffer, " (acc_device_nvidia)"); break; + default: break; + } + BUFFER_PRINTF(buffer, "\n"); + + // acc_get_property, acc_get_property_string introduced with OpenACC 2.6 +#if _OPENACC >= 201711 + long int mem = acc_get_property( device_num, devicetype, acc_property_memory); + long int free_mem = acc_get_property( device_num, devicetype, acc_property_free_memory); + const char *property_name = acc_get_property_string(device_num, devicetype, acc_property_name); + const char *property_vendor = acc_get_property_string(device_num, devicetype, acc_property_vendor ); + const char *property_driver = acc_get_property_string(device_num, devicetype, acc_property_driver ); + // if (mem > 0) + { + BUFFER_PRINTF(buffer, " Memory on OpenACC device: %li\n", mem); + } + // if (free_mem > 0) + { + BUFFER_PRINTF(buffer, " Free Memory on OpenACC device: %li\n", free_mem); + } + // if (property_name != NULL) + { + BUFFER_PRINTF(buffer, " OpenACC device name: %s\n", property_name); + } + // if (property_vendor != NULL) + { + BUFFER_PRINTF(buffer, " OpenACC device vendor: %s\n", property_vendor); + } + // if (property_driver != NULL) + { + BUFFER_PRINTF(buffer, " OpenACC device driver: %s\n", property_driver); + } +#endif + return buffer; +} + +int atlas_acc_get_num_devices() { + return acc_get_num_devices(acc_get_device_type()); +} +} diff --git a/src/atlas_acc_support/atlas_acc.h b/src/atlas_acc_support/atlas_acc.h new file mode 100644 index 000000000..37c928b95 --- /dev/null +++ b/src/atlas_acc_support/atlas_acc.h @@ -0,0 +1,32 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + atlas_acc_device_host = 0, + atlas_acc_device_not_host = 1 +} atlas_acc_device_t; + +void atlas_acc_map_data(void* cpu_ptr, void* gpu_ptr, size_t bytes); +void atlas_acc_unmap_data(void* cpu_ptr); +int atlas_acc_is_present(void* cpu_ptr, size_t bytes); +void* atlas_acc_deviceptr(void* cpu_ptr); +atlas_acc_device_t atlas_acc_get_device_type(); +int atlas_acc_get_num_devices(); + +#ifdef __cplusplus +} +#endif diff --git a/src/atlas_acc_support/atlas_acc_map_data.c b/src/atlas_acc_support/atlas_acc_map_data.c deleted file mode 100644 index 6b8a148e7..000000000 --- a/src/atlas_acc_support/atlas_acc_map_data.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * (C) Copyright 2013 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - - -// This file needs to be compiled with an OpenACC capable C compiler that is -// compatible with the Fortran compiler - -#ifndef _OPENACC -#error atlas_acc_map_data must be compiled with OpenACC capable compiler -#endif - -#include - -void atlas_acc_map_data(void* cpu_ptr, void* gpu_ptr, unsigned long size) { - acc_map_data(cpu_ptr, gpu_ptr, size); -} - - -void atlas_acc_unmap_data(void* cpu_ptr) { - acc_unmap_data(cpu_ptr); -} diff --git a/src/atlas_f/CMakeLists.txt b/src/atlas_f/CMakeLists.txt index 3c3d396e0..2fd47e7d8 100644 --- a/src/atlas_f/CMakeLists.txt +++ b/src/atlas_f/CMakeLists.txt @@ -49,7 +49,7 @@ function(generate_fortran_bindings output filename) OUTPUT ${outfile} COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/c2f.py ${CMAKE_CURRENT_SOURCE_DIR}/${filename} -o ${outfile} -m ${_PAR_MODULE} - -t '{"idx_t":"${F_IDX}","gidx_t":"${F_GIDX}"}' + -t '{"idx_t":"${F_IDX}","gidx_t":"${F_GIDX}","atlas::idx_t":"${F_IDX}","atlas::gidx_t":"${F_GIDX}"}' DEPENDS ${filename} ) set_source_files_properties(${outfile} PROPERTIES GENERATED TRUE) endfunction() @@ -96,6 +96,7 @@ generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/mesh/actions/BuildPeriodicBo generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/mesh/actions/BuildHalo.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/mesh/actions/BuildEdges.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/mesh/actions/BuildDualMesh.h) +generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/mesh/actions/BuildNode2CellConnectivity.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/mesh/actions/WriteLoadBalanceReport.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/meshgenerator/detail/MeshGeneratorInterface.h MODULE atlas_meshgenerator_c_binding @@ -108,6 +109,9 @@ generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/field/State.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/field/detail/FieldInterface.h MODULE atlas_field_c_binding OUTPUT field_c_binding.f90 ) +generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/field/detail/MultiFieldInterface.h + MODULE atlas_multifield_c_binding + OUTPUT multifield_c_binding.f90 ) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/field/FieldSet.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/functionspace/detail/FunctionSpaceInterface.h MODULE atlas_functionspace_c_binding @@ -137,6 +141,10 @@ generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/redistribution/detail/Redist MODULE atlas_redistribution_c_binding OUTPUT redistribution_c_binding.f90) +generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/grid/detail/grid/StencilComputerInterface.h + MODULE atlas_grid_StencilComputer_c_binding + OUTPUT grid_StencilComputer_c_binding.f90 ) + if( atlas_HAVE_ATLAS_NUMERICS ) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/numerics/Nabla.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/numerics/Nabla.h) @@ -226,10 +234,12 @@ ecbuild_add_library( TARGET atlas_f field/atlas_FieldSet_module.fypp field/atlas_State_module.F90 field/atlas_Field_module.fypp + field/atlas_MultiField_module.F90 grid/atlas_Grid_module.F90 grid/atlas_GridDistribution_module.F90 grid/atlas_Vertical_module.F90 grid/atlas_Partitioner_module.F90 + grid/atlas_StencilComputer_module.F90 mesh/atlas_MeshBuilder_module.F90 mesh/atlas_MeshGenerator_module.F90 mesh/atlas_Mesh_module.F90 @@ -276,6 +286,14 @@ ecbuild_add_library( TARGET atlas_f $ ) +if( HAVE_ACC AND CMAKE_Fortran_COMPILER_ID MATCHES NVHPC ) + target_link_options( atlas_f INTERFACE + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> + $<$:SHELL:${ACC_LINK_OPTIONS}> ) +endif() + fckit_target_preprocess_fypp( atlas_f DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/atlas_f.h.in diff --git a/src/atlas_f/atlas_f.h.in b/src/atlas_f/atlas_f.h.in index cea48ee0e..3511bca12 100644 --- a/src/atlas_f/atlas_f.h.in +++ b/src/atlas_f/atlas_f.h.in @@ -18,6 +18,7 @@ #define ATLAS_HAVE_OMP @atlas_HAVE_OMP_Fortran@ #define ATLAS_HAVE_ACC @atlas_HAVE_ACC@ +#define ATLAS_HAVE_GPU @atlas_HAVE_GPU@ #define ATLAS_BITS_GLOBAL @ATLAS_BITS_GLOBAL@ #define ATLAS_BITS_LOCAL @ATLAS_BITS_LOCAL@ diff --git a/src/atlas_f/atlas_module.F90 b/src/atlas_f/atlas_module.F90 index 97ddb45ae..e21890629 100644 --- a/src/atlas_f/atlas_module.F90 +++ b/src/atlas_f/atlas_module.F90 @@ -89,6 +89,11 @@ module atlas_module & atlas_RegionalGrid use atlas_Vertical_module, only :& & atlas_Vertical +use atlas_StencilComputer_module, only: & + & atlas_StructuredGrid_ComputeNorth, & + & atlas_StructuredGrid_ComputeWest, & + & atlas_StructuredGrid_ComputeStencil, & + & atlas_StructuredGrid_Stencil use atlas_functionspace_EdgeColumns_module, only: & & atlas_functionspace_EdgeColumns use atlas_functionspace_CellColumns_module, only: & @@ -141,6 +146,7 @@ module atlas_module & atlas_build_halo, & & atlas_build_edges, & & atlas_build_pole_edges, & + & atlas_build_node_to_cell_connectivity, & & atlas_build_node_to_edge_connectivity, & & atlas_build_median_dual_mesh, & & atlas_write_load_balance_report, & diff --git a/src/atlas_f/field/atlas_FieldSet_module.fypp b/src/atlas_f/field/atlas_FieldSet_module.fypp index e337c958c..36a2a38da 100644 --- a/src/atlas_f/field/atlas_FieldSet_module.fypp +++ b/src/atlas_f/field/atlas_FieldSet_module.fypp @@ -51,7 +51,9 @@ contains procedure, private :: field_by_name procedure, private :: field_by_idx_int procedure, private :: field_by_idx_long - procedure, public :: add + procedure, public :: add_field + procedure, public :: add_fieldset + generic :: add => add_field, add_fieldset generic :: field => field_by_name, field_by_idx_int, field_by_idx_long procedure, public :: set_dirty @@ -213,7 +215,7 @@ function FieldSet__name(this) result(fieldset_name) fieldset_name = c_ptr_to_string(fieldset_name_c_str) end function FieldSet__name -subroutine add(this,field) +subroutine add_field(this,field) use atlas_fieldset_c_binding use atlas_Field_module, only: atlas_Field class(atlas_FieldSet), intent(in) :: this @@ -221,6 +223,13 @@ subroutine add(this,field) call atlas__FieldSet__add_field(this%CPTR_PGIBUG_A, field%CPTR_PGIBUG_A) end subroutine +subroutine add_fieldset(this,fset) + use atlas_fieldset_c_binding + class(atlas_FieldSet), intent(inout) :: this + class(atlas_FieldSet), intent(in) :: fset + call atlas__FieldSet__add_fieldset(this%CPTR_PGIBUG_A, fset%CPTR_PGIBUG_A) +end subroutine + function has(this,name) result(flag) use, intrinsic :: iso_c_binding, only: c_int use fckit_c_interop_module, only: c_str diff --git a/src/atlas_f/field/atlas_Field_module.fypp b/src/atlas_f/field/atlas_Field_module.fypp index c841a9a00..acaadc3a3 100644 --- a/src/atlas_f/field/atlas_Field_module.fypp +++ b/src/atlas_f/field/atlas_Field_module.fypp @@ -183,10 +183,17 @@ subroutine array_c_to_f_${dtype}$_r${rank}$(array_cptr,rank,shape_cptr,strides_c enddo eshape(rank) = shape(rank) call c_f_pointer ( array_cptr , tmp , shape=eshape ) - #{if rank == 1}# array_fptr => tmp(1:shape(1)) #{endif}# - #{if rank == 2}# array_fptr => tmp(1:shape(1),1:shape(2)) #{endif}# - #{if rank == 3}# array_fptr => tmp(1:shape(1),1:shape(2),1:shape(3)) #{endif}# - #{if rank == 4}# array_fptr => tmp(1:shape(1),1:shape(2),1:shape(3),1:shape(4)) #{endif}# + if (associated(tmp)) then + #{if rank == 1}# array_fptr => tmp(1:shape(1)) #{endif}# + #{if rank == 2}# array_fptr => tmp(1:shape(1),1:shape(2)) #{endif}# + #{if rank == 3}# array_fptr => tmp(1:shape(1),1:shape(2),1:shape(3)) #{endif}# + #{if rank == 4}# array_fptr => tmp(1:shape(1),1:shape(2),1:shape(3),1:shape(4)) #{endif}# + else + #{if rank == 1}# allocate(array_fptr(0)) #{endif}# + #{if rank == 2}# allocate(array_fptr(0,0)) #{endif}# + #{if rank == 3}# allocate(array_fptr(0,0,0)) #{endif}# + #{if rank == 4}# allocate(array_fptr(0,0,0,0)) #{endif}# + endif end subroutine !------------------------------------------------------------------------------- @@ -403,7 +410,7 @@ function atlas_Field__wrap_name_${dtype}$_r${rank}$(name,data) result(field) use fckit_c_interop_module, only : c_str type(atlas_Field) :: field character(len=*), intent(in) :: name - ${ftype}$, intent(in) :: data(${dim[rank]}$) + ${ftype}$, intent(in), target :: data(${dim[rank]}$) integer(c_int) :: shapef(${rank}$) integer(c_int) :: stridesf(${rank}$) #:if ftype != "logical" @@ -420,12 +427,13 @@ function atlas_Field__wrap_name_${dtype}$_r${rank}$(name,data) result(field) call field%return() end function function atlas_Field__wrap_${dtype}$_r${rank}$(data) result(field) + use, intrinsic :: iso_c_binding + use :: fckit_c_interop_module use atlas_field_c_binding use, intrinsic :: iso_c_binding, only : c_int, c_long, c_float, c_double - use fckit_c_interop_module, only : c_str use fckit_array_module, only : array_strides, array_view1d type(atlas_Field) :: field - ${ftype}$, intent(in) :: data(${dim[rank]}$) + ${ftype}$, intent(in), target :: data(${dim[rank]}$) integer(c_int) :: shapef(${rank}$) integer(c_int) :: stridesf(${rank}$) #:if ftype != "logical" @@ -436,7 +444,38 @@ function atlas_Field__wrap_${dtype}$_r${rank}$(data) result(field) data1d => array_view1d( data, int(0,c_int) ) #:endif shapef = shape(data) - stridesf = array_strides(data) + if( size(data)>0 ) then + !!! See issue https://github.com/ecmwf/atlas/pull/213 : problem with array_strides(data) with NAG compiler. +#:if rank == 4 + stridesf(1) = & + int(c_ptr_to_loc(c_loc(data(2,1,1,1)))-c_ptr_to_loc(c_loc(data(1,1,1,1))),c_int32_t)/int(8,c_int32_t) + stridesf(2) = & + int(c_ptr_to_loc(c_loc(data(1,2,1,1)))-c_ptr_to_loc(c_loc(data(1,1,1,1))),c_int32_t)/int(8,c_int32_t) + stridesf(3) = & + int(c_ptr_to_loc(c_loc(data(1,1,2,1)))-c_ptr_to_loc(c_loc(data(1,1,1,1))),c_int32_t)/int(8,c_int32_t) + stridesf(4) = & + int(c_ptr_to_loc(c_loc(data(1,1,1,2)))-c_ptr_to_loc(c_loc(data(1,1,1,1))),c_int32_t)/int(8,c_int32_t) +#:elif rank == 3 + stridesf(1) = & + int(c_ptr_to_loc(c_loc(data(2,1,1)))-c_ptr_to_loc(c_loc(data(1,1,1))),c_int32_t)/int(8,c_int32_t) + stridesf(2) = & + int(c_ptr_to_loc(c_loc(data(1,2,1)))-c_ptr_to_loc(c_loc(data(1,1,1))),c_int32_t)/int(8,c_int32_t) + stridesf(3) = & + int(c_ptr_to_loc(c_loc(data(1,1,2)))-c_ptr_to_loc(c_loc(data(1,1,1))),c_int32_t)/int(8,c_int32_t) +#:elif rank == 2 + stridesf(1) = & + int(c_ptr_to_loc(c_loc(data(2,1)))-c_ptr_to_loc(c_loc(data(1,1))),c_int32_t)/int(8,c_int32_t) + stridesf(2) = & + int(c_ptr_to_loc(c_loc(data(1,2)))-c_ptr_to_loc(c_loc(data(1,1))),c_int32_t)/int(8,c_int32_t) +#:elif rank == 1 + stridesf(1) = & + int(c_ptr_to_loc(c_loc(data(2)))-c_ptr_to_loc(c_loc(data(1))),c_int32_t)/int(8,c_int32_t) +#:else + stridesf = array_strides(data) +#:endif + else + stridesf = 0 + endif field = atlas_Field__cptr( & atlas__Field__wrap_${ctype}$_specf( c_str(""),data1d,size(shapef),shapef, stridesf) ) call field%return() diff --git a/src/atlas_f/field/atlas_MultiField_module.F90 b/src/atlas_f/field/atlas_MultiField_module.F90 new file mode 100644 index 000000000..57168efaa --- /dev/null +++ b/src/atlas_f/field/atlas_MultiField_module.F90 @@ -0,0 +1,166 @@ +! (C) Copyright 2013 ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +#include "atlas/atlas_f.h" + +module atlas_multifield_module + +use fckit_owned_object_module, only : fckit_owned_object +use atlas_Config_module, only: atlas_Config +use atlas_field_module, only: atlas_field, array_c_to_f +use atlas_fieldset_module, only: atlas_fieldset + +implicit none + +public :: atlas_MultiField + +private + +!------------------------------------------------------------------------------ +TYPE, extends(fckit_owned_object) :: atlas_MultiField + +! Purpose : +! ------- +! *MultiField* : Object containing field data and Metadata + +! Methods : +! ------- +! name : The name or tag this field was created with +! data : Return the field as a fortran array of specified shape +! Metadata : Return object that can contain a variety of Metadata + +! Author : +! ------ +! 20-Nov-2013 Willem Deconinck *ECMWF* +! 29-Aug-2023 Slavko Brdar *ECMWF* + +!------------------------------------------------------------------------------ +contains + procedure, public :: MultiField__fieldset + procedure, public :: MultiField__size + generic :: fieldset => MultiField__fieldset + generic :: size => MultiField__size + +#if FCKIT_FINAL_NOT_INHERITING + final :: atlas_MultiField__final_auto +#endif + +END TYPE + +interface atlas_MultiField + module procedure atlas_MultiField__cptr + module procedure atlas_MultiField__create + module procedure atlas_MultiField__create_names +end interface + +private :: fckit_owned_object +private :: atlas_Config + +!======================================================== +contains +!======================================================== + +!------------------------------------------------------------------------------- + +function atlas_MultiField__cptr(cptr) result(field) + use, intrinsic :: iso_c_binding, only : c_ptr + type(atlas_MultiField) :: field + type(c_ptr), intent(in) :: cptr + call field%reset_c_ptr( cptr ) + call field%return() +end function + +!------------------------------------------------------------------------------- + +function atlas_MultiField__create(params) result(field) + use atlas_multifield_c_binding + type(atlas_MultiField) :: field + class(atlas_Config), intent(in) :: params + field = atlas_MultiField__cptr( atlas__MultiField__create(params%CPTR_PGIBUG_B) ) + call field%return() +end function + +!------------------------------------------------------------------------------- + +function atlas_MultiField__create_names(kind, shape, field_names) result(field) + use, intrinsic :: iso_c_binding, only : c_char, c_int, c_int32_t, c_size_t + use atlas_multifield_c_binding + type(atlas_MultiField) :: field + integer(c_int), intent(in) :: kind + integer, intent(in) :: shape(:) + character(*), intent(in) :: field_names(:) + character(kind=c_char,len=:), allocatable :: flat_field_names + integer(c_size_t) :: length + integer(c_int32_t) :: ii + integer(c_int32_t), allocatable :: field_name_sizes(:) + + if (size(field_names) == 0 .or. size(shape) < 3) then + print *, "atlas_MultiField__create_names: must have at least one field name, and the size of shape", & + & " is minimum 3, e.g. [nproma,-1,nblk]" + stop -1 + end if + + length = len(field_names(1)) + allocate(field_name_sizes(size(field_names))) + field_name_sizes = len(field_names(:)) + + if (any(field_name_sizes /= length)) then + print *, "atlas_MultiField__create_names: field_names have to have same length in characters" + stop -1 + end if + + allocate(character(len=length*size(field_names) ) :: flat_field_names) + do ii = 1, size(field_names) + flat_field_names((ii-1)*length+1:ii*length) = field_names(ii) + enddo + + field = atlas_MultiField__cptr( atlas__MultiField__create_shape(kind, size(shape), shape, & + & flat_field_names, length, size(field_names,kind=c_size_t)) ) + call field%return() +end function + +!------------------------------------------------------------------------------- + +function MultiField__size(this) result(size) + use atlas_multifield_c_binding + class(atlas_MultiField), intent(in) :: this + integer :: size + size = atlas__MultiField__size(this%CPTR_PGIBUG_B) +end function + +!------------------------------------------------------------------------------- +function MultiField__fieldset(this) result(fset) + use, intrinsic :: iso_c_binding, only : c_ptr + use atlas_multifield_c_binding + class(atlas_MultiField), intent(in) :: this + type(c_ptr) :: fset_cptr + type(atlas_FieldSet) :: fset + fset_cptr = atlas__MultiField__fieldset(this%CPTR_PGIBUG_B) + fset = atlas_FieldSet( fset_cptr ) + call fset%return() +end function + +!------------------------------------------------------------------------------- + +#if FCKIT_FINAL_NOT_INHERITING +ATLAS_FINAL subroutine atlas_MultiField__final_auto(this) + type(atlas_MultiField), intent(inout) :: this +#if FCKIT_FINAL_DEBUGGING + write(0,*) "atlas_MultiField__final_auto" +#endif +#if FCKIT_FINAL_NOT_PROPAGATING + call this%final() +#endif + FCKIT_SUPPRESS_UNUSED( this ) +end subroutine +#endif + +!------------------------------------------------------------------------------- + +end module atlas_multifield_module + diff --git a/src/atlas_f/functionspace/atlas_functionspace_BlockStructuredColumns_module.F90 b/src/atlas_f/functionspace/atlas_functionspace_BlockStructuredColumns_module.F90 index f25f76321..de3770e28 100644 --- a/src/atlas_f/functionspace/atlas_functionspace_BlockStructuredColumns_module.F90 +++ b/src/atlas_f/functionspace/atlas_functionspace_BlockStructuredColumns_module.F90 @@ -89,6 +89,7 @@ module atlas_functionspace_BlockStructuredColumns_module procedure :: levels procedure :: block_begin procedure :: block_size + procedure :: nproma procedure :: nblks procedure :: xy @@ -554,6 +555,13 @@ function block_size(this,j) result(i) i = atlas__fs__BStructuredColumns__block_size(this%CPTR_PGIBUG_A,j-1) end function +function nproma(this) result(i) + use atlas_functionspace_BlockStructuredColumns_c_binding + integer(ATLAS_KIND_IDX) :: i + class(atlas_functionspace_BlockStructuredColumns), intent(in) :: this + i = atlas__fs__BStructuredColumns__nproma(this%CPTR_PGIBUG_A) +end function + function nblks(this) result(i) use atlas_functionspace_BlockStructuredColumns_c_binding integer(ATLAS_KIND_IDX) :: i diff --git a/src/atlas_f/grid/atlas_Grid_module.F90 b/src/atlas_f/grid/atlas_Grid_module.F90 index 42dd9e458..d5a50d62d 100644 --- a/src/atlas_f/grid/atlas_Grid_module.F90 +++ b/src/atlas_f/grid/atlas_Grid_module.F90 @@ -54,6 +54,7 @@ module atlas_Grid_module !------------------------------------------------------------------------------ contains + procedure :: name => atlas_Grid__name procedure :: size => atlas_Grid__size procedure :: spec => atlas_Grid__spec procedure :: uid @@ -554,7 +555,7 @@ function atlas_UnstructuredGrid__ctor_points( xy ) result(this) use fckit_array_module, only : array_strides, array_view1d use atlas_grid_unstructured_c_binding type(atlas_UnstructuredGrid) :: this - real(c_double), intent(in) :: xy(:,:) + real(c_double), intent(in), target :: xy(:,:) integer(c_int) :: shapef(2) integer(c_int) :: stridesf(2) real(c_double), pointer :: xy1d(:) @@ -751,6 +752,19 @@ function atlas_grid_ShiftedLat__ctor_int64(nlon,nlat) result(this) ! ----------------------------------------------------------------------------- ! Structured members +function atlas_Grid__name(this) result(name) + use atlas_grid_Grid_c_binding + use fckit_c_interop_module, only : c_ptr_to_string, c_ptr_free + use, intrinsic :: iso_c_binding, only : c_ptr + class(atlas_Grid), intent(in) :: this + character(len=:), allocatable :: name + type(c_ptr) :: name_c_str + integer :: size + call atlas__grid__Grid__name(this%CPTR_PGIBUG_A, name_c_str, size ) + name = c_ptr_to_string(name_c_str) + call c_ptr_free(name_c_str) +end function + function atlas_Grid__size(this) result(npts) use, intrinsic :: iso_c_binding, only: c_int use atlas_grid_Grid_c_binding diff --git a/src/atlas_f/grid/atlas_StencilComputer_module.F90 b/src/atlas_f/grid/atlas_StencilComputer_module.F90 new file mode 100644 index 000000000..e3bb5966f --- /dev/null +++ b/src/atlas_f/grid/atlas_StencilComputer_module.F90 @@ -0,0 +1,339 @@ +! (C) Copyright 2013 ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +#include "atlas/atlas_f.h" + +module atlas_StencilComputer_module + +use fckit_shared_object_module, only: fckit_shared_object, fckit_c_deleter +use atlas_kinds_module, only : ATLAS_KIND_IDX +use, intrinsic :: iso_c_binding, only : c_ptr, c_int + +implicit none + +private :: fckit_shared_object, fckit_c_deleter +private :: c_ptr, c_int +private :: ATLAS_KIND_IDX + +public :: atlas_StructuredGrid_ComputeNorth +public :: atlas_StructuredGrid_ComputeWest +public :: atlas_StructuredGrid_ComputeStencil +public :: atlas_StructuredGrid_Stencil + +private + +!------------------------------------------------------------------------------ +TYPE, extends(fckit_shared_object) :: atlas_StructuredGrid_ComputeNorth + +! Purpose : +! ------- +! *atlas_StructuredGrid_ComputeNorth* : To compute latitude index north of latitude + +! Methods : +! ------- + +! Author : +! ------ +! 9-Jan-2024 Willem Deconinck *ECMWF* + +!------------------------------------------------------------------------------ +contains + + procedure, private :: atlas_StructuredGrid_ComputeNorth__execute_real32 + procedure, private :: atlas_StructuredGrid_ComputeNorth__execute_real64 + generic :: execute => atlas_StructuredGrid_ComputeNorth__execute_real32, & + & atlas_StructuredGrid_ComputeNorth__execute_real64 + +#if FCKIT_FINAL_NOT_INHERITING + final :: atlas_StructuredGrid_ComputeNorth__final_auto +#endif + +END TYPE atlas_StructuredGrid_ComputeNorth + +interface atlas_StructuredGrid_ComputeNorth + module procedure atlas_StructuredGrid_ComputeNorth__ctor +end interface + +!------------------------------------------------------------------------------ + +!------------------------------------------------------------------------------ +TYPE, extends(fckit_shared_object) :: atlas_StructuredGrid_ComputeWest + +! Purpose : +! ------- +! *atlas_StructuredGrid_ComputeWest* : To compute longitude index west of longitude at given latitude index + +! Methods : +! ------- + +! Author : +! ------ +! 9-Jan-2024 Willem Deconinck *ECMWF* + +!------------------------------------------------------------------------------ +contains + + procedure, private :: atlas_StructuredGrid_ComputeWest__execute_real32 + procedure, private :: atlas_StructuredGrid_ComputeWest__execute_real64 + generic :: execute => atlas_StructuredGrid_ComputeWest__execute_real32, & + & atlas_StructuredGrid_ComputeWest__execute_real64 + +#if FCKIT_FINAL_NOT_INHERITING + final :: atlas_StructuredGrid_ComputeWest__final_auto +#endif + +END TYPE atlas_StructuredGrid_ComputeWest + +interface atlas_StructuredGrid_ComputeWest + module procedure atlas_StructuredGrid_ComputeWest__ctor +end interface + +!------------------------------------------------------------------------------ + +integer(c_int), parameter, private :: STENCIL_MAX_WIDTH = 6 !-> maximum 6x6 stencil + +TYPE atlas_StructuredGrid_Stencil + integer(ATLAS_KIND_IDX) :: width + integer(ATLAS_KIND_IDX) :: j_begin + integer(ATLAS_KIND_IDX) :: i_begin(STENCIL_MAX_WIDTH) ! on stack +contains + procedure, pass :: write => atlas_StructuredGrid_Stencil__write + generic, public :: write(FORMATTED) => write + procedure, public :: i => atlas_StructuredGrid_Stencil__i + procedure, public :: j => atlas_StructuredGrid_Stencil__j +END TYPE atlas_StructuredGrid_Stencil + +TYPE :: atlas_StructuredGrid_ComputeStencil + integer(c_int) :: halo + integer(ATLAS_KIND_IDX) :: stencil_width + integer(ATLAS_KIND_IDX) :: stencil_offset + type(atlas_StructuredGrid_ComputeNorth) :: compute_north + type(atlas_StructuredGrid_ComputeWest) :: compute_west +contains + procedure :: setup => atlas_StructuredGrid_ComputeStencil__setup + procedure :: execute => atlas_StructuredGrid_ComputeStencil__execute_real64 + + procedure :: assignment_operator => atlas_StructuredGrid_ComputeStencil__assignment + generic, public :: assignment(=) => assignment_operator + + procedure :: final => atlas_StructuredGrid_ComputeStencil__final +END TYPE + +! Better not use, use setup member function instead ! +!interface atlas_StructuredGrid_ComputeStencil +! module procedure atlas_StructuredGrid_ComputeStencil__ctor +!end interface + + +!======================================================== +contains +!======================================================== + + +! ----------------------------------------------------------------------------- +! Destructor + +#if FCKIT_FINAL_NOT_INHERITING +ATLAS_FINAL subroutine atlas_StructuredGrid_ComputeNorth__final_auto(this) + type(atlas_StructuredGrid_ComputeNorth), intent(inout) :: this +#if FCKIT_FINAL_NOT_PROPAGATING + call this%final() +#endif + FCKIT_SUPPRESS_UNUSED( this ) +end subroutine +#endif + +#if FCKIT_FINAL_NOT_INHERITING +ATLAS_FINAL subroutine atlas_StructuredGrid_ComputeWest__final_auto(this) + type(atlas_StructuredGrid_ComputeWest), intent(inout) :: this +#if FCKIT_FINAL_NOT_PROPAGATING + call this%final() +#endif + FCKIT_SUPPRESS_UNUSED( this ) +end subroutine +#endif + + +! ----------------------------------------------------------------------------- +! Constructors + +function atlas_StructuredGrid_ComputeNorth__ctor(grid, halo) result(this) + use, intrinsic :: iso_c_binding, only : c_int + use fckit_c_interop_module, only: c_str + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeNorth__new, atlas__grid__ComputeNorth__delete + use atlas_grid_module, only : atlas_StructuredGrid + implicit none + type(atlas_StructuredGrid_ComputeNorth) :: this + class(atlas_StructuredGrid), intent(in) :: grid + integer(c_int), intent(in) :: halo + call this%reset_c_ptr( atlas__grid__ComputeNorth__new(grid%CPTR_PGIBUG_B, halo), & + & fckit_c_deleter(atlas__grid__ComputeNorth__delete) ) + call this%return() +end function + +function atlas_StructuredGrid_ComputeWest__ctor(grid, halo) result(this) + use, intrinsic :: iso_c_binding, only : c_int + use fckit_c_interop_module, only: c_str + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeWest__new, atlas__grid__ComputeWest__delete + use atlas_grid_module, only : atlas_StructuredGrid + implicit none + type(atlas_StructuredGrid_ComputeWest) :: this + class(atlas_StructuredGrid), intent(in) :: grid + integer(c_int), intent(in) :: halo + call this%reset_c_ptr( atlas__grid__ComputeWest__new(grid%CPTR_PGIBUG_B, halo), & + & fckit_c_deleter(atlas__grid__ComputeWest__delete) ) + call this%return() +end function + +subroutine atlas_StructuredGrid_ComputeStencil__setup(this, grid, stencil_width) + use, intrinsic :: iso_c_binding, only : c_double + use atlas_grid_module, only : atlas_StructuredGrid + implicit none + class(atlas_StructuredGrid_ComputeStencil) :: this + class(atlas_StructuredGrid), intent(in) :: grid + integer(ATLAS_KIND_IDX), intent(in) :: stencil_width + this%stencil_width = stencil_width + this%halo = (stencil_width + 1) / 2 + this%stencil_offset = stencil_width - floor(real(stencil_width,c_double) / 2._c_double + 1._c_double, ATLAS_KIND_IDX) + this%compute_north = atlas_StructuredGrid_ComputeNorth(grid, this%halo) + this%compute_west = atlas_StructuredGrid_ComputeWest(grid, this%halo) +end subroutine + + + +! ---------------------------------------------------------------------------------------- + +function atlas_StructuredGrid_ComputeNorth__execute_real32(this, y) result(index) + use, intrinsic :: iso_c_binding, only : c_float + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeNorth__execute_real32 + implicit none + integer(ATLAS_KIND_IDX) :: index + class(atlas_StructuredGrid_ComputeNorth), intent(in) :: this + real(c_float), intent(in) :: y + index = atlas__grid__ComputeNorth__execute_real32(this%CPTR_PGIBUG_B, y) + 1 +end function + +function atlas_StructuredGrid_ComputeNorth__execute_real64(this, y) result(index) + use, intrinsic :: iso_c_binding, only : c_double + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeNorth__execute_real64 + implicit none + integer(ATLAS_KIND_IDX) :: index + class(atlas_StructuredGrid_ComputeNorth), intent(in) :: this + real(c_double), intent(in) :: y + index = atlas__grid__ComputeNorth__execute_real64(this%CPTR_PGIBUG_B, y) + 1 +end function +! ---------------------------------------------------------------------------------------- + +function atlas_StructuredGrid_ComputeWest__execute_real32(this, x, j) result(index) + use, intrinsic :: iso_c_binding, only : c_float + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeWest__execute_real32 + implicit none + integer(ATLAS_KIND_IDX) :: index + class(atlas_StructuredGrid_ComputeWest), intent(in) :: this + real(c_float), intent(in) :: x + integer(ATLAS_KIND_IDX), intent(in) :: j + index = atlas__grid__ComputeWest__execute_real32(this%CPTR_PGIBUG_B, x, j-int(1,ATLAS_KIND_IDX)) + 1 +end function + +function atlas_StructuredGrid_ComputeWest__execute_real64(this, x, j) result(index) + use, intrinsic :: iso_c_binding, only : c_double + use atlas_grid_StencilComputer_c_binding, only : atlas__grid__ComputeWest__execute_real64 + implicit none + integer(ATLAS_KIND_IDX) :: index + class(atlas_StructuredGrid_ComputeWest), intent(in) :: this + real(c_double), intent(in) :: x + integer(ATLAS_KIND_IDX), intent(in) :: j + index = atlas__grid__ComputeWest__execute_real64(this%CPTR_PGIBUG_B, x, j-int(1,ATLAS_KIND_IDX)) + 1 +end function +! ---------------------------------------------------------------------------------------- + + +subroutine atlas_StructuredGrid_ComputeStencil__execute_real64(this, x, y, stencil) + use, intrinsic :: iso_c_binding, only : c_double + implicit none + class(atlas_StructuredGrid_ComputeStencil), intent(in) :: this + real(c_double), intent(in) :: x, y + type(atlas_StructuredGrid_Stencil), intent(inout) :: stencil + + integer(ATLAS_KIND_IDX) :: jj + stencil%width = this%stencil_width + + stencil%j_begin = this%compute_north%execute(y) - this%stencil_offset + do jj = 1_ATLAS_KIND_IDX, this%stencil_width + stencil%i_begin(jj) = this%compute_west%execute(x, stencil%j_begin + jj - 1) - this%stencil_offset + enddo +end subroutine +! ---------------------------------------------------------------------------------------- + +subroutine atlas_StructuredGrid_Stencil__write (stencil, unit, iotype, v_list, iostat, iomsg) + implicit none + class(atlas_StructuredGrid_Stencil), intent(in) :: stencil + INTEGER, INTENT(IN) :: unit + CHARACTER(*), INTENT(IN) :: iotype + INTEGER, INTENT(IN) :: v_list(:) + INTEGER, INTENT(OUT) :: iostat + CHARACTER(*), INTENT(INOUT) :: iomsg + integer(ATLAS_KIND_IDX) :: jlat, jlon + do jlat = 1, stencil%width + write(unit,'(a,I0,a)',advance='no',IOSTAT=iostat) " j: ", stencil%j(jlat), " i = " + do jlon = 1, stencil%width + write(unit,'(I3)',advance='no',IOSTAT=iostat) stencil%i(jlon,jlat) + enddo + write(0,'(a)',IOSTAT=iostat) new_line('a') + enddo +end subroutine + +function atlas_StructuredGrid_Stencil__j(this, j_index) result(j) + integer(ATLAS_KIND_IDX) :: j + class(atlas_StructuredGrid_Stencil), intent(in) :: this + integer(ATLAS_KIND_IDX) :: j_index + j = this%j_begin + (j_index-1) +end function + +function atlas_StructuredGrid_Stencil__i(this, i_index, j_index) result(i) + integer(ATLAS_KIND_IDX) :: i + class(atlas_StructuredGrid_Stencil), intent(in) :: this + integer(ATLAS_KIND_IDX) :: i_index + integer(ATLAS_KIND_IDX) :: j_index + i = this%i_begin(j_index) + (i_index-1) +end function + +! ---------------------------------------------------------------------------------------- + +function atlas_StructuredGrid_ComputeStencil__ctor(grid, stencil_width) result(this) + use atlas_grid_module, only : atlas_StructuredGrid + implicit none + type(atlas_StructuredGrid_ComputeStencil) :: this + class(atlas_StructuredGrid), intent(in) :: grid + integer(ATLAS_KIND_IDX), intent(in) :: stencil_width + call this%setup(grid, stencil_width) +end function + + +subroutine atlas_StructuredGrid_ComputeStencil__assignment(this, other) +implicit none +class(atlas_StructuredGrid_ComputeStencil), intent(inout) :: this +class(atlas_StructuredGrid_ComputeStencil), intent(in) :: other +call this%final() +write(0,*) "owners = ", other%compute_north%owners() +this%compute_north = other%compute_north +this%compute_west = other%compute_west +this%stencil_width = other%stencil_width +this%stencil_offset = other%stencil_offset +this%halo = other%halo +end subroutine + +subroutine atlas_StructuredGrid_ComputeStencil__final(this) + class(atlas_StructuredGrid_ComputeStencil), intent(inout) :: this + call this%compute_north%final() + call this%compute_west%final() +end subroutine + +! ---------------------------------------------------------------------------------------- + +end module atlas_StencilComputer_module diff --git a/src/atlas_f/mesh/atlas_MeshBuilder_module.F90 b/src/atlas_f/mesh/atlas_MeshBuilder_module.F90 index 5b1fce4dd..7d0f6f4a5 100644 --- a/src/atlas_f/mesh/atlas_MeshBuilder_module.F90 +++ b/src/atlas_f/mesh/atlas_MeshBuilder_module.F90 @@ -87,7 +87,7 @@ function atlas_TriangularMeshBuilder__build(this, & real(c_double), contiguous, intent(in) :: x(:), y(:), lon(:), lat(:) integer, intent(in) :: nb_triags integer(ATLAS_KIND_GIDX), intent(in) :: triag_global_index(nb_triags) - integer(ATLAS_KIND_GIDX), intent(in) :: triag_nodes(3,nb_triags) + integer(ATLAS_KIND_GIDX), intent(in), target :: triag_nodes(3,nb_triags) integer(ATLAS_KIND_GIDX), pointer :: triag_nodes_1d(:) triag_nodes_1d => array_view1d( triag_nodes, int(0,ATLAS_KIND_GIDX) ) diff --git a/src/atlas_f/mesh/atlas_mesh_actions_module.F90 b/src/atlas_f/mesh/atlas_mesh_actions_module.F90 index 8da0c0fb9..aae559dd2 100644 --- a/src/atlas_f/mesh/atlas_mesh_actions_module.F90 +++ b/src/atlas_f/mesh/atlas_mesh_actions_module.F90 @@ -19,6 +19,7 @@ module atlas_mesh_actions_module public :: atlas_build_halo public :: atlas_build_edges public :: atlas_build_pole_edges +public :: atlas_build_node_to_cell_connectivity public :: atlas_build_node_to_edge_connectivity public :: atlas_build_median_dual_mesh public :: atlas_write_load_balance_report @@ -84,6 +85,13 @@ subroutine atlas_build_pole_edges(mesh) call atlas__build_pole_edges(mesh%CPTR_PGIBUG_A) end subroutine atlas_build_pole_edges +subroutine atlas_build_node_to_cell_connectivity(mesh) + use atlas_BuildNode2CellConnectivity_c_binding + use atlas_Mesh_module, only: atlas_Mesh + type(atlas_Mesh), intent(inout) :: mesh + call atlas__build_node_to_cell_connectivity(mesh%CPTR_PGIBUG_A) +end subroutine atlas_build_node_to_cell_connectivity + subroutine atlas_build_node_to_edge_connectivity(mesh) use atlas_BuildEdges_c_binding use atlas_Mesh_module, only: atlas_Mesh diff --git a/src/atlas_f/util/atlas_allocate_module.F90 b/src/atlas_f/util/atlas_allocate_module.F90 index 3cc1d3a7a..1bda17e81 100644 --- a/src/atlas_f/util/atlas_allocate_module.F90 +++ b/src/atlas_f/util/atlas_allocate_module.F90 @@ -231,7 +231,7 @@ subroutine atlas_deallocate_managedmem_real64_r1( A ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding real(c_double), pointer :: a(:) - call atlas__deallocate_managedmem( c_loc_real64(A(1)) ) + call atlas__deallocate_managedmem_double( c_loc_real64(A(1)), size(A,KIND=c_size_t) ) nullify( a ) end subroutine @@ -239,7 +239,7 @@ subroutine atlas_deallocate_managedmem_real32_r1( A ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding real(c_float), pointer :: a(:) - call atlas__deallocate_managedmem( c_loc_real32(A(1)) ) + call atlas__deallocate_managedmem_float( c_loc_real32(A(1)), size(A,KIND=c_size_t) ) nullify( a ) end subroutine @@ -247,7 +247,7 @@ subroutine atlas_deallocate_managedmem_int32_r1( A ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding integer(c_int), pointer :: a(:) - call atlas__deallocate_managedmem( c_loc_int32(A(1)) ) + call atlas__deallocate_managedmem_int( c_loc_int32(A(1)), size(A,KIND=c_size_t) ) nullify( a ) end subroutine @@ -255,7 +255,7 @@ subroutine atlas_deallocate_managedmem_int64_r1( A ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding integer(c_long), pointer :: a(:) - call atlas__deallocate_managedmem( c_loc_int64(A(1)) ) + call atlas__deallocate_managedmem_long( c_loc_int64(A(1)), size(A,KIND=c_size_t) ) nullify( a ) end subroutine @@ -263,7 +263,7 @@ subroutine atlas_deallocate_managedmem_real64_r2( A ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding real(c_double), pointer :: a(:,:) - call atlas__deallocate_managedmem( c_loc_real64(A(1,1)) ) + call atlas__deallocate_managedmem_double( c_loc_real64(A(1,1)), size(A,KIND=c_size_t) ) nullify( a ) end subroutine @@ -271,7 +271,7 @@ subroutine atlas_deallocate_managedmem_real32_r2( A ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding real(c_float), pointer :: a(:,:) - call atlas__deallocate_managedmem( c_loc_real32(A(1,1)) ) + call atlas__deallocate_managedmem_float( c_loc_real32(A(1,1)), size(A,KIND=c_size_t) ) nullify( a ) end subroutine @@ -279,7 +279,7 @@ subroutine atlas_deallocate_managedmem_int32_r2( A ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding integer(c_int), pointer :: a(:,:) - call atlas__deallocate_managedmem( c_loc_int32(A(1,1)) ) + call atlas__deallocate_managedmem_int( c_loc_int32(A(1,1)), size(A,KIND=c_size_t) ) nullify( a ) end subroutine @@ -287,7 +287,7 @@ subroutine atlas_deallocate_managedmem_int64_r2( A ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding integer(c_long), pointer :: a(:,:) - call atlas__deallocate_managedmem( c_loc_int64(A(1,1)) ) + call atlas__deallocate_managedmem_long( c_loc_int64(A(1,1)), size(A,KIND=c_size_t) ) nullify( a ) end subroutine diff --git a/src/sandbox/fortran_acc_fields/CMakeLists.txt b/src/sandbox/fortran_acc_fields/CMakeLists.txt index a3b49ad08..733960140 100644 --- a/src/sandbox/fortran_acc_fields/CMakeLists.txt +++ b/src/sandbox/fortran_acc_fields/CMakeLists.txt @@ -10,10 +10,8 @@ if( atlas_HAVE_ACC ) ecbuild_add_executable( TARGET atlas-acc-fields SOURCES atlas-acc-fields.F90 - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran LINKER_LANGUAGE Fortran NOINSTALL ) - target_compile_options( atlas-acc-fields PUBLIC ${ACC_Fortran_FLAGS} ) - target_link_libraries( atlas-acc-fields ${ACC_Fortran_FLAGS} ) endif() diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 33da2af95..cbbe7536a 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -38,29 +38,33 @@ endif() ###################################################### -# Macro atlas_add_cuda_test -# Envisioned to become part of ecbuild_add_test with a CUDA option -# in a future ecbuild release -macro( atlas_add_cuda_test ) +# Macro atlas_add_hic_test +macro( atlas_add_hic_test ) set( options "" ) set( single_value_args TARGET ) set( multi_value_args SOURCES LIBS ENVIRONMENT ) cmake_parse_arguments( _PAR "${options}" "${single_value_args}" "${multi_value_args}" ${_FIRST_ARG} ${ARGN} ) if(_PAR_UNPARSED_ARGUMENTS) - ecbuild_critical("Unknown keywords given to atlas_add_cuda_test(): \"${_PAR_UNPARSED_ARGUMENTS}\"") + ecbuild_critical("Unknown keywords given to atlas_add_hic_test(): \"${_PAR_UNPARSED_ARGUMENTS}\"") endif() if( HAVE_CUDA AND HAVE_TESTS ) - ecbuild_debug("atlas_add_cuda_test: Adding test ${_PAR_TARGET}") + ecbuild_debug("atlas_add_hic_test: Adding test ${_PAR_TARGET}") list( APPEND _PAR_ENVIRONMENT "ATLAS_RUN_NGPUS=1" ) list( REMOVE_DUPLICATES _PAR_ENVIRONMENT ) list( APPEND _libs ${_PAR_LIBS} ) if( _libs ) - ecbuild_debug("atlas_add_cuda_test: Test ${_PAR_TARGET} explicitly links with libraries ${_libs}") + ecbuild_debug("atlas_add_hic_test: Test ${_PAR_TARGET} explicitly links with libraries ${_libs}") + endif() + + if( HAVE_CUDA ) + set_source_files_properties(${_PAR_SOURCES} PROPERTIES LANGUAGE CUDA) + elseif( HAVE_HIP ) + set_source_files_properties(${_PAR_SOURCES} PROPERTIES LANGUAGE HIP) endif() ecbuild_add_test( TARGET ${_PAR_TARGET} @@ -72,10 +76,6 @@ macro( atlas_add_cuda_test ) endif() endmacro() -if( HAVE_CUDA ) - set( ATLAS_TEST_ENVIRONMENT "ATLAS_RUN_NGPUS=1" ) -endif() - add_subdirectory( util ) add_subdirectory( runtime ) diff --git a/src/tests/acc/CMakeLists.txt b/src/tests/acc/CMakeLists.txt index 768805fff..c827dc65a 100644 --- a/src/tests/acc/CMakeLists.txt +++ b/src/tests/acc/CMakeLists.txt @@ -6,33 +6,25 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. -if( HAVE_CUDA AND HAVE_TESTS AND HAVE_FCTEST AND HAVE_ACC ) +if( HAVE_GPU AND HAVE_TESTS AND HAVE_FCTEST AND HAVE_ACC ) - string (REPLACE ";" " " ACC_Fortran_FLAGS_STR "${ACC_Fortran_FLAGS}") - - - set_source_files_properties( fctest_unified_memory_with_openacc.F90 PROPERTIES COMPILE_FLAGS ${ACC_Fortran_FLAGS_STR} ) add_fctest( TARGET atlas_test_unified_memory_with_openacc SOURCES fctest_unified_memory_with_openacc.F90 fctest_unified_memory_with_openacc_cxx.cc - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran hic LINKER_LANGUAGE Fortran ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 ) - target_link_libraries( atlas_test_unified_memory_with_openacc ${ACC_Fortran_FLAGS} CUDA::cudart ) - set_tests_properties( atlas_test_unified_memory_with_openacc PROPERTIES LABELS "gpu;acc") add_fctest( TARGET atlas_test_connectivity_openacc SOURCES fctest_connectivity_openacc.F90 - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran LINKER_LANGUAGE Fortran ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 ) - target_link_libraries( atlas_test_connectivity_openacc ${ACC_Fortran_FLAGS} ) - set_target_properties( atlas_test_connectivity_openacc PROPERTIES COMPILE_FLAGS "${ACC_Fortran_FLAGS_STR}" ) set_tests_properties ( atlas_test_connectivity_openacc PROPERTIES LABELS "gpu;acc") endif() diff --git a/src/tests/acc/fctest_unified_memory_with_openacc_cxx.cc b/src/tests/acc/fctest_unified_memory_with_openacc_cxx.cc index 47c0e7dc5..4ded2d78e 100644 --- a/src/tests/acc/fctest_unified_memory_with_openacc_cxx.cc +++ b/src/tests/acc/fctest_unified_memory_with_openacc_cxx.cc @@ -8,10 +8,10 @@ * nor does it submit to any jurisdiction. */ -#include +#include "hic/hic.h" extern "C" { void allocate_unified_impl(double** a, int N) { - cudaMallocManaged(a, N * sizeof(double)); + HIC_CALL(hicMallocManaged(a, N * sizeof(double))); } } diff --git a/src/tests/array/CMakeLists.txt b/src/tests/array/CMakeLists.txt index 3a5583223..880e46ba6 100644 --- a/src/tests/array/CMakeLists.txt +++ b/src/tests/array/CMakeLists.txt @@ -61,23 +61,28 @@ if( CMAKE_BUILD_TYPE MATCHES "DEBUG" ) set ( CMAKE_NVCC_FLAGS "-G" ) endif() -atlas_add_cuda_test( +atlas_add_hic_test( TARGET atlas_test_array_kernel - SOURCES test_array_kernel.cu + SOURCES test_array_kernel.hic LIBS atlas ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) -atlas_add_cuda_test( +atlas_add_hic_test( TARGET atlas_test_vector_kernel - SOURCES test_vector_kernel.cu + SOURCES test_vector_kernel.hic LIBS atlas ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) -atlas_add_cuda_test( +atlas_add_hic_test( TARGET atlas_test_svector_kernel - SOURCES test_svector_kernel.cu + SOURCES test_svector_kernel.hic LIBS atlas ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_array_view_variant + SOURCES test_array_view_variant.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) diff --git a/src/tests/array/test_array.cc b/src/tests/array/test_array.cc index 9ce8cbec9..63f630c98 100644 --- a/src/tests/array/test_array.cc +++ b/src/tests/array/test_array.cc @@ -19,6 +19,8 @@ #include "atlas/array/gridtools/GridToolsMakeView.h" #endif +#include "atlas/parallel/acc/acc.h" + using namespace atlas::array; namespace atlas { @@ -555,8 +557,14 @@ CASE("test_wrap") { CASE("test_acc_map") { Array* ds = Array::create(2, 3, 4); - EXPECT_NO_THROW(ds->accMap()); - EXPECT(ds->accMapped() == ATLAS_HAVE_ACC); + EXPECT_NO_THROW(ds->allocateDevice()); + if( ds->deviceAllocated() ) { + EXPECT_NO_THROW(ds->accMap()); + EXPECT_EQ(ds->accMapped(), std::min(acc::devices(),1)); + } + else { + Log::warning() << "WARNING: Array could not be allocated on device, so acc_map could not be tested" << std::endl; + } delete ds; } diff --git a/src/tests/array/test_array_kernel.cu b/src/tests/array/test_array_kernel.hic similarity index 92% rename from src/tests/array/test_array_kernel.cu rename to src/tests/array/test_array_kernel.hic index 8f3ba678a..14935f46c 100644 --- a/src/tests/array/test_array_kernel.cu +++ b/src/tests/array/test_array_kernel.hic @@ -8,7 +8,7 @@ * does it submit to any jurisdiction. */ -#include +#include "hic/hic.h" #include "tests/AtlasTestEnvironment.h" #include "atlas/array.h" #include "atlas/array/MakeView.h" @@ -21,10 +21,10 @@ namespace test { #define REQUIRE_CUDA_SUCCESS(msg) \ do { \ - cudaError_t err = cudaPeekAtLastError(); \ - if (err != cudaSuccess ) { \ + hicError_t err = hicPeekAtLastError(); \ + if (err != hicSuccess ) { \ throw eckit::testing::TestException("REQUIRE_CUDA_SUCCESS ["+std::string(msg)+"] failed:\n"\ - + std::string(cudaGetErrorString(err)) , Here()); \ + + std::string(hicGetErrorString(err)) , Here()); \ } \ } while(false) @@ -70,7 +70,7 @@ CASE( "test_array" ) REQUIRE_CUDA_SUCCESS("kernel_ex"); - cudaDeviceSynchronize(); + hicDeviceSynchronize(); ds->updateHost(); @@ -108,7 +108,7 @@ CASE( "test_array_loop" ) REQUIRE_CUDA_SUCCESS("loop_kernel_ex"); - cudaDeviceSynchronize(); + hicDeviceSynchronize(); ds->updateHost(); diff --git a/src/tests/array/test_array_view_variant.cc b/src/tests/array/test_array_view_variant.cc new file mode 100644 index 000000000..f6eaeeacf --- /dev/null +++ b/src/tests/array/test_array_view_variant.cc @@ -0,0 +1,131 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include +#include + +#include "atlas/array.h" +#include "atlas/array/ArrayViewVariant.h" +#include "eckit/utils/Overloaded.h" +#include "tests/AtlasTestEnvironment.h" + +namespace atlas { +namespace test { + +using namespace array; + +CASE("test variant assignment") { + auto array1 = array::ArrayT(2); + auto array2 = array::ArrayT(2, 3); + auto array3 = array::ArrayT(2, 3, 4); + const auto& arrayRef = array1; + + array1.allocateDevice(); + array2.allocateDevice(); + array3.allocateDevice(); + + auto view1 = make_view_variant(array1); + auto view2 = make_view_variant(array2); + auto view3 = make_view_variant(array3); + auto view4 = make_view_variant(arrayRef); + + const auto hostView1 = make_host_view_variant(array1); + const auto hostView2 = make_host_view_variant(array2); + const auto hostView3 = make_host_view_variant(array3); + const auto hostView4 = make_host_view_variant(arrayRef); + + auto deviceView1 = make_device_view_variant(array1); + auto deviceView2 = make_device_view_variant(array2); + auto deviceView3 = make_device_view_variant(array3); + auto deviceView4 = make_device_view_variant(arrayRef); + + const auto visitVariants = [](auto& var1, auto& var2, auto var3, auto var4) { + std::visit( + [](auto view) { + EXPECT((is_rank<1>(view))); + EXPECT((is_value_type(view))); + EXPECT((is_non_const_value_type(view))); + }, + var1); + + std::visit( + [](auto view) { + EXPECT((is_rank<2>(view))); + EXPECT((is_value_type(view))); + EXPECT((is_non_const_value_type(view))); + }, + var2); + + std::visit( + [](auto view) { + EXPECT((is_rank<3>(view))); + EXPECT((is_value_type(view))); + EXPECT((is_non_const_value_type(view))); + }, + var3); + + std::visit( + [](auto view) { + EXPECT((is_rank<1>(view))); + EXPECT((is_value_type(view))); + EXPECT((is_non_const_value_type(view))); + }, + var4); + }; + + visitVariants(view1, view2, view3, view4); + visitVariants(hostView1, hostView2, hostView3, hostView4); + visitVariants(deviceView1, deviceView2, deviceView3, deviceView4); +} + +CASE("test std::visit") { + auto array1 = ArrayT(10); + make_view(array1).assign({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + auto array2 = ArrayT(5, 2); + make_view(array2).assign({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + const auto var1 = make_view_variant(array1); + const auto var2 = make_view_variant(array2); + auto rank1Tested = false; + auto rank2Tested = false; + + const auto visitor = [&](auto view) { + if constexpr (is_rank<1>(view)) { + EXPECT((is_value_type(view))); + auto testValue = int{0}; + for (auto i = size_t{0}; i < view.size(); ++i) { + const auto value = view(i); + EXPECT_EQ(value, static_cast(testValue++)); + } + rank1Tested = true; + } else if constexpr (is_rank<2>(view)) { + EXPECT((is_value_type(view))); + auto testValue = int{0}; + for (auto i = idx_t{0}; i < view.shape(0); ++i) { + for (auto j = idx_t{0}; j < view.shape(1); ++j) { + const auto value = view(i, j); + EXPECT_EQ(value, static_cast(testValue++)); + } + } + rank2Tested = true; + } else { + // Test should not reach here. + EXPECT(false); + } + }; + + std::visit(visitor, var1); + EXPECT(rank1Tested); + std::visit(visitor, var2); + EXPECT(rank2Tested); +} + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { return atlas::test::run(argc, argv); } diff --git a/src/tests/array/test_svector_kernel.cu b/src/tests/array/test_svector_kernel.hic similarity index 84% rename from src/tests/array/test_svector_kernel.cu rename to src/tests/array/test_svector_kernel.hic index 79631133b..0865e34ff 100644 --- a/src/tests/array/test_svector_kernel.cu +++ b/src/tests/array/test_svector_kernel.hic @@ -8,7 +8,7 @@ * does it submit to any jurisdiction. */ -#include +#include "hic/hic.h" #include "atlas/library/config.h" #include "tests/AtlasTestEnvironment.h" @@ -45,17 +45,17 @@ CASE( "test_svector" ) EXPECT( list_ints.size() == 2); bool *result; - cudaError_t err = cudaMallocManaged(&result, sizeof(bool)); + hicError_t err = hicMallocManaged(&result, sizeof(bool)); - if(err != cudaSuccess) + if(err != hicSuccess) throw_AssertionFailed("failed to allocate GPU memory"); *result=true; kernel_exe<<<1,1>>>(list_ints.data(), list_ints.size(), 0, result); - cudaDeviceSynchronize(); + hicDeviceSynchronize(); - err = cudaGetLastError(); - if(err != cudaSuccess) + err = hicGetLastError(); + if(err != hicSuccess) throw_AssertionFailed("failed to execute kernel"); EXPECT( *result ); @@ -81,9 +81,9 @@ CASE( "test_svector_resize" ) EXPECT( list_ints.size() == 5); bool *result; - cudaError_t err = cudaMallocManaged(&result, sizeof(bool)); + hicError_t err = hicMallocManaged(&result, sizeof(bool)); - if(err != cudaSuccess) + if(err != hicSuccess) throw_AssertionFailed("failed to allocate GPU memory"); *result=true; @@ -92,10 +92,10 @@ CASE( "test_svector_resize" ) list_ints[4] = 4; kernel_exe<<<1,1>>>(list_ints.data(), list_ints.size(), 3, result); - cudaDeviceSynchronize(); + hicDeviceSynchronize(); - err = cudaGetLastError(); - if(err != cudaSuccess) + err = hicGetLastError(); + if(err != hicSuccess) throw_AssertionFailed("failed to execute kernel"); EXPECT( *result ); diff --git a/src/tests/array/test_vector_kernel.cu b/src/tests/array/test_vector_kernel.hic similarity index 89% rename from src/tests/array/test_vector_kernel.cu rename to src/tests/array/test_vector_kernel.hic index 18b0eb2b4..dc8442fbf 100644 --- a/src/tests/array/test_vector_kernel.cu +++ b/src/tests/array/test_vector_kernel.hic @@ -8,7 +8,7 @@ * does it submit to any jurisdiction. */ -#include +#include "hic/hic.h" #include "tests/AtlasTestEnvironment.h" @@ -85,13 +85,13 @@ CASE( "test_vector_kernel" ) VectorView list_ints_d = make_device_vector_view(list_ints); VectorView* list_ints_dp; - cudaMalloc((void**)(&list_ints_dp), sizeof(VectorView)); + hicMalloc((void**)(&list_ints_dp), sizeof(VectorView)); - cudaMemcpy(list_ints_dp, &list_ints_d, sizeof(VectorView), cudaMemcpyHostToDevice); + hicMemcpy(list_ints_dp, &list_ints_d, sizeof(VectorView), hicMemcpyHostToDevice); kernel_ex<<<1,1>>>(list_ints_dp); - if( cudaPeekAtLastError() != cudaSuccess) std::cout << "ERROR " << std::endl; + if( hicPeekAtLastError() != hicSuccess) std::cout << "ERROR " << std::endl; list_ints.updateHost(); EXPECT( list_ints_h[0]->val_ == 8 ); diff --git a/src/tests/field/CMakeLists.txt b/src/tests/field/CMakeLists.txt index 3382e576b..dbf51ee37 100644 --- a/src/tests/field/CMakeLists.txt +++ b/src/tests/field/CMakeLists.txt @@ -27,18 +27,29 @@ ecbuild_add_test( TARGET atlas_test_field_foreach ecbuild_add_test( TARGET atlas_test_field_acc SOURCES test_field_acc.cc - LIBS atlas + LIBS atlas OpenACC::OpenACC_CXX ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} - CONDITION atlas_HAVE_ACC + CONDITION atlas_HAVE_ACC AND OpenACC_CXX_Found ) if( TEST atlas_test_field_acc ) - target_compile_options( atlas_test_field_acc PRIVATE "${ACC_C_FLAGS}") - target_link_options( atlas_test_field_acc PRIVATE "${ACC_C_FLAGS}") set_tests_properties ( atlas_test_field_acc PROPERTIES LABELS "gpu;acc") endif() +ecbuild_add_test( TARGET atlas_test_multifield_ifs + SOURCES test_multifield_ifs.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + if( HAVE_FCTEST ) + add_fctest( TARGET atlas_fctest_multifield_ifs + LINKER_LANGUAGE Fortran + SOURCES fctest_multifield_ifs.F90 + LIBS atlas_f + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} + ) + add_fctest( TARGET atlas_fctest_field LINKER_LANGUAGE Fortran SOURCES fctest_field.F90 @@ -73,14 +84,11 @@ if( HAVE_FCTEST ) CONDITION atlas_HAVE_ACC LINKER_LANGUAGE Fortran SOURCES fctest_field_gpu.F90 external_acc_routine.F90 - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 ) if( TARGET atlas_fctest_field_device ) - target_compile_options( atlas_fctest_field_device PUBLIC ${ACC_Fortran_FLAGS} ) - target_link_libraries( atlas_fctest_field_device ${ACC_Fortran_FLAGS} ) - target_link_options( atlas_fctest_field_device PRIVATE "${ACC_Fortran_FLAGS}") set_tests_properties ( atlas_fctest_field_device PROPERTIES LABELS "gpu;acc") endif() @@ -88,14 +96,11 @@ if( HAVE_FCTEST ) CONDITION atlas_HAVE_ACC LINKER_LANGUAGE Fortran SOURCES fctest_field_wrap_gpu.F90 external_acc_routine.F90 - LIBS atlas_f + LIBS atlas_f OpenACC::OpenACC_Fortran ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 ) if( TARGET atlas_fctest_field_wrap_device ) - target_compile_options( atlas_fctest_field_wrap_device PUBLIC ${ACC_Fortran_FLAGS} ) - target_link_libraries( atlas_fctest_field_wrap_device ${ACC_Fortran_FLAGS} ) - target_link_options( atlas_fctest_field_wrap_device PRIVATE "${ACC_Fortran_FLAGS}") set_tests_properties ( atlas_fctest_field_wrap_device PROPERTIES LABELS "gpu;acc") endif() diff --git a/src/tests/field/fctest_field.F90 b/src/tests/field/fctest_field.F90 index 8ea98e09d..d934b4886 100644 --- a/src/tests/field/fctest_field.F90 +++ b/src/tests/field/fctest_field.F90 @@ -139,8 +139,8 @@ module fcta_Field_fixture TEST( test_fieldset ) implicit none - type(atlas_FieldSet) :: fieldset - type(atlas_Field) :: field + type(atlas_FieldSet) :: fieldset, fieldset_2 + type(atlas_Field) :: field, field_2 write(*,*) "test_fieldset starting" @@ -163,6 +163,15 @@ module fcta_Field_fixture FCTEST_CHECK_EQUAL( field%name(), "field_1" ) field = fieldset%field(3) FCTEST_CHECK_EQUAL( field%name(), "field_2" ) + + fieldset_2 = atlas_FieldSet() + call fieldset_2%add(fieldset) + field_2 = fieldset_2%field("field_0") + call field_2%rename("field_00") + FCTEST_CHECK(fieldset%has("field_00")) + field = fieldset%field("field_00") + FCTEST_CHECK_EQUAL(field%name(), "field_00") + call fieldset%final() write(0,*) "test_fieldset end" diff --git a/src/tests/field/fctest_field_host.F90 b/src/tests/field/fctest_field_host.F90 index 955dd159c..fcc2ed690 100644 --- a/src/tests/field/fctest_field_host.F90 +++ b/src/tests/field/fctest_field_host.F90 @@ -10,6 +10,7 @@ ! @author Willem Deconinck #include "fckit/fctest.h" +#include "atlas/atlas_f.h" ! ----------------------------------------------------------------------------- @@ -49,7 +50,9 @@ module fcta_Field_fxt call field%data(view) FCTEST_CHECK( .not. field%host_needs_update() ) +#if ! ATLAS_HAVE_GPU FCTEST_CHECK( .not. field%device_needs_update() ) +#endif call field%update_device() FCTEST_CHECK( .not. field%device_needs_update() ) diff --git a/src/tests/field/fctest_multifield_ifs.F90 b/src/tests/field/fctest_multifield_ifs.F90 new file mode 100644 index 000000000..e71798c02 --- /dev/null +++ b/src/tests/field/fctest_multifield_ifs.F90 @@ -0,0 +1,228 @@ +! (C) Copyright 2013 ECMWF. +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +! This File contains Unit Tests for testing the +! C++ / Fortran Interfaces to the Mesh Datastructure +! @author Willem Deconinck +! @author Slavko Brdar + +#include "fckit/fctest.h" + +! ----------------------------------------------------------------------------- + +module fcta_MultiField_fixture +use atlas_module +use atlas_multifield_module +use, intrinsic :: iso_c_binding +implicit none +end module + +! ----------------------------------------------------------------------------- + +TESTSUITE_WITH_FIXTURE(fctest_atlas_MultiField, fcta_MultiField_fixture) + +! ----------------------------------------------------------------------------- + +TESTSUITE_INIT + call atlas_library%initialise() +END_TESTSUITE_INIT + +! ----------------------------------------------------------------------------- + +TESTSUITE_FINALIZE + call atlas_library%finalise() +END_TESTSUITE_FINALIZE + +! ----------------------------------------------------------------------------- + +TEST( test_multifield ) + implicit none + + type(atlas_MultiField) :: mfield(2) + type(atlas_FieldSet) :: fieldset(2) + type(atlas_Field) :: field + type(atlas_config) :: config + integer, pointer :: fdata_int_2d(:,:) + real(c_float), pointer :: fdata_f2d(:,:), fdata_f3d(:,:,:) + real(c_double), pointer :: fdata_d3d(:,:,:) + + integer, parameter :: nproma = 16; + integer, parameter :: nlev = 1; + integer, parameter :: ngptot = 2000; + integer, parameter :: nblk = (ngptot + nproma - 1) / nproma + type(atlas_Config), allocatable :: field_configs(:) + integer :: i + character(len=64), parameter, dimension(5) :: var_names = [ character(64) :: & + "temperature", "pressure", "density", "clv", "wind_u" ] + + config = atlas_Config() + call config%set("type", "MultiFieldCreatorIFS"); + call config%set("ngptot", ngptot); + call config%set("nproma", nproma); + allocate(field_configs(size(var_names))) + do i = 1, size(var_names) + field_configs(i) = atlas_Config() + call field_configs(i)%set("name", trim(var_names(i))) + end do + call field_configs(4)%set("nvar", 4) ! clv has four subvariables + call config%set("fields", field_configs) + + call config%set("nlev", 0); ! surface fields + call config%set("datatype", "real32"); + mfield(1) = atlas_MultiField(config) + + call config%set("nlev", 4); ! fields are 3d + call config%set("datatype", "real64"); + mfield(2) = atlas_MultiField(config) + + fieldset(1) = mfield(1)%fieldset() + FCTEST_CHECK_EQUAL(mfield(1)%size(), 5) + FCTEST_CHECK_EQUAL(fieldset(1)%size(), 5) + + fieldset(2) = atlas_FieldSet() + call fieldset(2)%add(mfield(1)%fieldset()) + field = fieldset(2)%field("density") + call field%data(fdata_f2d) + fdata_f2d(1,1) = 2. + call field%rename("dens") + + ! check data access directly though multifield + call fieldset(1)%data("dens", fdata_f2d) + fdata_f2d(1,1) = 3. + + ! check access to the renamed variable + field = fieldset(1)%field("dens") + call field%data(fdata_f2d) + FCTEST_CHECK_EQUAL(fdata_f2d(1,1), 3._c_float) + + ! check dimesionality + fieldset(2) = mfield(2)%fieldset() + call fieldset(2)%data("density", fdata_d3d) + fdata_d3d(1,1,1) = 4. + fieldset(2) = atlas_FieldSet() + call fieldset(2)%add(mfield(2)%fieldset()) + field = fieldset(2)%field("density") + call field%data(fdata_d3d) + FCTEST_CHECK_EQUAL(fdata_d3d(1,1,1), 4._c_double) +END_TEST + + +TEST( test_multifield_array_direct_constructor ) + implicit none + + type(atlas_MultiField) :: mfield(2) + type(atlas_FieldSet) :: fieldset(2), fset + type(atlas_Field) :: field + type(atlas_config) :: config + real(c_float), pointer :: fdata_f2d(:,:), fdata_f3d(:,:,:) + real(c_double), pointer :: fdata_d3d(:,:,:) + + integer, parameter :: nproma = 16; + integer, parameter :: nlev = 100; + integer, parameter :: ngptot = 2000; + integer, parameter :: nblk = (ngptot + nproma - 1) / nproma + integer :: i + character(len=64), parameter, dimension(5) :: var_names = [ character(64) :: & + "temperature ", "pressure", "density", "clv", "wind_u" ] + +return + + ! surface fields + mfield(1) = atlas_MultiField(atlas_real(c_float), [nproma, -1, nblk], var_names) + + ! 3d fields + mfield(2) = atlas_MultiField(atlas_real(c_double), [nproma, nlev, -1, nblk], var_names) + + FCTEST_CHECK_EQUAL(mfield(1)%size(), 5) + + fieldset(1) = mfield(1)%fieldset() + call fieldset(1)%data("density", fdata_f2d) + fdata_f2d(1,1) = 3. + fieldset(2) = mfield(2)%fieldset() + call fieldset(2)%data("density", fdata_d3d) + fdata_d3d(1,1,1) = 4. + + fset = atlas_FieldSet() + call fset%add(mfield(1)%fieldset()) + call fset%add(mfield(2)%fieldset()) + +END_TEST + + +TEST( test_multifield_array_config_constuctor ) + implicit none + + type(atlas_MultiField) :: mfield(2) + type(atlas_FieldSet) :: fieldset(2) + type(atlas_Field) :: field + type(atlas_config) :: config + integer, pointer :: fdata_int_2d(:,:) + real(c_float), pointer :: fdata_f2d(:,:), fdata_f3d(:,:,:) + real(c_double), pointer :: fdata_d3d(:,:,:) + + integer, parameter :: nproma = 16; + integer, parameter :: nlev = 1; + integer, parameter :: nblk = 200; + type(atlas_Config), allocatable :: field_configs(:) + integer :: i + character(len=64), parameter, dimension(5) :: var_names = [ character(64) :: & + "temperature", "pressure", "density", "clv", "wind_u" ] + + config = atlas_Config() + call config%set("type", "MultiFieldCreatorArray"); + allocate(field_configs(size(var_names))) + do i = 1, size(var_names) + field_configs(i) = atlas_Config() + call field_configs(i)%set("name", trim(var_names(i))) + end do + call field_configs(4)%set("nvar", 5) ! clv has four subvariables + call config%set("fields", field_configs) + + ! surface fields + call config%set("shape", [nproma, -1, nblk]); + call config%set("datatype", "real32"); + mfield(1) = atlas_MultiField(config) + + ! fields are 3d + call config%set("shape", [nproma, nlev, -1, nblk]); + call config%set("datatype", "real64"); + mfield(2) = atlas_MultiField(config) + + fieldset(1) = mfield(1)%fieldset() + FCTEST_CHECK_EQUAL(mfield(1)%size(), 9) + FCTEST_CHECK_EQUAL(fieldset(1)%size(), 9) + + fieldset(2) = atlas_FieldSet() + call fieldset(2)%add(mfield(1)%fieldset()) + field = fieldset(2)%field("density") + call field%data(fdata_f2d) + fdata_f2d(1,1) = 2. + call field%rename("dens") + + ! check data access directly though multifield + call fieldset(1)%data("dens", fdata_f2d) + fdata_f2d(1,1) = 3. + + ! check access to the renamed variable + field = fieldset(1)%field("dens") + call field%data(fdata_f2d) + FCTEST_CHECK_EQUAL(fdata_f2d(1,1), 3._c_float) + + ! check dimesionality + fieldset(2) = mfield(2)%fieldset() + call fieldset(2)%data("density", fdata_d3d) + fdata_d3d(1,1,1) = 4. + fieldset(2) = atlas_FieldSet() + call fieldset(2)%add(mfield(2)%fieldset()) + field = fieldset(2)%field("density") + call field%data(fdata_d3d) + FCTEST_CHECK_EQUAL(fdata_d3d(1,1,1), 4._c_double) +END_TEST + +! ----------------------------------------------------------------------------- + +END_TESTSUITE diff --git a/src/tests/field/test_field_acc.cc b/src/tests/field/test_field_acc.cc index 99e2343b3..8dad81fbd 100644 --- a/src/tests/field/test_field_acc.cc +++ b/src/tests/field/test_field_acc.cc @@ -12,7 +12,7 @@ #error This file needs to be compiled with OpenACC support #endif -#include +#include "hic/hic.h" #include #include "atlas/field/Field.h" @@ -35,10 +35,10 @@ CASE("test_acc") { *c_ptr = 5; int* d_ptr; - cudaMalloc(&d_ptr, sizeof(int)); + HIC_CALL(hicMalloc(&d_ptr, sizeof(int))); acc_map_data(c_ptr, d_ptr, sizeof(int)); - cudaMemcpy(d_ptr, c_ptr, sizeof(int), cudaMemcpyHostToDevice); + HIC_CALL(hicMemcpy(d_ptr, c_ptr, sizeof(int), hicMemcpyHostToDevice)); #pragma acc kernels present(c_ptr) { @@ -47,7 +47,7 @@ CASE("test_acc") { EXPECT_EQ( *c_ptr, 5. ); - cudaMemcpy(c_ptr, d_ptr, sizeof(int), cudaMemcpyDeviceToHost); + HIC_CALL(hicMemcpy(c_ptr, d_ptr, sizeof(int), hicMemcpyDeviceToHost)); EXPECT_EQ( *c_ptr, 2. ); } diff --git a/src/tests/field/test_fieldset.cc b/src/tests/field/test_fieldset.cc index 500256cd3..474fd5b5b 100644 --- a/src/tests/field/test_fieldset.cc +++ b/src/tests/field/test_fieldset.cc @@ -56,9 +56,53 @@ CASE("test_rename") { EXPECT(!fieldset.has("field_1")); EXPECT_EQ(fieldset.field(1).name(),std::string("")); EXPECT(fieldset.has("[1]")); +} + +CASE("test_fieldset_concatenation") { + FieldSet fieldset_1; + FieldSet fieldset_2; + auto field_1 = fieldset_1.add(Field("0", make_datatype(), array::make_shape(10,4))); + auto field_1_v = array::make_view(field_1); + field_1_v(1,1) = 2.; + + fieldset_2.add(fieldset_1); + auto field_2 = fieldset_2.field("0"); + field_2.rename(""); + auto field_2_v = array::make_view(field_2); + field_2_v(1,1) = 1.; + + EXPECT(!fieldset_2.has("")); + EXPECT_EQ(fieldset_2.field(0).name(), ""); + EXPECT_EQ(field_1_v(1,1), 1.); +} + +CASE("test_duplicate_name_throws") { + FieldSet fieldset; + auto field_0 = fieldset.add(Field("0", make_datatype(), array::make_shape(10,4))); + auto field_1 = fieldset.add(Field("0", make_datatype(), array::make_shape(10,5))); // same name as field_0, uh-oh ! + auto field_2 = fieldset.add(Field("2", make_datatype(), array::make_shape(10,6))); + Field f; + EXPECT_NO_THROW(f = fieldset["2"]); // OK + EXPECT_EQ(f.shape(1), 6); + + EXPECT_THROWS(f = fieldset["0"]); // ambiguous because field_0 and field_1 have same name, should throw + field_1.rename("1"); // fix ambiguity between field_0 and field_1 + EXPECT_NO_THROW(f = fieldset["0"]); // no longer ambiguous + EXPECT_EQ(f.shape(1), 4); // to be sure that we got the right field + EXPECT_NO_THROW(f = fieldset["1"]); // no longer ambiguous + EXPECT_EQ(f.shape(1), 5); // to be sure that we got the right field + + field_2.rename("0"); // Introduce new ambiguity between field_0 and field_2 + EXPECT_THROWS(f = fieldset["0"]); // ambiguous because field_0 and field_2 have same name, should throw + field_2.rename("2"); // fix ambiguity + EXPECT_NO_THROW(f = fieldset["0"]); // no longer ambiguous + EXPECT_EQ(f.shape(1), 4); // to be sure we got the right field + EXPECT_NO_THROW(f = fieldset["2"]); // no longer ambiguous + EXPECT_EQ(f.shape(1), 6); // to be sure we got the right field } + //----------------------------------------------------------------------------- } // namespace test diff --git a/src/tests/field/test_multifield_ifs.cc b/src/tests/field/test_multifield_ifs.cc new file mode 100644 index 000000000..e4539bc28 --- /dev/null +++ b/src/tests/field/test_multifield_ifs.cc @@ -0,0 +1,413 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include +#include + +#include "eckit/config/YAMLConfiguration.h" + +#include "atlas/array/ArrayView.h" +#include "atlas/array/MakeView.h" +#include "atlas/field/Field.h" +#include "atlas/field/MultiField.h" +#include "atlas/field/MultiFieldCreatorArray.h" +#include "atlas/field/detail/MultiFieldImpl.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" + +#include "tests/AtlasTestEnvironment.h" + +using namespace atlas::field; + +namespace atlas { +namespace test { + +const std::vector make_shape(const std::initializer_list& list) { + return std::vector(list); +} + +CASE("multifield_generator") { + EXPECT(MultiFieldCreatorFactory::has("MultiFieldCreatorIFS")); + std::unique_ptr MultiFieldCreatorIFS(MultiFieldCreatorFactory::build("MultiFieldCreatorIFS")); + + EXPECT(MultiFieldCreatorFactory::has("MultiFieldCreatorArray")); + std::unique_ptr MultiFieldCreatorArray(MultiFieldCreatorFactory::build("MultiFieldCreatorArray")); +} + + +CASE("multifield_ifs_create") { + using Value = float; + int nproma = 16; + int nlev = 100; + int ngptot = 2000; + + auto json = [&]() -> std::string { + util::Config p; + p.set("type", "MultiFieldCreatorIFS"); + p.set("ngptot", ngptot); + p.set("nproma", nproma); + p.set("nlev", nlev); + p.set("datatype", array::make_datatype().str()); + p.set("fields", std::vector{ + util::Config("name", "temperature"), + util::Config("name", "pressure"), + util::Config("name", "density"), + util::Config("name", "clv")("nvar", 5), // 'clv' with 5 subvariables + util::Config("name", "wind_u") + }); + return p.json(); + }; + + SECTION("Print configuration") { + Log::info() << "json = " << json() << std::endl; + } + + SECTION("test") { + MultiField multifield{eckit::YAMLConfiguration{json()}}; + + const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 5); + EXPECT_EQ(nvar, 9); + + EXPECT_EQ(multifield.size(), 5); + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("pressure")); + EXPECT(multifield.has("density")); + EXPECT(multifield.has("clv")); + EXPECT(multifield.has("wind_u")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("pressure") << std::endl; + Log::info() << multifield.field("density") << std::endl; + Log::info() << multifield.field("clv") << std::endl; + Log::info() << multifield.field("wind_u") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto pres = array::make_view(multifield.field("pressure")); + auto dens = array::make_view(multifield.field("density")); + auto clv = array::make_view(multifield.field("clv")); // note rank 4 + auto wind_u = array::make_view(multifield.field("wind_u")); + + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[1].name(), "pressure"); + EXPECT_EQ(multifield[2].name(), "density"); + EXPECT_EQ(multifield[3].name(), "clv"); + EXPECT_EQ(multifield[4].name(), "wind_u"); + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma * nlev; + auto level_stride = nproma; + auto nproma_stride = 1; + + temp(1, 2, 3) = 4; + pres(5, 6, 7) = 8; + dens(9, 10, 11) = 12; + clv(13, 2, 14, 15) = 16; + wind_u(17, 18, 3) = 19; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), level_stride); + EXPECT_EQ(temp.stride(2), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nlev * nproma); + + EXPECT_EQ(clv.stride(0), block_stride); + EXPECT_EQ(clv.stride(1), field_stride); + EXPECT_EQ(clv.stride(2), level_stride); + EXPECT_EQ(clv.stride(3), nproma_stride); + + EXPECT_EQ(clv.size(), nblk * 5 * nlev * nproma); + + + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), level_stride); + EXPECT_EQ(multiarray.stride(3), nproma_stride); + + EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); + EXPECT_EQ(multiarray(5, 1, 6, 7), 8.); + EXPECT_EQ(multiarray(9, 2, 10, 11), 12.); + EXPECT_EQ(multiarray(13, 5, 14, 15), 16.); + EXPECT_EQ(multiarray(17, 8, 18, 3), 19.); + + EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); + } + + // access FieldSet through MultiField + auto fieldset = multifield->fieldset(); + auto field_v = array::make_view(fieldset.field("temperature")); + EXPECT_EQ(fieldset.size(), 5); + EXPECT(fieldset.has("temperature")); + EXPECT(fieldset.has("wind_u")); + EXPECT_EQ(field_v(1,2,3), 4); + } + + SECTION("test registry") { + { + Field field = MultiField {eckit::YAMLConfiguration{json()}}.field("temperature"); + auto temp = array::make_view(field); + } + } +} + +//----------------------------------------------------------------------------- + +CASE("multifield_array_create") { + using Value = float; + int nproma = 16; + int nlev = 100; + int ngptot = 2000; + + const int nblks = (ngptot + nproma - 1) / nproma; + const std::vector var_names = {"temperature", "pressure", "density", "clv", "wind_u"}; + + auto json = [&]() -> std::string { + util::Config p; + p.set("type", "MultiFieldCreatorArray"); + p.set("shape", {nblks, -1, nlev, nproma}); + p.set("datatype", array::make_datatype().str()); + p.set("fields", std::vector{ + util::Config("name", "temperature"), // + util::Config("name", "pressure"), // + util::Config("name", "density"), // + util::Config("name", "clv")("nvar", 5), // + util::Config("name", "wind_u") // + }); + return p.json(); + }; + + SECTION("test_MultiFieldArray_noconfig_3d") { + int nlev = 3; + const std::vector vshape = make_shape({nblks, -1, nlev, nproma}); + MultiField multifield(array::make_datatype(), vshape, var_names); + + const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); + nlev = multifield.array().shape(2); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 5); + EXPECT_EQ(nvar, 5); + + EXPECT_EQ(multifield.size(), 5); + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("clv")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("clv") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto clv = array::make_view(multifield.field("clv")); + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[3].name(), "clv"); + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma * nlev; + auto level_stride = nproma; + auto nproma_stride = 1; + + temp(1, 2, 3) = 4; + clv(13, 2, 14) = 16; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), level_stride); + EXPECT_EQ(temp.stride(2), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nlev * nproma); + + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), level_stride); + EXPECT_EQ(multiarray.stride(3), nproma_stride); + + EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); + EXPECT_EQ(multiarray(13, 3, 2, 14), 16.); + + EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); + } + + // access FieldSet through MultiField + auto fieldset = multifield->fieldset(); + auto field_v = array::make_view(fieldset.field("temperature")); + EXPECT_EQ(fieldset.size(), 5); + EXPECT(fieldset.has("temperature")); + EXPECT(fieldset.has("wind_u")); + EXPECT_EQ(field_v(1,2,3), 4); + } + + SECTION("test_MultiFieldArray_noconfig_2d") { + const std::vector vshape = make_shape({nblks, -1, nproma}); + MultiField multifield(array::make_datatype(), vshape, var_names); + + const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); + nlev = multifield.array().shape(2); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 5); + EXPECT_EQ(nvar, 5); + + EXPECT_EQ(multifield.size(), 5); + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("clv")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("clv") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto clv = array::make_view(multifield.field("clv")); + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[3].name(), "clv"); + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma; + auto nproma_stride = 1; + + temp(1, 3) = 4; + clv(13, 14) = 16; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nproma); + + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), nproma_stride); + + EXPECT_EQ(multiarray(1, 0, 3), 4.); + EXPECT_EQ(multiarray(13, 3, 14), 16.); + + EXPECT_EQ(multiarray.size(), nblk * nvar * nproma); + } + + // access FieldSet through MultiField + auto fieldset = multifield->fieldset(); + auto field_v = array::make_view(fieldset.field("temperature")); + EXPECT_EQ(fieldset.size(), 5); + EXPECT(fieldset.has("temperature")); + EXPECT(fieldset.has("wind_u")); + EXPECT_EQ(field_v(1,3), 4); + } + + SECTION("Print configuration") { + Log::info() << "json = " << json() << std::endl; + } + + SECTION("test_MultiFieldArray_config") { + MultiField multifield{eckit::YAMLConfiguration{json()}}; + + const auto nblk = multifield.array().shape(0); + const auto nvar = multifield.array().shape(1); + const auto nfld = multifield.size(); + EXPECT_EQ(nfld, 9); + EXPECT_EQ(nvar, 9); + + EXPECT_EQ(multifield.size(), 9); + EXPECT(multifield.has("temperature")); + EXPECT(multifield.has("pressure")); + EXPECT(multifield.has("density")); + EXPECT(multifield.has("clv_0")); + EXPECT(multifield.has("wind_u")); + + Log::info() << multifield.field("temperature") << std::endl; + Log::info() << multifield.field("pressure") << std::endl; + Log::info() << multifield.field("density") << std::endl; + Log::info() << multifield.field("clv_0") << std::endl; + Log::info() << multifield.field("wind_u") << std::endl; + + auto temp = array::make_view(multifield.field("temperature")); + auto pres = array::make_view(multifield.field("pressure")); + auto dens = array::make_view(multifield.field("density")); + auto clv = array::make_view(multifield.field("clv_0")); // note rank 4 + auto wind_u = array::make_view(multifield.field("wind_u")); + + EXPECT_EQ(multifield[0].name(), "temperature"); + EXPECT_EQ(multifield[1].name(), "pressure"); + EXPECT_EQ(multifield[2].name(), "density"); + EXPECT_EQ(multifield[3].name(), "clv_0"); + EXPECT_EQ(multifield[8].name(), "wind_u"); + + auto block_stride = multifield.array().stride(0); + auto field_stride = nproma * nlev; + auto level_stride = nproma; + auto nproma_stride = 1; + + temp(1, 2, 3) = 4; + pres(5, 6, 7) = 8; + dens(9, 10, 11) = 12; + clv(13, 14, 15) = 16; + wind_u(17, 18, 3) = 19; + + EXPECT_EQ(temp.stride(0), block_stride); + EXPECT_EQ(temp.stride(1), level_stride); + EXPECT_EQ(temp.stride(2), nproma_stride); + EXPECT_EQ(temp.size(), nblk * nlev * nproma); + + EXPECT_EQ(clv.stride(0), block_stride); + EXPECT_EQ(clv.stride(1), level_stride); + EXPECT_EQ(clv.stride(2), nproma_stride); + + EXPECT_EQ(clv.size(), nblk * nlev * nproma); + + + // Advanced usage, to access underlying array. This should only be used + // in a driver and not be exposed to algorithms. + { + auto multiarray = array::make_view(multifield); + EXPECT_EQ(multiarray.stride(0), block_stride); + EXPECT_EQ(multiarray.stride(1), field_stride); + EXPECT_EQ(multiarray.stride(2), level_stride); + EXPECT_EQ(multiarray.stride(3), nproma_stride); + + EXPECT_EQ(multiarray(1, 0, 2, 3), 4.); + EXPECT_EQ(multiarray(5, 1, 6, 7), 8.); + EXPECT_EQ(multiarray(9, 2, 10, 11), 12.); + EXPECT_EQ(multiarray(13, 3, 14, 15), 16.); + EXPECT_EQ(multiarray(17, 8, 18, 3), 19.); + + EXPECT_EQ(multiarray.size(), nblk * nvar * nlev * nproma); + } + + // access FieldSet through MultiField + auto fieldset = multifield->fieldset(); + auto field_v = array::make_view(fieldset.field("temperature")); + EXPECT_EQ(fieldset.size(), 9); + EXPECT(fieldset.has("temperature")); + EXPECT(fieldset.has("wind_u")); + EXPECT_EQ(field_v(1,2,3), 4); + } + + SECTION("test registry") { + { + Field field = MultiField {eckit::YAMLConfiguration{json()}}.field("temperature"); + auto temp = array::make_view(field); + } + } +} + +//----------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} diff --git a/src/tests/grid/fctest_stretchedrotatedgaussiangrid.F90 b/src/tests/grid/fctest_stretchedrotatedgaussiangrid.F90 index 125322d8d..e4cf2e484 100644 --- a/src/tests/grid/fctest_stretchedrotatedgaussiangrid.F90 +++ b/src/tests/grid/fctest_stretchedrotatedgaussiangrid.F90 @@ -44,886 +44,1701 @@ real(dp), parameter :: angle = 180.0_dp integer :: i, j, jglo - lonlat( 1: 120) = [ & - & 2.00000000000000000_dp, 48.46708828700425187_dp, 1.17799068975270704_dp, 48.37772379915030996_dp, & - & 0.44418972240584109_dp, 48.11926090196744354_dp, -0.12512266986032569_dp, 47.71926346185548340_dp, & - & -0.47479391972813606_dp, 47.21956618120344018_dp, -0.57569351104372468_dp, 46.67109320702403608_dp, & - & -0.42528132740345748_dp, 46.12813070368466839_dp, -0.04494770792091035_dp, 45.64287023699112211_dp, & - & 0.52444137510688571_dp, 45.26073815098433784_dp, 1.22762737065264282_dp, 45.01674523281888440_dp, & - & 2.00000000000000000_dp, 44.93291171299576092_dp, 2.77237262934735629_dp, 45.01674523281888440_dp, & - & 3.47555862489311407_dp, 45.26073815098433784_dp, 4.04494770792090996_dp, 45.64287023699112211_dp, & - & 4.42528132740345725_dp, 46.12813070368466839_dp, 4.57569351104372490_dp, 46.67109320702403608_dp, & - & 4.47479391972813634_dp, 47.21956618120344018_dp, 4.12512266986032561_dp, 47.71926346185548340_dp, & - & 3.55581027759415846_dp, 48.11926090196744354_dp, 2.82200931024729318_dp, 48.37772379915030996_dp, & - & 2.00000000000000000_dp, 50.76274729016172671_dp, 0.52357145823104279_dp, 50.64425941434981837_dp, & - & -0.85309402263029532_dp, 50.29690558565773273_dp, -2.04123891610972796_dp, 49.74407157963349846_dp, & - & -2.97152383528954323_dp, 49.02183412771946536_dp, -3.59837298636155323_dp, 48.17535393470686955_dp, & - & -3.90029306374494222_dp, 47.25496955383284359_dp, -3.87730059701688790_dp, 46.31256836954069911_dp, & - & -3.54692456688646551_dp, 45.39855083314470363_dp, -2.93998927128814502_dp, 44.55946699558614910_dp, & - & -2.09688983996385936_dp, 43.83625800397906147_dp, -1.06462721424400342_dp, 43.26298044031561574_dp, & - & 0.10541754825032876_dp, 42.86589673132546352_dp, 1.35912400589531490_dp, 42.66284629701862485_dp, & - & 2.64087599410468910_dp, 42.66284629701862485_dp, 3.89458245174967033_dp, 42.86589673132546352_dp, & - & 5.06462721424400453_dp, 43.26298044031561574_dp, 6.09688983996385936_dp, 43.83625800397906147_dp, & - & 6.93998927128814458_dp, 44.55946699558614910_dp, 7.54692456688646551_dp, 45.39855083314470363_dp, & - & 7.87730059701688656_dp, 46.31256836954069911_dp, 7.90029306374494134_dp, 47.25496955383284359_dp, & - & 7.59837298636155190_dp, 48.17535393470686955_dp, 6.97152383528954367_dp, 49.02183412771946536_dp, & - & 6.04123891610972930_dp, 49.74407157963349846_dp, 4.85309402263029188_dp, 50.29690558565773273_dp, & - & 3.47642854176895577_dp, 50.64425941434981837_dp, 2.00000000000000089_dp, 53.08763788054714894_dp, & - & -0.06435502534210064_dp, 52.94800110445672203_dp, -2.01378910799337474_dp, 52.53707313311167582_dp, & - & -3.74622775989939782_dp, 51.87782553848720113_dp, -5.18197003975669812_dp, 51.00557820511744467_dp, & - & -6.26810804950859701_dp, 49.96440199105025926_dp, -6.97806564551988018_dp, 48.80336562297891589_dp, & - & -7.30784824781774223_dp, 47.57322688923063225_dp, -7.27090590027329053_dp, 46.32385288536406875_dp, & - & -6.89307303713296715_dp, 45.10238649522262477_dp, -6.20837670733706215_dp, 43.95202523375940729_dp, & - & -5.25594876339865014_dp, 42.91123363953519743_dp, -4.07794770476515911_dp, 42.01322983391511201_dp ] - lonlat( 121: 240) = [ & - & -2.71826031811739588_dp, 41.28563184510834105_dp, -1.22173788476928480_dp, 40.75019560382368411_dp, & - & 0.36623707967922525_dp, 40.42261301497919845_dp, 1.99999999999999933_dp, 40.31236211945285675_dp, & - & 3.63376292032077286_dp, 40.42261301497919845_dp, 5.22173788476928369_dp, 40.75019560382368411_dp, & - & 6.71826031811739455_dp, 41.28563184510834105_dp, 8.07794770476515822_dp, 42.01322983391511201_dp, & - & 9.25594876339864925_dp, 42.91123363953519743_dp, 10.20837670733705949_dp, 43.95202523375940729_dp, & - & 10.89307303713296804_dp, 45.10238649522262477_dp, 11.27090590027328965_dp, 46.32385288536406875_dp, & - & 11.30784824781774311_dp, 47.57322688923063225_dp, 10.97806564551988195_dp, 48.80336562297891589_dp, & - & 10.26810804950859790_dp, 49.96440199105025926_dp, 9.18197003975669901_dp, 51.00557820511744467_dp, & - & 7.74622775989939960_dp, 51.87782553848720113_dp, 6.01378910799337607_dp, 52.53707313311167582_dp, & - & 4.06435502534210169_dp, 52.94800110445672203_dp, 2.00000000000000089_dp, 55.44054043583248159_dp, & - & -0.39391037845720578_dp, 55.31115535850048559_dp, -2.68766853851381882_dp, 54.92865613654313961_dp, & - & -4.79116029542177202_dp, 54.30937174533021050_dp, -6.63193584333677855_dp, 53.47856301011842106_dp, & - & -8.15903975007321769_dp, 52.46802323745790630_dp, -9.34306219743110411_dp, 51.31353838046217675_dp, & - & -10.17346129570704782_dp, 50.05261748260089405_dp, -10.65454278958191026_dp, 48.72271255422421632_dp, & - & -10.80126342211737800_dp, 47.35996933207513848_dp, -10.63557388477568999_dp, 45.99843704779454612_dp, & - & -10.18359521918176824_dp, 44.66961909752426152_dp, -9.47363915243209931_dp, 43.40224634648851065_dp, & - & -8.53494002107633598_dp, 42.22217718966415845_dp, -7.39692139773956647_dp, 41.15235690381517486_dp, & - & -6.08882962579569487_dp, 40.21279467492543347_dp, -4.63959771096107954_dp, 39.42053684981195261_dp, & - & -3.07783882564465205_dp, 38.78962916423042628_dp, -1.43190111561111988_dp, 38.33106975566338548_dp, & - & 0.27005804914242593_dp, 38.05275970261258323_dp, 1.99999999999999956_dp, 37.95945956416753120_dp, & - & 3.72994195085757241_dp, 38.05275970261258323_dp, 5.43190111561111788_dp, 38.33106975566338548_dp, & - & 7.07783882564465028_dp, 38.78962916423042628_dp, 8.63959771096107865_dp, 39.42053684981195261_dp, & - & 10.08882962579569309_dp, 40.21279467492543347_dp, 11.39692139773956647_dp, 41.15235690381517486_dp, & - & 12.53494002107633243_dp, 42.22217718966415845_dp, 13.47363915243209576_dp, 43.40224634648851065_dp, & - & 14.18359521918176824_dp, 44.66961909752426152_dp, 14.63557388477568999_dp, 45.99843704779454612_dp, & - & 14.80126342211737445_dp, 47.35996933207513848_dp, 14.65454278958191381_dp, 48.72271255422421632_dp, & - & 14.17346129570704782_dp, 50.05261748260089405_dp, 13.34306219743110766_dp, 51.31353838046217675_dp, & - & 12.15903975007321769_dp, 52.46802323745790630_dp, 10.63193584333677855_dp, 53.47856301011842106_dp, & - & 8.79116029542177380_dp, 54.30937174533021050_dp, 6.68766853851382059_dp, 54.92865613654313961_dp, & - & 4.39391037845720867_dp, 55.31115535850048559_dp, 2.00000000000000222_dp, 57.82915094012270174_dp ] - lonlat( 241: 360) = [ & - & -0.88089731877775757_dp, 57.69078537163564846_dp, -3.64890062321652664_dp, 57.28147795233979167_dp, & - & -6.20331014929099656_dp, 56.61787273415653488_dp, -8.46463908845755597_dp, 55.72551502002679058_dp, & - & -10.37873026754043693_dp, 54.63620962052475960_dp, -11.91616209561276740_dp, 53.38530347804498888_dp, & - & -13.06845153476416321_dp, 52.00935252683229493_dp, -13.84288249522568925_dp, 50.54439040009851425_dp, & - & -14.25738892575702366_dp, 49.02480894996942595_dp, -14.33628744864260440_dp, 47.48274104170999976_dp, & - & -14.10711920596712332_dp, 45.94779768044404733_dp, -13.59853603019004176_dp, 44.44702279967973624_dp, & - & -12.83902541924944174_dp, 43.00496062724707258_dp, -11.85624296375186937_dp, 41.64376435793402464_dp, & - & -10.67674963753560036_dp, 40.38330297070848474_dp, -9.32599718132065902_dp, 39.24124360712294646_dp, & - & -7.82844927012504055_dp, 38.23310085983992224_dp, -6.20776260081896769_dp, 37.37225322309888043_dp, & - & -4.48697963071903061_dp, 36.66993230518373537_dp, -2.68870484500073204_dp, 36.13519327177002793_dp, & - & -0.83525097543211291_dp, 35.77487607444932394_dp, 1.05124791602986001_dp, 35.59356675116022473_dp, & - & 2.94875208397013866_dp, 35.59356675116022473_dp, 4.83525097543211047_dp, 35.77487607444932394_dp, & - & 6.68870484500073026_dp, 36.13519327177002793_dp, 8.48697963071903061_dp, 36.66993230518373537_dp, & - & 10.20776260081896680_dp, 37.37225322309888043_dp, 11.82844927012503966_dp, 38.23310085983992224_dp, & - & 13.32599718132065725_dp, 39.24124360712293225_dp, 14.67674963753560391_dp, 40.38330297070848474_dp, & - & 15.85624296375187114_dp, 41.64376435793402464_dp, 16.83902541924944174_dp, 43.00496062724707258_dp, & - & 17.59853603019003998_dp, 44.44702279967973624_dp, 18.10711920596712687_dp, 45.94779768044404733_dp, & - & 18.33628744864260796_dp, 47.48274104170998555_dp, 18.25738892575703076_dp, 49.02480894996942595_dp, & - & 17.84288249522569458_dp, 50.54439040009851425_dp, 17.06845153476416499_dp, 52.00935252683229493_dp, & - & 15.91616209561277273_dp, 53.38530347804498888_dp, 14.37873026754043870_dp, 54.63620962052475960_dp, & - & 12.46463908845756130_dp, 55.72551502002679058_dp, 10.20331014929099922_dp, 56.61787273415653488_dp, & - & 7.64890062321653019_dp, 57.28147795233979167_dp, 4.88089731877776067_dp, 57.69078537163564846_dp, & - & 2.00000000000000222_dp, 60.26295411645657651_dp, -1.52099396835736300_dp, 60.10440004572244277_dp, & - & -4.89637618755807136_dp, 59.63591573610307250_dp, -7.99912115224313958_dp, 58.87795079119066344_dp, & - & -10.73370655373201643_dp, 57.86135900031617751_dp, -13.04067793585433499_dp, 56.62367212326139310_dp, & - & -14.89384545876526111_dp, 55.20548791890712437_dp, -16.29319722451073105_dp, 53.64760968250063655_dp, & - & -17.25667322722224029_dp, 51.98914605534769606_dp, -17.81285996059914467_dp, 50.26647329190694080_dp, & - & -17.99546453290829007_dp, 48.51282952087984768_dp, -17.83961562141898938_dp, 46.75830290430587155_dp, & - & -17.37966075266355759_dp, 45.03002540480565585_dp, -16.64804001511417297_dp, 43.35244484209054860_dp, & - & -15.67486490585672243_dp, 41.74759899443786537_dp, -14.48792157318536411_dp, 40.23535159216241652_dp ] - lonlat( 361: 480) = [ & - & -13.11290455740948957_dp, 38.83357308646102979_dp, -11.57375501535206475_dp, 37.55826279697890158_dp, & - & -9.89302536639832653_dp, 36.42361680044041350_dp, -8.09222434325374884_dp, 35.44205010622848562_dp, & - & -6.19211732580419838_dp, 34.62418375193039566_dp, -4.21297051898843300_dp, 33.97880825794451454_dp, & - & -2.17473678976558649_dp, 33.51283479810019372_dp, -0.09718755711549092_dp, 33.23124463343540214_dp, & - & 1.99999999999999889_dp, 33.13704588354342917_dp, 4.09718755711548877_dp, 33.23124463343540214_dp, & - & 6.17473678976558382_dp, 33.51283479810019372_dp, 8.21297051898843122_dp, 33.97880825794451454_dp, & - & 10.19211732580419572_dp, 34.62418375193039566_dp, 12.09222434325374707_dp, 35.44205010622848562_dp, & - & 13.89302536639832830_dp, 36.42361680044041350_dp, 15.57375501535206297_dp, 37.55826279697889447_dp, & - & 17.11290455740948602_dp, 38.83357308646102268_dp, 18.48792157318536766_dp, 40.23535159216240942_dp, & - & 19.67486490585671888_dp, 41.74759899443786537_dp, 20.64804001511417653_dp, 43.35244484209054860_dp, & - & 21.37966075266355404_dp, 45.03002540480565585_dp, 21.83961562141898582_dp, 46.75830290430587155_dp, & - & 21.99546453290829007_dp, 48.51282952087984768_dp, 21.81285996059914822_dp, 50.26647329190694080_dp, & - & 21.25667322722224739_dp, 51.98914605534769606_dp, 20.29319722451073105_dp, 53.64760968250062234_dp, & - & 18.89384545876525934_dp, 55.20548791890712437_dp, 17.04067793585434032_dp, 56.62367212326139310_dp, & - & 14.73370655373202354_dp, 57.86135900031617751_dp, 11.99912115224314491_dp, 58.87795079119066344_dp, & - & 8.89637618755807580_dp, 59.63591573610307250_dp, 5.52099396835736744_dp, 60.10440004572244277_dp, & - & 2.00000000000000222_dp, 62.75242980718265784_dp, -1.60371952785031557_dp, 62.62270582496653049_dp, & - & -5.08984714396560278_dp, 62.23829052070698253_dp, -8.35401191716921510_dp, 61.61281120273996237_dp, & - & -11.31471471060360656_dp, 60.76704034614787986_dp, -13.91751559979281083_dp, 59.72658657388572578_dp, & - & -16.13405311103923978_dp, 58.51955862209560166_dp, -17.95764487214750815_dp, 57.17460858105235388_dp, & - & -19.39751805719553701_dp, 55.71953428872550518_dp, -20.47321926655622448_dp, 54.18043392376822709_dp, & - & -21.21002592824931199_dp, 52.58130404832083116_dp, -21.63559425749199150_dp, 50.94394459368972150_dp, & - & -21.77773936992417703_dp, 49.28804930270956675_dp, -21.66310777053267600_dp, 47.63139077670034283_dp, & - & -21.31648826424519072_dp, 45.99003977573201496_dp, -20.76054697060167342_dp, 44.37858250245571412_dp, & - & -20.01582559662803007_dp, 42.81031632956308641_dp, -19.10089088783405842_dp, 41.29741507659190347_dp, & - & -18.03256121225549791_dp, 39.85106125278488065_dp, -16.82616341777592339_dp, 38.48154618161254348_dp, & - & -15.49579145727241247_dp, 37.19834072287546434_dp, -14.05455017155123087_dp, 36.01014014726423795_dp, & - & -12.51477509868480809_dp, 34.92488704934376642_dp, -10.88822378743807739_dp, 33.94977626735131793_dp, & - & -9.18623692372254119_dp, 33.09124574576954103_dp, -7.41986936545087783_dp, 32.35495718584156322_dp, & - & -5.59999239238874669_dp, 31.74577019145995394_dp, -3.73736939611796615_dp, 31.26771342119434749_dp ] - lonlat( 481: 600) = [ & - & -1.84270801296844233_dp, 30.92395598105918353_dp, 0.07330759755401026_dp, 30.71678191817960268_dp, & - & 1.99999999999999889_dp, 30.64757019281734074_dp, 3.92669240244598639_dp, 30.71678191817960268_dp, & - & 5.84270801296843878_dp, 30.92395598105918353_dp, 7.73736939611796526_dp, 31.26771342119434749_dp, & - & 9.59999239238874225_dp, 31.74577019145995394_dp, 11.41986936545087516_dp, 32.35495718584156322_dp, & - & 13.18623692372254119_dp, 33.09124574576954103_dp, 14.88822378743807207_dp, 33.94977626735131793_dp, & - & 16.51477509868480809_dp, 34.92488704934376642_dp, 18.05455017155123087_dp, 36.01014014726423795_dp, & - & 19.49579145727241070_dp, 37.19834072287546434_dp, 20.82616341777592339_dp, 38.48154618161253637_dp, & - & 22.03256121225549791_dp, 39.85106125278488065_dp, 23.10089088783405842_dp, 41.29741507659190347_dp, & - & 24.01582559662803007_dp, 42.81031632956308641_dp, 24.76054697060167342_dp, 44.37858250245571412_dp, & - & 25.31648826424519072_dp, 45.99003977573201496_dp, 25.66310777053267600_dp, 47.63139077670033572_dp, & - & 25.77773936992417703_dp, 49.28804930270955964_dp, 25.63559425749199505_dp, 50.94394459368972150_dp, & - & 25.21002592824931199_dp, 52.58130404832083116_dp, 24.47321926655622448_dp, 54.18043392376822709_dp, & - & 23.39751805719553701_dp, 55.71953428872550518_dp, 21.95764487214750460_dp, 57.17460858105235388_dp, & - & 20.13405311103924333_dp, 58.51955862209560166_dp, 17.91751559979281438_dp, 59.72658657388572578_dp, & - & 15.31471471060361011_dp, 60.76704034614787986_dp, 12.35401191716921865_dp, 61.61281120273996237_dp, & - & 9.08984714396560634_dp, 62.23829052070698253_dp, 5.60371952785031979_dp, 62.62270582496653049_dp, & - & 2.00000000000000311_dp, 65.30899890463146562_dp, -2.55165837938126616_dp, 65.14506774788591770_dp, & - & -6.91821393310037980_dp, 64.66099291349783584_dp, -10.94248311853021427_dp, 63.87850997409253040_dp, & - & -14.51323316925390294_dp, 62.82970044453450953_dp, -17.56939664790982292_dp, 61.55243525845713748_dp, & - & -20.09325961331439458_dp, 60.08625295311978931_dp, -22.09844461997003506_dp, 58.46939890910893922_dp, & - & -23.61769060836897438_dp, 56.73712764036461920_dp, -24.69307609487012556_dp, 54.92100894435518654_dp, & - & -25.36933297612858951_dp, 53.04888396821036167_dp, -25.68983696232584535_dp, 51.14517010976478417_dp, & - & -25.69453569648834801_dp, 49.23130925676766623_dp, -25.41912613625677153_dp, 47.32623888788154431_dp, & - & -24.89496528133907205_dp, 45.44682484300304282_dp, -24.14937074714975296_dp, 43.60823042135866956_dp, & - & -23.20610004359021517_dp, 41.82421587722975431_dp, -22.08588698105332782_dp, 40.10737174655498194_dp, & - & -20.80696969977402944_dp, 38.46929323359992736_dp, -19.38557795842460507_dp, 36.92070387379718710_dp, & - & -17.83636590786354148_dp, 35.47153642700349963_dp, -16.17278641740254130_dp, 34.13097826525054046_dp, & - & -14.40740784814351017_dp, 32.90748779465315721_dp, -12.55217628249286932_dp, 31.80878785141789677_dp, & - & -10.61862699669389443_dp, 30.84184157026976436_dp, -8.61804920168179045_dp, 30.01281590885307438_dp, & - & -6.56160821625925728_dp, 29.32703776203888069_dp, -4.46042949045145676_dp, 28.78894734383437992_dp ] - lonlat( 601: 720) = [ & - & -2.32564933647368921_dp, 28.40205318377053345_dp, -0.16843782661930862_dp, 28.16889262543881500_dp, & - & 1.99999999999999889_dp, 28.09100109536852585_dp, 4.16843782661930629_dp, 28.16889262543881500_dp, & - & 6.32564933647368743_dp, 28.40205318377053345_dp, 8.46042949045145498_dp, 28.78894734383437992_dp, & - & 10.56160821625925372_dp, 29.32703776203888069_dp, 12.61804920168178867_dp, 30.01281590885307438_dp, & - & 14.61862699669389265_dp, 30.84184157026976436_dp, 16.55217628249286577_dp, 31.80878785141789677_dp, & - & 18.40740784814351017_dp, 32.90748779465315721_dp, 20.17278641740253775_dp, 34.13097826525054046_dp, & - & 21.83636590786353793_dp, 35.47153642700349963_dp, 23.38557795842460152_dp, 36.92070387379718710_dp, & - & 24.80696969977402588_dp, 38.46929323359992026_dp, 26.08588698105332426_dp, 40.10737174655498194_dp, & - & 27.20610004359021161_dp, 41.82421587722975431_dp, 28.14937074714974941_dp, 43.60823042135866956_dp, & - & 28.89496528133906494_dp, 45.44682484300304282_dp, 29.41912613625676798_dp, 47.32623888788153721_dp, & - & 29.69453569648834446_dp, 49.23130925676766623_dp, 29.68983696232584180_dp, 51.14517010976478417_dp, & - & 29.36933297612858951_dp, 53.04888396821034746_dp, 28.69307609487012911_dp, 54.92100894435518654_dp, & - & 27.61769060836897438_dp, 56.73712764036461920_dp, 26.09844461997003506_dp, 58.46939890910893922_dp, & - & 24.09325961331439814_dp, 60.08625295311978931_dp, 21.56939664790982292_dp, 61.55243525845713748_dp, & - & 18.51323316925390827_dp, 62.82970044453450953_dp, 14.94248311853021960_dp, 63.87850997409253040_dp, & - & 10.91821393310038779_dp, 64.66099291349783584_dp, 6.55165837938127193_dp, 65.14506774788591770_dp, & - & 2.00000000000000400_dp, 67.94514236453578349_dp, -3.38533547682125890_dp, 67.76325974975715383_dp, & - & -8.52326664933801226_dp, 67.22759281764989225_dp, -13.21138564172178143_dp, 66.36584339331807314_dp, & - & -17.31841683118042496_dp, 65.21781824976642383_dp, -20.78586461017365750_dp, 63.82887077001763032_dp, & - & -23.61278608277912383_dp, 62.24449065808524040_dp, -25.83517004248913196_dp, 60.50690475312734407_dp, & - & -27.50782877801382043_dp, 58.65352526302899605_dp, -28.69170289014789077_dp, 56.71665696607985296_dp, & - & -29.44632120220701665_dp, 54.72388646563504011_dp, -29.82602201650912477_dp, 52.69875031809839783_dp, & - & -29.87852862440772839_dp, 50.66145188613276673_dp, -29.64483525740958925_dp, 48.62951713112133945_dp, & - & -29.15973753507462618_dp, 46.61834908938176625_dp, -28.45262246278107554_dp, 44.64167567492752653_dp, & - & -27.54831327014242959_dp, 42.71190039889389567_dp, -26.46787020033523774_dp, 40.84037031204417190_dp, & - & -25.22930648122764552_dp, 39.03757551632936185_dp, -23.84820859564655038_dp, 37.31329283883029291_dp, & - & -22.33826403031283903_dp, 35.67668409468497970_dp, -20.71170528699355629_dp, 34.13635737784334623_dp, & - & -18.97968026705781242_dp, 32.70039821822412307_dp, -17.15255849246134900_dp, 31.37637626233231813_dp, & - & -15.24018123567453742_dp, 30.17133232687284305_dp, -13.25206214192900944_dp, 29.09175016726920759_dp, & - & -11.19754366142445079_dp, 28.14351701208745027_dp, -9.08591371200913400_dp, 27.33187675649334381_dp ] - lonlat( 721: 840) = [ & - & -6.92648650789847675_dp, 26.66137960320710931_dp, -4.72865139696445702_dp, 26.13583181560235502_dp, & - & -2.50189378344492486_dp, 25.75824904385734371_dp, -0.25579267831605296_dp, 25.53081635683327733_dp, & - & 1.99999999999999889_dp, 25.45485763546421154_dp, 4.25579267831605002_dp, 25.53081635683327733_dp, & - & 6.50189378344492308_dp, 25.75824904385734371_dp, 8.72865139696445347_dp, 26.13583181560235502_dp, & - & 10.92648650789847764_dp, 26.66137960320710931_dp, 13.08591371200913400_dp, 27.33187675649334381_dp, & - & 15.19754366142444724_dp, 28.14351701208745027_dp, 17.25206214192900589_dp, 29.09175016726920759_dp, & - & 19.24018123567453742_dp, 30.17133232687284305_dp, 21.15255849246134545_dp, 31.37637626233231813_dp, & - & 22.97968026705781242_dp, 32.70039821822412307_dp, 24.71170528699354918_dp, 34.13635737784334623_dp, & - & 26.33826403031282837_dp, 35.67668409468497259_dp, 27.84820859564654683_dp, 37.31329283883029291_dp, & - & 29.22930648122764197_dp, 39.03757551632935474_dp, 30.46787020033523419_dp, 40.84037031204417190_dp, & - & 31.54831327014242248_dp, 42.71190039889389567_dp, 32.45262246278107199_dp, 44.64167567492752653_dp, & - & 33.15973753507462618_dp, 46.61834908938176625_dp, 33.64483525740958214_dp, 48.62951713112133945_dp, & - & 33.87852862440772839_dp, 50.66145188613276673_dp, 33.82602201650912122_dp, 52.69875031809839783_dp, & - & 33.44632120220701665_dp, 54.72388646563502590_dp, 32.69170289014788011_dp, 56.71665696607985296_dp, & - & 31.50782877801381687_dp, 58.65352526302899605_dp, 29.83517004248913906_dp, 60.50690475312734407_dp, & - & 27.61278608277913094_dp, 62.24449065808524040_dp, 24.78586461017366460_dp, 63.82887077001763032_dp, & - & 21.31841683118043562_dp, 65.21781824976642383_dp, 17.21138564172179031_dp, 66.36584339331807314_dp, & - & 12.52326664933801936_dp, 67.22759281764989225_dp, 7.38533547682126645_dp, 67.76325974975715383_dp, & - & 2.00000000000000488_dp, 70.67459026401789401_dp, -4.83336207267744200_dp, 70.44359392189112157_dp, & - & -11.25145387811900299_dp, 69.76794994271870110_dp, -16.94303945184942606_dp, 68.69426490502790728_dp, & - & -21.74696599568290978_dp, 67.28571148577093197_dp, -25.63456841105837825_dp, 65.60910965714572285_dp, & - & -28.66225471925319823_dp, 63.72633555655123416_dp, -30.92626834358740595_dp, 61.69055635034530383_dp, & - & -32.53242307522426557_dp, 59.54571651024583190_dp, -33.58017141013023377_dp, 57.32758829402004608_dp, & - & -34.15616674923442986_dp, 55.06530104282313687_dp, -34.33296113360040636_dp, 52.78282385843581892_dp, & - & -34.17004600591189956_dp, 50.50021197832979425_dp, -33.71572773969587900_dp, 48.23458650173874673_dp, & - & -33.00911969333094476_dp, 46.00087796034757304_dp, -32.08195470333168231_dp, 43.81237943576640248_dp, & - & -30.96012666362175025_dp, 41.68115224019865650_dp, -29.66495985223924237_dp, 39.61831917500947498_dp, & - & -28.21423852322426740_dp, 37.63427208010365632_dp, -26.62303747247054986_dp, 35.73881343521247089_dp, & - & -24.90439178940930276_dp, 33.94124647006630369_dp, -23.06983782140640216_dp, 32.25042441190934994_dp, & - & -21.12985059268007149_dp, 30.67476685924403057_dp, -19.09419678103115459_dp, 29.22224954465700009_dp ] - lonlat( 841: 960) = [ & - & -16.97221728412494457_dp, 27.90037270541385794_dp, -14.77304947679039238_dp, 26.71611272875198750_dp, & - & -12.50579641562138100_dp, 25.67586152120415832_dp, -10.17964838758363832_dp, 24.78535803035031648_dp, & - & -7.80396120930039583_dp, 24.04961640006232315_dp, -5.38829543151647705_dp, 23.47285525677743223_dp, & - & -2.94242093509331726_dp, 23.05843251151437201_dp, -0.47629214130109782_dp, 22.80878975038074685_dp, & - & 1.99999999999999800_dp, 22.72540973598210812_dp, 4.47629214130109521_dp, 22.80878975038074685_dp, & - & 6.94242093509331326_dp, 23.05843251151437201_dp, 9.38829543151647350_dp, 23.47285525677743223_dp, & - & 11.80396120930039316_dp, 24.04961640006232315_dp, 14.17964838758363655_dp, 24.78535803035031648_dp, & - & 16.50579641562137567_dp, 25.67586152120415832_dp, 18.77304947679039060_dp, 26.71611272875198750_dp, & - & 20.97221728412493746_dp, 27.90037270541385439_dp, 23.09419678103115103_dp, 29.22224954465700009_dp, & - & 25.12985059268006793_dp, 30.67476685924402702_dp, 27.06983782140639860_dp, 32.25042441190934994_dp, & - & 28.90439178940930631_dp, 33.94124647006630369_dp, 30.62303747247054275_dp, 35.73881343521246379_dp, & - & 32.21423852322426029_dp, 37.63427208010365632_dp, 33.66495985223924237_dp, 39.61831917500946787_dp, & - & 34.96012666362174315_dp, 41.68115224019865650_dp, 36.08195470333167520_dp, 43.81237943576640248_dp, & - & 37.00911969333093765_dp, 46.00087796034757304_dp, 37.71572773969587189_dp, 48.23458650173873252_dp, & - & 38.17004600591188535_dp, 50.50021197832979425_dp, 38.33296113360039925_dp, 52.78282385843581892_dp, & - & 38.15616674923441565_dp, 55.06530104282313687_dp, 37.58017141013023377_dp, 57.32758829402004608_dp, & - & 36.53242307522427268_dp, 59.54571651024581769_dp, 34.92626834358741661_dp, 61.69055635034530383_dp, & - & 32.66225471925319823_dp, 63.72633555655123416_dp, 29.63456841105838535_dp, 65.60910965714572285_dp, & - & 25.74696599568291333_dp, 67.28571148577093197_dp, 20.94303945184943316_dp, 68.69426490502790728_dp, & - & 15.25145387811901188_dp, 69.76794994271870110_dp, 8.83336207267745088_dp, 70.44359392189112157_dp, & - & 2.00000000000000622_dp, 73.51255823132638056_dp, -6.80643587234800940_dp, 73.21444539538327945_dp, & - & -14.87413258170036379_dp, 72.35214877025040892_dp, -21.72462936947633594_dp, 71.00751554077180572_dp, & - & -27.20553396293055215_dp, 69.28213523998356038_dp, -31.39257932570032139_dp, 67.27265449907143591_dp, & - & -34.46283411458643542_dp, 65.05952649321666570_dp, -36.61111913565843423_dp, 62.70556553816149403_dp, & - & -38.01204327449661236_dp, 60.25870541573158334_dp, -38.80901819915838047_dp, 57.75559962131574565_dp, & - & -39.11518954384546731_dp, 55.22475594928482678_dp, -39.01828312042780311_dp, 52.68892136253921876_dp, & - & -38.58594887503603132_dp, 50.16679739569313057_dp, -37.87042151784342536_dp, 47.67424596736638875_dp, & - & -36.91224672041000332_dp, 45.22513042719018728_dp, -35.74315247327934486_dp, 42.83190113242096686_dp, & - & -34.38823113470312620_dp, 40.50600232791481403_dp, -32.86759479425492714_dp, 38.25815259440047811_dp, & - & -31.19763777135731431_dp, 36.09853399317181299_dp, -29.39200856069424006_dp, 34.03691345955112979_dp ] - lonlat( 961: 1080) = [ & - & -27.46236656888655148_dp, 32.08271233019573287_dp, -25.41897786066869003_dp, 30.24503490367625247_dp, & - & -23.27118821325056430_dp, 28.53266378578365448_dp, -21.02780000980340347_dp, 26.95402789495273055_dp, & - & -18.69737092792362887_dp, 25.51714801636046559_dp, -16.28844626261447814_dp, 24.22956442487017625_dp, & - & -13.80973254683687124_dp, 23.09825113914620331_dp, -11.27021753707410845_dp, 22.12952164603582617_dp, & - & -8.67924035399200910_dp, 21.32893128539883065_dp, -6.04651538477557082_dp, 20.70118176192879744_dp, & - & -3.38211423303386649_dp, 20.25003331964319386_dp, -0.69641128035122979_dp, 19.97822987242789239_dp, & - & 1.99999999999999756_dp, 19.88744176867360025_dp, 4.69641128035122613_dp, 19.97822987242789239_dp, & - & 7.38211423303386205_dp, 20.25003331964319386_dp, 10.04651538477556727_dp, 20.70118176192879744_dp, & - & 12.67924035399200378_dp, 21.32893128539883065_dp, 15.27021753707410667_dp, 22.12952164603582617_dp, & - & 17.80973254683686946_dp, 23.09825113914620331_dp, 20.28844626261447814_dp, 24.22956442487017270_dp, & - & 22.69737092792362532_dp, 25.51714801636046559_dp, 25.02780000980339992_dp, 26.95402789495272700_dp, & - & 27.27118821325056786_dp, 28.53266378578365448_dp, 29.41897786066868292_dp, 30.24503490367624536_dp, & - & 31.46236656888655503_dp, 32.08271233019573287_dp, 33.39200856069423651_dp, 34.03691345955112979_dp, & - & 35.19763777135730720_dp, 36.09853399317180589_dp, 36.86759479425493424_dp, 38.25815259440047811_dp, & - & 38.38823113470313331_dp, 40.50600232791481403_dp, 39.74315247327934486_dp, 42.83190113242095975_dp, & - & 40.91224672041000332_dp, 45.22513042719018728_dp, 41.87042151784342536_dp, 47.67424596736638165_dp, & - & 42.58594887503603132_dp, 50.16679739569313057_dp, 43.01828312042781022_dp, 52.68892136253921876_dp, & - & 43.11518954384546731_dp, 55.22475594928482678_dp, 42.80901819915838757_dp, 57.75559962131573144_dp, & - & 42.01204327449661946_dp, 60.25870541573157624_dp, 40.61111913565844134_dp, 62.70556553816149403_dp, & - & 38.46283411458643542_dp, 65.05952649321666570_dp, 35.39257932570031784_dp, 67.27265449907143591_dp, & - & 31.20553396293055926_dp, 69.28213523998356038_dp, 25.72462936947633949_dp, 71.00751554077180572_dp, & - & 18.87413258170037267_dp, 72.35214877025040892_dp, 10.80643587234802361_dp, 73.21444539538327945_dp, & - & 2.00000000000000844_dp, 76.47602837656471308_dp, -9.67376613190775814_dp, 76.07990221077976400_dp, & - & -19.91726594386999949_dp, 74.95625177955801632_dp, -28.02735732207738550_dp, 73.25702648451857613_dp, & - & -34.02194152863378918_dp, 71.14627956201684356_dp, -38.25028545833758642_dp, 68.75766348420627594_dp, & - & -41.10709162054251209_dp, 66.18773615332521842_dp, -42.92334661203305757_dp, 63.50367340927638793_dp, & - & -43.94927569193657035_dp, 60.75248226909388904_dp, -44.36709840828364548_dp, 57.96800639328736793_dp, & - & -44.30826862884845241_dp, 55.17560336970880286_dp, -43.86791757533286074_dp, 52.39514197266055362_dp, & - & -43.11549450115907689_dp, 49.64291142290011294_dp, -42.10226597817306526_dp, 46.93284831506371546_dp, & - & -40.86653894639825779_dp, 44.27733603455973110_dp, -39.43731045663169965_dp, 41.68773239540036002_dp ] - lonlat( 1081: 1200) = [ & - & -37.83684688715485578_dp, 39.17472027603687934_dp, -36.08253581911941410_dp, 36.74853923728758076_dp, & - & -34.18824090805679816_dp, 34.41913389699746517_dp, -32.16531367916817885_dp, 32.19624133969532664_dp, & - & -30.02336509370288198_dp, 30.08943159705432890_dp, -27.77086550998262382_dp, 28.10811023218007776_dp, & - & -25.41561850057673411_dp, 26.26148912292403992_dp, -22.96513811238918379_dp, 24.55852997196868515_dp, & - & -20.42694818201796281_dp, 23.00786444863284785_dp, -17.80881477377849720_dp, 21.61769490666709714_dp, & - & -15.11891780975726896_dp, 20.39568010883170501_dp, -12.36596498789072740_dp, 19.34881113120014717_dp, & - & -9.55924980839402672_dp, 18.48328342761256593_dp, -6.70865569654629645_dp, 17.80437171431731613_dp, & - & -3.82460955068694286_dp, 17.31631470128239414_dp, -0.91799021933643277_dp, 17.02221659443388191_dp, & - & 1.99999999999999756_dp, 16.92397162343524286_dp, 4.91799021933642866_dp, 17.02221659443388191_dp, & - & 7.82460955068693664_dp, 17.31631470128239414_dp, 10.70865569654629290_dp, 17.80437171431731613_dp, & - & 13.55924980839402139_dp, 18.48328342761256593_dp, 16.36596498789072740_dp, 19.34881113120014717_dp, & - & 19.11891780975726363_dp, 20.39568010883170501_dp, 21.80881477377850075_dp, 21.61769490666709714_dp, & - & 24.42694818201795925_dp, 23.00786444863284430_dp, 26.96513811238918024_dp, 24.55852997196867804_dp, & - & 29.41561850057672345_dp, 26.26148912292403637_dp, 31.77086550998261671_dp, 28.10811023218007065_dp, & - & 34.02336509370287132_dp, 30.08943159705432180_dp, 36.16531367916817885_dp, 32.19624133969532664_dp, & - & 38.18824090805679106_dp, 34.41913389699745807_dp, 40.08253581911941410_dp, 36.74853923728758076_dp, & - & 41.83684688715484867_dp, 39.17472027603687934_dp, 43.43731045663169255_dp, 41.68773239540036002_dp, & - & 44.86653894639826490_dp, 44.27733603455973110_dp, 46.10226597817306526_dp, 46.93284831506371546_dp, & - & 47.11549450115907689_dp, 49.64291142290011294_dp, 47.86791757533286784_dp, 52.39514197266055362_dp, & - & 48.30826862884845951_dp, 55.17560336970880286_dp, 48.36709840828365259_dp, 57.96800639328736793_dp, & - & 47.94927569193657746_dp, 60.75248226909387483_dp, 46.92334661203305757_dp, 63.50367340927638793_dp, & - & 45.10709162054252630_dp, 66.18773615332521842_dp, 42.25028545833760063_dp, 68.75766348420627594_dp, & - & 38.02194152863379628_dp, 71.14627956201684356_dp, 32.02735732207738550_dp, 73.25702648451857613_dp, & - & 23.91726594387000659_dp, 74.95625177955801632_dp, 13.67376613190777590_dp, 76.07990221077976400_dp, & - & 2.00000000000001155_dp, 79.58407807962322522_dp, -14.24038894313819092_dp, 79.03039199474196153_dp, & - & -27.34759503822197502_dp, 77.51867325981696411_dp, -36.54693285975560713_dp, 75.34972495190746145_dp, & - & -42.57629367046713753_dp, 72.78039612779681988_dp, -46.37308749001456221_dp, 69.97672254379963874_dp, & - & -48.64383991815974895_dp, 67.03853932314146391_dp, -49.85773173662808233_dp, 64.02688869849654907_dp, & - & -50.31863539857851464_dp, 60.98111330635318694_dp, -50.22511448054331140_dp, 57.92830381592441569_dp, & - & -49.70950811496560107_dp, 54.88845672164920586_dp, -48.86197695521289575_dp, 51.87735585154322138_dp ] - lonlat( 1201: 1320) = [ & - & -47.74524369263553325_dp, 48.90823812634897649_dp, -46.40377244059971673_dp, 45.99279145534946167_dp, & - & -44.86961926381133026_dp, 43.14177213432515856_dp, -43.16626004327478228_dp, 40.36539642075791789_dp, & - & -41.31116510791174079_dp, 37.67359181604234664_dp, -39.31758136998610098_dp, 35.07615645611910082_dp, & - & -37.19580324741198041_dp, 32.58285444844916867_dp, -34.95410730493897233_dp, 30.20346328869343466_dp, & - & -32.59946098845734497_dp, 27.94778271439298933_dp, -30.13807554901531915_dp, 25.82561045671564770_dp, & - & -27.57584735879238025_dp, 23.84668824688734645_dp, -24.91871466466540852_dp, 22.02062053097340311_dp, & - & -22.17294518233925871_dp, 20.35676830949395466_dp, -19.34536199373425802_dp, 18.86412113035556004_dp, & - & -16.44350999615492270_dp, 17.55115134744557182_dp, -13.47576217013061850_dp, 16.42565612762181360_dp, & - & -10.45136393475438830_dp, 15.49459412897927280_dp, -7.38041466390653600_dp, 14.76392503362292707_dp, & - & -4.27378779825515664_dp, 14.23846093632396936_dp, -1.14299451095196525_dp, 13.92173872937387458_dp, & - & 1.99999999999999756_dp, 13.81592192037676980_dp, 5.14299451095195970_dp, 13.92173872937387458_dp, & - & 8.27378779825515132_dp, 14.23846093632396936_dp, 11.38041466390652978_dp, 14.76392503362292707_dp, & - & 14.45136393475438119_dp, 15.49459412897927280_dp, 17.47576217013061495_dp, 16.42565612762181360_dp, & - & 20.44350999615491560_dp, 17.55115134744557182_dp, 23.34536199373425447_dp, 18.86412113035556004_dp, & - & 26.17294518233925515_dp, 20.35676830949395466_dp, 28.91871466466540852_dp, 22.02062053097340311_dp, & - & 31.57584735879237314_dp, 23.84668824688734645_dp, 34.13807554901531205_dp, 25.82561045671564415_dp, & - & 36.59946098845733786_dp, 27.94778271439298933_dp, 38.95410730493897233_dp, 30.20346328869342756_dp, & - & 41.19580324741196620_dp, 32.58285444844916867_dp, 43.31758136998610098_dp, 35.07615645611910082_dp, & - & 45.31116510791174079_dp, 37.67359181604234664_dp, 47.16626004327477517_dp, 40.36539642075791789_dp, & - & 48.86961926381131605_dp, 43.14177213432515856_dp, 50.40377244059970963_dp, 45.99279145534945457_dp, & - & 51.74524369263552614_dp, 48.90823812634896228_dp, 52.86197695521290285_dp, 51.87735585154322138_dp, & - & 53.70950811496559396_dp, 54.88845672164920586_dp, 54.22511448054331140_dp, 57.92830381592441569_dp, & - & 54.31863539857851464_dp, 60.98111330635318694_dp, 53.85773173662808233_dp, 64.02688869849654907_dp, & - & 52.64383991815974895_dp, 67.03853932314146391_dp, 50.37308749001457642_dp, 69.97672254379963874_dp, & - & 46.57629367046714464_dp, 72.78039612779681988_dp, 40.54693285975561423_dp, 75.34972495190746145_dp, & - & 31.34759503822198212_dp, 77.51867325981696411_dp, 18.24038894313820691_dp, 79.03039199474196153_dp, & - & 2.00000000000001998_dp, 82.85826214470431239_dp, -22.58690897303193523_dp, 82.01033616765489853_dp, & - & -38.96575654499159924_dp, 79.88802174923017674_dp, -48.17735288037921038_dp, 77.11452529323827321_dp, & - & -53.18593275925196195_dp, 74.03707888623914357_dp, -55.81801935702030448_dp, 70.81486693953937106_dp, & - & -57.03453256694833584_dp, 67.52503657439315532_dp, -57.35114187047393841_dp, 64.20957103321012482_dp ] - lonlat( 1321: 1440) = [ & - & -57.05957379451162126_dp, 60.89430451319173443_dp, -56.33378767939932885_dp, 57.59709162031844443_dp, & - & -55.28230228211860009_dp, 54.33161333463023368_dp, -53.97533567047437231_dp, 51.10929358481432416_dp, & - & -52.45964870701570248_dp, 47.94033352757975308_dp, -50.76707508825830928_dp, 44.83430401955754974_dp, & - & -48.91964996438991875_dp, 41.80050199367043007_dp, -46.93282539675249154_dp, 38.84817223653459450_dp, & - & -44.81756887288366897_dp, 35.98664677769215103_dp, -42.58178882927894193_dp, 33.22542941501406233_dp, & - & -40.23134383513492196_dp, 30.57423988556289629_dp, -37.77078829775713587_dp, 28.04302503644353806_dp, & - & -35.20394762734520100_dp, 25.64194033091114022_dp, -32.53437968696509586_dp, 23.38130282889364508_dp, & - & -29.76575655354334771_dp, 21.27151577227776258_dp, -26.90218550361062810_dp, 19.32296477964254322_dp, & - & -23.94847769357362566_dp, 17.54588626406437513_dp, -20.91036566952359976_dp, 15.95020993686824795_dp, & - & -17.79466590782826074_dp, 14.54537904344352306_dp, -14.60937983520193661_dp, 13.34015413091371371_dp, & - & -11.36372621835939434_dp, 12.34240842556757833_dp, -8.06809948196382720_dp, 11.55892497220584580_dp, & - & -4.73395228112668587_dp, 10.99520717814031734_dp, -1.37360609917944543_dp, 10.65531494200142504_dp, & - & 1.99999999999999756_dp, 10.54173785529561336_dp, 5.37360609917944032_dp, 10.65531494200142504_dp, & - & 8.73395228112667965_dp, 10.99520717814031734_dp, 12.06809948196382187_dp, 11.55892497220584580_dp, & - & 15.36372621835939078_dp, 12.34240842556757833_dp, 18.60937983520193484_dp, 13.34015413091371371_dp, & - & 21.79466590782825364_dp, 14.54537904344352306_dp, 24.91036566952359621_dp, 15.95020993686824795_dp, & - & 27.94847769357362210_dp, 17.54588626406437513_dp, 30.90218550361061745_dp, 19.32296477964254322_dp, & - & 33.76575655354334060_dp, 21.27151577227776258_dp, 36.53437968696508875_dp, 23.38130282889363798_dp, & - & 39.20394762734519389_dp, 25.64194033091114022_dp, 41.77078829775712165_dp, 28.04302503644353095_dp, & - & 44.23134383513491485_dp, 30.57423988556289629_dp, 46.58178882927893483_dp, 33.22542941501406233_dp, & - & 48.81756887288366187_dp, 35.98664677769215103_dp, 50.93282539675248444_dp, 38.84817223653458029_dp, & - & 52.91964996438991164_dp, 41.80050199367041586_dp, 54.76707508825830928_dp, 44.83430401955754263_dp, & - & 56.45964870701569538_dp, 47.94033352757973887_dp, 57.97533567047437231_dp, 51.10929358481431706_dp, & - & 59.28230228211859298_dp, 54.33161333463023368_dp, 60.33378767939932885_dp, 57.59709162031844443_dp, & - & 61.05957379451161415_dp, 60.89430451319173443_dp, 61.35114187047394552_dp, 64.20957103321012482_dp, & - & 61.03453256694833584_dp, 67.52503657439315532_dp, 59.81801935702030448_dp, 70.81486693953937106_dp, & - & 57.18593275925196906_dp, 74.03707888623914357_dp, 52.17735288037921038_dp, 77.11452529323827321_dp, & - & 42.96575654499162056_dp, 79.88802174923017674_dp, 26.58690897303196721_dp, 82.01033616765489853_dp, & - & 2.00000000000004352_dp, 86.32305404620944955_dp, -41.28583165458979920_dp, 84.76914638163376026_dp, & - & -57.58131858106001744_dp, 81.70467639663135628_dp, -63.56415050352948981_dp, 78.26774990878784877_dp ] - lonlat( 1441: 1560) = [ & - & -65.82714145548160900_dp, 74.71983727667711150_dp, -66.41516373728856593_dp, 71.13755089586129543_dp, & - & -66.11115128919476547_dp, 67.55274433031233627_dp, -65.27043361461275595_dp, 63.98289851135272244_dp, & - & -64.07470870335011170_dp, 60.43992354246546483_dp, -62.62500642980751309_dp, 56.93332642202356197_dp, & - & -60.98089250042357889_dp, 53.47155025248752480_dp, -59.17889520468775544_dp, 50.06261728049957327_dp, & - & -57.24193904387428233_dp, 46.71447239476329827_dp, -55.18452370168171939_dp, 43.43518400478754415_dp, & - & -53.01574257648425004_dp, 40.23307020728445593_dp, -50.74114071516557800_dp, 37.11678138456306897_dp, & - & -48.36392121341754802_dp, 34.09535373993877982_dp, -45.88577372237665486_dp, 31.17824007995576707_dp, & - & -43.30747895739605013_dp, 28.37531978094453677_dp, -40.62937881000767959_dp, 25.69688738457191945_dp, & - & -37.85176517580522670_dp, 23.15361781597290758_dp, -34.97521854780547557_dp, 20.75650546647104377_dp, & - & -32.00091302933286386_dp, 18.51677422677156670_dp, -28.93089420696503922_dp, 16.44575600838861718_dp, & - & -25.76832854644280602_dp, 14.55473640862136619_dp, -22.51771689758312078_dp, 12.85476800444173051_dp, & - & -19.18506021909563941_dp, 11.35645428598744111_dp, -15.77796305903055440_dp, 10.06971032975658353_dp, & - & -12.30566013367807265_dp, 9.00350968307662569_dp, -8.77895396829167218_dp, 8.16563014493928918_dp, & - & -5.21005712455211789_dp, 7.56241362477411272_dp, -1.61234064761579554_dp, 7.19855645524003407_dp, & - & 1.99999999999999734_dp, 7.07694595379053126_dp, 5.61234064761579088_dp, 7.19855645524003407_dp, & - & 9.21005712455211167_dp, 7.56241362477411272_dp, 12.77895396829166685_dp, 8.16563014493928918_dp, & - & 16.30566013367807088_dp, 9.00350968307662569_dp, 19.77796305903055085_dp, 10.06971032975658353_dp, & - & 23.18506021909563231_dp, 11.35645428598744111_dp, 26.51771689758312078_dp, 12.85476800444173051_dp, & - & 29.76832854644280246_dp, 14.55473640862136619_dp, 32.93089420696504277_dp, 16.44575600838861718_dp, & - & 36.00091302933286386_dp, 18.51677422677156670_dp, 38.97521854780546846_dp, 20.75650546647104022_dp, & - & 41.85176517580521960_dp, 23.15361781597290403_dp, 44.62937881000767248_dp, 25.69688738457191590_dp, & - & 47.30747895739604303_dp, 28.37531978094453322_dp, 49.88577372237665486_dp, 31.17824007995575997_dp, & - & 52.36392121341754091_dp, 34.09535373993877982_dp, 54.74114071516557800_dp, 37.11678138456306897_dp, & - & 57.01574257648425004_dp, 40.23307020728444883_dp, 59.18452370168172649_dp, 43.43518400478754415_dp, & - & 61.24193904387428944_dp, 46.71447239476329827_dp, 63.17889520468774833_dp, 50.06261728049956616_dp, & - & 64.98089250042356468_dp, 53.47155025248752480_dp, 66.62500642980749888_dp, 56.93332642202356197_dp, & - & 68.07470870335012592_dp, 60.43992354246546483_dp, 69.27043361461275595_dp, 63.98289851135272244_dp, & - & 70.11115128919477968_dp, 67.55274433031233627_dp, 70.41516373728855172_dp, 71.13755089586129543_dp, & - & 69.82714145548160900_dp, 74.71983727667711150_dp, 67.56415050352947560_dp, 78.26774990878784877_dp, & - & 61.58131858106001744_dp, 81.70467639663135628_dp, 45.28583165458984183_dp, 84.76914638163376026_dp ] - lonlat( 1561: 1680) = [ & - & -178.00000000002495426_dp, 89.99364961382421768_dp, -86.04670840724655534_dp, 86.14286053253547948_dp, & - & -83.94730489861873934_dp, 82.29066165213360762_dp, -81.87003129087820241_dp, 78.44836496333867615_dp, & - & -79.78661098119418682_dp, 74.62102007313750107_dp, -77.68757690134611948_dp, 70.81379178736982283_dp, & - & -75.56653776770212971_dp, 67.03200075627314902_dp, -73.41791965045068480_dp, 63.28116415502336878_dp, & - & -71.23639478559491067_dp, 59.56703711552955127_dp, -69.01668877482984499_dp, 55.89565494125256606_dp, & - & -66.75350545455997064_dp, 52.27337593848611164_dp, -64.44149951448400770_dp, 48.70692450151241104_dp, & - & -62.07527488350018530_dp, 45.20343384586799118_dp, -59.64940188352277772_dp, 41.77048747717444854_dp, & - & -57.15845171105518574_dp, 38.41615810166682365_dp, -54.59704918478396252_dp, 35.14904222145087687_dp, & - & -51.95994579953059400_dp, 31.97828811153562256_dp, -49.24211556220075892_dp, 28.91361425616476666_dp, & - & -46.43887601207384819_dp, 25.96531465470105005_dp, -43.54603622089217652_dp, 23.14424674184217068_dp, & - & -40.56007232189229228_dp, 20.46179708442632972_dp, -37.47832910130670570_dp, 17.92981963683196156_dp, & - & -34.29924330014637945_dp, 15.56054131855884570_dp, -31.02258051120933402_dp, 13.36643021383323848_dp, & - & -27.64967308093982012_dp, 11.36002298851478542_dp, -24.18364165385173337_dp, 9.55371034930798224_dp, & - & -20.62957864939022201_dp, 7.95948261839810733_dp, -16.99466904928961242_dp, 6.58864168091602309_dp, & - & -13.28822357553577405_dp, 5.45149036402020837_dp, -9.52160273595813855_dp, 4.55701513369246669_dp, & - & -5.70801791517022217_dp, 3.91258202080964690_dp, -1.86220744039441199_dp, 3.52366797872773274_dp, & - & 1.99999999999999734_dp, 3.39364961385021413_dp, 5.86220744039440689_dp, 3.52366797872773274_dp, & - & 9.70801791517021861_dp, 3.91258202080964690_dp, 13.52160273595813145_dp, 4.55701513369246669_dp, & - & 17.28822357553576694_dp, 5.45149036402020837_dp, 20.99466904928960886_dp, 6.58864168091602309_dp, & - & 24.62957864939021491_dp, 7.95948261839810733_dp, 28.18364165385172981_dp, 9.55371034930798224_dp, & - & 31.64967308093981302_dp, 11.36002298851478542_dp, 35.02258051120933402_dp, 13.36643021383323848_dp, & - & 38.29924330014637235_dp, 15.56054131855884570_dp, 41.47832910130669859_dp, 17.92981963683195801_dp, & - & 44.56007232189227807_dp, 20.46179708442632261_dp, 47.54603622089216231_dp, 23.14424674184217068_dp, & - & 50.43887601207384819_dp, 25.96531465470104649_dp, 53.24211556220075892_dp, 28.91361425616475955_dp, & - & 55.95994579953058690_dp, 31.97828811153562256_dp, 58.59704918478396252_dp, 35.14904222145087687_dp, & - & 61.15845171105517863_dp, 38.41615810166681655_dp, 63.64940188352278483_dp, 41.77048747717444854_dp, & - & 66.07527488350018530_dp, 45.20343384586799118_dp, 68.44149951448399349_dp, 48.70692450151241104_dp, & - & 70.75350545455997064_dp, 52.27337593848611164_dp, 73.01668877482984499_dp, 55.89565494125256606_dp, & - & 75.23639478559491067_dp, 59.56703711552955127_dp, 77.41791965045068480_dp, 63.28116415502336878_dp, & - & 79.56653776770211550_dp, 67.03200075627314902_dp, 81.68757690134611948_dp, 70.81379178736982283_dp ] - lonlat( 1681: 1800) = [ & - & 83.78661098119418682_dp, 74.62102007313750107_dp, 85.87003129087820241_dp, 78.44836496333867615_dp, & - & 87.94730489861875355_dp, 82.29066165213360762_dp, 90.04670840724658376_dp, 86.14286053253547948_dp, & - & -178.00000000000005684_dp, 86.05996156130565566_dp, -130.58473801020662108_dp, 84.39088722277173815_dp, & - & -110.19717929614465390_dp, 81.10053146183230410_dp, -100.10346048738929881_dp, 77.40995108179684792_dp, & - & -93.69886509454548218_dp, 73.59868574092435267_dp, -88.93011750021230455_dp, 69.74815881341463353_dp, & - & -85.00534896567366161_dp, 65.89187266757556927_dp, -81.55981529780905248_dp, 62.04795929474798299_dp, & - & -78.40205491898922219_dp, 58.22861013398448193_dp, -75.42099130058548440_dp, 54.44347433172315220_dp, & - & -72.54676554128549526_dp, 50.70110112272598712_dp, -69.73234087330638431_dp, 47.00963897352001908_dp, & - & -66.94410839385282941_dp, 43.37721635788606989_dp, -64.15676435641834985_dp, 39.81217237539897269_dp, & - & -61.35037195723166548_dp, 36.32321005821970061_dp, -58.50861449014277582_dp, 32.91950572923986584_dp, & - & -55.61773745881250619_dp, 29.61078975921034129_dp, -52.66591251909912330_dp, 26.40740499964346810_dp, & - & -49.64287529119710030_dp, 23.32034407821754840_dp, -46.53975201683548590_dp, 20.36126354510683711_dp, & - & -43.34902402209520034_dp, 17.54247066454981407_dp, -40.06459700293881809_dp, 14.87687712846902599_dp, & - & -36.68195074563113423_dp, 12.37791309905906445_dp, -33.19834740797973893_dp, 10.05939491571369260_dp, & - & -29.61307490596015057_dp, 7.93534078498541273_dp, -25.92769777693465372_dp, 6.01973109672258389_dp, & - & -22.14628266758476371_dp, 4.32621388981253130_dp, -18.27556116130716646_dp, 2.86776143955401652_dp, & - & -14.32499114491301384_dp, 1.65629062554436479_dp, -10.30668142622437244_dp, 0.70226688878528098_dp, & - & -6.23515434659414680_dp, 0.01431798130227213_dp, -2.12693783758673716_dp, -0.40111214042971233_dp, & - & 1.99999999999999689_dp, -0.54003843869414525_dp, 6.12693783758673050_dp, -0.40111214042971233_dp, & - & 10.23515434659414147_dp, 0.01431798130227213_dp, 14.30668142622436712_dp, 0.70226688878528098_dp, & - & 18.32499114491300318_dp, 1.65629062554436479_dp, 22.27556116130715580_dp, 2.86776143955401652_dp, & - & 26.14628266758476016_dp, 4.32621388981253130_dp, 29.92769777693464306_dp, 6.01973109672258389_dp, & - & 33.61307490596014702_dp, 7.93534078498541273_dp, 37.19834740797973183_dp, 10.05939491571369260_dp, & - & 40.68195074563112001_dp, 12.37791309905905912_dp, 44.06459700293881809_dp, 14.87687712846902421_dp, & - & 47.34902402209520034_dp, 17.54247066454981052_dp, 50.53975201683548590_dp, 20.36126354510683711_dp, & - & 53.64287529119709319_dp, 23.32034407821754485_dp, 56.66591251909911620_dp, 26.40740499964346455_dp, & - & 59.61773745881250619_dp, 29.61078975921033418_dp, 62.50861449014276872_dp, 32.91950572923986584_dp, & - & 65.35037195723165837_dp, 36.32321005821969351_dp, 68.15676435641833564_dp, 39.81217237539895848_dp, & - & 70.94410839385281520_dp, 43.37721635788606989_dp, 73.73234087330637010_dp, 47.00963897352000487_dp, & - & 76.54676554128549526_dp, 50.70110112272598002_dp, 79.42099130058548440_dp, 54.44347433172315220_dp ] - lonlat( 1801: 1920) = [ & - & 82.40205491898922219_dp, 58.22861013398448193_dp, 85.55981529780903827_dp, 62.04795929474798299_dp, & - & 89.00534896567364740_dp, 65.89187266757556927_dp, 92.93011750021230455_dp, 69.74815881341463353_dp, & - & 97.69886509454546797_dp, 73.59868574092435267_dp, 104.10346048738931302_dp, 77.40995108179684792_dp, & - & 114.19717929614462548_dp, 81.10053146183230410_dp, 134.58473801020659266_dp, 84.39088722277173815_dp, & - & -178.00000000000002842_dp, 81.83938189765240168_dp, -149.15668933029422760_dp, 80.85560324742941418_dp, & - & -128.64763601333984866_dp, 78.39753950634394641_dp, -115.34882263320223217_dp, 75.18923730101467129_dp, & - & -106.23509874248557594_dp, 71.62987479878029262_dp, -99.45761459629794388_dp, 67.90061168070624831_dp, & - & -94.04421718085441739_dp, 64.08831694761892095_dp, -89.47013935832345055_dp, 60.23965935256792648_dp, & - & -85.43460651588344490_dp, 56.38292751246400769_dp, -81.75424365182693975_dp, 52.53738247574246856_dp, & - & -78.31071017117777444_dp, 48.71761719224470966_dp, -75.02360369831293951_dp, 44.93576201889581512_dp, & - & -71.83567948061782715_dp, 41.20268880474466755_dp, -68.70439458886511375_dp, 37.52871692222674938_dp, & - & -65.59687011372152199_dp, 33.92405612452550656_dp, -62.48679120890344052_dp, 30.39910192691253599_dp, & - & -59.35245736607464551_dp, 26.96464262473804041_dp, -56.17554747078253996_dp, 23.63200834498021408_dp, & - & -52.94035098918124760_dp, 20.41317688045037926_dp, -49.63331929525543984_dp, 17.32084182287969298_dp, & - & -46.24284889074778704_dp, 14.36844259642658983_dp, -42.75924066894448572_dp, 11.57015196980180605_dp, & - & -39.17479644658766347_dp, 8.94081392759362537_dp, -35.48402100288603123_dp, 6.49582334126627892_dp, & - & -31.68389780238809195_dp, 4.25093893994812166_dp, -27.77420154083882053_dp, 2.22202305439570358_dp, & - & -23.75780275269854158_dp, 0.42470591907018640_dp, -19.64091162595336826_dp, -1.12602081331554005_dp, & - & -15.43320327160518524_dp, -2.41626750988024330_dp, -11.14776873556009384_dp, -3.43374257581262743_dp, & - & -6.80084816537008940_dp, -4.16824654595516098_dp, -2.41132592404073165_dp, -4.61212239418478553_dp, & - & 1.99999999999999689_dp, -4.76061810234754024_dp, 6.41132592404072632_dp, -4.61212239418478553_dp, & - & 10.80084816537008230_dp, -4.16824654595516098_dp, 15.14776873556009029_dp, -3.43374257581262743_dp, & - & 19.43320327160517991_dp, -2.41626750988024330_dp, 23.64091162595336115_dp, -1.12602081331554005_dp, & - & 27.75780275269853803_dp, 0.42470591907018640_dp, 31.77420154083881698_dp, 2.22202305439570358_dp, & - & 35.68389780238808129_dp, 4.25093893994812166_dp, 39.48402100288601702_dp, 6.49582334126627270_dp, & - & 43.17479644658765636_dp, 8.94081392759361826_dp, 46.75924066894447861_dp, 11.57015196980179894_dp, & - & 50.24284889074777283_dp, 14.36844259642658628_dp, 53.63331929525543273_dp, 17.32084182287968943_dp, & - & 56.94035098918124049_dp, 20.41317688045037571_dp, 60.17554747078253286_dp, 23.63200834498020697_dp, & - & 63.35245736607463840_dp, 26.96464262473803331_dp, 66.48679120890342631_dp, 30.39910192691252178_dp, & - & 69.59687011372152199_dp, 33.92405612452549946_dp, 72.70439458886511375_dp, 37.52871692222674227_dp ] - lonlat( 1921: 2040) = [ & - & 75.83567948061782715_dp, 41.20268880474466755_dp, 79.02360369831293951_dp, 44.93576201889580091_dp, & - & 82.31071017117776023_dp, 48.71761719224470255_dp, 85.75424365182693975_dp, 52.53738247574246145_dp, & - & 89.43460651588344490_dp, 56.38292751246400769_dp, 93.47013935832345055_dp, 60.23965935256792648_dp, & - & 98.04421718085441739_dp, 64.08831694761892095_dp, 103.45761459629794388_dp, 67.90061168070624831_dp, & - & 110.23509874248557594_dp, 71.62987479878029262_dp, 119.34882263320223217_dp, 75.18923730101467129_dp, & - & 132.64763601333982024_dp, 78.39753950634394641_dp, 153.15668933029419918_dp, 80.85560324742941418_dp, & - & -178.00000000000002842_dp, 77.29014606148251687_dp, -157.47781378002883912_dp, 76.59572545663588983_dp, & - & -140.18467631053607647_dp, 74.70285264202438213_dp, -126.87442892417496410_dp, 71.99165187875776439_dp, & - & -116.75095597801099245_dp, 68.78228497629521598_dp, -108.83745587761724494_dp, 65.27873175231235336_dp, & - & -102.40541928769302160_dp, 61.60190913800556700_dp, -96.97173315822051620_dp, 57.82466562901673512_dp, & - & -92.22157019495104180_dp, 53.99321139219064491_dp, -87.94629406419838347_dp, 50.13884820321026581_dp, & - & -84.00357763838492531_dp, 46.28434439562811065_dp, -80.29307189810650414_dp, 42.44749809608728697_dp, & - & -76.74158359961857911_dp, 38.64321178012773572_dp, -73.29390254081907585_dp, 34.88475663106463287_dp, & - & -69.90700610430916129_dp, 31.18458087158649406_dp, -66.54632400302145356_dp, 27.55485191826680591_dp, & - & -63.18329478816111333_dp, 24.00783659504740086_dp, -59.79375952890369916_dp, 20.55617726580612015_dp, & - & -56.35691981311008192_dp, 17.21309536720795919_dp, -52.85469423510679832_dp, 13.99253787292509621_dp, & - & -49.27137129744098587_dp, 10.90927185095380558_dp, -45.59349430020986915_dp, 7.97892504367260802_dp, & - & -41.80993456941939712_dp, 5.21796518077428839_dp, -37.91211819209929246_dp, 2.64360721178194558_dp, & - & -33.89437094593881739_dp, 0.27363605581641759_dp, -29.75433817509684786_dp, -1.87386656696368226_dp, & - & -25.49342310484703589_dp, -3.78089833576525525_dp, -21.11717183125961128_dp, -5.43001433583734983_dp, & - & -16.63552097548324937_dp, -6.80489321395614066_dp, -12.06282113870962114_dp, -7.89096231408011839_dp, & - & -7.41756212661334047_dp, -8.67603648837036268_dp, -2.72175805422022021_dp, -9.15091230991673399_dp, & - & 1.99999999999999645_dp, -9.30985393851741172_dp, 6.72175805422021444_dp, -9.15091230991673399_dp, & - & 11.41756212661333691_dp, -8.67603648837036268_dp, 16.06282113870961581_dp, -7.89096231408011839_dp, & - & 20.63552097548324582_dp, -6.80489321395614066_dp, 25.11717183125960418_dp, -5.43001433583734983_dp, & - & 29.49342310484702523_dp, -3.78089833576525525_dp, 33.75433817509684786_dp, -1.87386656696368226_dp, & - & 37.89437094593881028_dp, 0.27363605581641121_dp, 41.91211819209927114_dp, 2.64360721178193936_dp, & - & 45.80993456941938291_dp, 5.21796518077428217_dp, 49.59349430020984784_dp, 7.97892504367260180_dp, & - & 53.27137129744098587_dp, 10.90927185095380203_dp, 56.85469423510678411_dp, 13.99253787292509266_dp, & - & 60.35691981311008192_dp, 17.21309536720795563_dp, 63.79375952890369206_dp, 20.55617726580611659_dp ] - lonlat( 2041: 2160) = [ & - & 67.18329478816110623_dp, 24.00783659504739376_dp, 70.54632400302143935_dp, 27.55485191826679880_dp, & - & 73.90700610430916129_dp, 31.18458087158649050_dp, 77.29390254081907585_dp, 34.88475663106463287_dp, & - & 80.74158359961856490_dp, 38.64321178012773572_dp, 84.29307189810650414_dp, 42.44749809608728697_dp, & - & 88.00357763838491110_dp, 46.28434439562810354_dp, 91.94629406419836926_dp, 50.13884820321025160_dp, & - & 96.22157019495104180_dp, 53.99321139219064491_dp, 100.97173315822050199_dp, 57.82466562901673512_dp, & - & 106.40541928769302160_dp, 61.60190913800556700_dp, 112.83745587761723073_dp, 65.27873175231235336_dp, & - & 120.75095597801100666_dp, 68.78228497629521598_dp, 130.87442892417493567_dp, 71.99165187875776439_dp, & - & 144.18467631053604805_dp, 74.70285264202438213_dp, 161.47781378002881070_dp, 76.59572545663588983_dp, & - & -178.00000000000002842_dp, 72.36459406509962378_dp, -162.05644118521095720_dp, 71.82662636485281382_dp, & - & -147.60742176334812825_dp, 70.30250402254888797_dp, -135.36255278451204731_dp, 68.00113945880528377_dp, & - & -125.26976638159491984_dp, 65.14478524125108549_dp, -116.94158660124730886_dp, 61.91142197433249095_dp, & - & -109.95470750710923369_dp, 58.42738157410047961_dp, -103.95825647893495614_dp, 54.77900423463843538_dp, & - & -98.68725830824647005_dp, 51.02562354013957702_dp, -93.94726867835278483_dp, 47.20919167069192923_dp, & - & -89.59567144235911940_dp, 43.36062114047545890_dp, -85.52649044264862255_dp, 39.50382677214758331_dp, & - & -81.65938939021744147_dp, 35.65830698143167155_dp, -77.93201512996090230_dp, 31.84082402428231973_dp, & - & -74.29472888122153051_dp, 28.06652943785655552_dp, -70.70698212428176532_dp, 24.34974403770184992_dp, & - & -67.13481958525987636_dp, 20.70451849248450316_dp, -63.54916484650964748_dp, 17.14505004500528074_dp, & - & -59.92466443809041010_dp, 13.68599977255222377_dp, -56.23894695811429756_dp, 10.34273467530951152_dp, & - & -52.47220691225792422_dp, 7.13150506907927006_dp, -48.60705694133756083_dp, 4.06955754229704603_dp, & - & -44.62861191461713162_dp, 1.17517574056815666_dp, -40.52477649322946007_dp, -1.53236497867592147_dp, & - & -36.28670514160292981_dp, -4.03294710811187507_dp, -31.90939060170597941_dp, -6.30588392466837533_dp, & - & -27.39231472226267528_dp, -8.33033151194295662_dp, -22.74006765897273397_dp, -10.08584610023313921_dp, & - & -17.96281473836326725_dp, -11.55307649365679090_dp, -13.07647532339788654_dp, -12.71455858250937254_dp, & - & -8.10248720394762678_dp, -13.55555303790299426_dp, -3.06707279862612969_dp, -14.06484423228424063_dp, & - & 1.99999999999999600_dp, -14.23540593490034212_dp, 7.06707279862612125_dp, -14.06484423228424063_dp, & - & 12.10248720394761790_dp, -13.55555303790299426_dp, 17.07647532339788299_dp, -12.71455858250937254_dp, & - & 21.96281473836326015_dp, -11.55307649365679090_dp, 26.74006765897273041_dp, -10.08584610023313921_dp, & - & 31.39231472226266817_dp, -8.33033151194295662_dp, 35.90939060170597230_dp, -6.30588392466837533_dp, & - & 40.28670514160292271_dp, -4.03294710811188128_dp, 44.52477649322944586_dp, -1.53236497867592791_dp, & - & 48.62861191461712451_dp, 1.17517574056815022_dp, 52.60705694133754662_dp, 4.06955754229703981_dp ] - lonlat( 2161: 2280) = [ & - & 56.47220691225791711_dp, 7.13150506907926740_dp, 60.23894695811429756_dp, 10.34273467530950796_dp, & - & 63.92466443809040300_dp, 13.68599977255221667_dp, 67.54916484650964037_dp, 17.14505004500527363_dp, & - & 71.13481958525987636_dp, 20.70451849248449605_dp, 74.70698212428177953_dp, 24.34974403770184992_dp, & - & 78.29472888122153051_dp, 28.06652943785655197_dp, 81.93201512996088809_dp, 31.84082402428231262_dp, & - & 85.65938939021744147_dp, 35.65830698143166444_dp, 89.52649044264860834_dp, 39.50382677214758331_dp, & - & 93.59567144235911940_dp, 43.36062114047545180_dp, 97.94726867835277062_dp, 47.20919167069192213_dp, & - & 102.68725830824645584_dp, 51.02562354013956281_dp, 107.95825647893494192_dp, 54.77900423463843538_dp, & - & 113.95470750710921948_dp, 58.42738157410047961_dp, 120.94158660124729465_dp, 61.91142197433249095_dp, & - & 129.26976638159487720_dp, 65.14478524125108549_dp, 139.36255278451201889_dp, 68.00113945880528377_dp, & - & 151.60742176334809983_dp, 70.30250402254888797_dp, 166.05644118521092878_dp, 71.82662636485281382_dp, & - & -178.00000000000002842_dp, 67.00865712643569339_dp, -164.95645291666266985_dp, 66.56908368553206401_dp, & - & -152.69437932769912436_dp, 65.29863962109095610_dp, -141.70578215046154469_dp, 63.31961305621509695_dp, & - & -132.12712761110685733_dp, 60.78156430751474204_dp, -123.85428534979440940_dp, 57.82417470821774685_dp, & - & -116.68240638414135901_dp, 54.56141374911145192_dp, -110.39390839554111778_dp, 51.08045112569507751_dp, & - & -104.79614728017998004_dp, 47.44644408577498496_dp, -99.73088255981890882_dp, 43.70821480745370025_dp, & - & -95.07197748871266185_dp, 39.90302877816822757_dp, -90.71961138434764393_dp, 36.06017287638244539_dp, & - & -86.59435351833845118_dp, 32.20351875194761959_dp, -82.63216392910496211_dp, 28.35333972343855535_dp, & - & -78.78048521884275601_dp, 24.52760876444716587_dp, -74.99529352424694650_dp, 20.74294398926493699_dp, & - & -71.23891693218340038_dp, 17.01531594444593765_dp, -67.47845003929684538_dp, 13.36059248585088355_dp, & - & -63.68463403089357655_dp, 9.79496963802121101_dp, -59.83111160507219495_dp, 6.33531713186757539_dp, & - & -55.89399913408760057_dp, 2.99945241539763696_dp, -51.85174332801064168_dp, -0.19365520038940248_dp, & - & -47.68524592294902931_dp, -3.22375917751486485_dp, -43.37824652323212860_dp, -6.06930752874206636_dp, & - & -38.91794868987490474_dp, -8.70753794849322027_dp, -34.29585521437643081_dp, -11.11472732454917001_dp, & - & -29.50874372060167872_dp, -13.26662598462659126_dp, -24.55966518490711081_dp, -15.13909773195326558_dp, & - & -19.45879403720245193_dp, -16.70896594558295334_dp, -14.22391622121717880_dp, -17.95503295463868554_dp, & - & -8.88033504279663255_dp, -18.85919797266866027_dp, -3.46002704385166027_dp, -19.40755726556069405_dp, & - & 1.99999999999999645_dp, -19.59134287356426540_dp, 7.46002704385165316_dp, -19.40755726556069405_dp, & - & 12.88033504279662367_dp, -18.85919797266866027_dp, 18.22391622121717347_dp, -17.95503295463868554_dp, & - & 23.45879403720244127_dp, -16.70896594558295334_dp, 28.55966518490710015_dp, -15.13909773195326558_dp, & - & 33.50874372060167161_dp, -13.26662598462659126_dp, 38.29585521437642370_dp, -11.11472732454917534_dp ] - lonlat( 2281: 2400) = [ & - & 42.91794868987490474_dp, -8.70753794849322738_dp, 47.37824652323212149_dp, -6.06930752874207258_dp, & - & 51.68524592294901510_dp, -3.22375917751487107_dp, 55.85174332801063457_dp, -0.19365520038940884_dp, & - & 59.89399913408759346_dp, 2.99945241539763385_dp, 63.83111160507218784_dp, 6.33531713186756917_dp, & - & 67.68463403089357655_dp, 9.79496963802120568_dp, 71.47845003929684538_dp, 13.36059248585087822_dp, & - & 75.23891693218338617_dp, 17.01531594444593054_dp, 78.99529352424693229_dp, 20.74294398926492988_dp, & - & 82.78048521884274180_dp, 24.52760876444715521_dp, 86.63216392910494790_dp, 28.35333972343854825_dp, & - & 90.59435351833845118_dp, 32.20351875194761959_dp, 94.71961138434764393_dp, 36.06017287638243118_dp, & - & 99.07197748871266185_dp, 39.90302877816822047_dp, 103.73088255981890882_dp, 43.70821480745368603_dp, & - & 108.79614728017998004_dp, 47.44644408577497074_dp, 114.39390839554110357_dp, 51.08045112569504909_dp, & - & 120.68240638414135901_dp, 54.56141374911145192_dp, 127.85428534979438098_dp, 57.82417470821774685_dp, & - & 136.12712761110682891_dp, 60.78156430751474204_dp, 145.70578215046154469_dp, 63.31961305621509695_dp, & - & 156.69437932769909594_dp, 65.29863962109095610_dp, 168.95645291666264143_dp, 66.56908368553206401_dp, & - & -178.00000000000002842_dp, 61.16162880179430772_dp, -166.97884079028878546_dp, 60.79043532425956897_dp, & - & -156.40162010671977555_dp, 59.70518607311999659_dp, -146.59283359977425221_dp, 57.98141283097307053_dp, & - & -137.70772458687926587_dp, 55.72003625021811501_dp, -129.75638606304184464_dp, 53.02581189527144545_dp, & - & -122.65952710171549711_dp, 49.99368843662635697_dp, -116.29889149413011751_dp, 46.70342202473146642_dp, & - & -110.54933935400606515_dp, 43.21943287866469774_dp, -105.29473060869931089_dp, 39.59298941537671368_dp, & - & -100.43366587620559471_dp, 35.86497163386139420_dp, -95.88003723653146437_dp, 32.06843215731343832_dp, & - & -91.56137924921046078_dp, 28.23071203179445376_dp, -87.41655400532016529_dp, 24.37510563102819461_dp, & - & -83.39346098727328638_dp, 20.52214967325533124_dp, -79.44703540885105042_dp, 16.69062430422997423_dp, & - & -75.53760403336077900_dp, 12.89834287286322301_dp, -71.62959031828354739_dp, 9.16278970353470967_dp, & - & -67.69053915995506543_dp, 5.50164833296652400_dp, -63.69043463465567356_dp, 1.93324784086672841_dp, & - & -59.60129716846515180_dp, -1.52305829704237294_dp, -55.39706222812606740_dp, -4.84657887125734721_dp, & - & -51.05375641269629483_dp, -8.01504773475606314_dp, -46.54999441522540593_dp, -11.00448645615599474_dp, & - & -41.86781613095839560_dp, -13.78920372951384543_dp, -36.99386004100858116_dp, -16.34197985921976581_dp, & - & -31.92081946357624034_dp, -18.63449057116782015_dp, -26.64904852681421588_dp, -20.63801909702448611_dp, & - & -21.18808138888132930_dp, -22.32448212182582026_dp, -15.55772615655294366_dp, -23.66774871950555692_dp, & - & -9.78834011215971245_dp, -24.64516330804185174_dp, -3.91994134356475854_dp, -25.23910692117754806_dp, & - & 1.99999999999999556_dp, -25.43837119820565107_dp, 7.91994134356474966_dp, -25.23910692117754806_dp, & - & 13.78834011215970534_dp, -24.64516330804185174_dp, 19.55772615655293123_dp, -23.66774871950555692_dp ] - lonlat( 2401: 2520) = [ & - & 25.18808138888131865_dp, -22.32448212182582026_dp, 30.64904852681421232_dp, -20.63801909702448611_dp, & - & 35.92081946357622968_dp, -18.63449057116782015_dp, 40.99386004100855985_dp, -16.34197985921976581_dp, & - & 45.86781613095838850_dp, -13.78920372951385076_dp, 50.54999441522539882_dp, -11.00448645615600007_dp, & - & 55.05375641269628773_dp, -8.01504773475606846_dp, 59.39706222812605318_dp, -4.84657887125735343_dp, & - & 63.60129716846514469_dp, -1.52305829704237916_dp, 67.69043463465565935_dp, 1.93324784086672197_dp, & - & 71.69053915995505122_dp, 5.50164833296651778_dp, 75.62959031828353318_dp, 9.16278970353470434_dp, & - & 79.53760403336076479_dp, 12.89834287286321590_dp, 83.44703540885105042_dp, 16.69062430422996712_dp, & - & 87.39346098727328638_dp, 20.52214967325532413_dp, 91.41655400532015108_dp, 24.37510563102818750_dp, & - & 95.56137924921046078_dp, 28.23071203179444666_dp, 99.88003723653145016_dp, 32.06843215731343832_dp, & - & 104.43366587620558050_dp, 35.86497163386137288_dp, 109.29473060869932510_dp, 39.59298941537669947_dp, & - & 114.54933935400606515_dp, 43.21943287866469774_dp, 120.29889149413010330_dp, 46.70342202473146642_dp, & - & 126.65952710171551132_dp, 49.99368843662635697_dp, 133.75638606304187306_dp, 53.02581189527144545_dp, & - & 141.70772458687926587_dp, 55.72003625021811501_dp, 150.59283359977422379_dp, 57.98141283097307053_dp, & - & 160.40162010671977555_dp, 59.70518607311999659_dp, 170.97884079028878546_dp, 60.79043532425956897_dp, & - & -178.00000000000002842_dp, 54.75649468439711143_dp, -168.49285933765114009_dp, 54.43639475342744305_dp, & - & -159.25249258416440057_dp, 53.49372597518600259_dp, -150.49218436749950456_dp, 51.97706503648645082_dp, & - & -142.34201331309026273_dp, 49.95526128262039123_dp, -134.84908760812987794_dp, 47.50530446205828383_dp, & - & -127.99715883033752561_dp, 44.70273256173389598_dp, -121.73118977168118704_dp, 41.61600262291977259_dp, & - & -115.97779813807071037_dp, 38.30427835916992052_dp, -110.65884380836621403_dp, 34.81740352705595143_dp, & - & -105.69899804029306267_dp, 31.19696754875727152_dp, -101.02917319698202903_dp, 27.47775433026597725_dp, & - & -96.58749981932022877_dp, 23.68920100972959730_dp, -92.31902021078286680_dp, 19.85670886149115688_dp, & - & -88.17480572169651509_dp, 16.00276430383187076_dp, -84.11088800598805904_dp, 12.14788119395578825_dp, & - & -80.08720437104130951_dp, 8.31139512800513991_dp, -76.06665407661381550_dp, 4.51214364124032752_dp, & - & -72.01431282527251199_dp, 0.76906207426875217_dp, -67.89683474738467339_dp, -2.89828246890512542_dp, & - & -63.68207179147747610_dp, -6.46920580966549608_dp, -59.33895182482493880_dp, -9.92146778444055499_dp, & - & -54.83767345094592827_dp, -13.23089104518994574_dp, -50.15029168417838434_dp, -16.37106576006626568_dp, & - & -45.25177543408179304_dp, -19.31318879069117145_dp, -40.12160159745184274_dp, -22.02610835172851722_dp, & - & -34.74589265732009835_dp, -24.47666503668304117_dp, -29.11998552133064422_dp, -26.63042818683534207_dp, & - & -23.25113130544623985_dp, -28.45290874783366064_dp, -17.16079514785620930_dp, -29.91127029421519978_dp, & - & -10.88583393507997776_dp, -30.97645080660139172_dp, -4.47781225763253765_dp, -31.62546289221583962_dp ] - lonlat( 2521: 2640) = [ & - & 1.99999999999999534_dp, -31.84350531560284736_dp, 8.47781225763252877_dp, -31.62546289221583962_dp, & - & 14.88583393507996711_dp, -30.97645080660139172_dp, 21.16079514785619509_dp, -29.91127029421519978_dp, & - & 27.25113130544623274_dp, -28.45290874783366064_dp, 33.11998552133064067_dp, -26.63042818683534207_dp, & - & 38.74589265732009835_dp, -24.47666503668304117_dp, 44.12160159745183563_dp, -22.02610835172851722_dp, & - & 49.25177543408178593_dp, -19.31318879069118211_dp, 54.15029168417837724_dp, -16.37106576006627279_dp, & - & 58.83767345094592827_dp, -13.23089104518995285_dp, 63.33895182482493169_dp, -9.92146778444056032_dp, & - & 67.68207179147748320_dp, -6.46920580966550229_dp, 71.89683474738467339_dp, -2.89828246890513164_dp, & - & 76.01431282527251199_dp, 0.76906207426874584_dp, 80.06665407661380129_dp, 4.51214364124032308_dp, & - & 84.08720437104132372_dp, 8.31139512800513458_dp, 88.11088800598805904_dp, 12.14788119395578470_dp, & - & 92.17480572169650088_dp, 16.00276430383186366_dp, 96.31902021078285259_dp, 19.85670886149114978_dp, & - & 100.58749981932022877_dp, 23.68920100972959020_dp, 105.02917319698201482_dp, 27.47775433026597014_dp, & - & 109.69899804029304846_dp, 31.19696754875725375_dp, 114.65884380836618561_dp, 34.81740352705593722_dp, & - & 119.97779813807069615_dp, 38.30427835916992052_dp, 125.73118977168117283_dp, 41.61600262291977259_dp, & - & 131.99715883033749719_dp, 44.70273256173389598_dp, 138.84908760812987794_dp, 47.50530446205828383_dp, & - & 146.34201331309026273_dp, 49.95526128262039123_dp, 154.49218436749950456_dp, 51.97706503648645082_dp, & - & 163.25249258416440057_dp, 53.49372597518600259_dp, 172.49285933765111167_dp, 54.43639475342744305_dp, & - & -178.00000000000002842_dp, 47.72124472298390430_dp, -169.69217969866491558_dp, 47.44157216781381692_dp, & - & -161.55074086228117380_dp, 46.61395503653132266_dp, -153.71667350566602295_dp, 45.27051130755705799_dp, & - & -146.28885177460344380_dp, 43.45866798082070659_dp, -139.31958252044236701_dp, 41.23439386179803279_dp, & - & -132.82020109922686402_dp, 38.65601141858345358_dp, -126.77168357131760956_dp, 35.77969749489390949_dp, & - & -121.13576891124588997_dp, 32.65687547770472321_dp, -115.86406835852179142_dp, 29.33315055785289260_dp, & - & -110.90442707083994378_dp, 25.84827445710250160_dp, -106.20481280215612685_dp, 22.23668760419472790_dp, & - & -101.71535755141638901_dp, 18.52832281285149207_dp, -97.38917071735673403_dp, 14.74948212031801020_dp, & - & -93.18240447461121789_dp, 10.92369066480776674_dp, -89.05390189983984328_dp, 7.07248905242842607_dp, & - & -84.96463936229969249_dp, 3.21615784383777559_dp, -80.87709425893895343_dp, -0.62561625366165152_dp, & - & -76.75462189677602964_dp, -4.43311662588922051_dp, -72.56090370117536281_dp, -8.18599910867232161_dp, & - & -68.25952713288315010_dp, -11.86270778471483389_dp, -63.81377209114523907_dp, -15.43989911427394368_dp, & - & -59.18670666190919150_dp, -18.89188184637555068_dp, -54.34173321358174746_dp, -22.19009961204913850_dp, & - & -49.24376548995358149_dp, -25.30271165018387691_dp, -43.86123932751760890_dp, -28.19436585459229860_dp, & - & -38.16912769952225659_dp, -30.82630445836841915_dp, -32.15298964123236658_dp, -33.15698491696964112_dp ] - lonlat( 2641: 2760) = [ & - & -25.81377244810833815_dp, -35.14341202945125531_dp, -19.17259140539265516_dp, -36.74332279213026453_dp, & - & -12.27414365511957151_dp, -37.91820250638207312_dp, -5.18708125479775184_dp, -38.63683300066042392_dp, & - & 1.99999999999999489_dp, -38.87875527701606160_dp, 9.18708125479774296_dp, -38.63683300066042392_dp, & - & 16.27414365511955907_dp, -37.91820250638207312_dp, 23.17259140539264095_dp, -36.74332279213026453_dp, & - & 29.81377244810832394_dp, -35.14341202945125531_dp, 36.15298964123235947_dp, -33.15698491696964112_dp, & - & 42.16912769952225659_dp, -30.82630445836841915_dp, 47.86123932751760179_dp, -28.19436585459229860_dp, & - & 53.24376548995357439_dp, -25.30271165018388402_dp, 58.34173321358174746_dp, -22.19009961204914205_dp, & - & 63.18670666190918439_dp, -18.89188184637555779_dp, 67.81377209114522486_dp, -15.43989911427395079_dp, & - & 72.25952713288313589_dp, -11.86270778471484277_dp, 76.56090370117536281_dp, -8.18599910867232872_dp, & - & 80.75462189677602964_dp, -4.43311662588922673_dp, 84.87709425893895343_dp, -0.62561625366165630_dp, & - & 88.96463936229967828_dp, 3.21615784383776981_dp, 93.05390189983984328_dp, 7.07248905242842163_dp, & - & 97.18240447461121789_dp, 10.92369066480776141_dp, 101.38917071735671982_dp, 14.74948212031800310_dp, & - & 105.71535755141637480_dp, 18.52832281285148497_dp, 110.20481280215611264_dp, 22.23668760419472079_dp, & - & 114.90442707083992957_dp, 25.84827445710249449_dp, 119.86406835852177721_dp, 29.33315055785288550_dp, & - & 125.13576891124587576_dp, 32.65687547770470900_dp, 130.77168357131759535_dp, 35.77969749489390949_dp, & - & 136.82020109922683559_dp, 38.65601141858345358_dp, 143.31958252044236701_dp, 41.23439386179803279_dp, & - & 150.28885177460347222_dp, 43.45866798082070659_dp, 157.71667350566602295_dp, 45.27051130755705799_dp, & - & 165.55074086228114538_dp, 46.61395503653132266_dp, 173.69217969866488716_dp, 47.44157216781381692_dp, & - & -178.00000000000002842_dp, 39.98177927524727693_dp, -170.20476659278793363_dp, 39.70189618399165710_dp, & - & -162.53782868216148927_dp, 38.87197455633481979_dp, -155.11038900941949237_dp, 37.51968052984662449_dp, & - & -148.00450116657361832_dp, 35.68659928106227852_dp, -141.26850797118672176_dp, 33.42299645292870736_dp, & - & -134.91921485966395267_dp, 30.78272820361293327_dp, -128.94811747553873715_dp, 27.81921722445741452_dp, & - & -123.32885205682140395_dp, 24.58281725765701253_dp, -118.02393786261990272_dp, 21.11944988981388605_dp, & - & -112.98994385952538266_dp, 17.47019805768939449_dp, -108.18096363521175363_dp, 13.67152082594358653_dp, & - & -103.55065416712271542_dp, 9.75582109859903390_dp, -99.05319914949195947_dp, 5.75218534812640492_dp, & - & -94.64352777118511995_dp, 1.68719008387413827_dp, -90.27704477012913742_dp, -2.41427530523583656_dp, & - & -85.90905356028720519_dp, -6.52818370181667529_dp, -81.49400025392820623_dp, -10.63053585607600837_dp, & - & -76.98463901984250413_dp, -14.69649678201448140_dp, -72.33122188826867216_dp, -18.69947873486281864_dp, & - & -67.48085419487966874_dp, -22.61014977269736193_dp, -62.37723925505212463_dp, -26.39536140379772533_dp, & - & -56.96117255247644096_dp, -30.01702215459737033_dp, -51.17233704527057370_dp, -33.43100825355479344_dp ] - lonlat( 2761: 2880) = [ & - & -44.95316063861177014_dp, -36.58631299796492442_dp, -38.25559890256131723_dp, -39.42480136235259636_dp, & - & -31.05142021090808413_dp, -41.88213387768102791_dp, -23.34546416576216643_dp, -43.89055765093644368_dp, & - & -15.18911080317680451_dp, -45.38412606933154336_dp, -6.68839576675081293_dp, -46.30624543028365991_dp, & - & 1.99999999999999445_dp, -46.61822072475268186_dp, 10.68839576675079961_dp, -46.30624543028365991_dp, & - & 19.18911080317679918_dp, -45.38412606933154336_dp, 27.34546416576215222_dp, -43.89055765093644368_dp, & - & 35.05142021090807702_dp, -41.88213387768102791_dp, 42.25559890256131013_dp, -39.42480136235259636_dp, & - & 48.95316063861175593_dp, -36.58631299796492442_dp, 55.17233704527057370_dp, -33.43100825355479344_dp, & - & 60.96117255247643385_dp, -30.01702215459737744_dp, 66.37723925505210332_dp, -26.39536140379773244_dp, & - & 71.48085419487965453_dp, -22.61014977269736548_dp, 76.33122188826865795_dp, -18.69947873486282575_dp, & - & 80.98463901984250413_dp, -14.69649678201448850_dp, 85.49400025392819202_dp, -10.63053585607601548_dp, & - & 89.90905356028720519_dp, -6.52818370181668062_dp, 94.27704477012913742_dp, -2.41427530523584188_dp, & - & 98.64352777118510573_dp, 1.68719008387413361_dp, 103.05319914949195947_dp, 5.75218534812639781_dp, & - & 107.55065416712271542_dp, 9.75582109859902680_dp, 112.18096363521173942_dp, 13.67152082594358120_dp, & - & 116.98994385952536845_dp, 17.47019805768938738_dp, 122.02393786261990272_dp, 21.11944988981387894_dp, & - & 127.32885205682140395_dp, 24.58281725765700187_dp, 132.94811747553873715_dp, 27.81921722445741452_dp, & - & 138.91921485966392424_dp, 30.78272820361293327_dp, 145.26850797118672176_dp, 33.42299645292870736_dp, & - & 152.00450116657361832_dp, 35.68659928106227852_dp, 159.11038900941949237_dp, 37.51968052984662449_dp, & - & 166.53782868216146085_dp, 38.87197455633481979_dp, 174.20476659278793363_dp, 39.70189618399165710_dp, & - & -178.00000000000002842_dp, 31.46720374958098887_dp, -171.12910064316457692_dp, 31.22051723140710067_dp, & - & -164.34108626834762390_dp, 30.48695101230147841_dp, -157.71017626763574526_dp, 29.28521267182222587_dp, & - & -151.29523174265207786_dp, 27.64411687523926986_dp, -145.13620894054648147_dp, 25.59967735271197498_dp, & - & -139.25381666270027381_dp, 23.19203115841211726_dp, -133.65157520973266969_dp, 20.46271167815037728_dp, & - & -128.31914580979304219_dp, 17.45256126555711873_dp, -123.23592626204657563_dp, 14.20035582641498806_dp, & - & -118.37425008640165913_dp, 10.74206713847307348_dp, -113.70187269530693186_dp, 7.11062322287716331_dp, & - & -109.18367588372132104_dp, 3.33602075627005945_dp, -104.78265670493266271_dp, -0.55433148808729127_dp, & - & -100.46031614728222792_dp, -4.53512373056416163_dp, -96.17656220612794016_dp, -8.58256192749418290_dp, & - & -91.88921878897831164_dp, -12.67371220268469223_dp, -87.55320457280681978_dp, -16.78577161643179849_dp, & - & -83.11942634360052296_dp, -20.89524495725206421_dp, -78.53343020155088539_dp, -24.97699141478169338_dp, & - & -73.73388667252179118_dp, -29.00309226896919412_dp, -68.65107920217621995_dp, -32.94148403257255353_dp, & - & -63.20576633421556068_dp, -36.75430974219704439_dp, -57.30916699148811233_dp, -40.39598469592144880_dp ] - lonlat( 2881: 3000) = [ & - & -50.86545369174167774_dp, -43.81109166541019562_dp, -43.77902429867569367_dp, -46.93247902115529513_dp, & - & -35.96960367385943158_dp, -49.68040408706252720_dp, -27.39769711559288723_dp, -51.96422430511800172_dp, & - & -18.09872704828334378_dp, -53.68864395349498864_dp, -8.21451927262478065_dp, -54.76594013073759015_dp, & - & 1.99999999999999289_dp, -55.13279625041898413_dp, 12.21451927262476822_dp, -54.76594013073759015_dp, & - & 22.09872704828332601_dp, -53.68864395349498864_dp, 31.39769711559288012_dp, -51.96422430511800172_dp, & - & 39.96960367385942448_dp, -49.68040408706252720_dp, 47.77902429867568657_dp, -46.93247902115529513_dp, & - & 54.86545369174167064_dp, -43.81109166541019562_dp, 61.30916699148811233_dp, -40.39598469592146301_dp, & - & 67.20576633421555357_dp, -36.75430974219705149_dp, 72.65107920217620574_dp, -32.94148403257256064_dp, & - & 77.73388667252179118_dp, -29.00309226896920123_dp, 82.53343020155088539_dp, -24.97699141478170048_dp, & - & 87.11942634360052296_dp, -20.89524495725207132_dp, 91.55320457280680557_dp, -16.78577161643180560_dp, & - & 95.88921878897831164_dp, -12.67371220268469756_dp, 100.17656220612792595_dp, -8.58256192749419000_dp, & - & 104.46031614728221371_dp, -4.53512373056416607_dp, 108.78265670493264849_dp, -0.55433148808729760_dp, & - & 113.18367588372132104_dp, 3.33602075627005323_dp, 117.70187269530691765_dp, 7.11062322287715620_dp, & - & 122.37425008640165913_dp, 10.74206713847306816_dp, 127.23592626204656142_dp, 14.20035582641498095_dp, & - & 132.31914580979304219_dp, 17.45256126555711162_dp, 137.65157520973264127_dp, 20.46271167815036662_dp, & - & 143.25381666270027381_dp, 23.19203115841211726_dp, 149.13620894054645305_dp, 25.59967735271197498_dp, & - & 155.29523174265207786_dp, 27.64411687523926986_dp, 161.71017626763577368_dp, 29.28521267182222587_dp, & - & 168.34108626834759548_dp, 30.48695101230147841_dp, 175.12910064316457692_dp, 31.22051723140710067_dp, & - & -178.00000000000002842_dp, 22.11837781753018817_dp, -170.46891148411387462_dp, 21.78042227826199095_dp, & - & -163.04151372648448159_dp, 20.77699151155475477_dp, -155.80845268931921055_dp, 19.13794220572576776_dp, & - & -148.83768595785252842_dp, 16.90865838596958781_dp, -142.17005258660481104_dp, 14.14485028302004999_dp, & - & -135.81978239976487544_dp, 10.90731557455581147_dp, -129.77826339847271697_dp, 7.25759760165333567_dp, & - & -124.01906809075352101_dp, 3.25496397729521814_dp, -118.50268451884778642_dp, -1.04529436283411115_dp, & - & -113.18006743498142441_dp, -5.59247248797413210_dp, -107.99465974304708027_dp, -10.34025377357217934_dp, & - & -102.88281945262124850_dp, -15.24594064653213898_dp, -97.77265218881962028_dp, -20.26925377022462982_dp, & - & -92.58116190590757810_dp, -25.37072818781790673_dp, -87.20945067766288616_dp, -30.50961194071436111_dp, & - & -81.53547443418635510_dp, -35.64102404029979709_dp, -75.40370496549049051_dp, -40.71192873614232610_dp, & - & -68.61131289029849256_dp, -45.65520975111465418_dp, -60.89227178252910733_dp, -50.38082342875217989_dp, & - & -51.90710322947828104_dp, -54.76300160634039571_dp, -41.26325159838054901_dp, -58.62403790832019723_dp, & - & -28.62428016143747200_dp, -61.72174914658255318_dp, -13.98037118830461445_dp, -63.76341508305664263_dp ] - lonlat( 3001: 3120) = [ & - & 1.99999999999999090_dp, -64.48162218246977773_dp, 17.98037118830459491_dp, -63.76341508305664263_dp, & - & 32.62428016143745424_dp, -61.72174914658255318_dp, 45.26325159838053480_dp, -58.62403790832019723_dp, & - & 55.90710322947827393_dp, -54.76300160634039571_dp, 64.89227178252910733_dp, -50.38082342875217989_dp, & - & 72.61131289029847835_dp, -45.65520975111467550_dp, 79.40370496549049051_dp, -40.71192873614232610_dp, & - & 85.53547443418635510_dp, -35.64102404029981841_dp, 91.20945067766287195_dp, -30.50961194071436822_dp, & - & 96.58116190590757810_dp, -25.37072818781791383_dp, 101.77265218881962028_dp, -20.26925377022463692_dp, & - & 106.88281945262124850_dp, -15.24594064653214609_dp, 111.99465974304708027_dp, -10.34025377357218289_dp, & - & 117.18006743498143862_dp, -5.59247248797413921_dp, 122.50268451884780063_dp, -1.04529436283411759_dp, & - & 128.01906809075350679_dp, 3.25496397729521192_dp, 133.77826339847268855_dp, 7.25759760165332857_dp, & - & 139.81978239976487544_dp, 10.90731557455580436_dp, 146.17005258660478262_dp, 14.14485028302004999_dp, & - & 152.83768595785250000_dp, 16.90865838596958781_dp, 159.80845268931921055_dp, 19.13794220572576776_dp, & - & 167.04151372648448159_dp, 20.77699151155475477_dp, 174.46891148411387462_dp, 21.78042227826199095_dp, & - & -178.00000000000002842_dp, 11.90033356154828503_dp, -171.03550378237531504_dp, 11.56695807154089728_dp, & - & -164.15259832130166728_dp, 10.57562338306961180_dp, -157.42382961455143686_dp, 8.95164656051215069_dp, & - & -150.90567761562240889_dp, 6.73398631258200187_dp, -144.63474532961075170_dp, 3.97127289925851068_dp, & - & -138.62718020698264354_dp, 0.71762564121834338_dp, -132.88045216690298389_dp, -2.97103149043460757_dp, & - & -127.37626453526326031_dp, -7.03975713563777195_dp, -122.08348616854385682_dp, -11.43638098615609344_dp, & - & -116.96030366998827787_dp, -16.11238508810905401_dp, -111.95506961066591600_dp, -21.02304593053227677_dp, & - & -107.00543419047215821_dp, -26.12699149209596428_dp, -102.03522828445495918_dp, -31.38525381109429446_dp, & - & -96.94814060146437384_dp, -36.75977477892173084_dp, -91.61632617812561818_dp, -42.21112686087874266_dp, & - & -85.86029859026153588_dp, -47.69485768882572785_dp, -79.41299264536502278_dp, -53.15514845029933610_dp, & - & -71.85455169648275842_dp, -58.51289100414830102_dp, -62.49613000707146426_dp, -63.64173578517077345_dp, & - & -50.20367738593296281_dp, -68.31866526945105988_dp, -33.33636828365784055_dp, -72.13161644510194037_dp, & - & -10.78607194201629582_dp, -74.39304559691805707_dp, 14.78607194201627273_dp, -74.39304559691805707_dp, & - & 37.33636828365781213_dp, -72.13161644510194037_dp, 54.20367738593294149_dp, -68.31866526945105988_dp, & - & 66.49613000707145716_dp, -63.64173578517077345_dp, 75.85455169648274421_dp, -58.51289100414830102_dp, & - & 83.41299264536502278_dp, -53.15514845029933610_dp, 89.86029859026150746_dp, -47.69485768882574916_dp, & - & 95.61632617812561818_dp, -42.21112686087874266_dp, 100.94814060146437384_dp, -36.75977477892173084_dp, & - & 106.03522828445494497_dp, -31.38525381109429446_dp, 111.00543419047215821_dp, -26.12699149209597138_dp, & - & 115.95506961066591600_dp, -21.02304593053228388_dp, 120.96030366998827787_dp, -16.11238508810905756_dp ] - lonlat( 3121: 3240) = [ & - & 126.08348616854387103_dp, -11.43638098615609699_dp, 131.37626453526326031_dp, -7.03975713563777816_dp, & - & 136.88045216690298389_dp, -2.97103149043461379_dp, 142.62718020698264354_dp, 0.71762564121833694_dp, & - & 148.63474532961075170_dp, 3.97127289925851068_dp, 154.90567761562243732_dp, 6.73398631258200187_dp, & - & 161.42382961455140844_dp, 8.95164656051215069_dp, 168.15259832130169571_dp, 10.57562338306961180_dp, & - & 175.03550378237531504_dp, 11.56695807154089728_dp, -178.00000000000002842_dp, 0.81833312891115306_dp, & - & -171.37485237931247184_dp, 0.46152536384328174_dp, -164.82591385569978115_dp, -0.60006016949342578_dp, & - & -158.42170502724641779_dp, -2.34091526057736310_dp, -152.21700460768090579_dp, -4.72161659803607048_dp, & - & -146.24942513902283281_dp, -7.69266569608215889_dp, -140.53869867413175143_dp, -11.19861017149900739_dp, & - & -135.08801440266989857_dp, -15.18177122287804615_dp, -129.88641650799127092_dp, -19.58517424494497305_dp, & - & -124.91129483912591525_dp, -24.35455549606792047_dp, -120.13018318901130499_dp, -29.43951176908901957_dp, & - & -115.50120511177955507_dp, -34.79395188944583595_dp, -110.97138028465980142_dp, -40.37601690822956613_dp, & - & -106.47134737780879732_dp, -46.14758359783259323_dp, -101.90317607822893820_dp, -52.07335345715244301_dp, & - & -97.11263541171206271_dp, -58.11928872485960085_dp, -91.82057104813961246_dp, -64.24948773689963843_dp, & - & -85.42499621356618889_dp, -70.41813086842684299_dp, -76.27768438369074033_dp, -76.54083121758115738_dp, & - & -57.87324357773111672_dp, -82.33459525449163152_dp, 1.99999999999996114_dp, -85.78166687108868871_dp, & - & 61.87324357773107408_dp, -82.33459525449163152_dp, 80.27768438369074033_dp, -76.54083121758115738_dp, & - & 89.42499621356618889_dp, -70.41813086842684299_dp, 95.82057104813959825_dp, -64.24948773689963843_dp, & - & 101.11263541171206271_dp, -58.11928872485960085_dp, 105.90317607822892398_dp, -52.07335345715245722_dp, & - & 110.47134737780879732_dp, -46.14758359783259323_dp, 114.97138028465980142_dp, -40.37601690822956613_dp, & - & 119.50120511177955507_dp, -34.79395188944583595_dp, 124.13018318901127657_dp, -29.43951176908902667_dp, & - & 128.91129483912592946_dp, -24.35455549606792403_dp, 133.88641650799127092_dp, -19.58517424494497661_dp, & - & 139.08801440266989857_dp, -15.18177122287804970_dp, 144.53869867413172301_dp, -11.19861017149901272_dp, & - & 150.24942513902280439_dp, -7.69266569608215889_dp, 156.21700460768090579_dp, -4.72161659803607048_dp, & - & 162.42170502724644621_dp, -2.34091526057736310_dp, 168.82591385569978115_dp, -0.60006016949342578_dp, & - & 175.37485237931247184_dp, 0.46152536384328174_dp, -178.00000000000000000_dp, -11.06408152044379101_dp, & - & -171.33870321353637678_dp, -11.51266748991093891_dp, -164.78020742076523675_dp, -12.84550987892848539_dp, & - & -158.41822389716838870_dp, -15.02545489467462403_dp, -152.33085313434585828_dp, -17.99542592709287803_dp, & - & -146.57821873782722832_dp, -21.68429473238734673_dp, -141.20455296082769792_dp, -26.01294410398721979_dp, & - & -136.24424010430539056_dp, -30.89942669541638764_dp, -131.73150447900647464_dp, -36.26259289105208694_dp, & - & -127.71480673751507595_dp, -42.02394287044848653_dp, -124.28024682035200499_dp, -48.10755262562803125_dp ] - lonlat( 3241: 3360) = [ & - & -121.59604777627559713_dp, -54.43751136022502379_dp, -120.01228809564838684_dp, -60.93070105520797597_dp, & - & -120.32470998353322500_dp, -67.47696540683870126_dp, -124.60898756414846389_dp, -73.87349687490876704_dp, & - & -139.22778135121401988_dp, -79.54230031515007227_dp, -178.00000000000000000_dp, -82.33591847955618448_dp, & - & 143.22778135121404830_dp, -79.54230031515007227_dp, 128.60898756414846389_dp, -73.87349687490876704_dp, & - & 124.32470998353322500_dp, -67.47696540683870126_dp, 124.01228809564837263_dp, -60.93070105520797597_dp, & - & 125.59604777627561134_dp, -54.43751136022503090_dp, 128.28024682035200499_dp, -48.10755262562803125_dp, & - & 131.71480673751509016_dp, -42.02394287044850074_dp, 135.73150447900647464_dp, -36.26259289105208694_dp, & - & 140.24424010430539056_dp, -30.89942669541638764_dp, 145.20455296082769792_dp, -26.01294410398722334_dp, & - & 150.57821873782722832_dp, -21.68429473238734673_dp, 156.33085313434585828_dp, -17.99542592709287803_dp, & - & 162.41822389716838870_dp, -15.02545489467462403_dp, 168.78020742076520833_dp, -12.84550987892848539_dp, & - & 175.33870321353637678_dp, -11.51266748991093891_dp, -178.00000000000000000_dp, -23.60656536858034116_dp, & - & -172.31484199740026497_dp, -24.06074499849094650_dp, -166.76206145225543764_dp, -25.40835290264588053_dp, & - & -161.47033176340409000_dp, -27.60594661847371611_dp, -156.56387974300261590_dp, -30.58514629912493277_dp, & - & -152.16715015384193066_dp, -34.25717258827405942_dp, -148.41689072479888978_dp, -38.51667916926085411_dp, & - & -145.48413942807309240_dp, -43.24305393544465659_dp, -143.61056018478930696_dp, -48.29703252527734492_dp, & - & -143.16669319413330186_dp, -53.50931894776815057_dp, -144.73891790293430404_dp, -58.65522080377154168_dp, & - & -149.21912766680259210_dp, -63.40582055258879279_dp, -157.69810487935359333_dp, -67.25441721659638006_dp, & - & -170.53118545218148938_dp, -69.49384617858525814_dp, 174.53118545218146096_dp, -69.49384617858525814_dp, & - & 161.69810487935359333_dp, -67.25441721659638006_dp, 153.21912766680256368_dp, -63.40582055258877858_dp, & - & 148.73891790293430404_dp, -58.65522080377152747_dp, 147.16669319413330186_dp, -53.50931894776815767_dp, & - & 147.61056018478930696_dp, -48.29703252527734492_dp, 149.48413942807309240_dp, -43.24305393544464948_dp, & - & 152.41689072479886136_dp, -38.51667916926086122_dp, 156.16715015384193066_dp, -34.25717258827403811_dp, & - & 160.56387974300261590_dp, -30.58514629912493277_dp, 165.47033176340409000_dp, -27.60594661847371611_dp, & - & 170.76206145225546607_dp, -25.40835290264587698_dp, 176.31484199740026497_dp, -24.06074499849094295_dp, & - & -178.00000000000000000_dp, -36.54741287921620341_dp, -174.09065200901946469_dp, -36.97055608025468132_dp, & - & -170.42266827604018431_dp, -38.21219158951615213_dp, -167.24101930549508666_dp, -40.18968456371464271_dp, & - & -164.79950790329061761_dp, -42.76713193279059766_dp, -163.36621335583259906_dp, -45.75617589753742465_dp, & - & -163.22080918295719698_dp, -48.91487419033276751_dp, -164.62326642581021474_dp, -51.94637057143930292_dp, & - & -167.72066876086483944_dp, -54.50623527875517738_dp, -172.37525674045562596_dp, -56.23764348052387874_dp, & - & -178.00000000000000000_dp, -56.85258712078380228_dp, 176.37525674045562596_dp, -56.23764348052387874_dp ] - lonlat( 3361: 3376) = [ & - & 171.72066876086483944_dp, -54.50623527875517738_dp, 168.62326642581021474_dp, -51.94637057143930292_dp, & - & 167.22080918295719698_dp, -48.91487419033276751_dp, 167.36621335583259906_dp, -45.75617589753742465_dp, & - & 168.79950790329058918_dp, -42.76713193279060476_dp, 171.24101930549508666_dp, -40.18968456371464271_dp, & - & 174.42266827604018431_dp, -38.21219158951615213_dp, 178.09065200901943626_dp, -36.97055608025468132_dp ] + lonlat = [ 1.99999999999999600_dp, 44.93291171299576092_dp, & + & 2.77237262934735273_dp, 45.01674523281888440_dp, & + & 3.47555862489311007_dp, 45.26073815098433784_dp, & + & 4.04494770792090641_dp, 45.64287023699112211_dp, & + & 4.42528132740345281_dp, 46.12813070368466839_dp, & + & 4.57569351104372046_dp, 46.67109320702403608_dp, & + & 4.47479391972813190_dp, 47.21956618120344018_dp, & + & 4.12512266986032206_dp, 47.71926346185548340_dp, & + & 3.55581027759415491_dp, 48.11926090196744354_dp, & + & 2.82200931024728874_dp, 48.37772379915030996_dp, & + & 1.99999999999999556_dp, 48.46708828700425187_dp, & + & 1.17799068975270305_dp, 48.37772379915030996_dp, & + & 0.44418972240583693_dp, 48.11926090196744354_dp, & + & -0.12512266986032983_dp, 47.71926346185548340_dp, & + & -0.47479391972814028_dp, 47.21956618120344018_dp, & + & -0.57569351104372879_dp, 46.67109320702403608_dp, & + & -0.42528132740346147_dp, 46.12813070368466839_dp, & + & -0.04494770792091433_dp, 45.64287023699112211_dp, & + & 0.52444137510688182_dp, 45.26073815098433784_dp, & + & 1.22762737065263883_dp, 45.01674523281888440_dp, & + & 1.99999999999999556_dp, 42.63725270983827897_dp, & + & 3.27473332072834422_dp, 42.73934376403467184_dp, & + & 4.49349321557291415_dp, 43.04109625964867547_dp, & + & 5.60127364516099924_dp, 43.52904513460186564_dp, & + & 6.54514794659825672_dp, 44.18107605134951399_dp, & + & 7.27568782798168190_dp, 44.96693492678289061_dp, & + & 7.74887351496657040_dp, 45.84898874281991965_dp, & + & 7.92867515230084408_dp, 46.78330526703850012_dp, & + & 7.79040894364469949_dp, 47.72115423047301164_dp, & + & 7.32476598915540844_dp, 48.61105462603097038_dp, & + & 6.54204071489340944_dp, 49.40147266886311428_dp, & + & 5.47559260124316793_dp, 50.04417716102324221_dp, & + & 4.18315343237871584_dp, 50.49806364409691639_dp, & + & 2.74458547194322389_dp, 50.73299668020899134_dp, & + & 1.25541452805676457_dp, 50.73299668020899134_dp, & + & -0.18315343237872406_dp, 50.49806364409691639_dp, & + & -1.47559260124317815_dp, 50.04417716102324221_dp, & + & -2.54204071489341921_dp, 49.40147266886311428_dp, & + & -3.32476598915541599_dp, 48.61105462603097038_dp, & + & -3.79040894364470748_dp, 47.72115423047301164_dp, & + & -3.92867515230085296_dp, 46.78330526703850012_dp, & + & -3.74887351496657928_dp, 45.84898874281991965_dp, & + & -3.27568782798168812_dp, 44.96693492678289061_dp, & + & -2.54514794659826427_dp, 44.18107605134951399_dp, & + & -1.60127364516100812_dp, 43.52904513460186564_dp, & + & -0.49349321557292075_dp, 43.04109625964867547_dp, & + & 0.72526667927164967_dp, 42.73934376403467184_dp, & + & 1.99999999999999556_dp, 40.31236211945285675_dp, & + & 3.63376292032076975_dp, 40.42261301497919845_dp, & + & 5.22173788476928014_dp, 40.75019560382368411_dp, & + & 6.71826031811739099_dp, 41.28563184510834105_dp, & + & 8.07794770476515467_dp, 42.01322983391511201_dp, & + & 9.25594876339864392_dp, 42.91123363953519743_dp, & + & 10.20837670733705771_dp, 43.95202523375940729_dp, & + & 10.89307303713296271_dp, 45.10238649522262477_dp, & + & 11.27090590027328432_dp, 46.32385288536406875_dp, & + & 11.30784824781773779_dp, 47.57322688923063225_dp, & + & 10.97806564551987840_dp, 48.80336562297891589_dp, & + & 10.26810804950859612_dp, 49.96440199105025926_dp, & + & 9.18197003975669723_dp, 51.00557820511744467_dp, & + & 7.74622775989939338_dp, 51.87782553848720823_dp, & + & 6.01378910799337074_dp, 52.53707313311167582_dp, & + & 4.06435502534209725_dp, 52.94800110445672203_dp, & + & 1.99999999999999600_dp, 53.08763788054714894_dp, & + & -0.06435502534210560_dp, 52.94800110445672203_dp, & + & -2.01378910799337962_dp, 52.53707313311167582_dp, & + & -3.74622775989940182_dp, 51.87782553848720823_dp, & + & -5.18197003975670256_dp, 51.00557820511744467_dp, & + & -6.26810804950860057_dp, 49.96440199105025926_dp, & + & -6.97806564551988551_dp, 48.80336562297891589_dp, & + & -7.30784824781774667_dp, 47.57322688923063225_dp, & + & -7.27090590027329498_dp, 46.32385288536406875_dp, & + & -6.89307303713297248_dp, 45.10238649522262477_dp, & + & -6.20837670733706481_dp, 43.95202523375940729_dp, & + & -5.25594876339865458_dp, 42.91123363953519743_dp, & + & -4.07794770476516355_dp, 42.01322983391511201_dp, & + & -2.71826031811739988_dp, 41.28563184510834105_dp, & + & -1.22173788476928813_dp, 40.75019560382368411_dp, & + & 0.36623707967922192_dp, 40.42261301497919845_dp, & + & 1.99999999999999600_dp, 37.95945956416753120_dp, & + & 3.72994195085756930_dp, 38.05275970261258323_dp, & + & 5.43190111561111433_dp, 38.33106975566338548_dp, & + & 7.07783882564464850_dp, 38.78962916423042628_dp, & + & 8.63959771096107509_dp, 39.42053684981195261_dp, & + & 10.08882962579568954_dp, 40.21279467492543347_dp, & + & 11.39692139773956114_dp, 41.15235690381517486_dp, & + & 12.53494002107632888_dp, 42.22217718966415845_dp, & + & 13.47363915243209043_dp, 43.40224634648851065_dp, & + & 14.18359521918176469_dp, 44.66961909752426152_dp, & + & 14.63557388477568644_dp, 45.99843704779454612_dp, & + & 14.80126342211737089_dp, 47.35996933207513848_dp, & + & 14.65454278958190670_dp, 48.72271255422421632_dp, & + & 14.17346129570704250_dp, 50.05261748260089405_dp, & + & 13.34306219743110233_dp, 51.31353838046217675_dp, & + & 12.15903975007321236_dp, 52.46802323745790630_dp, & + & 10.63193584333677499_dp, 53.47856301011842106_dp, & + & 8.79116029542177024_dp, 54.30937174533021761_dp, & + & 6.68766853851381526_dp, 54.92865613654313961_dp, & + & 4.39391037845720334_dp, 55.31115535850048559_dp, & + & 1.99999999999999600_dp, 55.44054043583248159_dp, & + & -0.39391037845721066_dp, 55.31115535850048559_dp, & + & -2.68766853851382415_dp, 54.92865613654313961_dp, & + & -4.79116029542177646_dp, 54.30937174533021761_dp, & + & -6.63193584333678299_dp, 53.47856301011842106_dp, & + & -8.15903975007321947_dp, 52.46802323745790630_dp, & + & -9.34306219743110766_dp, 51.31353838046217675_dp, & + & -10.17346129570704960_dp, 50.05261748260089405_dp, & + & -10.65454278958191559_dp, 48.72271255422421632_dp, & + & -10.80126342211738155_dp, 47.35996933207513848_dp, & + & -10.63557388477569354_dp, 45.99843704779454612_dp, & + & -10.18359521918177357_dp, 44.66961909752426152_dp, & + & -9.47363915243210286_dp, 43.40224634648851065_dp, & + & -8.53494002107633776_dp, 42.22217718966415845_dp, & + & -7.39692139773957003_dp, 41.15235690381517486_dp, & + & -6.08882962579569842_dp, 40.21279467492543347_dp, & + & -4.63959771096108220_dp, 39.42053684981195261_dp, & + & -3.07783882564465516_dp, 38.78962916423042628_dp, & + & -1.43190111561112299_dp, 38.33106975566338548_dp, & + & 0.27005804914242265_dp, 38.05275970261258323_dp, & + & 1.99999999999999600_dp, 35.57084905987729684_dp, & + & 3.89475329148972449_dp, 35.66164776728690811_dp, & + & 5.76748883913266752_dp, 35.93289207465146262_dp, & + & 7.59612881396671380_dp, 36.38113516165864070_dp, & + & 9.35846198524136241_dp, 37.00065953311393940_dp, & + & 11.03204592267791284_dp, 37.78351258180431671_dp, & + & 12.59407748901272228_dp, 38.71954642363609622_dp, & + & 14.02123063562953753_dp, 39.79645326718445375_dp, & + & 15.28946940417034384_dp, 40.99978674800681233_dp, & + & 16.37385639149118433_dp, 42.31296002482778107_dp, & + & 17.24839405994832120_dp, 43.71721334442865725_dp, & + & 17.88595991605807711_dp, 45.19154778206701195_dp, & + & 18.25842850603561374_dp, 46.71262882888417067_dp, & + & 18.33711398818940452_dp, 48.25467471095996785_dp, & + & 18.09371358173445188_dp, 49.78936139305309894_dp, & + & 17.50197280996727400_dp, 51.28580055994029152_dp, & + & 16.54030106040206149_dp, 52.71067834344860614_dp, & + & 15.19549149366584473_dp, 54.02867680201524081_dp, & + & 13.46747450557555048_dp, 55.20332460427978560_dp, & + & 11.37460140494845717_dp, 56.19841400024800038_dp, & + & 8.95834334902436602_dp, 56.98004544675784899_dp, & + & 6.28571777452580616_dp, 57.51919556269155720_dp, & + & 3.44765611568870733_dp, 57.79446760315521914_dp, & + & 0.55234388431128512_dp, 57.79446760315521914_dp, & + & -2.28571777452581371_dp, 57.51919556269155720_dp, & + & -4.95834334902437313_dp, 56.98004544675784899_dp, & + & -7.37460140494846428_dp, 56.19841400024800038_dp, & + & -9.46747450557555759_dp, 55.20332460427978560_dp, & + & -11.19549149366585006_dp, 54.02867680201524081_dp, & + & -12.54030106040206860_dp, 52.71067834344860614_dp, & + & -13.50197280996728288_dp, 51.28580055994029152_dp, & + & -14.09371358173445721_dp, 49.78936139305309894_dp, & + & -14.33711398818941340_dp, 48.25467471095996785_dp, & + & -14.25842850603561907_dp, 46.71262882888417067_dp, & + & -13.88595991605808244_dp, 45.19154778206701195_dp, & + & -13.24839405994833008_dp, 43.71721334442865725_dp, & + & -12.37385639149119498_dp, 42.31296002482778107_dp, & + & -11.28946940417035272_dp, 40.99978674800681233_dp, & + & -10.02123063562954464_dp, 39.79645326718445375_dp, & + & -8.59407748901273116_dp, 38.71954642363609622_dp, & + & -7.03204592267792172_dp, 37.78351258180431671_dp, & + & -5.35846198524136952_dp, 37.00065953311393940_dp, & + & -3.59612881396672091_dp, 36.38113516165864070_dp, & + & -1.76748883913267574_dp, 35.93289207465146262_dp, & + & 0.10524670851026705_dp, 35.66164776728690811_dp, & + & 1.99999999999999600_dp, 33.13704588354342917_dp, & + & 4.09718755711548610_dp, 33.23124463343540214_dp, & + & 6.17473678976558027_dp, 33.51283479810019372_dp, & + & 8.21297051898842767_dp, 33.97880825794451454_dp, & + & 10.19211732580419394_dp, 34.62418375193038855_dp, & + & 12.09222434325374529_dp, 35.44205010622848562_dp, & + & 13.89302536639832120_dp, 36.42361680044041350_dp, & + & 15.57375501535205942_dp, 37.55826279697889447_dp, & + & 17.11290455740948246_dp, 38.83357308646102268_dp, & + & 18.48792157318536056_dp, 40.23535159216240942_dp, & + & 19.67486490585671888_dp, 41.74759899443786537_dp, & + & 20.64804001511417653_dp, 43.35244484209054860_dp, & + & 21.37966075266355404_dp, 45.03002540480565585_dp, & + & 21.83961562141898582_dp, 46.75830290430587155_dp, & + & 21.99546453290828296_dp, 48.51282952087984768_dp, & + & 21.81285996059914467_dp, 50.26647329190694080_dp, & + & 21.25667322722224029_dp, 51.98914605534769606_dp, & + & 20.29319722451073105_dp, 53.64760968250063655_dp, & + & 18.89384545876525578_dp, 55.20548791890712437_dp, & + & 17.04067793585433321_dp, 56.62367212326139310_dp, & + & 14.73370655373201643_dp, 57.86135900031617751_dp, & + & 11.99912115224313958_dp, 58.87795079119066344_dp, & + & 8.89637618755807047_dp, 59.63591573610307250_dp, & + & 5.52099396835736211_dp, 60.10440004572244277_dp, & + & 1.99999999999999645_dp, 60.26295411645657651_dp, & + & -1.52099396835736900_dp, 60.10440004572244277_dp, & + & -4.89637618755807846_dp, 59.63591573610307250_dp, & + & -7.99912115224314757_dp, 58.87795079119066344_dp, & + & -10.73370655373202531_dp, 57.86135900031617751_dp, & + & -13.04067793585434032_dp, 56.62367212326139310_dp, & + & -14.89384545876526467_dp, 55.20548791890712437_dp, & + & -16.29319722451073460_dp, 53.64760968250063655_dp, & + & -17.25667322722224739_dp, 51.98914605534769606_dp, & + & -17.81285996059915178_dp, 50.26647329190694080_dp, & + & -17.99546453290829362_dp, 48.51282952087984768_dp, & + & -17.83961562141899648_dp, 46.75830290430587155_dp, & + & -17.37966075266356114_dp, 45.03002540480565585_dp, & + & -16.64804001511417653_dp, 43.35244484209054860_dp, & + & -15.67486490585672598_dp, 41.74759899443786537_dp, & + & -14.48792157318536944_dp, 40.23535159216240942_dp, & + & -13.11290455740949135_dp, 38.83357308646102268_dp, & + & -11.57375501535207007_dp, 37.55826279697889447_dp, & + & -9.89302536639833185_dp, 36.42361680044041350_dp, & + & -8.09222434325375239_dp, 35.44205010622848562_dp, & + & -6.19211732580420193_dp, 34.62418375193038855_dp, & + & -4.21297051898843566_dp, 33.97880825794451454_dp, & + & -2.17473678976558960_dp, 33.51283479810019372_dp, & + & -0.09718755711549353_dp, 33.23124463343540214_dp, & + & 1.99999999999999600_dp, 30.64757019281734074_dp, & + & 3.92669240244598505_dp, 30.71678191817960268_dp, & + & 5.84270801296843700_dp, 30.92395598105918353_dp, & + & 7.73736939611796171_dp, 31.26771342119434749_dp, & + & 9.59999239238874047_dp, 31.74577019145994683_dp, & + & 11.41986936545086984_dp, 32.35495718584155611_dp, & + & 13.18623692372253586_dp, 33.09124574576952682_dp, & + & 14.88822378743806851_dp, 33.94977626735131793_dp, & + & 16.51477509868480453_dp, 34.92488704934376642_dp, & + & 18.05455017155122732_dp, 36.01014014726423795_dp, & + & 19.49579145727240714_dp, 37.19834072287546434_dp, & + & 20.82616341777591984_dp, 38.48154618161253637_dp, & + & 22.03256121225549435_dp, 39.85106125278488065_dp, & + & 23.10089088783405131_dp, 41.29741507659190347_dp, & + & 24.01582559662802652_dp, 42.81031632956308641_dp, & + & 24.76054697060166987_dp, 44.37858250245571412_dp, & + & 25.31648826424518717_dp, 45.99003977573201496_dp, & + & 25.66310777053267245_dp, 47.63139077670033572_dp, & + & 25.77773936992416637_dp, 49.28804930270956675_dp, & + & 25.63559425749198795_dp, 50.94394459368972150_dp, & + & 25.21002592824931199_dp, 52.58130404832083116_dp, & + & 24.47321926655621738_dp, 54.18043392376822709_dp, & + & 23.39751805719552991_dp, 55.71953428872550518_dp, & + & 21.95764487214750105_dp, 57.17460858105235388_dp, & + & 20.13405311103923623_dp, 58.51955862209560166_dp, & + & 17.91751559979281083_dp, 59.72658657388572578_dp, & + & 15.31471471060360479_dp, 60.76704034614789407_dp, & + & 12.35401191716921154_dp, 61.61281120273996237_dp, & + & 9.08984714396560278_dp, 62.23829052070698253_dp, & + & 5.60371952785031446_dp, 62.62270582496653049_dp, & + & 1.99999999999999556_dp, 62.75242980718265784_dp, & + & -1.60371952785032224_dp, 62.62270582496653049_dp, & + & -5.08984714396560900_dp, 62.23829052070698253_dp, & + & -8.35401191716922042_dp, 61.61281120273996237_dp, & + & -11.31471471060361367_dp, 60.76704034614789407_dp, & + & -13.91751559979281616_dp, 59.72658657388572578_dp, & + & -16.13405311103924689_dp, 58.51955862209560166_dp, & + & -17.95764487214750815_dp, 57.17460858105235388_dp, & + & -19.39751805719554056_dp, 55.71953428872550518_dp, & + & -20.47321926655623159_dp, 54.18043392376822709_dp, & + & -21.21002592824931554_dp, 52.58130404832083116_dp, & + & -21.63559425749199860_dp, 50.94394459368972150_dp, & + & -21.77773936992418058_dp, 49.28804930270956675_dp, & + & -21.66310777053267955_dp, 47.63139077670033572_dp, & + & -21.31648826424519783_dp, 45.99003977573201496_dp, & + & -20.76054697060167697_dp, 44.37858250245571412_dp, & + & -20.01582559662803362_dp, 42.81031632956308641_dp, & + & -19.10089088783406197_dp, 41.29741507659190347_dp, & + & -18.03256121225550146_dp, 39.85106125278488065_dp, & + & -16.82616341777592694_dp, 38.48154618161253637_dp, & + & -15.49579145727241603_dp, 37.19834072287546434_dp, & + & -14.05455017155123620_dp, 36.01014014726423795_dp, & + & -12.51477509868481341_dp, 34.92488704934376642_dp, & + & -10.88822378743808095_dp, 33.94977626735131793_dp, & + & -9.18623692372254474_dp, 33.09124574576952682_dp, & + & -7.41986936545088049_dp, 32.35495718584155611_dp, & + & -5.59999239238874935_dp, 31.74577019145994683_dp, & + & -3.73736939611796970_dp, 31.26771342119434749_dp, & + & -1.84270801296844478_dp, 30.92395598105918353_dp, & + & 0.07330759755400772_dp, 30.71678191817960268_dp, & + & 1.99999999999999600_dp, 28.09100109536852230_dp, & + & 4.16843782661930362_dp, 28.16889262543881145_dp, & + & 6.32564933647368566_dp, 28.40205318377053345_dp, & + & 8.46042949045145143_dp, 28.78894734383437992_dp, & + & 10.56160821625925195_dp, 29.32703776203888069_dp, & + & 12.61804920168178690_dp, 30.01281590885307438_dp, & + & 14.61862699669389087_dp, 30.84184157026976436_dp, & + & 16.55217628249286577_dp, 31.80878785141789322_dp, & + & 18.40740784814350306_dp, 32.90748779465315010_dp, & + & 20.17278641740253420_dp, 34.13097826525053335_dp, & + & 21.83636590786353437_dp, 35.47153642700349963_dp, & + & 23.38557795842459797_dp, 36.92070387379718710_dp, & + & 24.80696969977402233_dp, 38.46929323359992026_dp, & + & 26.08588698105331716_dp, 40.10737174655498194_dp, & + & 27.20610004359020806_dp, 41.82421587722975431_dp, & + & 28.14937074714974585_dp, 43.60823042135866956_dp, & + & 28.89496528133906494_dp, 45.44682484300304282_dp, & + & 29.41912613625676443_dp, 47.32623888788154431_dp, & + & 29.69453569648833735_dp, 49.23130925676766623_dp, & + & 29.68983696232584180_dp, 51.14517010976478417_dp, & + & 29.36933297612858240_dp, 53.04888396821036167_dp, & + & 28.69307609487012201_dp, 54.92100894435518654_dp, & + & 27.61769060836896728_dp, 56.73712764036463341_dp, & + & 26.09844461997003151_dp, 58.46939890910893922_dp, & + & 24.09325961331439458_dp, 60.08625295311978931_dp, & + & 21.56939664790981936_dp, 61.55243525845713748_dp, & + & 18.51323316925390117_dp, 62.82970044453450953_dp, & + & 14.94248311853021427_dp, 63.87850997409253040_dp, & + & 10.91821393310038246_dp, 64.66099291349783584_dp, & + & 6.55165837938126572_dp, 65.14506774788591770_dp, & + & 1.99999999999999645_dp, 65.30899890463147983_dp, & + & -2.55165837938127327_dp, 65.14506774788591770_dp, & + & -6.91821393310038868_dp, 64.66099291349783584_dp, & + & -10.94248311853022138_dp, 63.87850997409253040_dp, & + & -14.51323316925390650_dp, 62.82970044453450953_dp, & + & -17.56939664790982647_dp, 61.55243525845713748_dp, & + & -20.09325961331440169_dp, 60.08625295311978931_dp, & + & -22.09844461997003862_dp, 58.46939890910893922_dp, & + & -23.61769060836897793_dp, 56.73712764036463341_dp, & + & -24.69307609487013266_dp, 54.92100894435518654_dp, & + & -25.36933297612859306_dp, 53.04888396821036167_dp, & + & -25.68983696232585245_dp, 51.14517010976478417_dp, & + & -25.69453569648835156_dp, 49.23130925676766623_dp, & + & -25.41912613625677508_dp, 47.32623888788154431_dp, & + & -24.89496528133907205_dp, 45.44682484300304282_dp, & + & -24.14937074714975651_dp, 43.60823042135866956_dp, & + & -23.20610004359021872_dp, 41.82421587722975431_dp, & + & -22.08588698105333137_dp, 40.10737174655498194_dp, & + & -20.80696969977403299_dp, 38.46929323359992026_dp, & + & -19.38557795842460862_dp, 36.92070387379718710_dp, & + & -17.83636590786354148_dp, 35.47153642700349963_dp, & + & -16.17278641740254486_dp, 34.13097826525053335_dp, & + & -14.40740784814351372_dp, 32.90748779465315010_dp, & + & -12.55217628249287110_dp, 31.80878785141789322_dp, & + & -10.61862699669389798_dp, 30.84184157026976436_dp, & + & -8.61804920168179400_dp, 30.01281590885307438_dp, & + & -6.56160821625925994_dp, 29.32703776203888069_dp, & + & -4.46042949045146031_dp, 28.78894734383437992_dp, & + & -2.32564933647369187_dp, 28.40205318377053345_dp, & + & -0.16843782661931131_dp, 28.16889262543881145_dp, & + & 1.99999999999999600_dp, 25.45485763546421154_dp, & + & 4.25579267831604824_dp, 25.53081635683327377_dp, & + & 6.50189378344491864_dp, 25.75824904385734015_dp, & + & 8.72865139696445169_dp, 26.13583181560235502_dp, & + & 10.92648650789847409_dp, 26.66137960320710931_dp, & + & 13.08591371200912867_dp, 27.33187675649334025_dp, & + & 15.19754366142444368_dp, 28.14351701208744316_dp, & + & 17.25206214192900589_dp, 29.09175016726920759_dp, & + & 19.24018123567453742_dp, 30.17133232687284305_dp, & + & 21.15255849246134545_dp, 31.37637626233231813_dp, & + & 22.97968026705780886_dp, 32.70039821822412307_dp, & + & 24.71170528699354918_dp, 34.13635737784334623_dp, & + & 26.33826403031282837_dp, 35.67668409468497259_dp, & + & 27.84820859564654683_dp, 37.31329283883029291_dp, & + & 29.22930648122764197_dp, 39.03757551632935474_dp, & + & 30.46787020033522708_dp, 40.84037031204417190_dp, & + & 31.54831327014242248_dp, 42.71190039889389567_dp, & + & 32.45262246278107199_dp, 44.64167567492752653_dp, & + & 33.15973753507462618_dp, 46.61834908938177335_dp, & + & 33.64483525740958214_dp, 48.62951713112133945_dp, & + & 33.87852862440772839_dp, 50.66145188613276673_dp, & + & 33.82602201650912122_dp, 52.69875031809839783_dp, & + & 33.44632120220701665_dp, 54.72388646563504011_dp, & + & 32.69170289014788011_dp, 56.71665696607986717_dp, & + & 31.50782877801381332_dp, 58.65352526302899605_dp, & + & 29.83517004248913196_dp, 60.50690475312735828_dp, & + & 27.61278608277912383_dp, 62.24449065808524040_dp, & + & 24.78586461017365750_dp, 63.82887077001763032_dp, & + & 21.31841683118042496_dp, 65.21781824976642383_dp, & + & 17.21138564172178320_dp, 66.36584339331807314_dp, & + & 12.52326664933801226_dp, 67.22759281764989225_dp, & + & 7.38533547682125846_dp, 67.76325974975715383_dp, & + & 1.99999999999999600_dp, 67.94514236453578349_dp, & + & -3.38533547682126690_dp, 67.76325974975715383_dp, & + & -8.52326664933801936_dp, 67.22759281764989225_dp, & + & -13.21138564172179031_dp, 66.36584339331807314_dp, & + & -17.31841683118043562_dp, 65.21781824976642383_dp, & + & -20.78586461017366460_dp, 63.82887077001763032_dp, & + & -23.61278608277912738_dp, 62.24449065808524040_dp, & + & -25.83517004248913906_dp, 60.50690475312735828_dp, & + & -27.50782877801382398_dp, 58.65352526302899605_dp, & + & -28.69170289014789077_dp, 56.71665696607986717_dp, & + & -29.44632120220701665_dp, 54.72388646563504011_dp, & + & -29.82602201650912477_dp, 52.69875031809839783_dp, & + & -29.87852862440772839_dp, 50.66145188613276673_dp, & + & -29.64483525740958925_dp, 48.62951713112133945_dp, & + & -29.15973753507462618_dp, 46.61834908938177335_dp, & + & -28.45262246278107554_dp, 44.64167567492752653_dp, & + & -27.54831327014242959_dp, 42.71190039889389567_dp, & + & -26.46787020033523774_dp, 40.84037031204417190_dp, & + & -25.22930648122764907_dp, 39.03757551632935474_dp, & + & -23.84820859564655038_dp, 37.31329283883029291_dp, & + & -22.33826403031283903_dp, 35.67668409468497259_dp, & + & -20.71170528699355629_dp, 34.13635737784334623_dp, & + & -18.97968026705781242_dp, 32.70039821822412307_dp, & + & -17.15255849246134900_dp, 31.37637626233231813_dp, & + & -15.24018123567454097_dp, 30.17133232687284305_dp, & + & -13.25206214192901299_dp, 29.09175016726920759_dp, & + & -11.19754366142445434_dp, 28.14351701208744316_dp, & + & -9.08591371200913933_dp, 27.33187675649334025_dp, & + & -6.92648650789848030_dp, 26.66137960320710931_dp, & + & -4.72865139696445969_dp, 26.13583181560235502_dp, & + & -2.50189378344492708_dp, 25.75824904385734015_dp, & + & -0.25579267831605540_dp, 25.53081635683327377_dp, & + & 1.99999999999999556_dp, 22.72540973598210456_dp, & + & 4.47629214130109165_dp, 22.80878975038074330_dp, & + & 6.94242093509331060_dp, 23.05843251151436846_dp, & + & 9.38829543151647350_dp, 23.47285525677742868_dp, & + & 11.80396120930039139_dp, 24.04961640006232315_dp, & + & 14.17964838758363122_dp, 24.78535803035031293_dp, & + & 16.50579641562137567_dp, 25.67586152120415477_dp, & + & 18.77304947679039060_dp, 26.71611272875198395_dp, & + & 20.97221728412494102_dp, 27.90037270541385439_dp, & + & 23.09419678103115103_dp, 29.22224954465700009_dp, & + & 25.12985059268006793_dp, 30.67476685924402702_dp, & + & 27.06983782140639860_dp, 32.25042441190934994_dp, & + & 28.90439178940929921_dp, 33.94124647006630369_dp, & + & 30.62303747247054275_dp, 35.73881343521246379_dp, & + & 32.21423852322426029_dp, 37.63427208010365632_dp, & + & 33.66495985223924237_dp, 39.61831917500946787_dp, & + & 34.96012666362174315_dp, 41.68115224019865650_dp, & + & 36.08195470333167520_dp, 43.81237943576640248_dp, & + & 37.00911969333093765_dp, 46.00087796034757304_dp, & + & 37.71572773969587189_dp, 48.23458650173874673_dp, & + & 38.17004600591188535_dp, 50.50021197832979425_dp, & + & 38.33296113360039925_dp, 52.78282385843581892_dp, & + & 38.15616674923441565_dp, 55.06530104282313687_dp, & + & 37.58017141013022666_dp, 57.32758829402004608_dp, & + & 36.53242307522426557_dp, 59.54571651024583190_dp, & + & 34.92626834358740950_dp, 61.69055635034530383_dp, & + & 32.66225471925319113_dp, 63.72633555655123416_dp, & + & 29.63456841105837825_dp, 65.60910965714572285_dp, & + & 25.74696599568290623_dp, 67.28571148577094618_dp, & + & 20.94303945184942251_dp, 68.69426490502790728_dp, & + & 15.25145387811900122_dp, 69.76794994271870110_dp, & + & 8.83336207267744022_dp, 70.44359392189112157_dp, & + & 1.99999999999999600_dp, 70.67459026401789401_dp, & + & -4.83336207267745088_dp, 70.44359392189112157_dp, & + & -11.25145387811901188_dp, 69.76794994271870110_dp, & + & -16.94303945184943316_dp, 68.69426490502790728_dp, & + & -21.74696599568291333_dp, 67.28571148577094618_dp, & + & -25.63456841105838890_dp, 65.60910965714572285_dp, & + & -28.66225471925320534_dp, 63.72633555655123416_dp, & + & -30.92626834358741306_dp, 61.69055635034530383_dp, & + & -32.53242307522427978_dp, 59.54571651024583190_dp, & + & -33.58017141013024087_dp, 57.32758829402004608_dp, & + & -34.15616674923442986_dp, 55.06530104282313687_dp, & + & -34.33296113360040636_dp, 52.78282385843581892_dp, & + & -34.17004600591189956_dp, 50.50021197832979425_dp, & + & -33.71572773969587900_dp, 48.23458650173874673_dp, & + & -33.00911969333094476_dp, 46.00087796034757304_dp, & + & -32.08195470333168231_dp, 43.81237943576640248_dp, & + & -30.96012666362175025_dp, 41.68115224019865650_dp, & + & -29.66495985223924237_dp, 39.61831917500946787_dp, & + & -28.21423852322426740_dp, 37.63427208010365632_dp, & + & -26.62303747247054631_dp, 35.73881343521246379_dp, & + & -24.90439178940930276_dp, 33.94124647006630369_dp, & + & -23.06983782140640216_dp, 32.25042441190934994_dp, & + & -21.12985059268007149_dp, 30.67476685924402702_dp, & + & -19.09419678103115459_dp, 29.22224954465700009_dp, & + & -16.97221728412494102_dp, 27.90037270541385439_dp, & + & -14.77304947679039948_dp, 26.71611272875198395_dp, & + & -12.50579641562138455_dp, 25.67586152120415477_dp, & + & -10.17964838758364365_dp, 24.78535803035031293_dp, & + & -7.80396120930039938_dp, 24.04961640006232315_dp, & + & -5.38829543151647972_dp, 23.47285525677742868_dp, & + & -2.94242093509332037_dp, 23.05843251151436846_dp, & + & -0.47629214130110020_dp, 22.80878975038074330_dp, & + & 1.99999999999999600_dp, 19.88744176867359670_dp, & + & 4.69641128035122435_dp, 19.97822987242788884_dp, & + & 7.38211423303385939_dp, 20.25003331964319031_dp, & + & 10.04651538477556549_dp, 20.70118176192879389_dp, & + & 12.67924035399200378_dp, 21.32893128539882710_dp, & + & 15.27021753707410312_dp, 22.12952164603582617_dp, & + & 17.80973254683686946_dp, 23.09825113914619976_dp, & + & 20.28844626261447459_dp, 24.22956442487017270_dp, & + & 22.69737092792362532_dp, 25.51714801636046204_dp, & + & 25.02780000980339636_dp, 26.95402789495272344_dp, & + & 27.27118821325056430_dp, 28.53266378578365448_dp, & + & 29.41897786066868292_dp, 30.24503490367624536_dp, & + & 31.46236656888654792_dp, 32.08271233019573287_dp, & + & 33.39200856069422940_dp, 34.03691345955112979_dp, & + & 35.19763777135730720_dp, 36.09853399317180589_dp, & + & 36.86759479425492714_dp, 38.25815259440047811_dp, & + & 38.38823113470312620_dp, 40.50600232791481403_dp, & + & 39.74315247327933776_dp, 42.83190113242096686_dp, & + & 40.91224672040999621_dp, 45.22513042719018728_dp, & + & 41.87042151784342536_dp, 47.67424596736638165_dp, & + & 42.58594887503602422_dp, 50.16679739569313057_dp, & + & 43.01828312042781022_dp, 52.68892136253921876_dp, & + & 43.11518954384546731_dp, 55.22475594928482678_dp, & + & 42.80901819915838757_dp, 57.75559962131574565_dp, & + & 42.01204327449660525_dp, 60.25870541573158334_dp, & + & 40.61111913565843423_dp, 62.70556553816149403_dp, & + & 38.46283411458643542_dp, 65.05952649321666570_dp, & + & 35.39257932570031784_dp, 67.27265449907143591_dp, & + & 31.20553396293055215_dp, 69.28213523998356038_dp, & + & 25.72462936947633239_dp, 71.00751554077180572_dp, & + & 18.87413258170036201_dp, 72.35214877025040892_dp, & + & 10.80643587234801295_dp, 73.21444539538327945_dp, & + & 1.99999999999999600_dp, 73.51255823132640899_dp, & + & -6.80643587234802006_dp, 73.21444539538327945_dp, & + & -14.87413258170037622_dp, 72.35214877025040892_dp, & + & -21.72462936947634304_dp, 71.00751554077180572_dp, & + & -27.20553396293055570_dp, 69.28213523998356038_dp, & + & -31.39257932570032139_dp, 67.27265449907143591_dp, & + & -34.46283411458643542_dp, 65.05952649321666570_dp, & + & -36.61111913565844134_dp, 62.70556553816149403_dp, & + & -38.01204327449661946_dp, 60.25870541573158334_dp, & + & -38.80901819915838047_dp, 57.75559962131574565_dp, & + & -39.11518954384547442_dp, 55.22475594928482678_dp, & + & -39.01828312042781022_dp, 52.68892136253921876_dp, & + & -38.58594887503603843_dp, 50.16679739569313057_dp, & + & -37.87042151784342536_dp, 47.67424596736638165_dp, & + & -36.91224672041000332_dp, 45.22513042719018728_dp, & + & -35.74315247327935197_dp, 42.83190113242096686_dp, & + & -34.38823113470313331_dp, 40.50600232791481403_dp, & + & -32.86759479425493424_dp, 38.25815259440047811_dp, & + & -31.19763777135731431_dp, 36.09853399317180589_dp, & + & -29.39200856069424717_dp, 34.03691345955112979_dp, & + & -27.46236656888655858_dp, 32.08271233019573287_dp, & + & -25.41897786066869003_dp, 30.24503490367624536_dp, & + & -23.27118821325056786_dp, 28.53266378578365448_dp, & + & -21.02780000980340702_dp, 26.95402789495272344_dp, & + & -18.69737092792362887_dp, 25.51714801636046204_dp, & + & -16.28844626261448170_dp, 24.22956442487017270_dp, & + & -13.80973254683687301_dp, 23.09825113914619976_dp, & + & -11.27021753707411023_dp, 22.12952164603582617_dp, & + & -8.67924035399200910_dp, 21.32893128539882710_dp, & + & -6.04651538477557171_dp, 20.70118176192879389_dp, & + & -3.38211423303386916_dp, 20.25003331964319031_dp, & + & -0.69641128035123179_dp, 19.97822987242788884_dp, & + & 1.99999999999999556_dp, 16.92397162343523931_dp, & + & 4.91799021933642688_dp, 17.02221659443387480_dp, & + & 7.82460955068693664_dp, 17.31631470128239059_dp, & + & 10.70865569654628935_dp, 17.80437171431731258_dp, & + & 13.55924980839402139_dp, 18.48328342761256238_dp, & + & 16.36596498789072385_dp, 19.34881113120014362_dp, & + & 19.11891780975726363_dp, 20.39568010883170501_dp, & + & 21.80881477377849720_dp, 21.61769490666709714_dp, & + & 24.42694818201795570_dp, 23.00786444863284430_dp, & + & 26.96513811238917668_dp, 24.55852997196867804_dp, & + & 29.41561850057672345_dp, 26.26148912292403637_dp, & + & 31.77086550998261671_dp, 28.10811023218007065_dp, & + & 34.02336509370287132_dp, 30.08943159705432180_dp, & + & 36.16531367916817175_dp, 32.19624133969532664_dp, & + & 38.18824090805679106_dp, 34.41913389699745807_dp, & + & 40.08253581911941410_dp, 36.74853923728758076_dp, & + & 41.83684688715484867_dp, 39.17472027603687934_dp, & + & 43.43731045663169255_dp, 41.68773239540036002_dp, & + & 44.86653894639825779_dp, 44.27733603455973110_dp, & + & 46.10226597817306526_dp, 46.93284831506371546_dp, & + & 47.11549450115907689_dp, 49.64291142290011294_dp, & + & 47.86791757533286074_dp, 52.39514197266055362_dp, & + & 48.30826862884845241_dp, 55.17560336970880286_dp, & + & 48.36709840828364548_dp, 57.96800639328736793_dp, & + & 47.94927569193657035_dp, 60.75248226909388904_dp, & + & 46.92334661203305046_dp, 63.50367340927639503_dp, & + & 45.10709162054252630_dp, 66.18773615332521842_dp, & + & 42.25028545833759352_dp, 68.75766348420629015_dp, & + & 38.02194152863378918_dp, 71.14627956201685777_dp, & + & 32.02735732207737840_dp, 73.25702648451857613_dp, & + & 23.91726594386999949_dp, 74.95625177955801632_dp, & + & 13.67376613190776347_dp, 76.07990221077976400_dp, & + & 1.99999999999999600_dp, 76.47602837656475572_dp, & + & -9.67376613190777235_dp, 76.07990221077976400_dp, & + & -19.91726594387001015_dp, 74.95625177955801632_dp, & + & -28.02735732207739261_dp, 73.25702648451857613_dp, & + & -34.02194152863380339_dp, 71.14627956201685777_dp, & + & -38.25028545833759352_dp, 68.75766348420629015_dp, & + & -41.10709162054251919_dp, 66.18773615332521842_dp, & + & -42.92334661203306467_dp, 63.50367340927639503_dp, & + & -43.94927569193657746_dp, 60.75248226909388904_dp, & + & -44.36709840828364548_dp, 57.96800639328736793_dp, & + & -44.30826862884845241_dp, 55.17560336970880286_dp, & + & -43.86791757533286784_dp, 52.39514197266055362_dp, & + & -43.11549450115907689_dp, 49.64291142290011294_dp, & + & -42.10226597817306526_dp, 46.93284831506371546_dp, & + & -40.86653894639826490_dp, 44.27733603455973110_dp, & + & -39.43731045663169965_dp, 41.68773239540036002_dp, & + & -37.83684688715485578_dp, 39.17472027603687934_dp, & + & -36.08253581911941410_dp, 36.74853923728758076_dp, & + & -34.18824090805679816_dp, 34.41913389699745807_dp, & + & -32.16531367916818596_dp, 32.19624133969532664_dp, & + & -30.02336509370288908_dp, 30.08943159705432180_dp, & + & -27.77086550998262737_dp, 28.10811023218007065_dp, & + & -25.41561850057673766_dp, 26.26148912292403637_dp, & + & -22.96513811238918734_dp, 24.55852997196867804_dp, & + & -20.42694818201796636_dp, 23.00786444863284430_dp, & + & -17.80881477377850075_dp, 21.61769490666709714_dp, & + & -15.11891780975727251_dp, 20.39568010883170501_dp, & + & -12.36596498789073095_dp, 19.34881113120014362_dp, & + & -9.55924980839402849_dp, 18.48328342761256238_dp, & + & -6.70865569654630001_dp, 17.80437171431731258_dp, & + & -3.82460955068694464_dp, 17.31631470128239059_dp, & + & -0.91799021933643477_dp, 17.02221659443387480_dp, & + & 1.99999999999999556_dp, 13.81592192037676625_dp, & + & 5.14299451095195792_dp, 13.92173872937387102_dp, & + & 8.27378779825514954_dp, 14.23846093632396581_dp, & + & 11.38041466390652978_dp, 14.76392503362292530_dp, & + & 14.45136393475438119_dp, 15.49459412897926924_dp, & + & 17.47576217013061139_dp, 16.42565612762181004_dp, & + & 20.44350999615491205_dp, 17.55115134744557182_dp, & + & 23.34536199373425092_dp, 18.86412113035556004_dp, & + & 26.17294518233925160_dp, 20.35676830949395466_dp, & + & 28.91871466466540141_dp, 22.02062053097340311_dp, & + & 31.57584735879237314_dp, 23.84668824688734645_dp, & + & 34.13807554901531205_dp, 25.82561045671564415_dp, & + & 36.59946098845733786_dp, 27.94778271439298933_dp, & + & 38.95410730493897233_dp, 30.20346328869342756_dp, & + & 41.19580324741197330_dp, 32.58285444844916867_dp, & + & 43.31758136998609388_dp, 35.07615645611910082_dp, & + & 45.31116510791174079_dp, 37.67359181604234664_dp, & + & 47.16626004327477517_dp, 40.36539642075791789_dp, & + & 48.86961926381131605_dp, 43.14177213432515856_dp, & + & 50.40377244059970963_dp, 45.99279145534946167_dp, & + & 51.74524369263552614_dp, 48.90823812634897649_dp, & + & 52.86197695521289575_dp, 51.87735585154322138_dp, & + & 53.70950811496559396_dp, 54.88845672164920586_dp, & + & 54.22511448054331140_dp, 57.92830381592441569_dp, & + & 54.31863539857850753_dp, 60.98111330635318694_dp, & + & 53.85773173662808233_dp, 64.02688869849654907_dp, & + & 52.64383991815974895_dp, 67.03853932314146391_dp, & + & 50.37308749001456931_dp, 69.97672254379963874_dp, & + & 46.57629367046713753_dp, 72.78039612779683409_dp, & + & 40.54693285975561423_dp, 75.34972495190746145_dp, & + & 31.34759503822197502_dp, 77.51867325981696411_dp, & + & 18.24038894313819625_dp, 79.03039199474196153_dp, & + & 1.99999999999999489_dp, 79.58407807962322522_dp, & + & -14.24038894313820691_dp, 79.03039199474196153_dp, & + & -27.34759503822198567_dp, 77.51867325981696411_dp, & + & -36.54693285975562134_dp, 75.34972495190746145_dp, & + & -42.57629367046714464_dp, 72.78039612779683409_dp, & + & -46.37308749001456931_dp, 69.97672254379963874_dp, & + & -48.64383991815974895_dp, 67.03853932314146391_dp, & + & -49.85773173662808233_dp, 64.02688869849654907_dp, & + & -50.31863539857852174_dp, 60.98111330635318694_dp, & + & -50.22511448054332561_dp, 57.92830381592441569_dp, & + & -49.70950811496560817_dp, 54.88845672164920586_dp, & + & -48.86197695521290996_dp, 51.87735585154322138_dp, & + & -47.74524369263553325_dp, 48.90823812634897649_dp, & + & -46.40377244059971673_dp, 45.99279145534946167_dp, & + & -44.86961926381133026_dp, 43.14177213432515856_dp, & + & -43.16626004327478938_dp, 40.36539642075791789_dp, & + & -41.31116510791174079_dp, 37.67359181604234664_dp, & + & -39.31758136998610098_dp, 35.07615645611910082_dp, & + & -37.19580324741197330_dp, 32.58285444844916867_dp, & + & -34.95410730493897233_dp, 30.20346328869342756_dp, & + & -32.59946098845735207_dp, 27.94778271439298933_dp, & + & -30.13807554901532626_dp, 25.82561045671564415_dp, & + & -27.57584735879238025_dp, 23.84668824688734645_dp, & + & -24.91871466466541207_dp, 22.02062053097340311_dp, & + & -22.17294518233926226_dp, 20.35676830949395466_dp, & + & -19.34536199373426157_dp, 18.86412113035556004_dp, & + & -16.44350999615492626_dp, 17.55115134744557182_dp, & + & -13.47576217013062028_dp, 16.42565612762181004_dp, & + & -10.45136393475438830_dp, 15.49459412897926924_dp, & + & -7.38041466390653600_dp, 14.76392503362292530_dp, & + & -4.27378779825515931_dp, 14.23846093632396581_dp, & + & -1.14299451095196680_dp, 13.92173872937387102_dp, & + & 1.99999999999999600_dp, 10.54173785529561158_dp, & + & 5.37360609917943943_dp, 10.65531494200142149_dp, & + & 8.73395228112667965_dp, 10.99520717814031556_dp, & + & 12.06809948196381832_dp, 11.55892497220584403_dp, & + & 15.36372621835939078_dp, 12.34240842556757656_dp, & + & 18.60937983520193129_dp, 13.34015413091371016_dp, & + & 21.79466590782825364_dp, 14.54537904344351951_dp, & + & 24.91036566952359266_dp, 15.95020993686824440_dp, & + & 27.94847769357361855_dp, 17.54588626406437513_dp, & + & 30.90218550361061745_dp, 19.32296477964253967_dp, & + & 33.76575655354334060_dp, 21.27151577227775903_dp, & + & 36.53437968696508875_dp, 23.38130282889363798_dp, & + & 39.20394762734519389_dp, 25.64194033091114022_dp, & + & 41.77078829775712165_dp, 28.04302503644353095_dp, & + & 44.23134383513491485_dp, 30.57423988556289629_dp, & + & 46.58178882927892772_dp, 33.22542941501406233_dp, & + & 48.81756887288366187_dp, 35.98664677769215103_dp, & + & 50.93282539675248444_dp, 38.84817223653458029_dp, & + & 52.91964996438991164_dp, 41.80050199367043007_dp, & + & 54.76707508825830928_dp, 44.83430401955754974_dp, & + & 56.45964870701569538_dp, 47.94033352757975308_dp, & + & 57.97533567047437231_dp, 51.10929358481432416_dp, & + & 59.28230228211859298_dp, 54.33161333463023368_dp, & + & 60.33378767939932885_dp, 57.59709162031844443_dp, & + & 61.05957379451161415_dp, 60.89430451319173443_dp, & + & 61.35114187047394552_dp, 64.20957103321013903_dp, & + & 61.03453256694833584_dp, 67.52503657439316953_dp, & + & 59.81801935702030448_dp, 70.81486693953938527_dp, & + & 57.18593275925195485_dp, 74.03707888623914357_dp, & + & 52.17735288037920327_dp, 77.11452529323830163_dp, & + & 42.96575654499160635_dp, 79.88802174923021937_dp, & + & 26.58690897303194944_dp, 82.01033616765489853_dp, & + & 1.99999999999999756_dp, 82.85826214470436923_dp, & + & -22.58690897303195300_dp, 82.01033616765489853_dp, & + & -38.96575654499161345_dp, 79.88802174923021937_dp, & + & -48.17735288037921748_dp, 77.11452529323830163_dp, & + & -53.18593275925197617_dp, 74.03707888623914357_dp, & + & -55.81801935702031159_dp, 70.81486693953938527_dp, & + & -57.03453256694834295_dp, 67.52503657439316953_dp, & + & -57.35114187047395262_dp, 64.20957103321013903_dp, & + & -57.05957379451162836_dp, 60.89430451319173443_dp, & + & -56.33378767939933596_dp, 57.59709162031844443_dp, & + & -55.28230228211860720_dp, 54.33161333463023368_dp, & + & -53.97533567047437231_dp, 51.10929358481432416_dp, & + & -52.45964870701570248_dp, 47.94033352757975308_dp, & + & -50.76707508825830928_dp, 44.83430401955754974_dp, & + & -48.91964996438991875_dp, 41.80050199367043007_dp, & + & -46.93282539675249154_dp, 38.84817223653458029_dp, & + & -44.81756887288366897_dp, 35.98664677769215103_dp, & + & -42.58178882927894193_dp, 33.22542941501406233_dp, & + & -40.23134383513492196_dp, 30.57423988556289629_dp, & + & -37.77078829775713587_dp, 28.04302503644353095_dp, & + & -35.20394762734520100_dp, 25.64194033091114022_dp, & + & -32.53437968696510296_dp, 23.38130282889363798_dp, & + & -29.76575655354335481_dp, 21.27151577227775903_dp, & + & -26.90218550361062810_dp, 19.32296477964253967_dp, & + & -23.94847769357362921_dp, 17.54588626406437513_dp, & + & -20.91036566952360332_dp, 15.95020993686824440_dp, & + & -17.79466590782826074_dp, 14.54537904344351951_dp, & + & -14.60937983520194017_dp, 13.34015413091371016_dp, & + & -11.36372621835939611_dp, 12.34240842556757656_dp, & + & -8.06809948196382898_dp, 11.55892497220584403_dp, & + & -4.73395228112668676_dp, 10.99520717814031556_dp, & + & -1.37360609917944720_dp, 10.65531494200142149_dp, & + & 1.99999999999999600_dp, 7.07694595379052860_dp, & + & 5.61234064761578821_dp, 7.19855645524002874_dp, & + & 9.21005712455210990_dp, 7.56241362477411005_dp, & + & 12.77895396829166508_dp, 8.16563014493928208_dp, & + & 16.30566013367806733_dp, 9.00350968307661859_dp, & + & 19.77796305903054730_dp, 10.06971032975657643_dp, & + & 23.18506021909562875_dp, 11.35645428598743756_dp, & + & 26.51771689758311723_dp, 12.85476800444172873_dp, & + & 29.76832854644279536_dp, 14.55473640862136264_dp, & + & 32.93089420696504277_dp, 16.44575600838861362_dp, & + & 36.00091302933286386_dp, 18.51677422677156315_dp, & + & 38.97521854780546846_dp, 20.75650546647104022_dp, & + & 41.85176517580521960_dp, 23.15361781597290403_dp, & + & 44.62937881000767248_dp, 25.69688738457191945_dp, & + & 47.30747895739604303_dp, 28.37531978094453677_dp, & + & 49.88577372237665486_dp, 31.17824007995576707_dp, & + & 52.36392121341754091_dp, 34.09535373993877982_dp, & + & 54.74114071516557090_dp, 37.11678138456306897_dp, & + & 57.01574257648425004_dp, 40.23307020728445593_dp, & + & 59.18452370168172649_dp, 43.43518400478754415_dp, & + & 61.24193904387428233_dp, 46.71447239476329827_dp, & + & 63.17889520468774833_dp, 50.06261728049957327_dp, & + & 64.98089250042356468_dp, 53.47155025248752480_dp, & + & 66.62500642980749888_dp, 56.93332642202356197_dp, & + & 68.07470870335012592_dp, 60.43992354246547904_dp, & + & 69.27043361461275595_dp, 63.98289851135272244_dp, & + & 70.11115128919476547_dp, 67.55274433031233627_dp, & + & 70.41516373728855172_dp, 71.13755089586132385_dp, & + & 69.82714145548160900_dp, 74.71983727667713993_dp, & + & 67.56415050352947560_dp, 78.26774990878784877_dp, & + & 61.58131858106001033_dp, 81.70467639663139892_dp, & + & 45.28583165458982052_dp, 84.76914638163376026_dp, & + & 1.99999999999999689_dp, 86.32305404620944955_dp, & + & -41.28583165458982762_dp, 84.76914638163376026_dp, & + & -57.58131858106002454_dp, 81.70467639663139892_dp, & + & -63.56415050352948981_dp, 78.26774990878784877_dp, & + & -65.82714145548162321_dp, 74.71983727667713993_dp, & + & -66.41516373728856593_dp, 71.13755089586132385_dp, & + & -66.11115128919477968_dp, 67.55274433031233627_dp, & + & -65.27043361461275595_dp, 63.98289851135272244_dp, & + & -64.07470870335011170_dp, 60.43992354246547904_dp, & + & -62.62500642980752019_dp, 56.93332642202356197_dp, & + & -60.98089250042357889_dp, 53.47155025248752480_dp, & + & -59.17889520468775544_dp, 50.06261728049957327_dp, & + & -57.24193904387428233_dp, 46.71447239476329827_dp, & + & -55.18452370168172649_dp, 43.43518400478754415_dp, & + & -53.01574257648425004_dp, 40.23307020728445593_dp, & + & -50.74114071516557800_dp, 37.11678138456306897_dp, & + & -48.36392121341754802_dp, 34.09535373993877982_dp, & + & -45.88577372237665486_dp, 31.17824007995576707_dp, & + & -43.30747895739605013_dp, 28.37531978094453677_dp, & + & -40.62937881000767959_dp, 25.69688738457191945_dp, & + & -37.85176517580522670_dp, 23.15361781597290403_dp, & + & -34.97521854780547557_dp, 20.75650546647104022_dp, & + & -32.00091302933286386_dp, 18.51677422677156315_dp, & + & -28.93089420696504632_dp, 16.44575600838861362_dp, & + & -25.76832854644280957_dp, 14.55473640862136264_dp, & + & -22.51771689758312434_dp, 12.85476800444172873_dp, & + & -19.18506021909564296_dp, 11.35645428598743756_dp, & + & -15.77796305903055796_dp, 10.06971032975657643_dp, & + & -12.30566013367807621_dp, 9.00350968307661859_dp, & + & -8.77895396829167574_dp, 8.16563014493928208_dp, & + & -5.21005712455211967_dp, 7.56241362477411005_dp, & + & -1.61234064761579732_dp, 7.19855645524002874_dp, & + & 1.99999999999999556_dp, 3.39364961385021102_dp, & + & 5.86220744039440511_dp, 3.52366797872772963_dp, & + & 9.70801791517021506_dp, 3.91258202080964379_dp, & + & 13.52160273595813145_dp, 4.55701513369246314_dp, & + & 17.28822357553576694_dp, 5.45149036402020482_dp, & + & 20.99466904928960531_dp, 6.58864168091601599_dp, & + & 24.62957864939021135_dp, 7.95948261839810378_dp, & + & 28.18364165385172626_dp, 9.55371034930797514_dp, & + & 31.64967308093981302_dp, 11.36002298851478365_dp, & + & 35.02258051120933402_dp, 13.36643021383323671_dp, & + & 38.29924330014637235_dp, 15.56054131855884215_dp, & + & 41.47832910130669859_dp, 17.92981963683195801_dp, & + & 44.56007232189227807_dp, 20.46179708442632261_dp, & + & 47.54603622089215520_dp, 23.14424674184217068_dp, & + & 50.43887601207384819_dp, 25.96531465470105005_dp, & + & 53.24211556220075892_dp, 28.91361425616476666_dp, & + & 55.95994579953058690_dp, 31.97828811153562256_dp, & + & 58.59704918478396252_dp, 35.14904222145087687_dp, & + & 61.15845171105517863_dp, 38.41615810166681655_dp, & + & 63.64940188352277062_dp, 41.77048747717444854_dp, & + & 66.07527488350018530_dp, 45.20343384586799118_dp, & + & 68.44149951448399349_dp, 48.70692450151241104_dp, & + & 70.75350545455997064_dp, 52.27337593848611164_dp, & + & 73.01668877482984499_dp, 55.89565494125256606_dp, & + & 75.23639478559491067_dp, 59.56703711552955127_dp, & + & 77.41791965045068480_dp, 63.28116415502339720_dp, & + & 79.56653776770211550_dp, 67.03200075627316323_dp, & + & 81.68757690134610527_dp, 70.81379178736983704_dp, & + & 83.78661098119418682_dp, 74.62102007313750107_dp, & + & 85.87003129087820241_dp, 78.44836496333871878_dp, & + & 87.94730489861872513_dp, 82.29066165213365025_dp, & + & 90.04670840724658376_dp, 86.14286053253557895_dp, & + & -177.99999999999985789_dp, 89.99364961382421768_dp, & + & -86.04670840724655534_dp, 86.14286053253557895_dp, & + & -83.94730489861876777_dp, 82.29066165213365025_dp, & + & -81.87003129087820241_dp, 78.44836496333871878_dp, & + & -79.78661098119418682_dp, 74.62102007313750107_dp, & + & -77.68757690134613370_dp, 70.81379178736983704_dp, & + & -75.56653776770212971_dp, 67.03200075627316323_dp, & + & -73.41791965045068480_dp, 63.28116415502339720_dp, & + & -71.23639478559491067_dp, 59.56703711552955127_dp, & + & -69.01668877482984499_dp, 55.89565494125256606_dp, & + & -66.75350545455997064_dp, 52.27337593848611164_dp, & + & -64.44149951448400770_dp, 48.70692450151241104_dp, & + & -62.07527488350018530_dp, 45.20343384586799118_dp, & + & -59.64940188352277772_dp, 41.77048747717444854_dp, & + & -57.15845171105518574_dp, 38.41615810166681655_dp, & + & -54.59704918478396252_dp, 35.14904222145087687_dp, & + & -51.95994579953059400_dp, 31.97828811153562256_dp, & + & -49.24211556220076602_dp, 28.91361425616476666_dp, & + & -46.43887601207384819_dp, 25.96531465470105005_dp, & + & -43.54603622089217652_dp, 23.14424674184217068_dp, & + & -40.56007232189229228_dp, 20.46179708442632261_dp, & + & -37.47832910130670570_dp, 17.92981963683195801_dp, & + & -34.29924330014637945_dp, 15.56054131855884215_dp, & + & -31.02258051120933402_dp, 13.36643021383323671_dp, & + & -27.64967308093982012_dp, 11.36002298851478365_dp, & + & -24.18364165385173337_dp, 9.55371034930797514_dp, & + & -20.62957864939022556_dp, 7.95948261839810378_dp, & + & -16.99466904928961597_dp, 6.58864168091601599_dp, & + & -13.28822357553577582_dp, 5.45149036402020482_dp, & + & -9.52160273595813855_dp, 4.55701513369246314_dp, & + & -5.70801791517022572_dp, 3.91258202080964379_dp, & + & -1.86220744039441377_dp, 3.52366797872772963_dp, & + & 1.99999999999999556_dp, -0.54003843869415158_dp, & + & 6.12693783758673050_dp, -0.40111214042971866_dp, & + & 10.23515434659413970_dp, 0.01431798130226895_dp, & + & 14.30668142622436534_dp, 0.70226688878527466_dp, & + & 18.32499114491300674_dp, 1.65629062554436168_dp, & + & 22.27556116130715580_dp, 2.86776143955401341_dp, & + & 26.14628266758476016_dp, 4.32621388981252775_dp, & + & 29.92769777693464306_dp, 6.01973109672258033_dp, & + & 33.61307490596014702_dp, 7.93534078498540918_dp, & + & 37.19834740797973183_dp, 10.05939491571368904_dp, & + & 40.68195074563112712_dp, 12.37791309905906267_dp, & + & 44.06459700293881809_dp, 14.87687712846902421_dp, & + & 47.34902402209520034_dp, 17.54247066454981052_dp, & + & 50.53975201683548590_dp, 20.36126354510683711_dp, & + & 53.64287529119709319_dp, 23.32034407821754485_dp, & + & 56.66591251909911620_dp, 26.40740499964346455_dp, & + & 59.61773745881250619_dp, 29.61078975921034129_dp, & + & 62.50861449014276872_dp, 32.91950572923986584_dp, & + & 65.35037195723165837_dp, 36.32321005821969351_dp, & + & 68.15676435641833564_dp, 39.81217237539897269_dp, & + & 70.94410839385281520_dp, 43.37721635788606989_dp, & + & 73.73234087330637010_dp, 47.00963897352001908_dp, & + & 76.54676554128549526_dp, 50.70110112272598712_dp, & + & 79.42099130058548440_dp, 54.44347433172315220_dp, & + & 82.40205491898922219_dp, 58.22861013398449614_dp, & + & 85.55981529780903827_dp, 62.04795929474799721_dp, & + & 89.00534896567364740_dp, 65.89187266757559769_dp, & + & 92.93011750021230455_dp, 69.74815881341463353_dp, & + & 97.69886509454546797_dp, 73.59868574092438109_dp, & + & 104.10346048738931302_dp, 77.40995108179684792_dp, & + & 114.19717929614462548_dp, 81.10053146183234674_dp, & + & 134.58473801020662108_dp, 84.39088722277180921_dp, & + & -178.00000000000002842_dp, 86.05996156130575514_dp, & + & -130.58473801020659266_dp, 84.39088722277180921_dp, & + & -110.19717929614465390_dp, 81.10053146183234674_dp, & + & -100.10346048738929881_dp, 77.40995108179684792_dp, & + & -93.69886509454548218_dp, 73.59868574092438109_dp, & + & -88.93011750021230455_dp, 69.74815881341463353_dp, & + & -85.00534896567366161_dp, 65.89187266757559769_dp, & + & -81.55981529780905248_dp, 62.04795929474799721_dp, & + & -78.40205491898922219_dp, 58.22861013398449614_dp, & + & -75.42099130058548440_dp, 54.44347433172315220_dp, & + & -72.54676554128549526_dp, 50.70110112272598712_dp, & + & -69.73234087330638431_dp, 47.00963897352001908_dp, & + & -66.94410839385282941_dp, 43.37721635788606989_dp, & + & -64.15676435641836406_dp, 39.81217237539897269_dp, & + & -61.35037195723166548_dp, 36.32321005821969351_dp, & + & -58.50861449014277582_dp, 32.91950572923986584_dp, & + & -55.61773745881251330_dp, 29.61078975921034129_dp, & + & -52.66591251909913041_dp, 26.40740499964346455_dp, & + & -49.64287529119710030_dp, 23.32034407821754485_dp, & + & -46.53975201683548590_dp, 20.36126354510683711_dp, & + & -43.34902402209520034_dp, 17.54247066454981052_dp, & + & -40.06459700293881809_dp, 14.87687712846902421_dp, & + & -36.68195074563112712_dp, 12.37791309905906267_dp, & + & -33.19834740797973893_dp, 10.05939491571368904_dp, & + & -29.61307490596015057_dp, 7.93534078498540918_dp, & + & -25.92769777693465727_dp, 6.01973109672258033_dp, & + & -22.14628266758476371_dp, 4.32621388981252775_dp, & + & -18.27556116130716291_dp, 2.86776143955401341_dp, & + & -14.32499114491301384_dp, 1.65629062554436168_dp, & + & -10.30668142622437244_dp, 0.70226688878527466_dp, & + & -6.23515434659414680_dp, 0.01431798130226895_dp, & + & -2.12693783758673804_dp, -0.40111214042971866_dp, & + & 1.99999999999999556_dp, -4.76061810234754645_dp, & + & 6.41132592404072454_dp, -4.61212239418478553_dp, & + & 10.80084816537008230_dp, -4.16824654595516098_dp, & + & 15.14776873556008674_dp, -3.43374257581263365_dp, & + & 19.43320327160517635_dp, -2.41626750988024641_dp, & + & 23.64091162595336115_dp, -1.12602081331554671_dp, & + & 27.75780275269853448_dp, 0.42470591907018324_dp, & + & 31.77420154083880988_dp, 2.22202305439570047_dp, & + & 35.68389780238808129_dp, 4.25093893994811900_dp, & + & 39.48402100288601702_dp, 6.49582334126627270_dp, & + & 43.17479644658765636_dp, 8.94081392759362181_dp, & + & 46.75924066894447861_dp, 11.57015196980179894_dp, & + & 50.24284889074777283_dp, 14.36844259642658628_dp, & + & 53.63331929525543273_dp, 17.32084182287968943_dp, & + & 56.94035098918124049_dp, 20.41317688045037571_dp, & + & 60.17554747078253286_dp, 23.63200834498020697_dp, & + & 63.35245736607463840_dp, 26.96464262473803686_dp, & + & 66.48679120890342631_dp, 30.39910192691253599_dp, & + & 69.59687011372152199_dp, 33.92405612452549946_dp, & + & 72.70439458886511375_dp, 37.52871692222674938_dp, & + & 75.83567948061782715_dp, 41.20268880474466755_dp, & + & 79.02360369831293951_dp, 44.93576201889581512_dp, & + & 82.31071017117776023_dp, 48.71761719224470966_dp, & + & 85.75424365182692554_dp, 52.53738247574246856_dp, & + & 89.43460651588344490_dp, 56.38292751246400769_dp, & + & 93.47013935832343634_dp, 60.23965935256792648_dp, & + & 98.04421718085441739_dp, 64.08831694761893516_dp, & + & 103.45761459629794388_dp, 67.90061168070626252_dp, & + & 110.23509874248559015_dp, 71.62987479878029262_dp, & + & 119.34882263320223217_dp, 75.18923730101467129_dp, & + & 132.64763601333982024_dp, 78.39753950634394641_dp, & + & 153.15668933029419918_dp, 80.85560324742944260_dp, & + & -178.00000000000002842_dp, 81.83938189765245852_dp, & + & -149.15668933029422760_dp, 80.85560324742944260_dp, & + & -128.64763601333984866_dp, 78.39753950634394641_dp, & + & -115.34882263320223217_dp, 75.18923730101467129_dp, & + & -106.23509874248556173_dp, 71.62987479878029262_dp, & + & -99.45761459629794388_dp, 67.90061168070626252_dp, & + & -94.04421718085441739_dp, 64.08831694761893516_dp, & + & -89.47013935832346476_dp, 60.23965935256792648_dp, & + & -85.43460651588344490_dp, 56.38292751246400769_dp, & + & -81.75424365182695396_dp, 52.53738247574246856_dp, & + & -78.31071017117777444_dp, 48.71761719224470966_dp, & + & -75.02360369831293951_dp, 44.93576201889581512_dp, & + & -71.83567948061784136_dp, 41.20268880474466755_dp, & + & -68.70439458886511375_dp, 37.52871692222674938_dp, & + & -65.59687011372152199_dp, 33.92405612452549946_dp, & + & -62.48679120890344052_dp, 30.39910192691253599_dp, & + & -59.35245736607464551_dp, 26.96464262473803686_dp, & + & -56.17554747078254707_dp, 23.63200834498020697_dp, & + & -52.94035098918124760_dp, 20.41317688045037571_dp, & + & -49.63331929525544695_dp, 17.32084182287968943_dp, & + & -46.24284889074779414_dp, 14.36844259642658628_dp, & + & -42.75924066894448572_dp, 11.57015196980179894_dp, & + & -39.17479644658766347_dp, 8.94081392759362181_dp, & + & -35.48402100288603123_dp, 6.49582334126627270_dp, & + & -31.68389780238809195_dp, 4.25093893994811900_dp, & + & -27.77420154083882409_dp, 2.22202305439570047_dp, & + & -23.75780275269854513_dp, 0.42470591907018324_dp, & + & -19.64091162595336826_dp, -1.12602081331554671_dp, & + & -15.43320327160518879_dp, -2.41626750988024641_dp, & + & -11.14776873556009562_dp, -3.43374257581263365_dp, & + & -6.80084816537009029_dp, -4.16824654595516098_dp, & + & -2.41132592404073298_dp, -4.61212239418478553_dp, & + & 1.99999999999999556_dp, -9.30985393851741705_dp, & + & 6.72175805422021355_dp, -9.15091230991673932_dp, & + & 11.41756212661333514_dp, -8.67603648837036268_dp, & + & 16.06282113870961226_dp, -7.89096231408012549_dp, & + & 20.63552097548324227_dp, -6.80489321395614688_dp, & + & 25.11717183125960418_dp, -5.43001433583735604_dp, & + & 29.49342310484702523_dp, -3.78089833576525836_dp, & + & 33.75433817509684786_dp, -1.87386656696368870_dp, & + & 37.89437094593881028_dp, 0.27363605581641121_dp, & + & 41.91211819209927114_dp, 2.64360721178193936_dp, & + & 45.80993456941938291_dp, 5.21796518077428217_dp, & + & 49.59349430020984784_dp, 7.97892504367260536_dp, & + & 53.27137129744098587_dp, 10.90927185095380025_dp, & + & 56.85469423510678411_dp, 13.99253787292509266_dp, & + & 60.35691981311008192_dp, 17.21309536720795919_dp, & + & 63.79375952890369206_dp, 20.55617726580611659_dp, & + & 67.18329478816110623_dp, 24.00783659504739731_dp, & + & 70.54632400302143935_dp, 27.55485191826680236_dp, & + & 73.90700610430916129_dp, 31.18458087158649406_dp, & + & 77.29390254081907585_dp, 34.88475663106463287_dp, & + & 80.74158359961856490_dp, 38.64321178012773572_dp, & + & 84.29307189810650414_dp, 42.44749809608728697_dp, & + & 88.00357763838491110_dp, 46.28434439562811065_dp, & + & 91.94629406419836926_dp, 50.13884820321026581_dp, & + & 96.22157019495104180_dp, 53.99321139219064491_dp, & + & 100.97173315822050199_dp, 57.82466562901674934_dp, & + & 106.40541928769302160_dp, 61.60190913800559542_dp, & + & 112.83745587761723073_dp, 65.27873175231235336_dp, & + & 120.75095597801100666_dp, 68.78228497629521598_dp, & + & 130.87442892417493567_dp, 71.99165187875782124_dp, & + & 144.18467631053604805_dp, 74.70285264202438213_dp, & + & 161.47781378002883912_dp, 76.59572545663588983_dp, & + & -178.00000000000002842_dp, 77.29014606148258792_dp, & + & -157.47781378002883912_dp, 76.59572545663588983_dp, & + & -140.18467631053607647_dp, 74.70285264202438213_dp, & + & -126.87442892417496410_dp, 71.99165187875782124_dp, & + & -116.75095597801099245_dp, 68.78228497629521598_dp, & + & -108.83745587761723073_dp, 65.27873175231235336_dp, & + & -102.40541928769302160_dp, 61.60190913800559542_dp, & + & -96.97173315822051620_dp, 57.82466562901674934_dp, & + & -92.22157019495104180_dp, 53.99321139219064491_dp, & + & -87.94629406419838347_dp, 50.13884820321026581_dp, & + & -84.00357763838492531_dp, 46.28434439562811065_dp, & + & -80.29307189810650414_dp, 42.44749809608728697_dp, & + & -76.74158359961857911_dp, 38.64321178012773572_dp, & + & -73.29390254081907585_dp, 34.88475663106463287_dp, & + & -69.90700610430916129_dp, 31.18458087158649406_dp, & + & -66.54632400302145356_dp, 27.55485191826680236_dp, & + & -63.18329478816111333_dp, 24.00783659504739731_dp, & + & -59.79375952890369916_dp, 20.55617726580611659_dp, & + & -56.35691981311008192_dp, 17.21309536720795919_dp, & + & -52.85469423510679832_dp, 13.99253787292509266_dp, & + & -49.27137129744098587_dp, 10.90927185095380025_dp, & + & -45.59349430020986915_dp, 7.97892504367260536_dp, & + & -41.80993456941939712_dp, 5.21796518077428217_dp, & + & -37.91211819209929246_dp, 2.64360721178193936_dp, & + & -33.89437094593881739_dp, 0.27363605581641121_dp, & + & -29.75433817509685497_dp, -1.87386656696368870_dp, & + & -25.49342310484703944_dp, -3.78089833576525836_dp, & + & -21.11717183125961128_dp, -5.43001433583735604_dp, & + & -16.63552097548325293_dp, -6.80489321395614688_dp, & + & -12.06282113870962114_dp, -7.89096231408012549_dp, & + & -7.41756212661334136_dp, -8.67603648837036268_dp, & + & -2.72175805422022110_dp, -9.15091230991673932_dp, & + & 1.99999999999999556_dp, -14.23540593490034212_dp, & + & 7.06707279862612125_dp, -14.06484423228424063_dp, & + & 12.10248720394761790_dp, -13.55555303790300137_dp, & + & 17.07647532339787944_dp, -12.71455858250937965_dp, & + & 21.96281473836326015_dp, -11.55307649365679623_dp, & + & 26.74006765897273041_dp, -10.08584610023314454_dp, & + & 31.39231472226267528_dp, -8.33033151194296018_dp, & + & 35.90939060170597230_dp, -6.30588392466837799_dp, & + & 40.28670514160292271_dp, -4.03294710811188395_dp, & + & 44.52477649322944586_dp, -1.53236497867592791_dp, & + & 48.62861191461712451_dp, 1.17517574056815022_dp, & + & 52.60705694133754662_dp, 4.06955754229703981_dp, & + & 56.47220691225791711_dp, 7.13150506907926562_dp, & + & 60.23894695811429756_dp, 10.34273467530950796_dp, & + & 63.92466443809040300_dp, 13.68599977255221845_dp, & + & 67.54916484650964037_dp, 17.14505004500527363_dp, & + & 71.13481958525987636_dp, 20.70451849248449960_dp, & + & 74.70698212428176532_dp, 24.34974403770184992_dp, & + & 78.29472888122153051_dp, 28.06652943785655552_dp, & + & 81.93201512996088809_dp, 31.84082402428231973_dp, & + & 85.65938939021744147_dp, 35.65830698143167155_dp, & + & 89.52649044264860834_dp, 39.50382677214758331_dp, & + & 93.59567144235911940_dp, 43.36062114047545890_dp, & + & 97.94726867835277062_dp, 47.20919167069192923_dp, & + & 102.68725830824645584_dp, 51.02562354013957702_dp, & + & 107.95825647893494192_dp, 54.77900423463843538_dp, & + & 113.95470750710921948_dp, 58.42738157410047961_dp, & + & 120.94158660124729465_dp, 61.91142197433249095_dp, & + & 129.26976638159490562_dp, 65.14478524125112813_dp, & + & 139.36255278451201889_dp, 68.00113945880528377_dp, & + & 151.60742176334809983_dp, 70.30250402254893061_dp, & + & 166.05644118521092878_dp, 71.82662636485281382_dp, & + & -178.00000000000002842_dp, 72.36459406509962378_dp, & + & -162.05644118521095720_dp, 71.82662636485281382_dp, & + & -147.60742176334812825_dp, 70.30250402254893061_dp, & + & -135.36255278451201889_dp, 68.00113945880528377_dp, & + & -125.26976638159489141_dp, 65.14478524125112813_dp, & + & -116.94158660124729465_dp, 61.91142197433249095_dp, & + & -109.95470750710923369_dp, 58.42738157410047961_dp, & + & -103.95825647893495614_dp, 54.77900423463843538_dp, & + & -98.68725830824647005_dp, 51.02562354013957702_dp, & + & -93.94726867835278483_dp, 47.20919167069192923_dp, & + & -89.59567144235911940_dp, 43.36062114047545890_dp, & + & -85.52649044264862255_dp, 39.50382677214758331_dp, & + & -81.65938939021744147_dp, 35.65830698143167155_dp, & + & -77.93201512996090230_dp, 31.84082402428231973_dp, & + & -74.29472888122153051_dp, 28.06652943785655552_dp, & + & -70.70698212428177953_dp, 24.34974403770184992_dp, & + & -67.13481958525987636_dp, 20.70451849248449960_dp, & + & -63.54916484650964748_dp, 17.14505004500527363_dp, & + & -59.92466443809041010_dp, 13.68599977255221845_dp, & + & -56.23894695811430466_dp, 10.34273467530950796_dp, & + & -52.47220691225792422_dp, 7.13150506907926562_dp, & + & -48.60705694133756083_dp, 4.06955754229703981_dp, & + & -44.62861191461713162_dp, 1.17517574056815022_dp, & + & -40.52477649322946007_dp, -1.53236497867592791_dp, & + & -36.28670514160293692_dp, -4.03294710811188395_dp, & + & -31.90939060170597941_dp, -6.30588392466837799_dp, & + & -27.39231472226267883_dp, -8.33033151194296018_dp, & + & -22.74006765897273752_dp, -10.08584610023314454_dp, & + & -17.96281473836326725_dp, -11.55307649365679623_dp, & + & -13.07647532339788832_dp, -12.71455858250937965_dp, & + & -8.10248720394762678_dp, -13.55555303790300137_dp, & + & -3.06707279862612925_dp, -14.06484423228424063_dp, & + & 1.99999999999999600_dp, -19.59134287356426896_dp, & + & 7.46002704385165138_dp, -19.40755726556070115_dp, & + & 12.88033504279662367_dp, -18.85919797266866738_dp, & + & 18.22391622121717347_dp, -17.95503295463869264_dp, & + & 23.45879403720244127_dp, -16.70896594558295334_dp, & + & 28.55966518490710015_dp, -15.13909773195327269_dp, & + & 33.50874372060166451_dp, -13.26662598462660014_dp, & + & 38.29585521437642370_dp, -11.11472732454917889_dp, & + & 42.91794868987490474_dp, -8.70753794849322738_dp, & + & 47.37824652323212149_dp, -6.06930752874207258_dp, & + & 51.68524592294901510_dp, -3.22375917751487107_dp, & + & 55.85174332801063457_dp, -0.19365520038940884_dp, & + & 59.89399913408759346_dp, 2.99945241539763385_dp, & + & 63.83111160507218784_dp, 6.33531713186757095_dp, & + & 67.68463403089357655_dp, 9.79496963802120746_dp, & + & 71.47845003929684538_dp, 13.36059248585088000_dp, & + & 75.23891693218338617_dp, 17.01531594444593409_dp, & + & 78.99529352424693229_dp, 20.74294398926493699_dp, & + & 82.78048521884274180_dp, 24.52760876444716587_dp, & + & 86.63216392910494790_dp, 28.35333972343855180_dp, & + & 90.59435351833845118_dp, 32.20351875194761959_dp, & + & 94.71961138434764393_dp, 36.06017287638244539_dp, & + & 99.07197748871266185_dp, 39.90302877816822757_dp, & + & 103.73088255981890882_dp, 43.70821480745370025_dp, & + & 108.79614728017998004_dp, 47.44644408577498496_dp, & + & 114.39390839554110357_dp, 51.08045112569507751_dp, & + & 120.68240638414135901_dp, 54.56141374911145192_dp, & + & 127.85428534979440940_dp, 57.82417470821775396_dp, & + & 136.12712761110682891_dp, 60.78156430751474204_dp, & + & 145.70578215046154469_dp, 63.31961305621511116_dp, & + & 156.69437932769909594_dp, 65.29863962109097031_dp, & + & 168.95645291666264143_dp, 66.56908368553207822_dp, & + & -178.00000000000002842_dp, 67.00865712643570760_dp, & + & -164.95645291666266985_dp, 66.56908368553207822_dp, & + & -152.69437932769912436_dp, 65.29863962109097031_dp, & + & -141.70578215046154469_dp, 63.31961305621511116_dp, & + & -132.12712761110685733_dp, 60.78156430751474204_dp, & + & -123.85428534979440940_dp, 57.82417470821775396_dp, & + & -116.68240638414138743_dp, 54.56141374911145192_dp, & + & -110.39390839554111778_dp, 51.08045112569507751_dp, & + & -104.79614728017998004_dp, 47.44644408577498496_dp, & + & -99.73088255981890882_dp, 43.70821480745370025_dp, & + & -95.07197748871266185_dp, 39.90302877816822757_dp, & + & -90.71961138434764393_dp, 36.06017287638244539_dp, & + & -86.59435351833845118_dp, 32.20351875194761959_dp, & + & -82.63216392910496211_dp, 28.35333972343855180_dp, & + & -78.78048521884275601_dp, 24.52760876444716587_dp, & + & -74.99529352424694650_dp, 20.74294398926493699_dp, & + & -71.23891693218340038_dp, 17.01531594444593409_dp, & + & -67.47845003929684538_dp, 13.36059248585088000_dp, & + & -63.68463403089357655_dp, 9.79496963802120746_dp, & + & -59.83111160507219495_dp, 6.33531713186757095_dp, & + & -55.89399913408760057_dp, 2.99945241539763385_dp, & + & -51.85174332801064168_dp, -0.19365520038940884_dp, & + & -47.68524592294902931_dp, -3.22375917751487107_dp, & + & -43.37824652323212860_dp, -6.06930752874207258_dp, & + & -38.91794868987490474_dp, -8.70753794849322738_dp, & + & -34.29585521437643081_dp, -11.11472732454917889_dp, & + & -29.50874372060167872_dp, -13.26662598462660014_dp, & + & -24.55966518490711081_dp, -15.13909773195327269_dp, & + & -19.45879403720245193_dp, -16.70896594558295334_dp, & + & -14.22391622121718058_dp, -17.95503295463869264_dp, & + & -8.88033504279663255_dp, -18.85919797266866738_dp, & + & -3.46002704385166071_dp, -19.40755726556070115_dp, & + & 1.99999999999999556_dp, -25.43837119820565817_dp, & + & 7.91994134356474966_dp, -25.23910692117755517_dp, & + & 13.78834011215970534_dp, -24.64516330804185529_dp, & + & 19.55772615655293478_dp, -23.66774871950556403_dp, & + & 25.18808138888132220_dp, -22.32448212182583092_dp, & + & 30.64904852681420522_dp, -20.63801909702449677_dp, & + & 35.92081946357622968_dp, -18.63449057116782726_dp, & + & 40.99386004100856695_dp, -16.34197985921977292_dp, & + & 45.86781613095838850_dp, -13.78920372951385431_dp, & + & 50.54999441522539882_dp, -11.00448645615600007_dp, & + & 55.05375641269628773_dp, -8.01504773475606846_dp, & + & 59.39706222812605318_dp, -4.84657887125735343_dp, & + & 63.60129716846514469_dp, -1.52305829704237761_dp, & + & 67.69043463465565935_dp, 1.93324784086672352_dp, & + & 71.69053915995505122_dp, 5.50164833296652045_dp, & + & 75.62959031828353318_dp, 9.16278970353470612_dp, & + & 79.53760403336076479_dp, 12.89834287286321945_dp, & + & 83.44703540885105042_dp, 16.69062430422997068_dp, & + & 87.39346098727328638_dp, 20.52214967325533124_dp, & + & 91.41655400532015108_dp, 24.37510563102819106_dp, & + & 95.56137924921046078_dp, 28.23071203179445021_dp, & + & 99.88003723653145016_dp, 32.06843215731343832_dp, & + & 104.43366587620558050_dp, 35.86497163386139420_dp, & + & 109.29473060869932510_dp, 39.59298941537671368_dp, & + & 114.54933935400606515_dp, 43.21943287866469774_dp, & + & 120.29889149413013172_dp, 46.70342202473147353_dp, & + & 126.65952710171551132_dp, 49.99368843662634987_dp, & + & 133.75638606304184464_dp, 53.02581189527145256_dp, & + & 141.70772458687929429_dp, 55.72003625021812923_dp, & + & 150.59283359977422379_dp, 57.98141283097308474_dp, & + & 160.40162010671977555_dp, 59.70518607312001080_dp, & + & 170.97884079028881388_dp, 60.79043532425958318_dp, & + & -178.00000000000002842_dp, 61.16162880179432193_dp, & + & -166.97884079028878546_dp, 60.79043532425958318_dp, & + & -156.40162010671977555_dp, 59.70518607312001080_dp, & + & -146.59283359977425221_dp, 57.98141283097308474_dp, & + & -137.70772458687926587_dp, 55.72003625021812923_dp, & + & -129.75638606304184464_dp, 53.02581189527145256_dp, & + & -122.65952710171549711_dp, 49.99368843662634987_dp, & + & -116.29889149413011751_dp, 46.70342202473147353_dp, & + & -110.54933935400606515_dp, 43.21943287866469774_dp, & + & -105.29473060869931089_dp, 39.59298941537671368_dp, & + & -100.43366587620559471_dp, 35.86497163386139420_dp, & + & -95.88003723653146437_dp, 32.06843215731343832_dp, & + & -91.56137924921046078_dp, 28.23071203179445021_dp, & + & -87.41655400532016529_dp, 24.37510563102819106_dp, & + & -83.39346098727328638_dp, 20.52214967325533124_dp, & + & -79.44703540885105042_dp, 16.69062430422997068_dp, & + & -75.53760403336077900_dp, 12.89834287286321945_dp, & + & -71.62959031828354739_dp, 9.16278970353470612_dp, & + & -67.69053915995506543_dp, 5.50164833296652045_dp, & + & -63.69043463465567356_dp, 1.93324784086672352_dp, & + & -59.60129716846515180_dp, -1.52305829704237761_dp, & + & -55.39706222812606740_dp, -4.84657887125735343_dp, & + & -51.05375641269629483_dp, -8.01504773475606846_dp, & + & -46.54999441522540593_dp, -11.00448645615600007_dp, & + & -41.86781613095839560_dp, -13.78920372951385431_dp, & + & -36.99386004100857406_dp, -16.34197985921977292_dp, & + & -31.92081946357624389_dp, -18.63449057116782726_dp, & + & -26.64904852681421943_dp, -20.63801909702449677_dp, & + & -21.18808138888132575_dp, -22.32448212182583092_dp, & + & -15.55772615655294011_dp, -23.66774871950556403_dp, & + & -9.78834011215971245_dp, -24.64516330804185529_dp, & + & -3.91994134356475854_dp, -25.23910692117755517_dp, & + & 1.99999999999999600_dp, -31.84350531560286868_dp, & + & 8.47781225763252877_dp, -31.62546289221583962_dp, & + & 14.88583393507996711_dp, -30.97645080660140593_dp, & + & 21.16079514785619509_dp, -29.91127029421520689_dp, & + & 27.25113130544622919_dp, -28.45290874783366775_dp, & + & 33.11998552133064067_dp, -26.63042818683534207_dp, & + & 38.74589265732009835_dp, -24.47666503668304117_dp, & + & 44.12160159745183563_dp, -22.02610835172851722_dp, & + & 49.25177543408178593_dp, -19.31318879069118211_dp, & + & 54.15029168417838434_dp, -16.37106576006627279_dp, & + & 58.83767345094592827_dp, -13.23089104518995640_dp, & + & 63.33895182482493169_dp, -9.92146778444056032_dp, & + & 67.68207179147748320_dp, -6.46920580966549874_dp, & + & 71.89683474738467339_dp, -2.89828246890513030_dp, & + & 76.01431282527251199_dp, 0.76906207426874740_dp, & + & 80.06665407661380129_dp, 4.51214364124032485_dp, & + & 84.08720437104130951_dp, 8.31139512800513636_dp, & + & 88.11088800598805904_dp, 12.14788119395578647_dp, & + & 92.17480572169650088_dp, 16.00276430383186366_dp, & + & 96.31902021078285259_dp, 19.85670886149115333_dp, & + & 100.58749981932022877_dp, 23.68920100972959730_dp, & + & 105.02917319698201482_dp, 27.47775433026597725_dp, & + & 109.69899804029304846_dp, 31.19696754875727152_dp, & + & 114.65884380836618561_dp, 34.81740352705595143_dp, & + & 119.97779813807069615_dp, 38.30427835916992052_dp, & + & 125.73118977168117283_dp, 41.61600262291977259_dp, & + & 131.99715883033749719_dp, 44.70273256173389598_dp, & + & 138.84908760812987794_dp, 47.50530446205828383_dp, & + & 146.34201331309026273_dp, 49.95526128262041254_dp, & + & 154.49218436749947614_dp, 51.97706503648645082_dp, & + & 163.25249258416440057_dp, 53.49372597518600259_dp, & + & 172.49285933765111167_dp, 54.43639475342744305_dp, & + & -178.00000000000002842_dp, 54.75649468439711143_dp, & + & -168.49285933765114009_dp, 54.43639475342744305_dp, & + & -159.25249258416440057_dp, 53.49372597518600259_dp, & + & -150.49218436749950456_dp, 51.97706503648645082_dp, & + & -142.34201331309029115_dp, 49.95526128262041254_dp, & + & -134.84908760812987794_dp, 47.50530446205828383_dp, & + & -127.99715883033752561_dp, 44.70273256173389598_dp, & + & -121.73118977168118704_dp, 41.61600262291977259_dp, & + & -115.97779813807071037_dp, 38.30427835916992052_dp, & + & -110.65884380836619982_dp, 34.81740352705595143_dp, & + & -105.69899804029306267_dp, 31.19696754875727152_dp, & + & -101.02917319698202903_dp, 27.47775433026597725_dp, & + & -96.58749981932022877_dp, 23.68920100972959730_dp, & + & -92.31902021078286680_dp, 19.85670886149115333_dp, & + & -88.17480572169651509_dp, 16.00276430383186366_dp, & + & -84.11088800598805904_dp, 12.14788119395578647_dp, & + & -80.08720437104132372_dp, 8.31139512800513636_dp, & + & -76.06665407661381550_dp, 4.51214364124032485_dp, & + & -72.01431282527251199_dp, 0.76906207426874740_dp, & + & -67.89683474738467339_dp, -2.89828246890513030_dp, & + & -63.68207179147749031_dp, -6.46920580966549874_dp, & + & -59.33895182482493880_dp, -9.92146778444056032_dp, & + & -54.83767345094592827_dp, -13.23089104518995640_dp, & + & -50.15029168417838434_dp, -16.37106576006627279_dp, & + & -45.25177543408179304_dp, -19.31318879069118211_dp, & + & -40.12160159745184274_dp, -22.02610835172851722_dp, & + & -34.74589265732009835_dp, -24.47666503668304117_dp, & + & -29.11998552133064422_dp, -26.63042818683534207_dp, & + & -23.25113130544624340_dp, -28.45290874783366775_dp, & + & -17.16079514785620930_dp, -29.91127029421520689_dp, & + & -10.88583393507997776_dp, -30.97645080660140593_dp, & + & -4.47781225763253765_dp, -31.62546289221583962_dp, & + & 1.99999999999999556_dp, -38.87875527701606160_dp, & + & 9.18708125479774296_dp, -38.63683300066042392_dp, & + & 16.27414365511956262_dp, -37.91820250638208734_dp, & + & 23.17259140539264095_dp, -36.74332279213027164_dp, & + & 29.81377244810832394_dp, -35.14341202945125531_dp, & + & 36.15298964123235947_dp, -33.15698491696964822_dp, & + & 42.16912769952225659_dp, -30.82630445836842625_dp, & + & 47.86123932751760179_dp, -28.19436585459229860_dp, & + & 53.24376548995357439_dp, -25.30271165018388402_dp, & + & 58.34173321358173325_dp, -22.19009961204914561_dp, & + & 63.18670666190918439_dp, -18.89188184637555779_dp, & + & 67.81377209114522486_dp, -15.43989911427395079_dp, & + & 72.25952713288313589_dp, -11.86270778471484277_dp, & + & 76.56090370117536281_dp, -8.18599910867232694_dp, & + & 80.75462189677602964_dp, -4.43311662588922584_dp, & + & 84.87709425893895343_dp, -0.62561625366165385_dp, & + & 88.96463936229967828_dp, 3.21615784383777248_dp, & + & 93.05390189983984328_dp, 7.07248905242842518_dp, & + & 97.18240447461120368_dp, 10.92369066480776318_dp, & + & 101.38917071735671982_dp, 14.74948212031801020_dp, & + & 105.71535755141637480_dp, 18.52832281285148497_dp, & + & 110.20481280215611264_dp, 22.23668760419472790_dp, & + & 114.90442707083992957_dp, 25.84827445710250160_dp, & + & 119.86406835852177721_dp, 29.33315055785289260_dp, & + & 125.13576891124590418_dp, 32.65687547770472321_dp, & + & 130.77168357131759535_dp, 35.77969749489390949_dp, & + & 136.82020109922683559_dp, 38.65601141858346068_dp, & + & 143.31958252044233859_dp, 41.23439386179803279_dp, & + & 150.28885177460347222_dp, 43.45866798082070659_dp, & + & 157.71667350566602295_dp, 45.27051130755705799_dp, & + & 165.55074086228117380_dp, 46.61395503653132977_dp, & + & 173.69217969866488716_dp, 47.44157216781382402_dp, & + & -178.00000000000002842_dp, 47.72124472298391851_dp, & + & -169.69217969866491558_dp, 47.44157216781382402_dp, & + & -161.55074086228117380_dp, 46.61395503653132977_dp, & + & -153.71667350566602295_dp, 45.27051130755705799_dp, & + & -146.28885177460344380_dp, 43.45866798082070659_dp, & + & -139.31958252044236701_dp, 41.23439386179803279_dp, & + & -132.82020109922686402_dp, 38.65601141858346068_dp, & + & -126.77168357131760956_dp, 35.77969749489390949_dp, & + & -121.13576891124588997_dp, 32.65687547770472321_dp, & + & -115.86406835852179142_dp, 29.33315055785289260_dp, & + & -110.90442707083994378_dp, 25.84827445710250160_dp, & + & -106.20481280215612685_dp, 22.23668760419472790_dp, & + & -101.71535755141638901_dp, 18.52832281285148497_dp, & + & -97.38917071735673403_dp, 14.74948212031801020_dp, & + & -93.18240447461121789_dp, 10.92369066480776318_dp, & + & -89.05390189983984328_dp, 7.07248905242842518_dp, & + & -84.96463936229969249_dp, 3.21615784383777248_dp, & + & -80.87709425893895343_dp, -0.62561625366165385_dp, & + & -76.75462189677602964_dp, -4.43311662588922584_dp, & + & -72.56090370117536281_dp, -8.18599910867232694_dp, & + & -68.25952713288315010_dp, -11.86270778471484277_dp, & + & -63.81377209114523907_dp, -15.43989911427395079_dp, & + & -59.18670666190919150_dp, -18.89188184637555779_dp, & + & -54.34173321358174746_dp, -22.19009961204914561_dp, & + & -49.24376548995358149_dp, -25.30271165018388402_dp, & + & -43.86123932751760890_dp, -28.19436585459229860_dp, & + & -38.16912769952225659_dp, -30.82630445836842625_dp, & + & -32.15298964123236658_dp, -33.15698491696964822_dp, & + & -25.81377244810833815_dp, -35.14341202945125531_dp, & + & -19.17259140539265516_dp, -36.74332279213027164_dp, & + & -12.27414365511956973_dp, -37.91820250638208734_dp, & + & -5.18708125479775184_dp, -38.63683300066042392_dp, & + & 1.99999999999999600_dp, -46.61822072475269607_dp, & + & 10.68839576675080139_dp, -46.30624543028367412_dp, & + & 19.18911080317679918_dp, -45.38412606933155757_dp, & + & 27.34546416576215222_dp, -43.89055765093645078_dp, & + & 35.05142021090807702_dp, -41.88213387768103502_dp, & + & 42.25559890256131013_dp, -39.42480136235260346_dp, & + & 48.95316063861176303_dp, -36.58631299796493153_dp, & + & 55.17233704527057370_dp, -33.43100825355479344_dp, & + & 60.96117255247643385_dp, -30.01702215459737744_dp, & + & 66.37723925505210332_dp, -26.39536140379773599_dp, & + & 71.48085419487965453_dp, -22.61014977269736548_dp, & + & 76.33122188826865795_dp, -18.69947873486282575_dp, & + & 80.98463901984250413_dp, -14.69649678201448850_dp, & + & 85.49400025392819202_dp, -10.63053585607601548_dp, & + & 89.90905356028720519_dp, -6.52818370181667884_dp, & + & 94.27704477012913742_dp, -2.41427530523583922_dp, & + & 98.64352777118510573_dp, 1.68719008387413583_dp, & + & 103.05319914949195947_dp, 5.75218534812640314_dp, & + & 107.55065416712271542_dp, 9.75582109859903390_dp, & + & 112.18096363521173942_dp, 13.67152082594358298_dp, & + & 116.98994385952536845_dp, 17.47019805768939449_dp, & + & 122.02393786261990272_dp, 21.11944988981388605_dp, & + & 127.32885205682140395_dp, 24.58281725765701253_dp, & + & 132.94811747553873715_dp, 27.81921722445741807_dp, & + & 138.91921485966395267_dp, 30.78272820361294038_dp, & + & 145.26850797118672176_dp, 33.42299645292871446_dp, & + & 152.00450116657361832_dp, 35.68659928106228563_dp, & + & 159.11038900941949237_dp, 37.51968052984662449_dp, & + & 166.53782868216146085_dp, 38.87197455633482690_dp, & + & 174.20476659278793363_dp, 39.70189618399167131_dp, & + & -178.00000000000002842_dp, 39.98177927524729114_dp, & + & -170.20476659278793363_dp, 39.70189618399167131_dp, & + & -162.53782868216148927_dp, 38.87197455633482690_dp, & + & -155.11038900941949237_dp, 37.51968052984662449_dp, & + & -148.00450116657361832_dp, 35.68659928106228563_dp, & + & -141.26850797118672176_dp, 33.42299645292871446_dp, & + & -134.91921485966395267_dp, 30.78272820361294038_dp, & + & -128.94811747553873715_dp, 27.81921722445741807_dp, & + & -123.32885205682140395_dp, 24.58281725765701253_dp, & + & -118.02393786261990272_dp, 21.11944988981388605_dp, & + & -112.98994385952538266_dp, 17.47019805768939449_dp, & + & -108.18096363521175363_dp, 13.67152082594358298_dp, & + & -103.55065416712271542_dp, 9.75582109859903390_dp, & + & -99.05319914949195947_dp, 5.75218534812640314_dp, & + & -94.64352777118511995_dp, 1.68719008387413583_dp, & + & -90.27704477012913742_dp, -2.41427530523583922_dp, & + & -85.90905356028720519_dp, -6.52818370181667884_dp, & + & -81.49400025392820623_dp, -10.63053585607601548_dp, & + & -76.98463901984250413_dp, -14.69649678201448850_dp, & + & -72.33122188826867216_dp, -18.69947873486282575_dp, & + & -67.48085419487966874_dp, -22.61014977269736548_dp, & + & -62.37723925505212463_dp, -26.39536140379773599_dp, & + & -56.96117255247644096_dp, -30.01702215459737744_dp, & + & -51.17233704527057370_dp, -33.43100825355479344_dp, & + & -44.95316063861176303_dp, -36.58631299796493153_dp, & + & -38.25559890256131723_dp, -39.42480136235260346_dp, & + & -31.05142021090808413_dp, -41.88213387768103502_dp, & + & -23.34546416576216643_dp, -43.89055765093645078_dp, & + & -15.18911080317680451_dp, -45.38412606933155757_dp, & + & -6.68839576675080849_dp, -46.30624543028367412_dp, & + & 1.99999999999999556_dp, -55.13279625041899834_dp, & + & 12.21451927262477000_dp, -54.76594013073760436_dp, & + & 22.09872704828332957_dp, -53.68864395349499574_dp, & + & 31.39769711559288012_dp, -51.96422430511800883_dp, & + & 39.96960367385942448_dp, -49.68040408706254141_dp, & + & 47.77902429867568657_dp, -46.93247902115530223_dp, & + & 54.86545369174167064_dp, -43.81109166541020983_dp, & + & 61.30916699148811233_dp, -40.39598469592146301_dp, & + & 67.20576633421555357_dp, -36.75430974219705860_dp, & + & 72.65107920217620574_dp, -32.94148403257256064_dp, & + & 77.73388667252179118_dp, -29.00309226896920123_dp, & + & 82.53343020155088539_dp, -24.97699141478170048_dp, & + & 87.11942634360052296_dp, -20.89524495725207132_dp, & + & 91.55320457280680557_dp, -16.78577161643180204_dp, & + & 95.88921878897831164_dp, -12.67371220268469578_dp, & + & 100.17656220612792595_dp, -8.58256192749418645_dp, & + & 104.46031614728221371_dp, -4.53512373056416251_dp, & + & 108.78265670493264849_dp, -0.55433148808729282_dp, & + & 113.18367588372132104_dp, 3.33602075627005812_dp, & + & 117.70187269530691765_dp, 7.11062322287716331_dp, & + & 122.37425008640165913_dp, 10.74206713847307348_dp, & + & 127.23592626204656142_dp, 14.20035582641498806_dp, & + & 132.31914580979304219_dp, 17.45256126555711873_dp, & + & 137.65157520973264127_dp, 20.46271167815037728_dp, & + & 143.25381666270027381_dp, 23.19203115841211726_dp, & + & 149.13620894054645305_dp, 25.59967735271198208_dp, & + & 155.29523174265207786_dp, 27.64411687523928052_dp, & + & 161.71017626763577368_dp, 29.28521267182223298_dp, & + & 168.34108626834759548_dp, 30.48695101230148552_dp, & + & 175.12910064316457692_dp, 31.22051723140710777_dp, & + & -178.00000000000002842_dp, 31.46720374958099598_dp, & + & -171.12910064316457692_dp, 31.22051723140710777_dp, & + & -164.34108626834762390_dp, 30.48695101230148552_dp, & + & -157.71017626763574526_dp, 29.28521267182223298_dp, & + & -151.29523174265207786_dp, 27.64411687523928052_dp, & + & -145.13620894054648147_dp, 25.59967735271198208_dp, & + & -139.25381666270027381_dp, 23.19203115841211726_dp, & + & -133.65157520973266969_dp, 20.46271167815037728_dp, & + & -128.31914580979304219_dp, 17.45256126555711873_dp, & + & -123.23592626204654721_dp, 14.20035582641498806_dp, & + & -118.37425008640165913_dp, 10.74206713847307348_dp, & + & -113.70187269530693186_dp, 7.11062322287716331_dp, & + & -109.18367588372132104_dp, 3.33602075627005812_dp, & + & -104.78265670493266271_dp, -0.55433148808729282_dp, & + & -100.46031614728222792_dp, -4.53512373056416251_dp, & + & -96.17656220612794016_dp, -8.58256192749418645_dp, & + & -91.88921878897831164_dp, -12.67371220268469578_dp, & + & -87.55320457280681978_dp, -16.78577161643180204_dp, & + & -83.11942634360052296_dp, -20.89524495725207132_dp, & + & -78.53343020155088539_dp, -24.97699141478170048_dp, & + & -73.73388667252179118_dp, -29.00309226896920123_dp, & + & -68.65107920217620574_dp, -32.94148403257256064_dp, & + & -63.20576633421556068_dp, -36.75430974219705860_dp, & + & -57.30916699148811233_dp, -40.39598469592146301_dp, & + & -50.86545369174167774_dp, -43.81109166541020983_dp, & + & -43.77902429867569367_dp, -46.93247902115530223_dp, & + & -35.96960367385943158_dp, -49.68040408706254141_dp, & + & -27.39769711559288723_dp, -51.96422430511800883_dp, & + & -18.09872704828334022_dp, -53.68864395349499574_dp, & + & -8.21451927262477888_dp, -54.76594013073760436_dp, & + & 1.99999999999999556_dp, -64.48162218246979194_dp, & + & 17.98037118830460201_dp, -63.76341508305665684_dp, & + & 32.62428016143745424_dp, -61.72174914658255318_dp, & + & 45.26325159838053480_dp, -58.62403790832021144_dp, & + & 55.90710322947827393_dp, -54.76300160634040992_dp, & + & 64.89227178252910733_dp, -50.38082342875218700_dp, & + & 72.61131289029847835_dp, -45.65520975111467550_dp, & + & 79.40370496549049051_dp, -40.71192873614232610_dp, & + & 85.53547443418635510_dp, -35.64102404029981841_dp, & + & 91.20945067766287195_dp, -30.50961194071436822_dp, & + & 96.58116190590757810_dp, -25.37072818781791383_dp, & + & 101.77265218881962028_dp, -20.26925377022463337_dp, & + & 106.88281945262124850_dp, -15.24594064653214254_dp, & + & 111.99465974304708027_dp, -10.34025377357217934_dp, & + & 117.18006743498143862_dp, -5.59247248797413388_dp, & + & 122.50268451884780063_dp, -1.04529436283411115_dp, & + & 128.01906809075350679_dp, 3.25496397729521814_dp, & + & 133.77826339847268855_dp, 7.25759760165333567_dp, & + & 139.81978239976487544_dp, 10.90731557455581502_dp, & + & 146.17005258660478262_dp, 14.14485028302005709_dp, & + & 152.83768595785250000_dp, 16.90865838596959492_dp, & + & 159.80845268931921055_dp, 19.13794220572577487_dp, & + & 167.04151372648448159_dp, 20.77699151155475477_dp, & + & 174.46891148411387462_dp, 21.78042227826199806_dp, & + & -178.00000000000002842_dp, 22.11837781753019527_dp, & + & -170.46891148411387462_dp, 21.78042227826199806_dp, & + & -163.04151372648448159_dp, 20.77699151155475477_dp, & + & -155.80845268931921055_dp, 19.13794220572577487_dp, & + & -148.83768595785252842_dp, 16.90865838596959492_dp, & + & -142.17005258660481104_dp, 14.14485028302005709_dp, & + & -135.81978239976487544_dp, 10.90731557455581502_dp, & + & -129.77826339847271697_dp, 7.25759760165333567_dp, & + & -124.01906809075352101_dp, 3.25496397729521814_dp, & + & -118.50268451884778642_dp, -1.04529436283411115_dp, & + & -113.18006743498142441_dp, -5.59247248797413388_dp, & + & -107.99465974304709448_dp, -10.34025377357217934_dp, & + & -102.88281945262124850_dp, -15.24594064653214254_dp, & + & -97.77265218881962028_dp, -20.26925377022463337_dp, & + & -92.58116190590757810_dp, -25.37072818781791383_dp, & + & -87.20945067766288616_dp, -30.50961194071436822_dp, & + & -81.53547443418635510_dp, -35.64102404029981841_dp, & + & -75.40370496549049051_dp, -40.71192873614232610_dp, & + & -68.61131289029849256_dp, -45.65520975111467550_dp, & + & -60.89227178252910733_dp, -50.38082342875218700_dp, & + & -51.90710322947828104_dp, -54.76300160634040992_dp, & + & -41.26325159838054901_dp, -58.62403790832021144_dp, & + & -28.62428016143746490_dp, -61.72174914658255318_dp, & + & -13.98037118830460557_dp, -63.76341508305665684_dp, & + & 1.99999999999999489_dp, -74.69966643845165777_dp, & + & 26.73698176730493614_dp, -73.50791820369254026_dp, & + & 46.45208209886830275_dp, -70.36956689494368788_dp, & + & 60.80977159628141493_dp, -66.05660901455554779_dp, & + & 71.45848324441054444_dp, -61.11675773175377913_dp, & + & 79.80759217874309286_dp, -55.85321872580696834_dp, & + & 86.74466514239207982_dp, -50.43213273346431436_dp, & + & 92.80540124701705906_dp, -44.95206516381727369_dp, & + & 98.32251211109890221_dp, -39.47845092120540045_dp, & + & 103.51336526710929320_dp, -34.06038011063251503_dp, & + & 108.52834753231333309_dp, -28.73921107936183361_dp, & + & 113.47768355265095863_dp, -23.55333544230752452_dp, & + & 118.44656525084906207_dp, -18.54103902309049090_dp, & + & 123.50369215326308847_dp, -13.74234202233444968_dp, & + & 128.70582948829564884_dp, -9.20020460514958671_dp, & + & 134.09971622507504208_dp, -4.96122571165357140_dp, & + & 139.72202213508867885_dp, -1.07580930674149267_dp, & + & 145.59779383610683112_dp, 2.40232548413760538_dp, & + & 151.73783261271663036_dp, 5.41746050329454931_dp, & + & 158.13565186637288207_dp, 7.91416414131925894_dp, & + & 164.76497871932286898_dp, 9.84055665005913660_dp, & + & 171.57900980622665088_dp, 11.15231427016150256_dp, & + & 178.51254101195692670_dp, 11.81685015330946520_dp, & + & -174.51254101195692670_dp, 11.81685015330946520_dp, & + & -167.57900980622667930_dp, 11.15231427016150256_dp, & + & -160.76497871932289740_dp, 9.84055665005913660_dp, & + & -154.13565186637291049_dp, 7.91416414131925894_dp, & + & -147.73783261271665879_dp, 5.41746050329454931_dp, & + & -141.59779383610683112_dp, 2.40232548413760538_dp, & + & -135.72202213508870727_dp, -1.07580930674149267_dp, & + & -130.09971622507501365_dp, -4.96122571165357140_dp, & + & -124.70582948829567727_dp, -9.20020460514958671_dp, & + & -119.50369215326308847_dp, -13.74234202233444968_dp, & + & -114.44656525084907628_dp, -18.54103902309049090_dp, & + & -109.47768355265095863_dp, -23.55333544230752452_dp, & + & -104.52834753231334730_dp, -28.73921107936183361_dp, & + & -99.51336526710929320_dp, -34.06038011063251503_dp, & + & -94.32251211109890221_dp, -39.47845092120540045_dp, & + & -88.80540124701705906_dp, -44.95206516381727369_dp, & + & -82.74466514239209403_dp, -50.43213273346431436_dp, & + & -75.80759217874310707_dp, -55.85321872580696834_dp, & + & -67.45848324441054444_dp, -61.11675773175377913_dp, & + & -56.80977159628143625_dp, -66.05660901455554779_dp, & + & -42.45208209886832407_dp, -70.36956689494368788_dp, & + & -22.73698176730495391_dp, -73.50791820369254026_dp, & + & 1.99999999999999600_dp, -85.78166687108885924_dp, & + & 61.87324357773108119_dp, -82.33459525449173100_dp, & + & 80.27768438369075454_dp, -76.54083121758115738_dp, & + & 89.42499621356618889_dp, -70.41813086842684299_dp, & + & 95.82057104813959825_dp, -64.24948773689966686_dp, & + & 101.11263541171206271_dp, -58.11928872485961506_dp, & + & 105.90317607822892398_dp, -52.07335345715245722_dp, & + & 110.47134737780878311_dp, -46.14758359783259323_dp, & + & 114.97138028465980142_dp, -40.37601690822956613_dp, & + & 119.50120511177952665_dp, -34.79395188944583595_dp, & + & 124.13018318901127657_dp, -29.43951176908901957_dp, & + & 128.91129483912590104_dp, -24.35455549606792047_dp, & + & 133.88641650799124250_dp, -19.58517424494497305_dp, & + & 139.08801440266989857_dp, -15.18177122287804615_dp, & + & 144.53869867413172301_dp, -11.19861017149900739_dp, & + & 150.24942513902280439_dp, -7.69266569608215622_dp, & + & 156.21700460768090579_dp, -4.72161659803606781_dp, & + & 162.42170502724644621_dp, -2.34091526057735644_dp, & + & 168.82591385569978115_dp, -0.60006016949341945_dp, & + & 175.37485237931247184_dp, 0.46152536384328491_dp, & + & -178.00000000000002842_dp, 0.81833312891115950_dp, & + & -171.37485237931247184_dp, 0.46152536384328491_dp, & + & -164.82591385569978115_dp, -0.60006016949341945_dp, & + & -158.42170502724641779_dp, -2.34091526057735644_dp, & + & -152.21700460768090579_dp, -4.72161659803606781_dp, & + & -146.24942513902283281_dp, -7.69266569608215622_dp, & + & -140.53869867413175143_dp, -11.19861017149900739_dp, & + & -135.08801440266989857_dp, -15.18177122287804615_dp, & + & -129.88641650799127092_dp, -19.58517424494497305_dp, & + & -124.91129483912591525_dp, -24.35455549606792047_dp, & + & -120.13018318901130499_dp, -29.43951176908901957_dp, & + & -115.50120511177955507_dp, -34.79395188944583595_dp, & + & -110.97138028465980142_dp, -40.37601690822956613_dp, & + & -106.47134737780881153_dp, -46.14758359783259323_dp, & + & -101.90317607822893820_dp, -52.07335345715245722_dp, & + & -97.11263541171207692_dp, -58.11928872485961506_dp, & + & -91.82057104813961246_dp, -64.24948773689966686_dp, & + & -85.42499621356618889_dp, -70.41813086842684299_dp, & + & -76.27768438369074033_dp, -76.54083121758115738_dp, & + & -57.87324357773110250_dp, -82.33459525449173100_dp, & + & -178.00000000000002842_dp, -82.33591847955618448_dp, & + & 143.22778135121401988_dp, -79.54230031515010069_dp, & + & 128.60898756414843547_dp, -73.87349687490876704_dp, & + & 124.32470998353319658_dp, -67.47696540683871547_dp, & + & 124.01228809564837263_dp, -60.93070105520797597_dp, & + & 125.59604777627561134_dp, -54.43751136022503090_dp, & + & 128.28024682035200499_dp, -48.10755262562803125_dp, & + & 131.71480673751509016_dp, -42.02394287044850074_dp, & + & 135.73150447900647464_dp, -36.26259289105208694_dp, & + & 140.24424010430539056_dp, -30.89942669541638764_dp, & + & 145.20455296082769792_dp, -26.01294410398721979_dp, & + & 150.57821873782722832_dp, -21.68429473238734317_dp, & + & 156.33085313434582986_dp, -17.99542592709287092_dp, & + & 162.41822389716838870_dp, -15.02545489467462048_dp, & + & 168.78020742076520833_dp, -12.84550987892848184_dp, & + & 175.33870321353637678_dp, -11.51266748991093536_dp, & + & -178.00000000000002842_dp, -11.06408152044378745_dp, & + & -171.33870321353637678_dp, -11.51266748991093536_dp, & + & -164.78020742076523675_dp, -12.84550987892848184_dp, & + & -158.41822389716841712_dp, -15.02545489467462048_dp, & + & -152.33085313434585828_dp, -17.99542592709287092_dp, & + & -146.57821873782722832_dp, -21.68429473238734317_dp, & + & -141.20455296082769792_dp, -26.01294410398721979_dp, & + & -136.24424010430539056_dp, -30.89942669541638764_dp, & + & -131.73150447900647464_dp, -36.26259289105208694_dp, & + & -127.71480673751507595_dp, -42.02394287044850074_dp, & + & -124.28024682035203341_dp, -48.10755262562803125_dp, & + & -121.59604777627559713_dp, -54.43751136022503090_dp, & + & -120.01228809564838684_dp, -60.93070105520797597_dp, & + & -120.32470998353322500_dp, -67.47696540683871547_dp, & + & -124.60898756414846389_dp, -73.87349687490876704_dp, & + & -139.22778135121404830_dp, -79.54230031515010069_dp, & + & -178.00000000000002842_dp, -69.79343463141964321_dp, & + & 167.63134400963940607_dp, -68.62304968612301082_dp, & + & 156.89871695437406629_dp, -65.48334135458465255_dp, & + & 150.54736374030309776_dp, -61.10651485448561715_dp, & + & 147.65287319536125210_dp, -56.10790502157546911_dp, & + & 147.17953110490091717_dp, -50.89570090116846757_dp, & + & 148.39568884343808008_dp, -45.73876083396746139_dp, & + & 150.83543074335267420_dp, -40.82969132012996027_dp, & + & 154.20166627754437627_dp, -36.32066069380964279_dp, & + & 158.29374271187359113_dp, -32.34078423455542861_dp, & + & 162.96127389198957758_dp, -29.00296899656860816_dp, & + & 168.07572003197756771_dp, -26.40464312032804983_dp, & + & 173.51374580520655400_dp, -24.62495078480371546_dp, & + & 179.14909640850049755_dp, -23.72034636702055366_dp, & + & -175.14909640850046912_dp, -23.72034636702055366_dp, & + & -169.51374580520655400_dp, -24.62495078480371546_dp, & + & -164.07572003197756771_dp, -26.40464312032806049_dp, & + & -158.96127389198957758_dp, -29.00296899656861527_dp, & + & -154.29374271187361956_dp, -32.34078423455542861_dp, & + & -150.20166627754440469_dp, -36.32066069380964279_dp, & + & -146.83543074335270262_dp, -40.82969132012996027_dp, & + & -144.39568884343808008_dp, -45.73876083396746139_dp, & + & -143.17953110490091717_dp, -50.89570090116849599_dp, & + & -143.65287319536128052_dp, -56.10790502157547621_dp, & + & -146.54736374030309776_dp, -61.10651485448561715_dp, & + & -152.89871695437406629_dp, -65.48334135458465255_dp, & + & -163.63134400963940607_dp, -68.62304968612301082_dp, & + & -178.00000000000002842_dp, -56.85258712078380228_dp, & + & 176.37525674045562596_dp, -56.23764348052387874_dp, & + & 171.72066876086481102_dp, -54.50623527875517738_dp, & + & 168.62326642581021474_dp, -51.94637057143930292_dp, & + & 167.22080918295719698_dp, -48.91487419033276751_dp, & + & 167.36621335583259906_dp, -45.75617589753742465_dp, & + & 168.79950790329058918_dp, -42.76713193279059766_dp, & + & 171.24101930549505823_dp, -40.18968456371464271_dp, & + & 174.42266827604018431_dp, -38.21219158951615213_dp, & + & 178.09065200901943626_dp, -36.97055608025468132_dp, & + & -178.00000000000002842_dp, -36.54741287921620341_dp, & + & -174.09065200901946469_dp, -36.97055608025468132_dp, & + & -170.42266827604018431_dp, -38.21219158951615213_dp, & + & -167.24101930549508666_dp, -40.18968456371464271_dp, & + & -164.79950790329061761_dp, -42.76713193279059766_dp, & + & -163.36621335583262749_dp, -45.75617589753742465_dp, & + & -163.22080918295719698_dp, -48.91487419033276751_dp, & + & -164.62326642581024316_dp, -51.94637057143930292_dp, & + & -167.72066876086483944_dp, -54.50623527875517738_dp, & + & -172.37525674045562596_dp, -56.23764348052387874_dp ] grid = atlas_ReducedGaussianGrid(nx, projection=atlas_RotatedSchmidtProjection(stretch,centre,angle) ) jglo = 0 do j = 1, grid%ny() do i = 1, grid%nx(j) - FCTEST_CHECK_CLOSE( grid%lonlat(i,j), ([lonlat(2*jglo+1),lonlat(2*jglo+2)]), 1.e-10_dp ); + FCTEST_CHECK_CLOSE( grid%lonlat(i,j), ([lonlat(2*jglo+1),lonlat(2*jglo+2)]), 1.e-9_dp ); jglo = jglo + 1 enddo enddo diff --git a/src/tests/grid/test_stretchedrotatedgaussian.cc b/src/tests/grid/test_stretchedrotatedgaussian.cc index 66ea4c7de..6c6d51042 100644 --- a/src/tests/grid/test_stretchedrotatedgaussian.cc +++ b/src/tests/grid/test_stretchedrotatedgaussian.cc @@ -12,851 +12,1694 @@ using namespace atlas::grid; namespace { const double lonlat_arp_t32c24[] = { - 2.00000000000000000, 48.46708828700425187, 1.17799068975270704, 48.37772379915030996, - 0.44418972240584109, 48.11926090196744354, -0.12512266986032569, 47.71926346185548340, - -0.47479391972813606, 47.21956618120344018, -0.57569351104372468, 46.67109320702403608, - -0.42528132740345748, 46.12813070368466839, -0.04494770792091035, 45.64287023699112211, - 0.52444137510688571, 45.26073815098433784, 1.22762737065264282, 45.01674523281888440, - 2.00000000000000000, 44.93291171299576092, 2.77237262934735629, 45.01674523281888440, - 3.47555862489311407, 45.26073815098433784, 4.04494770792090996, 45.64287023699112211, - 4.42528132740345725, 46.12813070368466839, 4.57569351104372490, 46.67109320702403608, - 4.47479391972813634, 47.21956618120344018, 4.12512266986032561, 47.71926346185548340, - 3.55581027759415846, 48.11926090196744354, 2.82200931024729318, 48.37772379915030996, - 2.00000000000000000, 50.76274729016172671, 0.52357145823104279, 50.64425941434981837, - -0.85309402263029532, 50.29690558565773273, -2.04123891610972796, 49.74407157963349846, - -2.97152383528954323, 49.02183412771946536, -3.59837298636155323, 48.17535393470686955, - -3.90029306374494222, 47.25496955383284359, -3.87730059701688790, 46.31256836954069911, - -3.54692456688646551, 45.39855083314470363, -2.93998927128814502, 44.55946699558614910, - -2.09688983996385936, 43.83625800397906147, -1.06462721424400342, 43.26298044031561574, - 0.10541754825032876, 42.86589673132546352, 1.35912400589531490, 42.66284629701862485, - 2.64087599410468910, 42.66284629701862485, 3.89458245174967033, 42.86589673132546352, - 5.06462721424400453, 43.26298044031561574, 6.09688983996385936, 43.83625800397906147, - 6.93998927128814458, 44.55946699558614910, 7.54692456688646551, 45.39855083314470363, - 7.87730059701688656, 46.31256836954069911, 7.90029306374494134, 47.25496955383284359, - 7.59837298636155190, 48.17535393470686955, 6.97152383528954367, 49.02183412771946536, - 6.04123891610972930, 49.74407157963349846, 4.85309402263029188, 50.29690558565773273, - 3.47642854176895577, 50.64425941434981837, 2.00000000000000089, 53.08763788054714894, - -0.06435502534210064, 52.94800110445672203, -2.01378910799337474, 52.53707313311167582, - -3.74622775989939782, 51.87782553848720113, -5.18197003975669812, 51.00557820511744467, - -6.26810804950859701, 49.96440199105025926, -6.97806564551988018, 48.80336562297891589, - -7.30784824781774223, 47.57322688923063225, -7.27090590027329053, 46.32385288536406875, - -6.89307303713296715, 45.10238649522262477, -6.20837670733706215, 43.95202523375940729, - -5.25594876339865014, 42.91123363953519743, -4.07794770476515911, 42.01322983391511201, - -2.71826031811739588, 41.28563184510834105, -1.22173788476928480, 40.75019560382368411, - 0.36623707967922525, 40.42261301497919845, 1.99999999999999933, 40.31236211945285675, - 3.63376292032077286, 40.42261301497919845, 5.22173788476928369, 40.75019560382368411, - 6.71826031811739455, 41.28563184510834105, 8.07794770476515822, 42.01322983391511201, - 9.25594876339864925, 42.91123363953519743, 10.20837670733705949, 43.95202523375940729, - 10.89307303713296804, 45.10238649522262477, 11.27090590027328965, 46.32385288536406875, - 11.30784824781774311, 47.57322688923063225, 10.97806564551988195, 48.80336562297891589, - 10.26810804950859790, 49.96440199105025926, 9.18197003975669901, 51.00557820511744467, - 7.74622775989939960, 51.87782553848720113, 6.01378910799337607, 52.53707313311167582, - 4.06435502534210169, 52.94800110445672203, 2.00000000000000089, 55.44054043583248159, - -0.39391037845720578, 55.31115535850048559, -2.68766853851381882, 54.92865613654313961, - -4.79116029542177202, 54.30937174533021050, -6.63193584333677855, 53.47856301011842106, - -8.15903975007321769, 52.46802323745790630, -9.34306219743110411, 51.31353838046217675, - -10.17346129570704782, 50.05261748260089405, -10.65454278958191026, 48.72271255422421632, - -10.80126342211737800, 47.35996933207513848, -10.63557388477568999, 45.99843704779454612, - -10.18359521918176824, 44.66961909752426152, -9.47363915243209931, 43.40224634648851065, - -8.53494002107633598, 42.22217718966415845, -7.39692139773956647, 41.15235690381517486, - -6.08882962579569487, 40.21279467492543347, -4.63959771096107954, 39.42053684981195261, - -3.07783882564465205, 38.78962916423042628, -1.43190111561111988, 38.33106975566338548, - 0.27005804914242593, 38.05275970261258323, 1.99999999999999956, 37.95945956416753120, - 3.72994195085757241, 38.05275970261258323, 5.43190111561111788, 38.33106975566338548, - 7.07783882564465028, 38.78962916423042628, 8.63959771096107865, 39.42053684981195261, - 10.08882962579569309, 40.21279467492543347, 11.39692139773956647, 41.15235690381517486, - 12.53494002107633243, 42.22217718966415845, 13.47363915243209576, 43.40224634648851065, - 14.18359521918176824, 44.66961909752426152, 14.63557388477568999, 45.99843704779454612, - 14.80126342211737445, 47.35996933207513848, 14.65454278958191381, 48.72271255422421632, - 14.17346129570704782, 50.05261748260089405, 13.34306219743110766, 51.31353838046217675, - 12.15903975007321769, 52.46802323745790630, 10.63193584333677855, 53.47856301011842106, - 8.79116029542177380, 54.30937174533021050, 6.68766853851382059, 54.92865613654313961, - 4.39391037845720867, 55.31115535850048559, 2.00000000000000222, 57.82915094012270174, - -0.88089731877775757, 57.69078537163564846, -3.64890062321652664, 57.28147795233979167, - -6.20331014929099656, 56.61787273415653488, -8.46463908845755597, 55.72551502002679058, - -10.37873026754043693, 54.63620962052475960, -11.91616209561276740, 53.38530347804498888, - -13.06845153476416321, 52.00935252683229493, -13.84288249522568925, 50.54439040009851425, - -14.25738892575702366, 49.02480894996942595, -14.33628744864260440, 47.48274104170999976, - -14.10711920596712332, 45.94779768044404733, -13.59853603019004176, 44.44702279967973624, - -12.83902541924944174, 43.00496062724707258, -11.85624296375186937, 41.64376435793402464, - -10.67674963753560036, 40.38330297070848474, -9.32599718132065902, 39.24124360712294646, - -7.82844927012504055, 38.23310085983992224, -6.20776260081896769, 37.37225322309888043, - -4.48697963071903061, 36.66993230518373537, -2.68870484500073204, 36.13519327177002793, - -0.83525097543211291, 35.77487607444932394, 1.05124791602986001, 35.59356675116022473, - 2.94875208397013866, 35.59356675116022473, 4.83525097543211047, 35.77487607444932394, - 6.68870484500073026, 36.13519327177002793, 8.48697963071903061, 36.66993230518373537, - 10.20776260081896680, 37.37225322309888043, 11.82844927012503966, 38.23310085983992224, - 13.32599718132065725, 39.24124360712293225, 14.67674963753560391, 40.38330297070848474, - 15.85624296375187114, 41.64376435793402464, 16.83902541924944174, 43.00496062724707258, - 17.59853603019003998, 44.44702279967973624, 18.10711920596712687, 45.94779768044404733, - 18.33628744864260796, 47.48274104170998555, 18.25738892575703076, 49.02480894996942595, - 17.84288249522569458, 50.54439040009851425, 17.06845153476416499, 52.00935252683229493, - 15.91616209561277273, 53.38530347804498888, 14.37873026754043870, 54.63620962052475960, - 12.46463908845756130, 55.72551502002679058, 10.20331014929099922, 56.61787273415653488, - 7.64890062321653019, 57.28147795233979167, 4.88089731877776067, 57.69078537163564846, - 2.00000000000000222, 60.26295411645657651, -1.52099396835736300, 60.10440004572244277, - -4.89637618755807136, 59.63591573610307250, -7.99912115224313958, 58.87795079119066344, - -10.73370655373201643, 57.86135900031617751, -13.04067793585433499, 56.62367212326139310, - -14.89384545876526111, 55.20548791890712437, -16.29319722451073105, 53.64760968250063655, - -17.25667322722224029, 51.98914605534769606, -17.81285996059914467, 50.26647329190694080, - -17.99546453290829007, 48.51282952087984768, -17.83961562141898938, 46.75830290430587155, - -17.37966075266355759, 45.03002540480565585, -16.64804001511417297, 43.35244484209054860, - -15.67486490585672243, 41.74759899443786537, -14.48792157318536411, 40.23535159216241652, - -13.11290455740948957, 38.83357308646102979, -11.57375501535206475, 37.55826279697890158, - -9.89302536639832653, 36.42361680044041350, -8.09222434325374884, 35.44205010622848562, - -6.19211732580419838, 34.62418375193039566, -4.21297051898843300, 33.97880825794451454, - -2.17473678976558649, 33.51283479810019372, -0.09718755711549092, 33.23124463343540214, - 1.99999999999999889, 33.13704588354342917, 4.09718755711548877, 33.23124463343540214, - 6.17473678976558382, 33.51283479810019372, 8.21297051898843122, 33.97880825794451454, - 10.19211732580419572, 34.62418375193039566, 12.09222434325374707, 35.44205010622848562, - 13.89302536639832830, 36.42361680044041350, 15.57375501535206297, 37.55826279697889447, - 17.11290455740948602, 38.83357308646102268, 18.48792157318536766, 40.23535159216240942, - 19.67486490585671888, 41.74759899443786537, 20.64804001511417653, 43.35244484209054860, - 21.37966075266355404, 45.03002540480565585, 21.83961562141898582, 46.75830290430587155, - 21.99546453290829007, 48.51282952087984768, 21.81285996059914822, 50.26647329190694080, - 21.25667322722224739, 51.98914605534769606, 20.29319722451073105, 53.64760968250062234, - 18.89384545876525934, 55.20548791890712437, 17.04067793585434032, 56.62367212326139310, - 14.73370655373202354, 57.86135900031617751, 11.99912115224314491, 58.87795079119066344, - 8.89637618755807580, 59.63591573610307250, 5.52099396835736744, 60.10440004572244277, - 2.00000000000000222, 62.75242980718265784, -1.60371952785031557, 62.62270582496653049, - -5.08984714396560278, 62.23829052070698253, -8.35401191716921510, 61.61281120273996237, - -11.31471471060360656, 60.76704034614787986, -13.91751559979281083, 59.72658657388572578, - -16.13405311103923978, 58.51955862209560166, -17.95764487214750815, 57.17460858105235388, - -19.39751805719553701, 55.71953428872550518, -20.47321926655622448, 54.18043392376822709, - -21.21002592824931199, 52.58130404832083116, -21.63559425749199150, 50.94394459368972150, - -21.77773936992417703, 49.28804930270956675, -21.66310777053267600, 47.63139077670034283, - -21.31648826424519072, 45.99003977573201496, -20.76054697060167342, 44.37858250245571412, - -20.01582559662803007, 42.81031632956308641, -19.10089088783405842, 41.29741507659190347, - -18.03256121225549791, 39.85106125278488065, -16.82616341777592339, 38.48154618161254348, - -15.49579145727241247, 37.19834072287546434, -14.05455017155123087, 36.01014014726423795, - -12.51477509868480809, 34.92488704934376642, -10.88822378743807739, 33.94977626735131793, - -9.18623692372254119, 33.09124574576954103, -7.41986936545087783, 32.35495718584156322, - -5.59999239238874669, 31.74577019145995394, -3.73736939611796615, 31.26771342119434749, - -1.84270801296844233, 30.92395598105918353, 0.07330759755401026, 30.71678191817960268, - 1.99999999999999889, 30.64757019281734074, 3.92669240244598639, 30.71678191817960268, - 5.84270801296843878, 30.92395598105918353, 7.73736939611796526, 31.26771342119434749, - 9.59999239238874225, 31.74577019145995394, 11.41986936545087516, 32.35495718584156322, - 13.18623692372254119, 33.09124574576954103, 14.88822378743807207, 33.94977626735131793, - 16.51477509868480809, 34.92488704934376642, 18.05455017155123087, 36.01014014726423795, - 19.49579145727241070, 37.19834072287546434, 20.82616341777592339, 38.48154618161253637, - 22.03256121225549791, 39.85106125278488065, 23.10089088783405842, 41.29741507659190347, - 24.01582559662803007, 42.81031632956308641, 24.76054697060167342, 44.37858250245571412, - 25.31648826424519072, 45.99003977573201496, 25.66310777053267600, 47.63139077670033572, - 25.77773936992417703, 49.28804930270955964, 25.63559425749199505, 50.94394459368972150, - 25.21002592824931199, 52.58130404832083116, 24.47321926655622448, 54.18043392376822709, - 23.39751805719553701, 55.71953428872550518, 21.95764487214750460, 57.17460858105235388, - 20.13405311103924333, 58.51955862209560166, 17.91751559979281438, 59.72658657388572578, - 15.31471471060361011, 60.76704034614787986, 12.35401191716921865, 61.61281120273996237, - 9.08984714396560634, 62.23829052070698253, 5.60371952785031979, 62.62270582496653049, - 2.00000000000000311, 65.30899890463146562, -2.55165837938126616, 65.14506774788591770, - -6.91821393310037980, 64.66099291349783584, -10.94248311853021427, 63.87850997409253040, - -14.51323316925390294, 62.82970044453450953, -17.56939664790982292, 61.55243525845713748, - -20.09325961331439458, 60.08625295311978931, -22.09844461997003506, 58.46939890910893922, - -23.61769060836897438, 56.73712764036461920, -24.69307609487012556, 54.92100894435518654, - -25.36933297612858951, 53.04888396821036167, -25.68983696232584535, 51.14517010976478417, - -25.69453569648834801, 49.23130925676766623, -25.41912613625677153, 47.32623888788154431, - -24.89496528133907205, 45.44682484300304282, -24.14937074714975296, 43.60823042135866956, - -23.20610004359021517, 41.82421587722975431, -22.08588698105332782, 40.10737174655498194, - -20.80696969977402944, 38.46929323359992736, -19.38557795842460507, 36.92070387379718710, - -17.83636590786354148, 35.47153642700349963, -16.17278641740254130, 34.13097826525054046, - -14.40740784814351017, 32.90748779465315721, -12.55217628249286932, 31.80878785141789677, - -10.61862699669389443, 30.84184157026976436, -8.61804920168179045, 30.01281590885307438, - -6.56160821625925728, 29.32703776203888069, -4.46042949045145676, 28.78894734383437992, - -2.32564933647368921, 28.40205318377053345, -0.16843782661930862, 28.16889262543881500, - 1.99999999999999889, 28.09100109536852585, 4.16843782661930629, 28.16889262543881500, - 6.32564933647368743, 28.40205318377053345, 8.46042949045145498, 28.78894734383437992, - 10.56160821625925372, 29.32703776203888069, 12.61804920168178867, 30.01281590885307438, - 14.61862699669389265, 30.84184157026976436, 16.55217628249286577, 31.80878785141789677, - 18.40740784814351017, 32.90748779465315721, 20.17278641740253775, 34.13097826525054046, - 21.83636590786353793, 35.47153642700349963, 23.38557795842460152, 36.92070387379718710, - 24.80696969977402588, 38.46929323359992026, 26.08588698105332426, 40.10737174655498194, - 27.20610004359021161, 41.82421587722975431, 28.14937074714974941, 43.60823042135866956, - 28.89496528133906494, 45.44682484300304282, 29.41912613625676798, 47.32623888788153721, - 29.69453569648834446, 49.23130925676766623, 29.68983696232584180, 51.14517010976478417, - 29.36933297612858951, 53.04888396821034746, 28.69307609487012911, 54.92100894435518654, - 27.61769060836897438, 56.73712764036461920, 26.09844461997003506, 58.46939890910893922, - 24.09325961331439814, 60.08625295311978931, 21.56939664790982292, 61.55243525845713748, - 18.51323316925390827, 62.82970044453450953, 14.94248311853021960, 63.87850997409253040, - 10.91821393310038779, 64.66099291349783584, 6.55165837938127193, 65.14506774788591770, - 2.00000000000000400, 67.94514236453578349, -3.38533547682125890, 67.76325974975715383, - -8.52326664933801226, 67.22759281764989225, -13.21138564172178143, 66.36584339331807314, - -17.31841683118042496, 65.21781824976642383, -20.78586461017365750, 63.82887077001763032, - -23.61278608277912383, 62.24449065808524040, -25.83517004248913196, 60.50690475312734407, - -27.50782877801382043, 58.65352526302899605, -28.69170289014789077, 56.71665696607985296, - -29.44632120220701665, 54.72388646563504011, -29.82602201650912477, 52.69875031809839783, - -29.87852862440772839, 50.66145188613276673, -29.64483525740958925, 48.62951713112133945, - -29.15973753507462618, 46.61834908938176625, -28.45262246278107554, 44.64167567492752653, - -27.54831327014242959, 42.71190039889389567, -26.46787020033523774, 40.84037031204417190, - -25.22930648122764552, 39.03757551632936185, -23.84820859564655038, 37.31329283883029291, - -22.33826403031283903, 35.67668409468497970, -20.71170528699355629, 34.13635737784334623, - -18.97968026705781242, 32.70039821822412307, -17.15255849246134900, 31.37637626233231813, - -15.24018123567453742, 30.17133232687284305, -13.25206214192900944, 29.09175016726920759, - -11.19754366142445079, 28.14351701208745027, -9.08591371200913400, 27.33187675649334381, - -6.92648650789847675, 26.66137960320710931, -4.72865139696445702, 26.13583181560235502, - -2.50189378344492486, 25.75824904385734371, -0.25579267831605296, 25.53081635683327733, - 1.99999999999999889, 25.45485763546421154, 4.25579267831605002, 25.53081635683327733, - 6.50189378344492308, 25.75824904385734371, 8.72865139696445347, 26.13583181560235502, - 10.92648650789847764, 26.66137960320710931, 13.08591371200913400, 27.33187675649334381, - 15.19754366142444724, 28.14351701208745027, 17.25206214192900589, 29.09175016726920759, - 19.24018123567453742, 30.17133232687284305, 21.15255849246134545, 31.37637626233231813, - 22.97968026705781242, 32.70039821822412307, 24.71170528699354918, 34.13635737784334623, - 26.33826403031282837, 35.67668409468497259, 27.84820859564654683, 37.31329283883029291, - 29.22930648122764197, 39.03757551632935474, 30.46787020033523419, 40.84037031204417190, - 31.54831327014242248, 42.71190039889389567, 32.45262246278107199, 44.64167567492752653, - 33.15973753507462618, 46.61834908938176625, 33.64483525740958214, 48.62951713112133945, - 33.87852862440772839, 50.66145188613276673, 33.82602201650912122, 52.69875031809839783, - 33.44632120220701665, 54.72388646563502590, 32.69170289014788011, 56.71665696607985296, - 31.50782877801381687, 58.65352526302899605, 29.83517004248913906, 60.50690475312734407, - 27.61278608277913094, 62.24449065808524040, 24.78586461017366460, 63.82887077001763032, - 21.31841683118043562, 65.21781824976642383, 17.21138564172179031, 66.36584339331807314, - 12.52326664933801936, 67.22759281764989225, 7.38533547682126645, 67.76325974975715383, - 2.00000000000000488, 70.67459026401789401, -4.83336207267744200, 70.44359392189112157, - -11.25145387811900299, 69.76794994271870110, -16.94303945184942606, 68.69426490502790728, - -21.74696599568290978, 67.28571148577093197, -25.63456841105837825, 65.60910965714572285, - -28.66225471925319823, 63.72633555655123416, -30.92626834358740595, 61.69055635034530383, - -32.53242307522426557, 59.54571651024583190, -33.58017141013023377, 57.32758829402004608, - -34.15616674923442986, 55.06530104282313687, -34.33296113360040636, 52.78282385843581892, - -34.17004600591189956, 50.50021197832979425, -33.71572773969587900, 48.23458650173874673, - -33.00911969333094476, 46.00087796034757304, -32.08195470333168231, 43.81237943576640248, - -30.96012666362175025, 41.68115224019865650, -29.66495985223924237, 39.61831917500947498, - -28.21423852322426740, 37.63427208010365632, -26.62303747247054986, 35.73881343521247089, - -24.90439178940930276, 33.94124647006630369, -23.06983782140640216, 32.25042441190934994, - -21.12985059268007149, 30.67476685924403057, -19.09419678103115459, 29.22224954465700009, - -16.97221728412494457, 27.90037270541385794, -14.77304947679039238, 26.71611272875198750, - -12.50579641562138100, 25.67586152120415832, -10.17964838758363832, 24.78535803035031648, - -7.80396120930039583, 24.04961640006232315, -5.38829543151647705, 23.47285525677743223, - -2.94242093509331726, 23.05843251151437201, -0.47629214130109782, 22.80878975038074685, - 1.99999999999999800, 22.72540973598210812, 4.47629214130109521, 22.80878975038074685, - 6.94242093509331326, 23.05843251151437201, 9.38829543151647350, 23.47285525677743223, - 11.80396120930039316, 24.04961640006232315, 14.17964838758363655, 24.78535803035031648, - 16.50579641562137567, 25.67586152120415832, 18.77304947679039060, 26.71611272875198750, - 20.97221728412493746, 27.90037270541385439, 23.09419678103115103, 29.22224954465700009, - 25.12985059268006793, 30.67476685924402702, 27.06983782140639860, 32.25042441190934994, - 28.90439178940930631, 33.94124647006630369, 30.62303747247054275, 35.73881343521246379, - 32.21423852322426029, 37.63427208010365632, 33.66495985223924237, 39.61831917500946787, - 34.96012666362174315, 41.68115224019865650, 36.08195470333167520, 43.81237943576640248, - 37.00911969333093765, 46.00087796034757304, 37.71572773969587189, 48.23458650173873252, - 38.17004600591188535, 50.50021197832979425, 38.33296113360039925, 52.78282385843581892, - 38.15616674923441565, 55.06530104282313687, 37.58017141013023377, 57.32758829402004608, - 36.53242307522427268, 59.54571651024581769, 34.92626834358741661, 61.69055635034530383, - 32.66225471925319823, 63.72633555655123416, 29.63456841105838535, 65.60910965714572285, - 25.74696599568291333, 67.28571148577093197, 20.94303945184943316, 68.69426490502790728, - 15.25145387811901188, 69.76794994271870110, 8.83336207267745088, 70.44359392189112157, - 2.00000000000000622, 73.51255823132638056, -6.80643587234800940, 73.21444539538327945, - -14.87413258170036379, 72.35214877025040892, -21.72462936947633594, 71.00751554077180572, - -27.20553396293055215, 69.28213523998356038, -31.39257932570032139, 67.27265449907143591, - -34.46283411458643542, 65.05952649321666570, -36.61111913565843423, 62.70556553816149403, - -38.01204327449661236, 60.25870541573158334, -38.80901819915838047, 57.75559962131574565, - -39.11518954384546731, 55.22475594928482678, -39.01828312042780311, 52.68892136253921876, - -38.58594887503603132, 50.16679739569313057, -37.87042151784342536, 47.67424596736638875, - -36.91224672041000332, 45.22513042719018728, -35.74315247327934486, 42.83190113242096686, - -34.38823113470312620, 40.50600232791481403, -32.86759479425492714, 38.25815259440047811, - -31.19763777135731431, 36.09853399317181299, -29.39200856069424006, 34.03691345955112979, - -27.46236656888655148, 32.08271233019573287, -25.41897786066869003, 30.24503490367625247, - -23.27118821325056430, 28.53266378578365448, -21.02780000980340347, 26.95402789495273055, - -18.69737092792362887, 25.51714801636046559, -16.28844626261447814, 24.22956442487017625, - -13.80973254683687124, 23.09825113914620331, -11.27021753707410845, 22.12952164603582617, - -8.67924035399200910, 21.32893128539883065, -6.04651538477557082, 20.70118176192879744, - -3.38211423303386649, 20.25003331964319386, -0.69641128035122979, 19.97822987242789239, - 1.99999999999999756, 19.88744176867360025, 4.69641128035122613, 19.97822987242789239, - 7.38211423303386205, 20.25003331964319386, 10.04651538477556727, 20.70118176192879744, - 12.67924035399200378, 21.32893128539883065, 15.27021753707410667, 22.12952164603582617, - 17.80973254683686946, 23.09825113914620331, 20.28844626261447814, 24.22956442487017270, - 22.69737092792362532, 25.51714801636046559, 25.02780000980339992, 26.95402789495272700, - 27.27118821325056786, 28.53266378578365448, 29.41897786066868292, 30.24503490367624536, - 31.46236656888655503, 32.08271233019573287, 33.39200856069423651, 34.03691345955112979, - 35.19763777135730720, 36.09853399317180589, 36.86759479425493424, 38.25815259440047811, - 38.38823113470313331, 40.50600232791481403, 39.74315247327934486, 42.83190113242095975, - 40.91224672041000332, 45.22513042719018728, 41.87042151784342536, 47.67424596736638165, - 42.58594887503603132, 50.16679739569313057, 43.01828312042781022, 52.68892136253921876, - 43.11518954384546731, 55.22475594928482678, 42.80901819915838757, 57.75559962131573144, - 42.01204327449661946, 60.25870541573157624, 40.61111913565844134, 62.70556553816149403, - 38.46283411458643542, 65.05952649321666570, 35.39257932570031784, 67.27265449907143591, - 31.20553396293055926, 69.28213523998356038, 25.72462936947633949, 71.00751554077180572, - 18.87413258170037267, 72.35214877025040892, 10.80643587234802361, 73.21444539538327945, - 2.00000000000000844, 76.47602837656471308, -9.67376613190775814, 76.07990221077976400, - -19.91726594386999949, 74.95625177955801632, -28.02735732207738550, 73.25702648451857613, - -34.02194152863378918, 71.14627956201684356, -38.25028545833758642, 68.75766348420627594, - -41.10709162054251209, 66.18773615332521842, -42.92334661203305757, 63.50367340927638793, - -43.94927569193657035, 60.75248226909388904, -44.36709840828364548, 57.96800639328736793, - -44.30826862884845241, 55.17560336970880286, -43.86791757533286074, 52.39514197266055362, - -43.11549450115907689, 49.64291142290011294, -42.10226597817306526, 46.93284831506371546, - -40.86653894639825779, 44.27733603455973110, -39.43731045663169965, 41.68773239540036002, - -37.83684688715485578, 39.17472027603687934, -36.08253581911941410, 36.74853923728758076, - -34.18824090805679816, 34.41913389699746517, -32.16531367916817885, 32.19624133969532664, - -30.02336509370288198, 30.08943159705432890, -27.77086550998262382, 28.10811023218007776, - -25.41561850057673411, 26.26148912292403992, -22.96513811238918379, 24.55852997196868515, - -20.42694818201796281, 23.00786444863284785, -17.80881477377849720, 21.61769490666709714, - -15.11891780975726896, 20.39568010883170501, -12.36596498789072740, 19.34881113120014717, - -9.55924980839402672, 18.48328342761256593, -6.70865569654629645, 17.80437171431731613, - -3.82460955068694286, 17.31631470128239414, -0.91799021933643277, 17.02221659443388191, - 1.99999999999999756, 16.92397162343524286, 4.91799021933642866, 17.02221659443388191, - 7.82460955068693664, 17.31631470128239414, 10.70865569654629290, 17.80437171431731613, - 13.55924980839402139, 18.48328342761256593, 16.36596498789072740, 19.34881113120014717, - 19.11891780975726363, 20.39568010883170501, 21.80881477377850075, 21.61769490666709714, - 24.42694818201795925, 23.00786444863284430, 26.96513811238918024, 24.55852997196867804, - 29.41561850057672345, 26.26148912292403637, 31.77086550998261671, 28.10811023218007065, - 34.02336509370287132, 30.08943159705432180, 36.16531367916817885, 32.19624133969532664, - 38.18824090805679106, 34.41913389699745807, 40.08253581911941410, 36.74853923728758076, - 41.83684688715484867, 39.17472027603687934, 43.43731045663169255, 41.68773239540036002, - 44.86653894639826490, 44.27733603455973110, 46.10226597817306526, 46.93284831506371546, - 47.11549450115907689, 49.64291142290011294, 47.86791757533286784, 52.39514197266055362, - 48.30826862884845951, 55.17560336970880286, 48.36709840828365259, 57.96800639328736793, - 47.94927569193657746, 60.75248226909387483, 46.92334661203305757, 63.50367340927638793, - 45.10709162054252630, 66.18773615332521842, 42.25028545833760063, 68.75766348420627594, - 38.02194152863379628, 71.14627956201684356, 32.02735732207738550, 73.25702648451857613, - 23.91726594387000659, 74.95625177955801632, 13.67376613190777590, 76.07990221077976400, - 2.00000000000001155, 79.58407807962322522, -14.24038894313819092, 79.03039199474196153, - -27.34759503822197502, 77.51867325981696411, -36.54693285975560713, 75.34972495190746145, - -42.57629367046713753, 72.78039612779681988, -46.37308749001456221, 69.97672254379963874, - -48.64383991815974895, 67.03853932314146391, -49.85773173662808233, 64.02688869849654907, - -50.31863539857851464, 60.98111330635318694, -50.22511448054331140, 57.92830381592441569, - -49.70950811496560107, 54.88845672164920586, -48.86197695521289575, 51.87735585154322138, - -47.74524369263553325, 48.90823812634897649, -46.40377244059971673, 45.99279145534946167, - -44.86961926381133026, 43.14177213432515856, -43.16626004327478228, 40.36539642075791789, - -41.31116510791174079, 37.67359181604234664, -39.31758136998610098, 35.07615645611910082, - -37.19580324741198041, 32.58285444844916867, -34.95410730493897233, 30.20346328869343466, - -32.59946098845734497, 27.94778271439298933, -30.13807554901531915, 25.82561045671564770, - -27.57584735879238025, 23.84668824688734645, -24.91871466466540852, 22.02062053097340311, - -22.17294518233925871, 20.35676830949395466, -19.34536199373425802, 18.86412113035556004, - -16.44350999615492270, 17.55115134744557182, -13.47576217013061850, 16.42565612762181360, - -10.45136393475438830, 15.49459412897927280, -7.38041466390653600, 14.76392503362292707, - -4.27378779825515664, 14.23846093632396936, -1.14299451095196525, 13.92173872937387458, - 1.99999999999999756, 13.81592192037676980, 5.14299451095195970, 13.92173872937387458, - 8.27378779825515132, 14.23846093632396936, 11.38041466390652978, 14.76392503362292707, - 14.45136393475438119, 15.49459412897927280, 17.47576217013061495, 16.42565612762181360, - 20.44350999615491560, 17.55115134744557182, 23.34536199373425447, 18.86412113035556004, - 26.17294518233925515, 20.35676830949395466, 28.91871466466540852, 22.02062053097340311, - 31.57584735879237314, 23.84668824688734645, 34.13807554901531205, 25.82561045671564415, - 36.59946098845733786, 27.94778271439298933, 38.95410730493897233, 30.20346328869342756, - 41.19580324741196620, 32.58285444844916867, 43.31758136998610098, 35.07615645611910082, - 45.31116510791174079, 37.67359181604234664, 47.16626004327477517, 40.36539642075791789, - 48.86961926381131605, 43.14177213432515856, 50.40377244059970963, 45.99279145534945457, - 51.74524369263552614, 48.90823812634896228, 52.86197695521290285, 51.87735585154322138, - 53.70950811496559396, 54.88845672164920586, 54.22511448054331140, 57.92830381592441569, - 54.31863539857851464, 60.98111330635318694, 53.85773173662808233, 64.02688869849654907, - 52.64383991815974895, 67.03853932314146391, 50.37308749001457642, 69.97672254379963874, - 46.57629367046714464, 72.78039612779681988, 40.54693285975561423, 75.34972495190746145, - 31.34759503822198212, 77.51867325981696411, 18.24038894313820691, 79.03039199474196153, - 2.00000000000001998, 82.85826214470431239, -22.58690897303193523, 82.01033616765489853, - -38.96575654499159924, 79.88802174923017674, -48.17735288037921038, 77.11452529323827321, - -53.18593275925196195, 74.03707888623914357, -55.81801935702030448, 70.81486693953937106, - -57.03453256694833584, 67.52503657439315532, -57.35114187047393841, 64.20957103321012482, - -57.05957379451162126, 60.89430451319173443, -56.33378767939932885, 57.59709162031844443, - -55.28230228211860009, 54.33161333463023368, -53.97533567047437231, 51.10929358481432416, - -52.45964870701570248, 47.94033352757975308, -50.76707508825830928, 44.83430401955754974, - -48.91964996438991875, 41.80050199367043007, -46.93282539675249154, 38.84817223653459450, - -44.81756887288366897, 35.98664677769215103, -42.58178882927894193, 33.22542941501406233, - -40.23134383513492196, 30.57423988556289629, -37.77078829775713587, 28.04302503644353806, - -35.20394762734520100, 25.64194033091114022, -32.53437968696509586, 23.38130282889364508, - -29.76575655354334771, 21.27151577227776258, -26.90218550361062810, 19.32296477964254322, - -23.94847769357362566, 17.54588626406437513, -20.91036566952359976, 15.95020993686824795, - -17.79466590782826074, 14.54537904344352306, -14.60937983520193661, 13.34015413091371371, - -11.36372621835939434, 12.34240842556757833, -8.06809948196382720, 11.55892497220584580, - -4.73395228112668587, 10.99520717814031734, -1.37360609917944543, 10.65531494200142504, - 1.99999999999999756, 10.54173785529561336, 5.37360609917944032, 10.65531494200142504, - 8.73395228112667965, 10.99520717814031734, 12.06809948196382187, 11.55892497220584580, - 15.36372621835939078, 12.34240842556757833, 18.60937983520193484, 13.34015413091371371, - 21.79466590782825364, 14.54537904344352306, 24.91036566952359621, 15.95020993686824795, - 27.94847769357362210, 17.54588626406437513, 30.90218550361061745, 19.32296477964254322, - 33.76575655354334060, 21.27151577227776258, 36.53437968696508875, 23.38130282889363798, - 39.20394762734519389, 25.64194033091114022, 41.77078829775712165, 28.04302503644353095, - 44.23134383513491485, 30.57423988556289629, 46.58178882927893483, 33.22542941501406233, - 48.81756887288366187, 35.98664677769215103, 50.93282539675248444, 38.84817223653458029, - 52.91964996438991164, 41.80050199367041586, 54.76707508825830928, 44.83430401955754263, - 56.45964870701569538, 47.94033352757973887, 57.97533567047437231, 51.10929358481431706, - 59.28230228211859298, 54.33161333463023368, 60.33378767939932885, 57.59709162031844443, - 61.05957379451161415, 60.89430451319173443, 61.35114187047394552, 64.20957103321012482, - 61.03453256694833584, 67.52503657439315532, 59.81801935702030448, 70.81486693953937106, - 57.18593275925196906, 74.03707888623914357, 52.17735288037921038, 77.11452529323827321, - 42.96575654499162056, 79.88802174923017674, 26.58690897303196721, 82.01033616765489853, - 2.00000000000004352, 86.32305404620944955, -41.28583165458979920, 84.76914638163376026, - -57.58131858106001744, 81.70467639663135628, -63.56415050352948981, 78.26774990878784877, - -65.82714145548160900, 74.71983727667711150, -66.41516373728856593, 71.13755089586129543, - -66.11115128919476547, 67.55274433031233627, -65.27043361461275595, 63.98289851135272244, - -64.07470870335011170, 60.43992354246546483, -62.62500642980751309, 56.93332642202356197, - -60.98089250042357889, 53.47155025248752480, -59.17889520468775544, 50.06261728049957327, - -57.24193904387428233, 46.71447239476329827, -55.18452370168171939, 43.43518400478754415, - -53.01574257648425004, 40.23307020728445593, -50.74114071516557800, 37.11678138456306897, - -48.36392121341754802, 34.09535373993877982, -45.88577372237665486, 31.17824007995576707, - -43.30747895739605013, 28.37531978094453677, -40.62937881000767959, 25.69688738457191945, - -37.85176517580522670, 23.15361781597290758, -34.97521854780547557, 20.75650546647104377, - -32.00091302933286386, 18.51677422677156670, -28.93089420696503922, 16.44575600838861718, - -25.76832854644280602, 14.55473640862136619, -22.51771689758312078, 12.85476800444173051, - -19.18506021909563941, 11.35645428598744111, -15.77796305903055440, 10.06971032975658353, - -12.30566013367807265, 9.00350968307662569, -8.77895396829167218, 8.16563014493928918, - -5.21005712455211789, 7.56241362477411272, -1.61234064761579554, 7.19855645524003407, - 1.99999999999999734, 7.07694595379053126, 5.61234064761579088, 7.19855645524003407, - 9.21005712455211167, 7.56241362477411272, 12.77895396829166685, 8.16563014493928918, - 16.30566013367807088, 9.00350968307662569, 19.77796305903055085, 10.06971032975658353, - 23.18506021909563231, 11.35645428598744111, 26.51771689758312078, 12.85476800444173051, - 29.76832854644280246, 14.55473640862136619, 32.93089420696504277, 16.44575600838861718, - 36.00091302933286386, 18.51677422677156670, 38.97521854780546846, 20.75650546647104022, - 41.85176517580521960, 23.15361781597290403, 44.62937881000767248, 25.69688738457191590, - 47.30747895739604303, 28.37531978094453322, 49.88577372237665486, 31.17824007995575997, - 52.36392121341754091, 34.09535373993877982, 54.74114071516557800, 37.11678138456306897, - 57.01574257648425004, 40.23307020728444883, 59.18452370168172649, 43.43518400478754415, - 61.24193904387428944, 46.71447239476329827, 63.17889520468774833, 50.06261728049956616, - 64.98089250042356468, 53.47155025248752480, 66.62500642980749888, 56.93332642202356197, - 68.07470870335012592, 60.43992354246546483, 69.27043361461275595, 63.98289851135272244, - 70.11115128919477968, 67.55274433031233627, 70.41516373728855172, 71.13755089586129543, - 69.82714145548160900, 74.71983727667711150, 67.56415050352947560, 78.26774990878784877, - 61.58131858106001744, 81.70467639663135628, 45.28583165458984183, 84.76914638163376026, - -178.00000000002495426, 89.99364961382421768, -86.04670840724655534, 86.14286053253547948, - -83.94730489861873934, 82.29066165213360762, -81.87003129087820241, 78.44836496333867615, - -79.78661098119418682, 74.62102007313750107, -77.68757690134611948, 70.81379178736982283, - -75.56653776770212971, 67.03200075627314902, -73.41791965045068480, 63.28116415502336878, - -71.23639478559491067, 59.56703711552955127, -69.01668877482984499, 55.89565494125256606, - -66.75350545455997064, 52.27337593848611164, -64.44149951448400770, 48.70692450151241104, - -62.07527488350018530, 45.20343384586799118, -59.64940188352277772, 41.77048747717444854, - -57.15845171105518574, 38.41615810166682365, -54.59704918478396252, 35.14904222145087687, - -51.95994579953059400, 31.97828811153562256, -49.24211556220075892, 28.91361425616476666, - -46.43887601207384819, 25.96531465470105005, -43.54603622089217652, 23.14424674184217068, - -40.56007232189229228, 20.46179708442632972, -37.47832910130670570, 17.92981963683196156, - -34.29924330014637945, 15.56054131855884570, -31.02258051120933402, 13.36643021383323848, - -27.64967308093982012, 11.36002298851478542, -24.18364165385173337, 9.55371034930798224, - -20.62957864939022201, 7.95948261839810733, -16.99466904928961242, 6.58864168091602309, - -13.28822357553577405, 5.45149036402020837, -9.52160273595813855, 4.55701513369246669, - -5.70801791517022217, 3.91258202080964690, -1.86220744039441199, 3.52366797872773274, - 1.99999999999999734, 3.39364961385021413, 5.86220744039440689, 3.52366797872773274, - 9.70801791517021861, 3.91258202080964690, 13.52160273595813145, 4.55701513369246669, - 17.28822357553576694, 5.45149036402020837, 20.99466904928960886, 6.58864168091602309, - 24.62957864939021491, 7.95948261839810733, 28.18364165385172981, 9.55371034930798224, - 31.64967308093981302, 11.36002298851478542, 35.02258051120933402, 13.36643021383323848, - 38.29924330014637235, 15.56054131855884570, 41.47832910130669859, 17.92981963683195801, - 44.56007232189227807, 20.46179708442632261, 47.54603622089216231, 23.14424674184217068, - 50.43887601207384819, 25.96531465470104649, 53.24211556220075892, 28.91361425616475955, - 55.95994579953058690, 31.97828811153562256, 58.59704918478396252, 35.14904222145087687, - 61.15845171105517863, 38.41615810166681655, 63.64940188352278483, 41.77048747717444854, - 66.07527488350018530, 45.20343384586799118, 68.44149951448399349, 48.70692450151241104, - 70.75350545455997064, 52.27337593848611164, 73.01668877482984499, 55.89565494125256606, - 75.23639478559491067, 59.56703711552955127, 77.41791965045068480, 63.28116415502336878, - 79.56653776770211550, 67.03200075627314902, 81.68757690134611948, 70.81379178736982283, - 83.78661098119418682, 74.62102007313750107, 85.87003129087820241, 78.44836496333867615, - 87.94730489861875355, 82.29066165213360762, 90.04670840724658376, 86.14286053253547948, - -178.00000000000005684, 86.05996156130565566, -130.58473801020662108, 84.39088722277173815, - -110.19717929614465390, 81.10053146183230410, -100.10346048738929881, 77.40995108179684792, - -93.69886509454548218, 73.59868574092435267, -88.93011750021230455, 69.74815881341463353, - -85.00534896567366161, 65.89187266757556927, -81.55981529780905248, 62.04795929474798299, - -78.40205491898922219, 58.22861013398448193, -75.42099130058548440, 54.44347433172315220, - -72.54676554128549526, 50.70110112272598712, -69.73234087330638431, 47.00963897352001908, - -66.94410839385282941, 43.37721635788606989, -64.15676435641834985, 39.81217237539897269, - -61.35037195723166548, 36.32321005821970061, -58.50861449014277582, 32.91950572923986584, - -55.61773745881250619, 29.61078975921034129, -52.66591251909912330, 26.40740499964346810, - -49.64287529119710030, 23.32034407821754840, -46.53975201683548590, 20.36126354510683711, - -43.34902402209520034, 17.54247066454981407, -40.06459700293881809, 14.87687712846902599, - -36.68195074563113423, 12.37791309905906445, -33.19834740797973893, 10.05939491571369260, - -29.61307490596015057, 7.93534078498541273, -25.92769777693465372, 6.01973109672258389, - -22.14628266758476371, 4.32621388981253130, -18.27556116130716646, 2.86776143955401652, - -14.32499114491301384, 1.65629062554436479, -10.30668142622437244, 0.70226688878528098, - -6.23515434659414680, 0.01431798130227213, -2.12693783758673716, -0.40111214042971233, - 1.99999999999999689, -0.54003843869414525, 6.12693783758673050, -0.40111214042971233, - 10.23515434659414147, 0.01431798130227213, 14.30668142622436712, 0.70226688878528098, - 18.32499114491300318, 1.65629062554436479, 22.27556116130715580, 2.86776143955401652, - 26.14628266758476016, 4.32621388981253130, 29.92769777693464306, 6.01973109672258389, - 33.61307490596014702, 7.93534078498541273, 37.19834740797973183, 10.05939491571369260, - 40.68195074563112001, 12.37791309905905912, 44.06459700293881809, 14.87687712846902421, - 47.34902402209520034, 17.54247066454981052, 50.53975201683548590, 20.36126354510683711, - 53.64287529119709319, 23.32034407821754485, 56.66591251909911620, 26.40740499964346455, - 59.61773745881250619, 29.61078975921033418, 62.50861449014276872, 32.91950572923986584, - 65.35037195723165837, 36.32321005821969351, 68.15676435641833564, 39.81217237539895848, - 70.94410839385281520, 43.37721635788606989, 73.73234087330637010, 47.00963897352000487, - 76.54676554128549526, 50.70110112272598002, 79.42099130058548440, 54.44347433172315220, - 82.40205491898922219, 58.22861013398448193, 85.55981529780903827, 62.04795929474798299, - 89.00534896567364740, 65.89187266757556927, 92.93011750021230455, 69.74815881341463353, - 97.69886509454546797, 73.59868574092435267, 104.10346048738931302, 77.40995108179684792, - 114.19717929614462548, 81.10053146183230410, 134.58473801020659266, 84.39088722277173815, - -178.00000000000002842, 81.83938189765240168, -149.15668933029422760, 80.85560324742941418, - -128.64763601333984866, 78.39753950634394641, -115.34882263320223217, 75.18923730101467129, - -106.23509874248557594, 71.62987479878029262, -99.45761459629794388, 67.90061168070624831, - -94.04421718085441739, 64.08831694761892095, -89.47013935832345055, 60.23965935256792648, - -85.43460651588344490, 56.38292751246400769, -81.75424365182693975, 52.53738247574246856, - -78.31071017117777444, 48.71761719224470966, -75.02360369831293951, 44.93576201889581512, - -71.83567948061782715, 41.20268880474466755, -68.70439458886511375, 37.52871692222674938, - -65.59687011372152199, 33.92405612452550656, -62.48679120890344052, 30.39910192691253599, - -59.35245736607464551, 26.96464262473804041, -56.17554747078253996, 23.63200834498021408, - -52.94035098918124760, 20.41317688045037926, -49.63331929525543984, 17.32084182287969298, - -46.24284889074778704, 14.36844259642658983, -42.75924066894448572, 11.57015196980180605, - -39.17479644658766347, 8.94081392759362537, -35.48402100288603123, 6.49582334126627892, - -31.68389780238809195, 4.25093893994812166, -27.77420154083882053, 2.22202305439570358, - -23.75780275269854158, 0.42470591907018640, -19.64091162595336826, -1.12602081331554005, - -15.43320327160518524, -2.41626750988024330, -11.14776873556009384, -3.43374257581262743, - -6.80084816537008940, -4.16824654595516098, -2.41132592404073165, -4.61212239418478553, - 1.99999999999999689, -4.76061810234754024, 6.41132592404072632, -4.61212239418478553, - 10.80084816537008230, -4.16824654595516098, 15.14776873556009029, -3.43374257581262743, - 19.43320327160517991, -2.41626750988024330, 23.64091162595336115, -1.12602081331554005, - 27.75780275269853803, 0.42470591907018640, 31.77420154083881698, 2.22202305439570358, - 35.68389780238808129, 4.25093893994812166, 39.48402100288601702, 6.49582334126627270, - 43.17479644658765636, 8.94081392759361826, 46.75924066894447861, 11.57015196980179894, - 50.24284889074777283, 14.36844259642658628, 53.63331929525543273, 17.32084182287968943, - 56.94035098918124049, 20.41317688045037571, 60.17554747078253286, 23.63200834498020697, - 63.35245736607463840, 26.96464262473803331, 66.48679120890342631, 30.39910192691252178, - 69.59687011372152199, 33.92405612452549946, 72.70439458886511375, 37.52871692222674227, - 75.83567948061782715, 41.20268880474466755, 79.02360369831293951, 44.93576201889580091, - 82.31071017117776023, 48.71761719224470255, 85.75424365182693975, 52.53738247574246145, - 89.43460651588344490, 56.38292751246400769, 93.47013935832345055, 60.23965935256792648, - 98.04421718085441739, 64.08831694761892095, 103.45761459629794388, 67.90061168070624831, - 110.23509874248557594, 71.62987479878029262, 119.34882263320223217, 75.18923730101467129, - 132.64763601333982024, 78.39753950634394641, 153.15668933029419918, 80.85560324742941418, - -178.00000000000002842, 77.29014606148251687, -157.47781378002883912, 76.59572545663588983, - -140.18467631053607647, 74.70285264202438213, -126.87442892417496410, 71.99165187875776439, - -116.75095597801099245, 68.78228497629521598, -108.83745587761724494, 65.27873175231235336, - -102.40541928769302160, 61.60190913800556700, -96.97173315822051620, 57.82466562901673512, - -92.22157019495104180, 53.99321139219064491, -87.94629406419838347, 50.13884820321026581, - -84.00357763838492531, 46.28434439562811065, -80.29307189810650414, 42.44749809608728697, - -76.74158359961857911, 38.64321178012773572, -73.29390254081907585, 34.88475663106463287, - -69.90700610430916129, 31.18458087158649406, -66.54632400302145356, 27.55485191826680591, - -63.18329478816111333, 24.00783659504740086, -59.79375952890369916, 20.55617726580612015, - -56.35691981311008192, 17.21309536720795919, -52.85469423510679832, 13.99253787292509621, - -49.27137129744098587, 10.90927185095380558, -45.59349430020986915, 7.97892504367260802, - -41.80993456941939712, 5.21796518077428839, -37.91211819209929246, 2.64360721178194558, - -33.89437094593881739, 0.27363605581641759, -29.75433817509684786, -1.87386656696368226, - -25.49342310484703589, -3.78089833576525525, -21.11717183125961128, -5.43001433583734983, - -16.63552097548324937, -6.80489321395614066, -12.06282113870962114, -7.89096231408011839, - -7.41756212661334047, -8.67603648837036268, -2.72175805422022021, -9.15091230991673399, - 1.99999999999999645, -9.30985393851741172, 6.72175805422021444, -9.15091230991673399, - 11.41756212661333691, -8.67603648837036268, 16.06282113870961581, -7.89096231408011839, - 20.63552097548324582, -6.80489321395614066, 25.11717183125960418, -5.43001433583734983, - 29.49342310484702523, -3.78089833576525525, 33.75433817509684786, -1.87386656696368226, - 37.89437094593881028, 0.27363605581641121, 41.91211819209927114, 2.64360721178193936, - 45.80993456941938291, 5.21796518077428217, 49.59349430020984784, 7.97892504367260180, - 53.27137129744098587, 10.90927185095380203, 56.85469423510678411, 13.99253787292509266, - 60.35691981311008192, 17.21309536720795563, 63.79375952890369206, 20.55617726580611659, - 67.18329478816110623, 24.00783659504739376, 70.54632400302143935, 27.55485191826679880, - 73.90700610430916129, 31.18458087158649050, 77.29390254081907585, 34.88475663106463287, - 80.74158359961856490, 38.64321178012773572, 84.29307189810650414, 42.44749809608728697, - 88.00357763838491110, 46.28434439562810354, 91.94629406419836926, 50.13884820321025160, - 96.22157019495104180, 53.99321139219064491, 100.97173315822050199, 57.82466562901673512, - 106.40541928769302160, 61.60190913800556700, 112.83745587761723073, 65.27873175231235336, - 120.75095597801100666, 68.78228497629521598, 130.87442892417493567, 71.99165187875776439, - 144.18467631053604805, 74.70285264202438213, 161.47781378002881070, 76.59572545663588983, - -178.00000000000002842, 72.36459406509962378, -162.05644118521095720, 71.82662636485281382, - -147.60742176334812825, 70.30250402254888797, -135.36255278451204731, 68.00113945880528377, - -125.26976638159491984, 65.14478524125108549, -116.94158660124730886, 61.91142197433249095, - -109.95470750710923369, 58.42738157410047961, -103.95825647893495614, 54.77900423463843538, - -98.68725830824647005, 51.02562354013957702, -93.94726867835278483, 47.20919167069192923, - -89.59567144235911940, 43.36062114047545890, -85.52649044264862255, 39.50382677214758331, - -81.65938939021744147, 35.65830698143167155, -77.93201512996090230, 31.84082402428231973, - -74.29472888122153051, 28.06652943785655552, -70.70698212428176532, 24.34974403770184992, - -67.13481958525987636, 20.70451849248450316, -63.54916484650964748, 17.14505004500528074, - -59.92466443809041010, 13.68599977255222377, -56.23894695811429756, 10.34273467530951152, - -52.47220691225792422, 7.13150506907927006, -48.60705694133756083, 4.06955754229704603, - -44.62861191461713162, 1.17517574056815666, -40.52477649322946007, -1.53236497867592147, - -36.28670514160292981, -4.03294710811187507, -31.90939060170597941, -6.30588392466837533, - -27.39231472226267528, -8.33033151194295662, -22.74006765897273397, -10.08584610023313921, - -17.96281473836326725, -11.55307649365679090, -13.07647532339788654, -12.71455858250937254, - -8.10248720394762678, -13.55555303790299426, -3.06707279862612969, -14.06484423228424063, - 1.99999999999999600, -14.23540593490034212, 7.06707279862612125, -14.06484423228424063, - 12.10248720394761790, -13.55555303790299426, 17.07647532339788299, -12.71455858250937254, - 21.96281473836326015, -11.55307649365679090, 26.74006765897273041, -10.08584610023313921, - 31.39231472226266817, -8.33033151194295662, 35.90939060170597230, -6.30588392466837533, - 40.28670514160292271, -4.03294710811188128, 44.52477649322944586, -1.53236497867592791, - 48.62861191461712451, 1.17517574056815022, 52.60705694133754662, 4.06955754229703981, - 56.47220691225791711, 7.13150506907926740, 60.23894695811429756, 10.34273467530950796, - 63.92466443809040300, 13.68599977255221667, 67.54916484650964037, 17.14505004500527363, - 71.13481958525987636, 20.70451849248449605, 74.70698212428177953, 24.34974403770184992, - 78.29472888122153051, 28.06652943785655197, 81.93201512996088809, 31.84082402428231262, - 85.65938939021744147, 35.65830698143166444, 89.52649044264860834, 39.50382677214758331, - 93.59567144235911940, 43.36062114047545180, 97.94726867835277062, 47.20919167069192213, - 102.68725830824645584, 51.02562354013956281, 107.95825647893494192, 54.77900423463843538, - 113.95470750710921948, 58.42738157410047961, 120.94158660124729465, 61.91142197433249095, - 129.26976638159487720, 65.14478524125108549, 139.36255278451201889, 68.00113945880528377, - 151.60742176334809983, 70.30250402254888797, 166.05644118521092878, 71.82662636485281382, - -178.00000000000002842, 67.00865712643569339, -164.95645291666266985, 66.56908368553206401, - -152.69437932769912436, 65.29863962109095610, -141.70578215046154469, 63.31961305621509695, - -132.12712761110685733, 60.78156430751474204, -123.85428534979440940, 57.82417470821774685, - -116.68240638414135901, 54.56141374911145192, -110.39390839554111778, 51.08045112569507751, - -104.79614728017998004, 47.44644408577498496, -99.73088255981890882, 43.70821480745370025, - -95.07197748871266185, 39.90302877816822757, -90.71961138434764393, 36.06017287638244539, - -86.59435351833845118, 32.20351875194761959, -82.63216392910496211, 28.35333972343855535, - -78.78048521884275601, 24.52760876444716587, -74.99529352424694650, 20.74294398926493699, - -71.23891693218340038, 17.01531594444593765, -67.47845003929684538, 13.36059248585088355, - -63.68463403089357655, 9.79496963802121101, -59.83111160507219495, 6.33531713186757539, - -55.89399913408760057, 2.99945241539763696, -51.85174332801064168, -0.19365520038940248, - -47.68524592294902931, -3.22375917751486485, -43.37824652323212860, -6.06930752874206636, - -38.91794868987490474, -8.70753794849322027, -34.29585521437643081, -11.11472732454917001, - -29.50874372060167872, -13.26662598462659126, -24.55966518490711081, -15.13909773195326558, - -19.45879403720245193, -16.70896594558295334, -14.22391622121717880, -17.95503295463868554, - -8.88033504279663255, -18.85919797266866027, -3.46002704385166027, -19.40755726556069405, - 1.99999999999999645, -19.59134287356426540, 7.46002704385165316, -19.40755726556069405, - 12.88033504279662367, -18.85919797266866027, 18.22391622121717347, -17.95503295463868554, - 23.45879403720244127, -16.70896594558295334, 28.55966518490710015, -15.13909773195326558, - 33.50874372060167161, -13.26662598462659126, 38.29585521437642370, -11.11472732454917534, - 42.91794868987490474, -8.70753794849322738, 47.37824652323212149, -6.06930752874207258, - 51.68524592294901510, -3.22375917751487107, 55.85174332801063457, -0.19365520038940884, - 59.89399913408759346, 2.99945241539763385, 63.83111160507218784, 6.33531713186756917, - 67.68463403089357655, 9.79496963802120568, 71.47845003929684538, 13.36059248585087822, - 75.23891693218338617, 17.01531594444593054, 78.99529352424693229, 20.74294398926492988, - 82.78048521884274180, 24.52760876444715521, 86.63216392910494790, 28.35333972343854825, - 90.59435351833845118, 32.20351875194761959, 94.71961138434764393, 36.06017287638243118, - 99.07197748871266185, 39.90302877816822047, 103.73088255981890882, 43.70821480745368603, - 108.79614728017998004, 47.44644408577497074, 114.39390839554110357, 51.08045112569504909, - 120.68240638414135901, 54.56141374911145192, 127.85428534979438098, 57.82417470821774685, - 136.12712761110682891, 60.78156430751474204, 145.70578215046154469, 63.31961305621509695, - 156.69437932769909594, 65.29863962109095610, 168.95645291666264143, 66.56908368553206401, - -178.00000000000002842, 61.16162880179430772, -166.97884079028878546, 60.79043532425956897, - -156.40162010671977555, 59.70518607311999659, -146.59283359977425221, 57.98141283097307053, - -137.70772458687926587, 55.72003625021811501, -129.75638606304184464, 53.02581189527144545, - -122.65952710171549711, 49.99368843662635697, -116.29889149413011751, 46.70342202473146642, - -110.54933935400606515, 43.21943287866469774, -105.29473060869931089, 39.59298941537671368, - -100.43366587620559471, 35.86497163386139420, -95.88003723653146437, 32.06843215731343832, - -91.56137924921046078, 28.23071203179445376, -87.41655400532016529, 24.37510563102819461, - -83.39346098727328638, 20.52214967325533124, -79.44703540885105042, 16.69062430422997423, - -75.53760403336077900, 12.89834287286322301, -71.62959031828354739, 9.16278970353470967, - -67.69053915995506543, 5.50164833296652400, -63.69043463465567356, 1.93324784086672841, - -59.60129716846515180, -1.52305829704237294, -55.39706222812606740, -4.84657887125734721, - -51.05375641269629483, -8.01504773475606314, -46.54999441522540593, -11.00448645615599474, - -41.86781613095839560, -13.78920372951384543, -36.99386004100858116, -16.34197985921976581, - -31.92081946357624034, -18.63449057116782015, -26.64904852681421588, -20.63801909702448611, - -21.18808138888132930, -22.32448212182582026, -15.55772615655294366, -23.66774871950555692, - -9.78834011215971245, -24.64516330804185174, -3.91994134356475854, -25.23910692117754806, - 1.99999999999999556, -25.43837119820565107, 7.91994134356474966, -25.23910692117754806, - 13.78834011215970534, -24.64516330804185174, 19.55772615655293123, -23.66774871950555692, - 25.18808138888131865, -22.32448212182582026, 30.64904852681421232, -20.63801909702448611, - 35.92081946357622968, -18.63449057116782015, 40.99386004100855985, -16.34197985921976581, - 45.86781613095838850, -13.78920372951385076, 50.54999441522539882, -11.00448645615600007, - 55.05375641269628773, -8.01504773475606846, 59.39706222812605318, -4.84657887125735343, - 63.60129716846514469, -1.52305829704237916, 67.69043463465565935, 1.93324784086672197, - 71.69053915995505122, 5.50164833296651778, 75.62959031828353318, 9.16278970353470434, - 79.53760403336076479, 12.89834287286321590, 83.44703540885105042, 16.69062430422996712, - 87.39346098727328638, 20.52214967325532413, 91.41655400532015108, 24.37510563102818750, - 95.56137924921046078, 28.23071203179444666, 99.88003723653145016, 32.06843215731343832, - 104.43366587620558050, 35.86497163386137288, 109.29473060869932510, 39.59298941537669947, - 114.54933935400606515, 43.21943287866469774, 120.29889149413010330, 46.70342202473146642, - 126.65952710171551132, 49.99368843662635697, 133.75638606304187306, 53.02581189527144545, - 141.70772458687926587, 55.72003625021811501, 150.59283359977422379, 57.98141283097307053, - 160.40162010671977555, 59.70518607311999659, 170.97884079028878546, 60.79043532425956897, - -178.00000000000002842, 54.75649468439711143, -168.49285933765114009, 54.43639475342744305, - -159.25249258416440057, 53.49372597518600259, -150.49218436749950456, 51.97706503648645082, - -142.34201331309026273, 49.95526128262039123, -134.84908760812987794, 47.50530446205828383, - -127.99715883033752561, 44.70273256173389598, -121.73118977168118704, 41.61600262291977259, - -115.97779813807071037, 38.30427835916992052, -110.65884380836621403, 34.81740352705595143, - -105.69899804029306267, 31.19696754875727152, -101.02917319698202903, 27.47775433026597725, - -96.58749981932022877, 23.68920100972959730, -92.31902021078286680, 19.85670886149115688, - -88.17480572169651509, 16.00276430383187076, -84.11088800598805904, 12.14788119395578825, - -80.08720437104130951, 8.31139512800513991, -76.06665407661381550, 4.51214364124032752, - -72.01431282527251199, 0.76906207426875217, -67.89683474738467339, -2.89828246890512542, - -63.68207179147747610, -6.46920580966549608, -59.33895182482493880, -9.92146778444055499, - -54.83767345094592827, -13.23089104518994574, -50.15029168417838434, -16.37106576006626568, - -45.25177543408179304, -19.31318879069117145, -40.12160159745184274, -22.02610835172851722, - -34.74589265732009835, -24.47666503668304117, -29.11998552133064422, -26.63042818683534207, - -23.25113130544623985, -28.45290874783366064, -17.16079514785620930, -29.91127029421519978, - -10.88583393507997776, -30.97645080660139172, -4.47781225763253765, -31.62546289221583962, - 1.99999999999999534, -31.84350531560284736, 8.47781225763252877, -31.62546289221583962, - 14.88583393507996711, -30.97645080660139172, 21.16079514785619509, -29.91127029421519978, - 27.25113130544623274, -28.45290874783366064, 33.11998552133064067, -26.63042818683534207, - 38.74589265732009835, -24.47666503668304117, 44.12160159745183563, -22.02610835172851722, - 49.25177543408178593, -19.31318879069118211, 54.15029168417837724, -16.37106576006627279, - 58.83767345094592827, -13.23089104518995285, 63.33895182482493169, -9.92146778444056032, - 67.68207179147748320, -6.46920580966550229, 71.89683474738467339, -2.89828246890513164, - 76.01431282527251199, 0.76906207426874584, 80.06665407661380129, 4.51214364124032308, - 84.08720437104132372, 8.31139512800513458, 88.11088800598805904, 12.14788119395578470, - 92.17480572169650088, 16.00276430383186366, 96.31902021078285259, 19.85670886149114978, - 100.58749981932022877, 23.68920100972959020, 105.02917319698201482, 27.47775433026597014, - 109.69899804029304846, 31.19696754875725375, 114.65884380836618561, 34.81740352705593722, - 119.97779813807069615, 38.30427835916992052, 125.73118977168117283, 41.61600262291977259, - 131.99715883033749719, 44.70273256173389598, 138.84908760812987794, 47.50530446205828383, - 146.34201331309026273, 49.95526128262039123, 154.49218436749950456, 51.97706503648645082, - 163.25249258416440057, 53.49372597518600259, 172.49285933765111167, 54.43639475342744305, - -178.00000000000002842, 47.72124472298390430, -169.69217969866491558, 47.44157216781381692, - -161.55074086228117380, 46.61395503653132266, -153.71667350566602295, 45.27051130755705799, - -146.28885177460344380, 43.45866798082070659, -139.31958252044236701, 41.23439386179803279, - -132.82020109922686402, 38.65601141858345358, -126.77168357131760956, 35.77969749489390949, - -121.13576891124588997, 32.65687547770472321, -115.86406835852179142, 29.33315055785289260, - -110.90442707083994378, 25.84827445710250160, -106.20481280215612685, 22.23668760419472790, - -101.71535755141638901, 18.52832281285149207, -97.38917071735673403, 14.74948212031801020, - -93.18240447461121789, 10.92369066480776674, -89.05390189983984328, 7.07248905242842607, - -84.96463936229969249, 3.21615784383777559, -80.87709425893895343, -0.62561625366165152, - -76.75462189677602964, -4.43311662588922051, -72.56090370117536281, -8.18599910867232161, - -68.25952713288315010, -11.86270778471483389, -63.81377209114523907, -15.43989911427394368, - -59.18670666190919150, -18.89188184637555068, -54.34173321358174746, -22.19009961204913850, - -49.24376548995358149, -25.30271165018387691, -43.86123932751760890, -28.19436585459229860, - -38.16912769952225659, -30.82630445836841915, -32.15298964123236658, -33.15698491696964112, - -25.81377244810833815, -35.14341202945125531, -19.17259140539265516, -36.74332279213026453, - -12.27414365511957151, -37.91820250638207312, -5.18708125479775184, -38.63683300066042392, - 1.99999999999999489, -38.87875527701606160, 9.18708125479774296, -38.63683300066042392, - 16.27414365511955907, -37.91820250638207312, 23.17259140539264095, -36.74332279213026453, - 29.81377244810832394, -35.14341202945125531, 36.15298964123235947, -33.15698491696964112, - 42.16912769952225659, -30.82630445836841915, 47.86123932751760179, -28.19436585459229860, - 53.24376548995357439, -25.30271165018388402, 58.34173321358174746, -22.19009961204914205, - 63.18670666190918439, -18.89188184637555779, 67.81377209114522486, -15.43989911427395079, - 72.25952713288313589, -11.86270778471484277, 76.56090370117536281, -8.18599910867232872, - 80.75462189677602964, -4.43311662588922673, 84.87709425893895343, -0.62561625366165630, - 88.96463936229967828, 3.21615784383776981, 93.05390189983984328, 7.07248905242842163, - 97.18240447461121789, 10.92369066480776141, 101.38917071735671982, 14.74948212031800310, - 105.71535755141637480, 18.52832281285148497, 110.20481280215611264, 22.23668760419472079, - 114.90442707083992957, 25.84827445710249449, 119.86406835852177721, 29.33315055785288550, - 125.13576891124587576, 32.65687547770470900, 130.77168357131759535, 35.77969749489390949, - 136.82020109922683559, 38.65601141858345358, 143.31958252044236701, 41.23439386179803279, - 150.28885177460347222, 43.45866798082070659, 157.71667350566602295, 45.27051130755705799, - 165.55074086228114538, 46.61395503653132266, 173.69217969866488716, 47.44157216781381692, - -178.00000000000002842, 39.98177927524727693, -170.20476659278793363, 39.70189618399165710, - -162.53782868216148927, 38.87197455633481979, -155.11038900941949237, 37.51968052984662449, - -148.00450116657361832, 35.68659928106227852, -141.26850797118672176, 33.42299645292870736, - -134.91921485966395267, 30.78272820361293327, -128.94811747553873715, 27.81921722445741452, - -123.32885205682140395, 24.58281725765701253, -118.02393786261990272, 21.11944988981388605, - -112.98994385952538266, 17.47019805768939449, -108.18096363521175363, 13.67152082594358653, - -103.55065416712271542, 9.75582109859903390, -99.05319914949195947, 5.75218534812640492, - -94.64352777118511995, 1.68719008387413827, -90.27704477012913742, -2.41427530523583656, - -85.90905356028720519, -6.52818370181667529, -81.49400025392820623, -10.63053585607600837, - -76.98463901984250413, -14.69649678201448140, -72.33122188826867216, -18.69947873486281864, - -67.48085419487966874, -22.61014977269736193, -62.37723925505212463, -26.39536140379772533, - -56.96117255247644096, -30.01702215459737033, -51.17233704527057370, -33.43100825355479344, - -44.95316063861177014, -36.58631299796492442, -38.25559890256131723, -39.42480136235259636, - -31.05142021090808413, -41.88213387768102791, -23.34546416576216643, -43.89055765093644368, - -15.18911080317680451, -45.38412606933154336, -6.68839576675081293, -46.30624543028365991, - 1.99999999999999445, -46.61822072475268186, 10.68839576675079961, -46.30624543028365991, - 19.18911080317679918, -45.38412606933154336, 27.34546416576215222, -43.89055765093644368, - 35.05142021090807702, -41.88213387768102791, 42.25559890256131013, -39.42480136235259636, - 48.95316063861175593, -36.58631299796492442, 55.17233704527057370, -33.43100825355479344, - 60.96117255247643385, -30.01702215459737744, 66.37723925505210332, -26.39536140379773244, - 71.48085419487965453, -22.61014977269736548, 76.33122188826865795, -18.69947873486282575, - 80.98463901984250413, -14.69649678201448850, 85.49400025392819202, -10.63053585607601548, - 89.90905356028720519, -6.52818370181668062, 94.27704477012913742, -2.41427530523584188, - 98.64352777118510573, 1.68719008387413361, 103.05319914949195947, 5.75218534812639781, - 107.55065416712271542, 9.75582109859902680, 112.18096363521173942, 13.67152082594358120, - 116.98994385952536845, 17.47019805768938738, 122.02393786261990272, 21.11944988981387894, - 127.32885205682140395, 24.58281725765700187, 132.94811747553873715, 27.81921722445741452, - 138.91921485966392424, 30.78272820361293327, 145.26850797118672176, 33.42299645292870736, - 152.00450116657361832, 35.68659928106227852, 159.11038900941949237, 37.51968052984662449, - 166.53782868216146085, 38.87197455633481979, 174.20476659278793363, 39.70189618399165710, - -178.00000000000002842, 31.46720374958098887, -171.12910064316457692, 31.22051723140710067, - -164.34108626834762390, 30.48695101230147841, -157.71017626763574526, 29.28521267182222587, - -151.29523174265207786, 27.64411687523926986, -145.13620894054648147, 25.59967735271197498, - -139.25381666270027381, 23.19203115841211726, -133.65157520973266969, 20.46271167815037728, - -128.31914580979304219, 17.45256126555711873, -123.23592626204657563, 14.20035582641498806, - -118.37425008640165913, 10.74206713847307348, -113.70187269530693186, 7.11062322287716331, - -109.18367588372132104, 3.33602075627005945, -104.78265670493266271, -0.55433148808729127, - -100.46031614728222792, -4.53512373056416163, -96.17656220612794016, -8.58256192749418290, - -91.88921878897831164, -12.67371220268469223, -87.55320457280681978, -16.78577161643179849, - -83.11942634360052296, -20.89524495725206421, -78.53343020155088539, -24.97699141478169338, - -73.73388667252179118, -29.00309226896919412, -68.65107920217621995, -32.94148403257255353, - -63.20576633421556068, -36.75430974219704439, -57.30916699148811233, -40.39598469592144880, - -50.86545369174167774, -43.81109166541019562, -43.77902429867569367, -46.93247902115529513, - -35.96960367385943158, -49.68040408706252720, -27.39769711559288723, -51.96422430511800172, - -18.09872704828334378, -53.68864395349498864, -8.21451927262478065, -54.76594013073759015, - 1.99999999999999289, -55.13279625041898413, 12.21451927262476822, -54.76594013073759015, - 22.09872704828332601, -53.68864395349498864, 31.39769711559288012, -51.96422430511800172, - 39.96960367385942448, -49.68040408706252720, 47.77902429867568657, -46.93247902115529513, - 54.86545369174167064, -43.81109166541019562, 61.30916699148811233, -40.39598469592146301, - 67.20576633421555357, -36.75430974219705149, 72.65107920217620574, -32.94148403257256064, - 77.73388667252179118, -29.00309226896920123, 82.53343020155088539, -24.97699141478170048, - 87.11942634360052296, -20.89524495725207132, 91.55320457280680557, -16.78577161643180560, - 95.88921878897831164, -12.67371220268469756, 100.17656220612792595, -8.58256192749419000, - 104.46031614728221371, -4.53512373056416607, 108.78265670493264849, -0.55433148808729760, - 113.18367588372132104, 3.33602075627005323, 117.70187269530691765, 7.11062322287715620, - 122.37425008640165913, 10.74206713847306816, 127.23592626204656142, 14.20035582641498095, - 132.31914580979304219, 17.45256126555711162, 137.65157520973264127, 20.46271167815036662, - 143.25381666270027381, 23.19203115841211726, 149.13620894054645305, 25.59967735271197498, - 155.29523174265207786, 27.64411687523926986, 161.71017626763577368, 29.28521267182222587, - 168.34108626834759548, 30.48695101230147841, 175.12910064316457692, 31.22051723140710067, - -178.00000000000002842, 22.11837781753018817, -170.46891148411387462, 21.78042227826199095, - -163.04151372648448159, 20.77699151155475477, -155.80845268931921055, 19.13794220572576776, - -148.83768595785252842, 16.90865838596958781, -142.17005258660481104, 14.14485028302004999, - -135.81978239976487544, 10.90731557455581147, -129.77826339847271697, 7.25759760165333567, - -124.01906809075352101, 3.25496397729521814, -118.50268451884778642, -1.04529436283411115, - -113.18006743498142441, -5.59247248797413210, -107.99465974304708027, -10.34025377357217934, - -102.88281945262124850, -15.24594064653213898, -97.77265218881962028, -20.26925377022462982, - -92.58116190590757810, -25.37072818781790673, -87.20945067766288616, -30.50961194071436111, - -81.53547443418635510, -35.64102404029979709, -75.40370496549049051, -40.71192873614232610, - -68.61131289029849256, -45.65520975111465418, -60.89227178252910733, -50.38082342875217989, - -51.90710322947828104, -54.76300160634039571, -41.26325159838054901, -58.62403790832019723, - -28.62428016143747200, -61.72174914658255318, -13.98037118830461445, -63.76341508305664263, - 1.99999999999999090, -64.48162218246977773, 17.98037118830459491, -63.76341508305664263, - 32.62428016143745424, -61.72174914658255318, 45.26325159838053480, -58.62403790832019723, - 55.90710322947827393, -54.76300160634039571, 64.89227178252910733, -50.38082342875217989, - 72.61131289029847835, -45.65520975111467550, 79.40370496549049051, -40.71192873614232610, - 85.53547443418635510, -35.64102404029981841, 91.20945067766287195, -30.50961194071436822, - 96.58116190590757810, -25.37072818781791383, 101.77265218881962028, -20.26925377022463692, - 106.88281945262124850, -15.24594064653214609, 111.99465974304708027, -10.34025377357218289, - 117.18006743498143862, -5.59247248797413921, 122.50268451884780063, -1.04529436283411759, - 128.01906809075350679, 3.25496397729521192, 133.77826339847268855, 7.25759760165332857, - 139.81978239976487544, 10.90731557455580436, 146.17005258660478262, 14.14485028302004999, - 152.83768595785250000, 16.90865838596958781, 159.80845268931921055, 19.13794220572576776, - 167.04151372648448159, 20.77699151155475477, 174.46891148411387462, 21.78042227826199095, - -178.00000000000002842, 11.90033356154828503, -171.03550378237531504, 11.56695807154089728, - -164.15259832130166728, 10.57562338306961180, -157.42382961455143686, 8.95164656051215069, - -150.90567761562240889, 6.73398631258200187, -144.63474532961075170, 3.97127289925851068, - -138.62718020698264354, 0.71762564121834338, -132.88045216690298389, -2.97103149043460757, - -127.37626453526326031, -7.03975713563777195, -122.08348616854385682, -11.43638098615609344, - -116.96030366998827787, -16.11238508810905401, -111.95506961066591600, -21.02304593053227677, - -107.00543419047215821, -26.12699149209596428, -102.03522828445495918, -31.38525381109429446, - -96.94814060146437384, -36.75977477892173084, -91.61632617812561818, -42.21112686087874266, - -85.86029859026153588, -47.69485768882572785, -79.41299264536502278, -53.15514845029933610, - -71.85455169648275842, -58.51289100414830102, -62.49613000707146426, -63.64173578517077345, - -50.20367738593296281, -68.31866526945105988, -33.33636828365784055, -72.13161644510194037, - -10.78607194201629582, -74.39304559691805707, 14.78607194201627273, -74.39304559691805707, - 37.33636828365781213, -72.13161644510194037, 54.20367738593294149, -68.31866526945105988, - 66.49613000707145716, -63.64173578517077345, 75.85455169648274421, -58.51289100414830102, - 83.41299264536502278, -53.15514845029933610, 89.86029859026150746, -47.69485768882574916, - 95.61632617812561818, -42.21112686087874266, 100.94814060146437384, -36.75977477892173084, - 106.03522828445494497, -31.38525381109429446, 111.00543419047215821, -26.12699149209597138, - 115.95506961066591600, -21.02304593053228388, 120.96030366998827787, -16.11238508810905756, - 126.08348616854387103, -11.43638098615609699, 131.37626453526326031, -7.03975713563777816, - 136.88045216690298389, -2.97103149043461379, 142.62718020698264354, 0.71762564121833694, - 148.63474532961075170, 3.97127289925851068, 154.90567761562243732, 6.73398631258200187, - 161.42382961455140844, 8.95164656051215069, 168.15259832130169571, 10.57562338306961180, - 175.03550378237531504, 11.56695807154089728, -178.00000000000002842, 0.81833312891115306, - -171.37485237931247184, 0.46152536384328174, -164.82591385569978115, -0.60006016949342578, - -158.42170502724641779, -2.34091526057736310, -152.21700460768090579, -4.72161659803607048, - -146.24942513902283281, -7.69266569608215889, -140.53869867413175143, -11.19861017149900739, - -135.08801440266989857, -15.18177122287804615, -129.88641650799127092, -19.58517424494497305, - -124.91129483912591525, -24.35455549606792047, -120.13018318901130499, -29.43951176908901957, - -115.50120511177955507, -34.79395188944583595, -110.97138028465980142, -40.37601690822956613, - -106.47134737780879732, -46.14758359783259323, -101.90317607822893820, -52.07335345715244301, - -97.11263541171206271, -58.11928872485960085, -91.82057104813961246, -64.24948773689963843, - -85.42499621356618889, -70.41813086842684299, -76.27768438369074033, -76.54083121758115738, - -57.87324357773111672, -82.33459525449163152, 1.99999999999996114, -85.78166687108868871, - 61.87324357773107408, -82.33459525449163152, 80.27768438369074033, -76.54083121758115738, - 89.42499621356618889, -70.41813086842684299, 95.82057104813959825, -64.24948773689963843, - 101.11263541171206271, -58.11928872485960085, 105.90317607822892398, -52.07335345715245722, - 110.47134737780879732, -46.14758359783259323, 114.97138028465980142, -40.37601690822956613, - 119.50120511177955507, -34.79395188944583595, 124.13018318901127657, -29.43951176908902667, - 128.91129483912592946, -24.35455549606792403, 133.88641650799127092, -19.58517424494497661, - 139.08801440266989857, -15.18177122287804970, 144.53869867413172301, -11.19861017149901272, - 150.24942513902280439, -7.69266569608215889, 156.21700460768090579, -4.72161659803607048, - 162.42170502724644621, -2.34091526057736310, 168.82591385569978115, -0.60006016949342578, - 175.37485237931247184, 0.46152536384328174, -178.00000000000000000, -11.06408152044379101, - -171.33870321353637678, -11.51266748991093891, -164.78020742076523675, -12.84550987892848539, - -158.41822389716838870, -15.02545489467462403, -152.33085313434585828, -17.99542592709287803, - -146.57821873782722832, -21.68429473238734673, -141.20455296082769792, -26.01294410398721979, - -136.24424010430539056, -30.89942669541638764, -131.73150447900647464, -36.26259289105208694, - -127.71480673751507595, -42.02394287044848653, -124.28024682035200499, -48.10755262562803125, - -121.59604777627559713, -54.43751136022502379, -120.01228809564838684, -60.93070105520797597, - -120.32470998353322500, -67.47696540683870126, -124.60898756414846389, -73.87349687490876704, - -139.22778135121401988, -79.54230031515007227, -178.00000000000000000, -82.33591847955618448, - 143.22778135121404830, -79.54230031515007227, 128.60898756414846389, -73.87349687490876704, - 124.32470998353322500, -67.47696540683870126, 124.01228809564837263, -60.93070105520797597, - 125.59604777627561134, -54.43751136022503090, 128.28024682035200499, -48.10755262562803125, - 131.71480673751509016, -42.02394287044850074, 135.73150447900647464, -36.26259289105208694, - 140.24424010430539056, -30.89942669541638764, 145.20455296082769792, -26.01294410398722334, - 150.57821873782722832, -21.68429473238734673, 156.33085313434585828, -17.99542592709287803, - 162.41822389716838870, -15.02545489467462403, 168.78020742076520833, -12.84550987892848539, - 175.33870321353637678, -11.51266748991093891, -178.00000000000000000, -23.60656536858034116, - -172.31484199740026497, -24.06074499849094650, -166.76206145225543764, -25.40835290264588053, - -161.47033176340409000, -27.60594661847371611, -156.56387974300261590, -30.58514629912493277, - -152.16715015384193066, -34.25717258827405942, -148.41689072479888978, -38.51667916926085411, - -145.48413942807309240, -43.24305393544465659, -143.61056018478930696, -48.29703252527734492, - -143.16669319413330186, -53.50931894776815057, -144.73891790293430404, -58.65522080377154168, - -149.21912766680259210, -63.40582055258879279, -157.69810487935359333, -67.25441721659638006, - -170.53118545218148938, -69.49384617858525814, 174.53118545218146096, -69.49384617858525814, - 161.69810487935359333, -67.25441721659638006, 153.21912766680256368, -63.40582055258877858, - 148.73891790293430404, -58.65522080377152747, 147.16669319413330186, -53.50931894776815767, - 147.61056018478930696, -48.29703252527734492, 149.48413942807309240, -43.24305393544464948, - 152.41689072479886136, -38.51667916926086122, 156.16715015384193066, -34.25717258827403811, - 160.56387974300261590, -30.58514629912493277, 165.47033176340409000, -27.60594661847371611, - 170.76206145225546607, -25.40835290264587698, 176.31484199740026497, -24.06074499849094295, - -178.00000000000000000, -36.54741287921620341, -174.09065200901946469, -36.97055608025468132, - -170.42266827604018431, -38.21219158951615213, -167.24101930549508666, -40.18968456371464271, - -164.79950790329061761, -42.76713193279059766, -163.36621335583259906, -45.75617589753742465, - -163.22080918295719698, -48.91487419033276751, -164.62326642581021474, -51.94637057143930292, - -167.72066876086483944, -54.50623527875517738, -172.37525674045562596, -56.23764348052387874, - -178.00000000000000000, -56.85258712078380228, 176.37525674045562596, -56.23764348052387874, - 171.72066876086483944, -54.50623527875517738, 168.62326642581021474, -51.94637057143930292, - 167.22080918295719698, -48.91487419033276751, 167.36621335583259906, -45.75617589753742465, - 168.79950790329058918, -42.76713193279060476, 171.24101930549508666, -40.18968456371464271, - 174.42266827604018431, -38.21219158951615213, 178.09065200901943626, -36.97055608025468132}; - + 1.99999999999999600, 44.93291171299576092, + 2.77237262934735273, 45.01674523281888440, + 3.47555862489311007, 45.26073815098433784, + 4.04494770792090641, 45.64287023699112211, + 4.42528132740345281, 46.12813070368466839, + 4.57569351104372046, 46.67109320702403608, + 4.47479391972813190, 47.21956618120344018, + 4.12512266986032206, 47.71926346185548340, + 3.55581027759415491, 48.11926090196744354, + 2.82200931024728874, 48.37772379915030996, + 1.99999999999999556, 48.46708828700425187, + 1.17799068975270305, 48.37772379915030996, + 0.44418972240583693, 48.11926090196744354, + -0.12512266986032983, 47.71926346185548340, + -0.47479391972814028, 47.21956618120344018, + -0.57569351104372879, 46.67109320702403608, + -0.42528132740346147, 46.12813070368466839, + -0.04494770792091433, 45.64287023699112211, + 0.52444137510688182, 45.26073815098433784, + 1.22762737065263883, 45.01674523281888440, + 1.99999999999999556, 42.63725270983827897, + 3.27473332072834422, 42.73934376403467184, + 4.49349321557291415, 43.04109625964867547, + 5.60127364516099924, 43.52904513460186564, + 6.54514794659825672, 44.18107605134951399, + 7.27568782798168190, 44.96693492678289061, + 7.74887351496657040, 45.84898874281991965, + 7.92867515230084408, 46.78330526703850012, + 7.79040894364469949, 47.72115423047301164, + 7.32476598915540844, 48.61105462603097038, + 6.54204071489340944, 49.40147266886311428, + 5.47559260124316793, 50.04417716102324221, + 4.18315343237871584, 50.49806364409691639, + 2.74458547194322389, 50.73299668020899134, + 1.25541452805676457, 50.73299668020899134, + -0.18315343237872406, 50.49806364409691639, + -1.47559260124317815, 50.04417716102324221, + -2.54204071489341921, 49.40147266886311428, + -3.32476598915541599, 48.61105462603097038, + -3.79040894364470748, 47.72115423047301164, + -3.92867515230085296, 46.78330526703850012, + -3.74887351496657928, 45.84898874281991965, + -3.27568782798168812, 44.96693492678289061, + -2.54514794659826427, 44.18107605134951399, + -1.60127364516100812, 43.52904513460186564, + -0.49349321557292075, 43.04109625964867547, + 0.72526667927164967, 42.73934376403467184, + 1.99999999999999556, 40.31236211945285675, + 3.63376292032076975, 40.42261301497919845, + 5.22173788476928014, 40.75019560382368411, + 6.71826031811739099, 41.28563184510834105, + 8.07794770476515467, 42.01322983391511201, + 9.25594876339864392, 42.91123363953519743, + 10.20837670733705771, 43.95202523375940729, + 10.89307303713296271, 45.10238649522262477, + 11.27090590027328432, 46.32385288536406875, + 11.30784824781773779, 47.57322688923063225, + 10.97806564551987840, 48.80336562297891589, + 10.26810804950859612, 49.96440199105025926, + 9.18197003975669723, 51.00557820511744467, + 7.74622775989939338, 51.87782553848720823, + 6.01378910799337074, 52.53707313311167582, + 4.06435502534209725, 52.94800110445672203, + 1.99999999999999600, 53.08763788054714894, + -0.06435502534210560, 52.94800110445672203, + -2.01378910799337962, 52.53707313311167582, + -3.74622775989940182, 51.87782553848720823, + -5.18197003975670256, 51.00557820511744467, + -6.26810804950860057, 49.96440199105025926, + -6.97806564551988551, 48.80336562297891589, + -7.30784824781774667, 47.57322688923063225, + -7.27090590027329498, 46.32385288536406875, + -6.89307303713297248, 45.10238649522262477, + -6.20837670733706481, 43.95202523375940729, + -5.25594876339865458, 42.91123363953519743, + -4.07794770476516355, 42.01322983391511201, + -2.71826031811739988, 41.28563184510834105, + -1.22173788476928813, 40.75019560382368411, + 0.36623707967922192, 40.42261301497919845, + 1.99999999999999600, 37.95945956416753120, + 3.72994195085756930, 38.05275970261258323, + 5.43190111561111433, 38.33106975566338548, + 7.07783882564464850, 38.78962916423042628, + 8.63959771096107509, 39.42053684981195261, + 10.08882962579568954, 40.21279467492543347, + 11.39692139773956114, 41.15235690381517486, + 12.53494002107632888, 42.22217718966415845, + 13.47363915243209043, 43.40224634648851065, + 14.18359521918176469, 44.66961909752426152, + 14.63557388477568644, 45.99843704779454612, + 14.80126342211737089, 47.35996933207513848, + 14.65454278958190670, 48.72271255422421632, + 14.17346129570704250, 50.05261748260089405, + 13.34306219743110233, 51.31353838046217675, + 12.15903975007321236, 52.46802323745790630, + 10.63193584333677499, 53.47856301011842106, + 8.79116029542177024, 54.30937174533021761, + 6.68766853851381526, 54.92865613654313961, + 4.39391037845720334, 55.31115535850048559, + 1.99999999999999600, 55.44054043583248159, + -0.39391037845721066, 55.31115535850048559, + -2.68766853851382415, 54.92865613654313961, + -4.79116029542177646, 54.30937174533021761, + -6.63193584333678299, 53.47856301011842106, + -8.15903975007321947, 52.46802323745790630, + -9.34306219743110766, 51.31353838046217675, + -10.17346129570704960, 50.05261748260089405, + -10.65454278958191559, 48.72271255422421632, + -10.80126342211738155, 47.35996933207513848, + -10.63557388477569354, 45.99843704779454612, + -10.18359521918177357, 44.66961909752426152, + -9.47363915243210286, 43.40224634648851065, + -8.53494002107633776, 42.22217718966415845, + -7.39692139773957003, 41.15235690381517486, + -6.08882962579569842, 40.21279467492543347, + -4.63959771096108220, 39.42053684981195261, + -3.07783882564465516, 38.78962916423042628, + -1.43190111561112299, 38.33106975566338548, + 0.27005804914242265, 38.05275970261258323, + 1.99999999999999600, 35.57084905987729684, + 3.89475329148972449, 35.66164776728690811, + 5.76748883913266752, 35.93289207465146262, + 7.59612881396671380, 36.38113516165864070, + 9.35846198524136241, 37.00065953311393940, + 11.03204592267791284, 37.78351258180431671, + 12.59407748901272228, 38.71954642363609622, + 14.02123063562953753, 39.79645326718445375, + 15.28946940417034384, 40.99978674800681233, + 16.37385639149118433, 42.31296002482778107, + 17.24839405994832120, 43.71721334442865725, + 17.88595991605807711, 45.19154778206701195, + 18.25842850603561374, 46.71262882888417067, + 18.33711398818940452, 48.25467471095996785, + 18.09371358173445188, 49.78936139305309894, + 17.50197280996727400, 51.28580055994029152, + 16.54030106040206149, 52.71067834344860614, + 15.19549149366584473, 54.02867680201524081, + 13.46747450557555048, 55.20332460427978560, + 11.37460140494845717, 56.19841400024800038, + 8.95834334902436602, 56.98004544675784899, + 6.28571777452580616, 57.51919556269155720, + 3.44765611568870733, 57.79446760315521914, + 0.55234388431128512, 57.79446760315521914, + -2.28571777452581371, 57.51919556269155720, + -4.95834334902437313, 56.98004544675784899, + -7.37460140494846428, 56.19841400024800038, + -9.46747450557555759, 55.20332460427978560, + -11.19549149366585006, 54.02867680201524081, + -12.54030106040206860, 52.71067834344860614, + -13.50197280996728288, 51.28580055994029152, + -14.09371358173445721, 49.78936139305309894, + -14.33711398818941340, 48.25467471095996785, + -14.25842850603561907, 46.71262882888417067, + -13.88595991605808244, 45.19154778206701195, + -13.24839405994833008, 43.71721334442865725, + -12.37385639149119498, 42.31296002482778107, + -11.28946940417035272, 40.99978674800681233, + -10.02123063562954464, 39.79645326718445375, + -8.59407748901273116, 38.71954642363609622, + -7.03204592267792172, 37.78351258180431671, + -5.35846198524136952, 37.00065953311393940, + -3.59612881396672091, 36.38113516165864070, + -1.76748883913267574, 35.93289207465146262, + 0.10524670851026705, 35.66164776728690811, + 1.99999999999999600, 33.13704588354342917, + 4.09718755711548610, 33.23124463343540214, + 6.17473678976558027, 33.51283479810019372, + 8.21297051898842767, 33.97880825794451454, + 10.19211732580419394, 34.62418375193038855, + 12.09222434325374529, 35.44205010622848562, + 13.89302536639832120, 36.42361680044041350, + 15.57375501535205942, 37.55826279697889447, + 17.11290455740948246, 38.83357308646102268, + 18.48792157318536056, 40.23535159216240942, + 19.67486490585671888, 41.74759899443786537, + 20.64804001511417653, 43.35244484209054860, + 21.37966075266355404, 45.03002540480565585, + 21.83961562141898582, 46.75830290430587155, + 21.99546453290828296, 48.51282952087984768, + 21.81285996059914467, 50.26647329190694080, + 21.25667322722224029, 51.98914605534769606, + 20.29319722451073105, 53.64760968250063655, + 18.89384545876525578, 55.20548791890712437, + 17.04067793585433321, 56.62367212326139310, + 14.73370655373201643, 57.86135900031617751, + 11.99912115224313958, 58.87795079119066344, + 8.89637618755807047, 59.63591573610307250, + 5.52099396835736211, 60.10440004572244277, + 1.99999999999999645, 60.26295411645657651, + -1.52099396835736900, 60.10440004572244277, + -4.89637618755807846, 59.63591573610307250, + -7.99912115224314757, 58.87795079119066344, + -10.73370655373202531, 57.86135900031617751, + -13.04067793585434032, 56.62367212326139310, + -14.89384545876526467, 55.20548791890712437, + -16.29319722451073460, 53.64760968250063655, + -17.25667322722224739, 51.98914605534769606, + -17.81285996059915178, 50.26647329190694080, + -17.99546453290829362, 48.51282952087984768, + -17.83961562141899648, 46.75830290430587155, + -17.37966075266356114, 45.03002540480565585, + -16.64804001511417653, 43.35244484209054860, + -15.67486490585672598, 41.74759899443786537, + -14.48792157318536944, 40.23535159216240942, + -13.11290455740949135, 38.83357308646102268, + -11.57375501535207007, 37.55826279697889447, + -9.89302536639833185, 36.42361680044041350, + -8.09222434325375239, 35.44205010622848562, + -6.19211732580420193, 34.62418375193038855, + -4.21297051898843566, 33.97880825794451454, + -2.17473678976558960, 33.51283479810019372, + -0.09718755711549353, 33.23124463343540214, + 1.99999999999999600, 30.64757019281734074, + 3.92669240244598505, 30.71678191817960268, + 5.84270801296843700, 30.92395598105918353, + 7.73736939611796171, 31.26771342119434749, + 9.59999239238874047, 31.74577019145994683, + 11.41986936545086984, 32.35495718584155611, + 13.18623692372253586, 33.09124574576952682, + 14.88822378743806851, 33.94977626735131793, + 16.51477509868480453, 34.92488704934376642, + 18.05455017155122732, 36.01014014726423795, + 19.49579145727240714, 37.19834072287546434, + 20.82616341777591984, 38.48154618161253637, + 22.03256121225549435, 39.85106125278488065, + 23.10089088783405131, 41.29741507659190347, + 24.01582559662802652, 42.81031632956308641, + 24.76054697060166987, 44.37858250245571412, + 25.31648826424518717, 45.99003977573201496, + 25.66310777053267245, 47.63139077670033572, + 25.77773936992416637, 49.28804930270956675, + 25.63559425749198795, 50.94394459368972150, + 25.21002592824931199, 52.58130404832083116, + 24.47321926655621738, 54.18043392376822709, + 23.39751805719552991, 55.71953428872550518, + 21.95764487214750105, 57.17460858105235388, + 20.13405311103923623, 58.51955862209560166, + 17.91751559979281083, 59.72658657388572578, + 15.31471471060360479, 60.76704034614789407, + 12.35401191716921154, 61.61281120273996237, + 9.08984714396560278, 62.23829052070698253, + 5.60371952785031446, 62.62270582496653049, + 1.99999999999999556, 62.75242980718265784, + -1.60371952785032224, 62.62270582496653049, + -5.08984714396560900, 62.23829052070698253, + -8.35401191716922042, 61.61281120273996237, + -11.31471471060361367, 60.76704034614789407, + -13.91751559979281616, 59.72658657388572578, + -16.13405311103924689, 58.51955862209560166, + -17.95764487214750815, 57.17460858105235388, + -19.39751805719554056, 55.71953428872550518, + -20.47321926655623159, 54.18043392376822709, + -21.21002592824931554, 52.58130404832083116, + -21.63559425749199860, 50.94394459368972150, + -21.77773936992418058, 49.28804930270956675, + -21.66310777053267955, 47.63139077670033572, + -21.31648826424519783, 45.99003977573201496, + -20.76054697060167697, 44.37858250245571412, + -20.01582559662803362, 42.81031632956308641, + -19.10089088783406197, 41.29741507659190347, + -18.03256121225550146, 39.85106125278488065, + -16.82616341777592694, 38.48154618161253637, + -15.49579145727241603, 37.19834072287546434, + -14.05455017155123620, 36.01014014726423795, + -12.51477509868481341, 34.92488704934376642, + -10.88822378743808095, 33.94977626735131793, + -9.18623692372254474, 33.09124574576952682, + -7.41986936545088049, 32.35495718584155611, + -5.59999239238874935, 31.74577019145994683, + -3.73736939611796970, 31.26771342119434749, + -1.84270801296844478, 30.92395598105918353, + 0.07330759755400772, 30.71678191817960268, + 1.99999999999999600, 28.09100109536852230, + 4.16843782661930362, 28.16889262543881145, + 6.32564933647368566, 28.40205318377053345, + 8.46042949045145143, 28.78894734383437992, + 10.56160821625925195, 29.32703776203888069, + 12.61804920168178690, 30.01281590885307438, + 14.61862699669389087, 30.84184157026976436, + 16.55217628249286577, 31.80878785141789322, + 18.40740784814350306, 32.90748779465315010, + 20.17278641740253420, 34.13097826525053335, + 21.83636590786353437, 35.47153642700349963, + 23.38557795842459797, 36.92070387379718710, + 24.80696969977402233, 38.46929323359992026, + 26.08588698105331716, 40.10737174655498194, + 27.20610004359020806, 41.82421587722975431, + 28.14937074714974585, 43.60823042135866956, + 28.89496528133906494, 45.44682484300304282, + 29.41912613625676443, 47.32623888788154431, + 29.69453569648833735, 49.23130925676766623, + 29.68983696232584180, 51.14517010976478417, + 29.36933297612858240, 53.04888396821036167, + 28.69307609487012201, 54.92100894435518654, + 27.61769060836896728, 56.73712764036463341, + 26.09844461997003151, 58.46939890910893922, + 24.09325961331439458, 60.08625295311978931, + 21.56939664790981936, 61.55243525845713748, + 18.51323316925390117, 62.82970044453450953, + 14.94248311853021427, 63.87850997409253040, + 10.91821393310038246, 64.66099291349783584, + 6.55165837938126572, 65.14506774788591770, + 1.99999999999999645, 65.30899890463147983, + -2.55165837938127327, 65.14506774788591770, + -6.91821393310038868, 64.66099291349783584, + -10.94248311853022138, 63.87850997409253040, + -14.51323316925390650, 62.82970044453450953, + -17.56939664790982647, 61.55243525845713748, + -20.09325961331440169, 60.08625295311978931, + -22.09844461997003862, 58.46939890910893922, + -23.61769060836897793, 56.73712764036463341, + -24.69307609487013266, 54.92100894435518654, + -25.36933297612859306, 53.04888396821036167, + -25.68983696232585245, 51.14517010976478417, + -25.69453569648835156, 49.23130925676766623, + -25.41912613625677508, 47.32623888788154431, + -24.89496528133907205, 45.44682484300304282, + -24.14937074714975651, 43.60823042135866956, + -23.20610004359021872, 41.82421587722975431, + -22.08588698105333137, 40.10737174655498194, + -20.80696969977403299, 38.46929323359992026, + -19.38557795842460862, 36.92070387379718710, + -17.83636590786354148, 35.47153642700349963, + -16.17278641740254486, 34.13097826525053335, + -14.40740784814351372, 32.90748779465315010, + -12.55217628249287110, 31.80878785141789322, + -10.61862699669389798, 30.84184157026976436, + -8.61804920168179400, 30.01281590885307438, + -6.56160821625925994, 29.32703776203888069, + -4.46042949045146031, 28.78894734383437992, + -2.32564933647369187, 28.40205318377053345, + -0.16843782661931131, 28.16889262543881145, + 1.99999999999999600, 25.45485763546421154, + 4.25579267831604824, 25.53081635683327377, + 6.50189378344491864, 25.75824904385734015, + 8.72865139696445169, 26.13583181560235502, + 10.92648650789847409, 26.66137960320710931, + 13.08591371200912867, 27.33187675649334025, + 15.19754366142444368, 28.14351701208744316, + 17.25206214192900589, 29.09175016726920759, + 19.24018123567453742, 30.17133232687284305, + 21.15255849246134545, 31.37637626233231813, + 22.97968026705780886, 32.70039821822412307, + 24.71170528699354918, 34.13635737784334623, + 26.33826403031282837, 35.67668409468497259, + 27.84820859564654683, 37.31329283883029291, + 29.22930648122764197, 39.03757551632935474, + 30.46787020033522708, 40.84037031204417190, + 31.54831327014242248, 42.71190039889389567, + 32.45262246278107199, 44.64167567492752653, + 33.15973753507462618, 46.61834908938177335, + 33.64483525740958214, 48.62951713112133945, + 33.87852862440772839, 50.66145188613276673, + 33.82602201650912122, 52.69875031809839783, + 33.44632120220701665, 54.72388646563504011, + 32.69170289014788011, 56.71665696607986717, + 31.50782877801381332, 58.65352526302899605, + 29.83517004248913196, 60.50690475312735828, + 27.61278608277912383, 62.24449065808524040, + 24.78586461017365750, 63.82887077001763032, + 21.31841683118042496, 65.21781824976642383, + 17.21138564172178320, 66.36584339331807314, + 12.52326664933801226, 67.22759281764989225, + 7.38533547682125846, 67.76325974975715383, + 1.99999999999999600, 67.94514236453578349, + -3.38533547682126690, 67.76325974975715383, + -8.52326664933801936, 67.22759281764989225, + -13.21138564172179031, 66.36584339331807314, + -17.31841683118043562, 65.21781824976642383, + -20.78586461017366460, 63.82887077001763032, + -23.61278608277912738, 62.24449065808524040, + -25.83517004248913906, 60.50690475312735828, + -27.50782877801382398, 58.65352526302899605, + -28.69170289014789077, 56.71665696607986717, + -29.44632120220701665, 54.72388646563504011, + -29.82602201650912477, 52.69875031809839783, + -29.87852862440772839, 50.66145188613276673, + -29.64483525740958925, 48.62951713112133945, + -29.15973753507462618, 46.61834908938177335, + -28.45262246278107554, 44.64167567492752653, + -27.54831327014242959, 42.71190039889389567, + -26.46787020033523774, 40.84037031204417190, + -25.22930648122764907, 39.03757551632935474, + -23.84820859564655038, 37.31329283883029291, + -22.33826403031283903, 35.67668409468497259, + -20.71170528699355629, 34.13635737784334623, + -18.97968026705781242, 32.70039821822412307, + -17.15255849246134900, 31.37637626233231813, + -15.24018123567454097, 30.17133232687284305, + -13.25206214192901299, 29.09175016726920759, + -11.19754366142445434, 28.14351701208744316, + -9.08591371200913933, 27.33187675649334025, + -6.92648650789848030, 26.66137960320710931, + -4.72865139696445969, 26.13583181560235502, + -2.50189378344492708, 25.75824904385734015, + -0.25579267831605540, 25.53081635683327377, + 1.99999999999999556, 22.72540973598210456, + 4.47629214130109165, 22.80878975038074330, + 6.94242093509331060, 23.05843251151436846, + 9.38829543151647350, 23.47285525677742868, + 11.80396120930039139, 24.04961640006232315, + 14.17964838758363122, 24.78535803035031293, + 16.50579641562137567, 25.67586152120415477, + 18.77304947679039060, 26.71611272875198395, + 20.97221728412494102, 27.90037270541385439, + 23.09419678103115103, 29.22224954465700009, + 25.12985059268006793, 30.67476685924402702, + 27.06983782140639860, 32.25042441190934994, + 28.90439178940929921, 33.94124647006630369, + 30.62303747247054275, 35.73881343521246379, + 32.21423852322426029, 37.63427208010365632, + 33.66495985223924237, 39.61831917500946787, + 34.96012666362174315, 41.68115224019865650, + 36.08195470333167520, 43.81237943576640248, + 37.00911969333093765, 46.00087796034757304, + 37.71572773969587189, 48.23458650173874673, + 38.17004600591188535, 50.50021197832979425, + 38.33296113360039925, 52.78282385843581892, + 38.15616674923441565, 55.06530104282313687, + 37.58017141013022666, 57.32758829402004608, + 36.53242307522426557, 59.54571651024583190, + 34.92626834358740950, 61.69055635034530383, + 32.66225471925319113, 63.72633555655123416, + 29.63456841105837825, 65.60910965714572285, + 25.74696599568290623, 67.28571148577094618, + 20.94303945184942251, 68.69426490502790728, + 15.25145387811900122, 69.76794994271870110, + 8.83336207267744022, 70.44359392189112157, + 1.99999999999999600, 70.67459026401789401, + -4.83336207267745088, 70.44359392189112157, + -11.25145387811901188, 69.76794994271870110, + -16.94303945184943316, 68.69426490502790728, + -21.74696599568291333, 67.28571148577094618, + -25.63456841105838890, 65.60910965714572285, + -28.66225471925320534, 63.72633555655123416, + -30.92626834358741306, 61.69055635034530383, + -32.53242307522427978, 59.54571651024583190, + -33.58017141013024087, 57.32758829402004608, + -34.15616674923442986, 55.06530104282313687, + -34.33296113360040636, 52.78282385843581892, + -34.17004600591189956, 50.50021197832979425, + -33.71572773969587900, 48.23458650173874673, + -33.00911969333094476, 46.00087796034757304, + -32.08195470333168231, 43.81237943576640248, + -30.96012666362175025, 41.68115224019865650, + -29.66495985223924237, 39.61831917500946787, + -28.21423852322426740, 37.63427208010365632, + -26.62303747247054631, 35.73881343521246379, + -24.90439178940930276, 33.94124647006630369, + -23.06983782140640216, 32.25042441190934994, + -21.12985059268007149, 30.67476685924402702, + -19.09419678103115459, 29.22224954465700009, + -16.97221728412494102, 27.90037270541385439, + -14.77304947679039948, 26.71611272875198395, + -12.50579641562138455, 25.67586152120415477, + -10.17964838758364365, 24.78535803035031293, + -7.80396120930039938, 24.04961640006232315, + -5.38829543151647972, 23.47285525677742868, + -2.94242093509332037, 23.05843251151436846, + -0.47629214130110020, 22.80878975038074330, + 1.99999999999999600, 19.88744176867359670, + 4.69641128035122435, 19.97822987242788884, + 7.38211423303385939, 20.25003331964319031, + 10.04651538477556549, 20.70118176192879389, + 12.67924035399200378, 21.32893128539882710, + 15.27021753707410312, 22.12952164603582617, + 17.80973254683686946, 23.09825113914619976, + 20.28844626261447459, 24.22956442487017270, + 22.69737092792362532, 25.51714801636046204, + 25.02780000980339636, 26.95402789495272344, + 27.27118821325056430, 28.53266378578365448, + 29.41897786066868292, 30.24503490367624536, + 31.46236656888654792, 32.08271233019573287, + 33.39200856069422940, 34.03691345955112979, + 35.19763777135730720, 36.09853399317180589, + 36.86759479425492714, 38.25815259440047811, + 38.38823113470312620, 40.50600232791481403, + 39.74315247327933776, 42.83190113242096686, + 40.91224672040999621, 45.22513042719018728, + 41.87042151784342536, 47.67424596736638165, + 42.58594887503602422, 50.16679739569313057, + 43.01828312042781022, 52.68892136253921876, + 43.11518954384546731, 55.22475594928482678, + 42.80901819915838757, 57.75559962131574565, + 42.01204327449660525, 60.25870541573158334, + 40.61111913565843423, 62.70556553816149403, + 38.46283411458643542, 65.05952649321666570, + 35.39257932570031784, 67.27265449907143591, + 31.20553396293055215, 69.28213523998356038, + 25.72462936947633239, 71.00751554077180572, + 18.87413258170036201, 72.35214877025040892, + 10.80643587234801295, 73.21444539538327945, + 1.99999999999999600, 73.51255823132640899, + -6.80643587234802006, 73.21444539538327945, + -14.87413258170037622, 72.35214877025040892, + -21.72462936947634304, 71.00751554077180572, + -27.20553396293055570, 69.28213523998356038, + -31.39257932570032139, 67.27265449907143591, + -34.46283411458643542, 65.05952649321666570, + -36.61111913565844134, 62.70556553816149403, + -38.01204327449661946, 60.25870541573158334, + -38.80901819915838047, 57.75559962131574565, + -39.11518954384547442, 55.22475594928482678, + -39.01828312042781022, 52.68892136253921876, + -38.58594887503603843, 50.16679739569313057, + -37.87042151784342536, 47.67424596736638165, + -36.91224672041000332, 45.22513042719018728, + -35.74315247327935197, 42.83190113242096686, + -34.38823113470313331, 40.50600232791481403, + -32.86759479425493424, 38.25815259440047811, + -31.19763777135731431, 36.09853399317180589, + -29.39200856069424717, 34.03691345955112979, + -27.46236656888655858, 32.08271233019573287, + -25.41897786066869003, 30.24503490367624536, + -23.27118821325056786, 28.53266378578365448, + -21.02780000980340702, 26.95402789495272344, + -18.69737092792362887, 25.51714801636046204, + -16.28844626261448170, 24.22956442487017270, + -13.80973254683687301, 23.09825113914619976, + -11.27021753707411023, 22.12952164603582617, + -8.67924035399200910, 21.32893128539882710, + -6.04651538477557171, 20.70118176192879389, + -3.38211423303386916, 20.25003331964319031, + -0.69641128035123179, 19.97822987242788884, + 1.99999999999999556, 16.92397162343523931, + 4.91799021933642688, 17.02221659443387480, + 7.82460955068693664, 17.31631470128239059, + 10.70865569654628935, 17.80437171431731258, + 13.55924980839402139, 18.48328342761256238, + 16.36596498789072385, 19.34881113120014362, + 19.11891780975726363, 20.39568010883170501, + 21.80881477377849720, 21.61769490666709714, + 24.42694818201795570, 23.00786444863284430, + 26.96513811238917668, 24.55852997196867804, + 29.41561850057672345, 26.26148912292403637, + 31.77086550998261671, 28.10811023218007065, + 34.02336509370287132, 30.08943159705432180, + 36.16531367916817175, 32.19624133969532664, + 38.18824090805679106, 34.41913389699745807, + 40.08253581911941410, 36.74853923728758076, + 41.83684688715484867, 39.17472027603687934, + 43.43731045663169255, 41.68773239540036002, + 44.86653894639825779, 44.27733603455973110, + 46.10226597817306526, 46.93284831506371546, + 47.11549450115907689, 49.64291142290011294, + 47.86791757533286074, 52.39514197266055362, + 48.30826862884845241, 55.17560336970880286, + 48.36709840828364548, 57.96800639328736793, + 47.94927569193657035, 60.75248226909388904, + 46.92334661203305046, 63.50367340927639503, + 45.10709162054252630, 66.18773615332521842, + 42.25028545833759352, 68.75766348420629015, + 38.02194152863378918, 71.14627956201685777, + 32.02735732207737840, 73.25702648451857613, + 23.91726594386999949, 74.95625177955801632, + 13.67376613190776347, 76.07990221077976400, + 1.99999999999999600, 76.47602837656475572, + -9.67376613190777235, 76.07990221077976400, + -19.91726594387001015, 74.95625177955801632, + -28.02735732207739261, 73.25702648451857613, + -34.02194152863380339, 71.14627956201685777, + -38.25028545833759352, 68.75766348420629015, + -41.10709162054251919, 66.18773615332521842, + -42.92334661203306467, 63.50367340927639503, + -43.94927569193657746, 60.75248226909388904, + -44.36709840828364548, 57.96800639328736793, + -44.30826862884845241, 55.17560336970880286, + -43.86791757533286784, 52.39514197266055362, + -43.11549450115907689, 49.64291142290011294, + -42.10226597817306526, 46.93284831506371546, + -40.86653894639826490, 44.27733603455973110, + -39.43731045663169965, 41.68773239540036002, + -37.83684688715485578, 39.17472027603687934, + -36.08253581911941410, 36.74853923728758076, + -34.18824090805679816, 34.41913389699745807, + -32.16531367916818596, 32.19624133969532664, + -30.02336509370288908, 30.08943159705432180, + -27.77086550998262737, 28.10811023218007065, + -25.41561850057673766, 26.26148912292403637, + -22.96513811238918734, 24.55852997196867804, + -20.42694818201796636, 23.00786444863284430, + -17.80881477377850075, 21.61769490666709714, + -15.11891780975727251, 20.39568010883170501, + -12.36596498789073095, 19.34881113120014362, + -9.55924980839402849, 18.48328342761256238, + -6.70865569654630001, 17.80437171431731258, + -3.82460955068694464, 17.31631470128239059, + -0.91799021933643477, 17.02221659443387480, + 1.99999999999999556, 13.81592192037676625, + 5.14299451095195792, 13.92173872937387102, + 8.27378779825514954, 14.23846093632396581, + 11.38041466390652978, 14.76392503362292530, + 14.45136393475438119, 15.49459412897926924, + 17.47576217013061139, 16.42565612762181004, + 20.44350999615491205, 17.55115134744557182, + 23.34536199373425092, 18.86412113035556004, + 26.17294518233925160, 20.35676830949395466, + 28.91871466466540141, 22.02062053097340311, + 31.57584735879237314, 23.84668824688734645, + 34.13807554901531205, 25.82561045671564415, + 36.59946098845733786, 27.94778271439298933, + 38.95410730493897233, 30.20346328869342756, + 41.19580324741197330, 32.58285444844916867, + 43.31758136998609388, 35.07615645611910082, + 45.31116510791174079, 37.67359181604234664, + 47.16626004327477517, 40.36539642075791789, + 48.86961926381131605, 43.14177213432515856, + 50.40377244059970963, 45.99279145534946167, + 51.74524369263552614, 48.90823812634897649, + 52.86197695521289575, 51.87735585154322138, + 53.70950811496559396, 54.88845672164920586, + 54.22511448054331140, 57.92830381592441569, + 54.31863539857850753, 60.98111330635318694, + 53.85773173662808233, 64.02688869849654907, + 52.64383991815974895, 67.03853932314146391, + 50.37308749001456931, 69.97672254379963874, + 46.57629367046713753, 72.78039612779683409, + 40.54693285975561423, 75.34972495190746145, + 31.34759503822197502, 77.51867325981696411, + 18.24038894313819625, 79.03039199474196153, + 1.99999999999999489, 79.58407807962322522, + -14.24038894313820691, 79.03039199474196153, + -27.34759503822198567, 77.51867325981696411, + -36.54693285975562134, 75.34972495190746145, + -42.57629367046714464, 72.78039612779683409, + -46.37308749001456931, 69.97672254379963874, + -48.64383991815974895, 67.03853932314146391, + -49.85773173662808233, 64.02688869849654907, + -50.31863539857852174, 60.98111330635318694, + -50.22511448054332561, 57.92830381592441569, + -49.70950811496560817, 54.88845672164920586, + -48.86197695521290996, 51.87735585154322138, + -47.74524369263553325, 48.90823812634897649, + -46.40377244059971673, 45.99279145534946167, + -44.86961926381133026, 43.14177213432515856, + -43.16626004327478938, 40.36539642075791789, + -41.31116510791174079, 37.67359181604234664, + -39.31758136998610098, 35.07615645611910082, + -37.19580324741197330, 32.58285444844916867, + -34.95410730493897233, 30.20346328869342756, + -32.59946098845735207, 27.94778271439298933, + -30.13807554901532626, 25.82561045671564415, + -27.57584735879238025, 23.84668824688734645, + -24.91871466466541207, 22.02062053097340311, + -22.17294518233926226, 20.35676830949395466, + -19.34536199373426157, 18.86412113035556004, + -16.44350999615492626, 17.55115134744557182, + -13.47576217013062028, 16.42565612762181004, + -10.45136393475438830, 15.49459412897926924, + -7.38041466390653600, 14.76392503362292530, + -4.27378779825515931, 14.23846093632396581, + -1.14299451095196680, 13.92173872937387102, + 1.99999999999999600, 10.54173785529561158, + 5.37360609917943943, 10.65531494200142149, + 8.73395228112667965, 10.99520717814031556, + 12.06809948196381832, 11.55892497220584403, + 15.36372621835939078, 12.34240842556757656, + 18.60937983520193129, 13.34015413091371016, + 21.79466590782825364, 14.54537904344351951, + 24.91036566952359266, 15.95020993686824440, + 27.94847769357361855, 17.54588626406437513, + 30.90218550361061745, 19.32296477964253967, + 33.76575655354334060, 21.27151577227775903, + 36.53437968696508875, 23.38130282889363798, + 39.20394762734519389, 25.64194033091114022, + 41.77078829775712165, 28.04302503644353095, + 44.23134383513491485, 30.57423988556289629, + 46.58178882927892772, 33.22542941501406233, + 48.81756887288366187, 35.98664677769215103, + 50.93282539675248444, 38.84817223653458029, + 52.91964996438991164, 41.80050199367043007, + 54.76707508825830928, 44.83430401955754974, + 56.45964870701569538, 47.94033352757975308, + 57.97533567047437231, 51.10929358481432416, + 59.28230228211859298, 54.33161333463023368, + 60.33378767939932885, 57.59709162031844443, + 61.05957379451161415, 60.89430451319173443, + 61.35114187047394552, 64.20957103321013903, + 61.03453256694833584, 67.52503657439316953, + 59.81801935702030448, 70.81486693953938527, + 57.18593275925195485, 74.03707888623914357, + 52.17735288037920327, 77.11452529323830163, + 42.96575654499160635, 79.88802174923021937, + 26.58690897303194944, 82.01033616765489853, + 1.99999999999999756, 82.85826214470436923, + -22.58690897303195300, 82.01033616765489853, + -38.96575654499161345, 79.88802174923021937, + -48.17735288037921748, 77.11452529323830163, + -53.18593275925197617, 74.03707888623914357, + -55.81801935702031159, 70.81486693953938527, + -57.03453256694834295, 67.52503657439316953, + -57.35114187047395262, 64.20957103321013903, + -57.05957379451162836, 60.89430451319173443, + -56.33378767939933596, 57.59709162031844443, + -55.28230228211860720, 54.33161333463023368, + -53.97533567047437231, 51.10929358481432416, + -52.45964870701570248, 47.94033352757975308, + -50.76707508825830928, 44.83430401955754974, + -48.91964996438991875, 41.80050199367043007, + -46.93282539675249154, 38.84817223653458029, + -44.81756887288366897, 35.98664677769215103, + -42.58178882927894193, 33.22542941501406233, + -40.23134383513492196, 30.57423988556289629, + -37.77078829775713587, 28.04302503644353095, + -35.20394762734520100, 25.64194033091114022, + -32.53437968696510296, 23.38130282889363798, + -29.76575655354335481, 21.27151577227775903, + -26.90218550361062810, 19.32296477964253967, + -23.94847769357362921, 17.54588626406437513, + -20.91036566952360332, 15.95020993686824440, + -17.79466590782826074, 14.54537904344351951, + -14.60937983520194017, 13.34015413091371016, + -11.36372621835939611, 12.34240842556757656, + -8.06809948196382898, 11.55892497220584403, + -4.73395228112668676, 10.99520717814031556, + -1.37360609917944720, 10.65531494200142149, + 1.99999999999999600, 7.07694595379052860, + 5.61234064761578821, 7.19855645524002874, + 9.21005712455210990, 7.56241362477411005, + 12.77895396829166508, 8.16563014493928208, + 16.30566013367806733, 9.00350968307661859, + 19.77796305903054730, 10.06971032975657643, + 23.18506021909562875, 11.35645428598743756, + 26.51771689758311723, 12.85476800444172873, + 29.76832854644279536, 14.55473640862136264, + 32.93089420696504277, 16.44575600838861362, + 36.00091302933286386, 18.51677422677156315, + 38.97521854780546846, 20.75650546647104022, + 41.85176517580521960, 23.15361781597290403, + 44.62937881000767248, 25.69688738457191945, + 47.30747895739604303, 28.37531978094453677, + 49.88577372237665486, 31.17824007995576707, + 52.36392121341754091, 34.09535373993877982, + 54.74114071516557090, 37.11678138456306897, + 57.01574257648425004, 40.23307020728445593, + 59.18452370168172649, 43.43518400478754415, + 61.24193904387428233, 46.71447239476329827, + 63.17889520468774833, 50.06261728049957327, + 64.98089250042356468, 53.47155025248752480, + 66.62500642980749888, 56.93332642202356197, + 68.07470870335012592, 60.43992354246547904, + 69.27043361461275595, 63.98289851135272244, + 70.11115128919476547, 67.55274433031233627, + 70.41516373728855172, 71.13755089586132385, + 69.82714145548160900, 74.71983727667713993, + 67.56415050352947560, 78.26774990878784877, + 61.58131858106001033, 81.70467639663139892, + 45.28583165458982052, 84.76914638163376026, + 1.99999999999999689, 86.32305404620944955, + -41.28583165458982762, 84.76914638163376026, + -57.58131858106002454, 81.70467639663139892, + -63.56415050352948981, 78.26774990878784877, + -65.82714145548162321, 74.71983727667713993, + -66.41516373728856593, 71.13755089586132385, + -66.11115128919477968, 67.55274433031233627, + -65.27043361461275595, 63.98289851135272244, + -64.07470870335011170, 60.43992354246547904, + -62.62500642980752019, 56.93332642202356197, + -60.98089250042357889, 53.47155025248752480, + -59.17889520468775544, 50.06261728049957327, + -57.24193904387428233, 46.71447239476329827, + -55.18452370168172649, 43.43518400478754415, + -53.01574257648425004, 40.23307020728445593, + -50.74114071516557800, 37.11678138456306897, + -48.36392121341754802, 34.09535373993877982, + -45.88577372237665486, 31.17824007995576707, + -43.30747895739605013, 28.37531978094453677, + -40.62937881000767959, 25.69688738457191945, + -37.85176517580522670, 23.15361781597290403, + -34.97521854780547557, 20.75650546647104022, + -32.00091302933286386, 18.51677422677156315, + -28.93089420696504632, 16.44575600838861362, + -25.76832854644280957, 14.55473640862136264, + -22.51771689758312434, 12.85476800444172873, + -19.18506021909564296, 11.35645428598743756, + -15.77796305903055796, 10.06971032975657643, + -12.30566013367807621, 9.00350968307661859, + -8.77895396829167574, 8.16563014493928208, + -5.21005712455211967, 7.56241362477411005, + -1.61234064761579732, 7.19855645524002874, + 1.99999999999999556, 3.39364961385021102, + 5.86220744039440511, 3.52366797872772963, + 9.70801791517021506, 3.91258202080964379, + 13.52160273595813145, 4.55701513369246314, + 17.28822357553576694, 5.45149036402020482, + 20.99466904928960531, 6.58864168091601599, + 24.62957864939021135, 7.95948261839810378, + 28.18364165385172626, 9.55371034930797514, + 31.64967308093981302, 11.36002298851478365, + 35.02258051120933402, 13.36643021383323671, + 38.29924330014637235, 15.56054131855884215, + 41.47832910130669859, 17.92981963683195801, + 44.56007232189227807, 20.46179708442632261, + 47.54603622089215520, 23.14424674184217068, + 50.43887601207384819, 25.96531465470105005, + 53.24211556220075892, 28.91361425616476666, + 55.95994579953058690, 31.97828811153562256, + 58.59704918478396252, 35.14904222145087687, + 61.15845171105517863, 38.41615810166681655, + 63.64940188352277062, 41.77048747717444854, + 66.07527488350018530, 45.20343384586799118, + 68.44149951448399349, 48.70692450151241104, + 70.75350545455997064, 52.27337593848611164, + 73.01668877482984499, 55.89565494125256606, + 75.23639478559491067, 59.56703711552955127, + 77.41791965045068480, 63.28116415502339720, + 79.56653776770211550, 67.03200075627316323, + 81.68757690134610527, 70.81379178736983704, + 83.78661098119418682, 74.62102007313750107, + 85.87003129087820241, 78.44836496333871878, + 87.94730489861872513, 82.29066165213365025, + 90.04670840724658376, 86.14286053253557895, + -177.99999999999985789, 89.99364961382421768, + -86.04670840724655534, 86.14286053253557895, + -83.94730489861876777, 82.29066165213365025, + -81.87003129087820241, 78.44836496333871878, + -79.78661098119418682, 74.62102007313750107, + -77.68757690134613370, 70.81379178736983704, + -75.56653776770212971, 67.03200075627316323, + -73.41791965045068480, 63.28116415502339720, + -71.23639478559491067, 59.56703711552955127, + -69.01668877482984499, 55.89565494125256606, + -66.75350545455997064, 52.27337593848611164, + -64.44149951448400770, 48.70692450151241104, + -62.07527488350018530, 45.20343384586799118, + -59.64940188352277772, 41.77048747717444854, + -57.15845171105518574, 38.41615810166681655, + -54.59704918478396252, 35.14904222145087687, + -51.95994579953059400, 31.97828811153562256, + -49.24211556220076602, 28.91361425616476666, + -46.43887601207384819, 25.96531465470105005, + -43.54603622089217652, 23.14424674184217068, + -40.56007232189229228, 20.46179708442632261, + -37.47832910130670570, 17.92981963683195801, + -34.29924330014637945, 15.56054131855884215, + -31.02258051120933402, 13.36643021383323671, + -27.64967308093982012, 11.36002298851478365, + -24.18364165385173337, 9.55371034930797514, + -20.62957864939022556, 7.95948261839810378, + -16.99466904928961597, 6.58864168091601599, + -13.28822357553577582, 5.45149036402020482, + -9.52160273595813855, 4.55701513369246314, + -5.70801791517022572, 3.91258202080964379, + -1.86220744039441377, 3.52366797872772963, + 1.99999999999999556, -0.54003843869415158, + 6.12693783758673050, -0.40111214042971866, + 10.23515434659413970, 0.01431798130226895, + 14.30668142622436534, 0.70226688878527466, + 18.32499114491300674, 1.65629062554436168, + 22.27556116130715580, 2.86776143955401341, + 26.14628266758476016, 4.32621388981252775, + 29.92769777693464306, 6.01973109672258033, + 33.61307490596014702, 7.93534078498540918, + 37.19834740797973183, 10.05939491571368904, + 40.68195074563112712, 12.37791309905906267, + 44.06459700293881809, 14.87687712846902421, + 47.34902402209520034, 17.54247066454981052, + 50.53975201683548590, 20.36126354510683711, + 53.64287529119709319, 23.32034407821754485, + 56.66591251909911620, 26.40740499964346455, + 59.61773745881250619, 29.61078975921034129, + 62.50861449014276872, 32.91950572923986584, + 65.35037195723165837, 36.32321005821969351, + 68.15676435641833564, 39.81217237539897269, + 70.94410839385281520, 43.37721635788606989, + 73.73234087330637010, 47.00963897352001908, + 76.54676554128549526, 50.70110112272598712, + 79.42099130058548440, 54.44347433172315220, + 82.40205491898922219, 58.22861013398449614, + 85.55981529780903827, 62.04795929474799721, + 89.00534896567364740, 65.89187266757559769, + 92.93011750021230455, 69.74815881341463353, + 97.69886509454546797, 73.59868574092438109, + 104.10346048738931302, 77.40995108179684792, + 114.19717929614462548, 81.10053146183234674, + 134.58473801020662108, 84.39088722277180921, + -178.00000000000002842, 86.05996156130575514, + -130.58473801020659266, 84.39088722277180921, + -110.19717929614465390, 81.10053146183234674, + -100.10346048738929881, 77.40995108179684792, + -93.69886509454548218, 73.59868574092438109, + -88.93011750021230455, 69.74815881341463353, + -85.00534896567366161, 65.89187266757559769, + -81.55981529780905248, 62.04795929474799721, + -78.40205491898922219, 58.22861013398449614, + -75.42099130058548440, 54.44347433172315220, + -72.54676554128549526, 50.70110112272598712, + -69.73234087330638431, 47.00963897352001908, + -66.94410839385282941, 43.37721635788606989, + -64.15676435641836406, 39.81217237539897269, + -61.35037195723166548, 36.32321005821969351, + -58.50861449014277582, 32.91950572923986584, + -55.61773745881251330, 29.61078975921034129, + -52.66591251909913041, 26.40740499964346455, + -49.64287529119710030, 23.32034407821754485, + -46.53975201683548590, 20.36126354510683711, + -43.34902402209520034, 17.54247066454981052, + -40.06459700293881809, 14.87687712846902421, + -36.68195074563112712, 12.37791309905906267, + -33.19834740797973893, 10.05939491571368904, + -29.61307490596015057, 7.93534078498540918, + -25.92769777693465727, 6.01973109672258033, + -22.14628266758476371, 4.32621388981252775, + -18.27556116130716291, 2.86776143955401341, + -14.32499114491301384, 1.65629062554436168, + -10.30668142622437244, 0.70226688878527466, + -6.23515434659414680, 0.01431798130226895, + -2.12693783758673804, -0.40111214042971866, + 1.99999999999999556, -4.76061810234754645, + 6.41132592404072454, -4.61212239418478553, + 10.80084816537008230, -4.16824654595516098, + 15.14776873556008674, -3.43374257581263365, + 19.43320327160517635, -2.41626750988024641, + 23.64091162595336115, -1.12602081331554671, + 27.75780275269853448, 0.42470591907018324, + 31.77420154083880988, 2.22202305439570047, + 35.68389780238808129, 4.25093893994811900, + 39.48402100288601702, 6.49582334126627270, + 43.17479644658765636, 8.94081392759362181, + 46.75924066894447861, 11.57015196980179894, + 50.24284889074777283, 14.36844259642658628, + 53.63331929525543273, 17.32084182287968943, + 56.94035098918124049, 20.41317688045037571, + 60.17554747078253286, 23.63200834498020697, + 63.35245736607463840, 26.96464262473803686, + 66.48679120890342631, 30.39910192691253599, + 69.59687011372152199, 33.92405612452549946, + 72.70439458886511375, 37.52871692222674938, + 75.83567948061782715, 41.20268880474466755, + 79.02360369831293951, 44.93576201889581512, + 82.31071017117776023, 48.71761719224470966, + 85.75424365182692554, 52.53738247574246856, + 89.43460651588344490, 56.38292751246400769, + 93.47013935832343634, 60.23965935256792648, + 98.04421718085441739, 64.08831694761893516, + 103.45761459629794388, 67.90061168070626252, + 110.23509874248559015, 71.62987479878029262, + 119.34882263320223217, 75.18923730101467129, + 132.64763601333982024, 78.39753950634394641, + 153.15668933029419918, 80.85560324742944260, + -178.00000000000002842, 81.83938189765245852, + -149.15668933029422760, 80.85560324742944260, + -128.64763601333984866, 78.39753950634394641, + -115.34882263320223217, 75.18923730101467129, + -106.23509874248556173, 71.62987479878029262, + -99.45761459629794388, 67.90061168070626252, + -94.04421718085441739, 64.08831694761893516, + -89.47013935832346476, 60.23965935256792648, + -85.43460651588344490, 56.38292751246400769, + -81.75424365182695396, 52.53738247574246856, + -78.31071017117777444, 48.71761719224470966, + -75.02360369831293951, 44.93576201889581512, + -71.83567948061784136, 41.20268880474466755, + -68.70439458886511375, 37.52871692222674938, + -65.59687011372152199, 33.92405612452549946, + -62.48679120890344052, 30.39910192691253599, + -59.35245736607464551, 26.96464262473803686, + -56.17554747078254707, 23.63200834498020697, + -52.94035098918124760, 20.41317688045037571, + -49.63331929525544695, 17.32084182287968943, + -46.24284889074779414, 14.36844259642658628, + -42.75924066894448572, 11.57015196980179894, + -39.17479644658766347, 8.94081392759362181, + -35.48402100288603123, 6.49582334126627270, + -31.68389780238809195, 4.25093893994811900, + -27.77420154083882409, 2.22202305439570047, + -23.75780275269854513, 0.42470591907018324, + -19.64091162595336826, -1.12602081331554671, + -15.43320327160518879, -2.41626750988024641, + -11.14776873556009562, -3.43374257581263365, + -6.80084816537009029, -4.16824654595516098, + -2.41132592404073298, -4.61212239418478553, + 1.99999999999999556, -9.30985393851741705, + 6.72175805422021355, -9.15091230991673932, + 11.41756212661333514, -8.67603648837036268, + 16.06282113870961226, -7.89096231408012549, + 20.63552097548324227, -6.80489321395614688, + 25.11717183125960418, -5.43001433583735604, + 29.49342310484702523, -3.78089833576525836, + 33.75433817509684786, -1.87386656696368870, + 37.89437094593881028, 0.27363605581641121, + 41.91211819209927114, 2.64360721178193936, + 45.80993456941938291, 5.21796518077428217, + 49.59349430020984784, 7.97892504367260536, + 53.27137129744098587, 10.90927185095380025, + 56.85469423510678411, 13.99253787292509266, + 60.35691981311008192, 17.21309536720795919, + 63.79375952890369206, 20.55617726580611659, + 67.18329478816110623, 24.00783659504739731, + 70.54632400302143935, 27.55485191826680236, + 73.90700610430916129, 31.18458087158649406, + 77.29390254081907585, 34.88475663106463287, + 80.74158359961856490, 38.64321178012773572, + 84.29307189810650414, 42.44749809608728697, + 88.00357763838491110, 46.28434439562811065, + 91.94629406419836926, 50.13884820321026581, + 96.22157019495104180, 53.99321139219064491, + 100.97173315822050199, 57.82466562901674934, + 106.40541928769302160, 61.60190913800559542, + 112.83745587761723073, 65.27873175231235336, + 120.75095597801100666, 68.78228497629521598, + 130.87442892417493567, 71.99165187875782124, + 144.18467631053604805, 74.70285264202438213, + 161.47781378002883912, 76.59572545663588983, + -178.00000000000002842, 77.29014606148258792, + -157.47781378002883912, 76.59572545663588983, + -140.18467631053607647, 74.70285264202438213, + -126.87442892417496410, 71.99165187875782124, + -116.75095597801099245, 68.78228497629521598, + -108.83745587761723073, 65.27873175231235336, + -102.40541928769302160, 61.60190913800559542, + -96.97173315822051620, 57.82466562901674934, + -92.22157019495104180, 53.99321139219064491, + -87.94629406419838347, 50.13884820321026581, + -84.00357763838492531, 46.28434439562811065, + -80.29307189810650414, 42.44749809608728697, + -76.74158359961857911, 38.64321178012773572, + -73.29390254081907585, 34.88475663106463287, + -69.90700610430916129, 31.18458087158649406, + -66.54632400302145356, 27.55485191826680236, + -63.18329478816111333, 24.00783659504739731, + -59.79375952890369916, 20.55617726580611659, + -56.35691981311008192, 17.21309536720795919, + -52.85469423510679832, 13.99253787292509266, + -49.27137129744098587, 10.90927185095380025, + -45.59349430020986915, 7.97892504367260536, + -41.80993456941939712, 5.21796518077428217, + -37.91211819209929246, 2.64360721178193936, + -33.89437094593881739, 0.27363605581641121, + -29.75433817509685497, -1.87386656696368870, + -25.49342310484703944, -3.78089833576525836, + -21.11717183125961128, -5.43001433583735604, + -16.63552097548325293, -6.80489321395614688, + -12.06282113870962114, -7.89096231408012549, + -7.41756212661334136, -8.67603648837036268, + -2.72175805422022110, -9.15091230991673932, + 1.99999999999999556, -14.23540593490034212, + 7.06707279862612125, -14.06484423228424063, + 12.10248720394761790, -13.55555303790300137, + 17.07647532339787944, -12.71455858250937965, + 21.96281473836326015, -11.55307649365679623, + 26.74006765897273041, -10.08584610023314454, + 31.39231472226267528, -8.33033151194296018, + 35.90939060170597230, -6.30588392466837799, + 40.28670514160292271, -4.03294710811188395, + 44.52477649322944586, -1.53236497867592791, + 48.62861191461712451, 1.17517574056815022, + 52.60705694133754662, 4.06955754229703981, + 56.47220691225791711, 7.13150506907926562, + 60.23894695811429756, 10.34273467530950796, + 63.92466443809040300, 13.68599977255221845, + 67.54916484650964037, 17.14505004500527363, + 71.13481958525987636, 20.70451849248449960, + 74.70698212428176532, 24.34974403770184992, + 78.29472888122153051, 28.06652943785655552, + 81.93201512996088809, 31.84082402428231973, + 85.65938939021744147, 35.65830698143167155, + 89.52649044264860834, 39.50382677214758331, + 93.59567144235911940, 43.36062114047545890, + 97.94726867835277062, 47.20919167069192923, + 102.68725830824645584, 51.02562354013957702, + 107.95825647893494192, 54.77900423463843538, + 113.95470750710921948, 58.42738157410047961, + 120.94158660124729465, 61.91142197433249095, + 129.26976638159490562, 65.14478524125112813, + 139.36255278451201889, 68.00113945880528377, + 151.60742176334809983, 70.30250402254893061, + 166.05644118521092878, 71.82662636485281382, + -178.00000000000002842, 72.36459406509962378, + -162.05644118521095720, 71.82662636485281382, + -147.60742176334812825, 70.30250402254893061, + -135.36255278451201889, 68.00113945880528377, + -125.26976638159489141, 65.14478524125112813, + -116.94158660124729465, 61.91142197433249095, + -109.95470750710923369, 58.42738157410047961, + -103.95825647893495614, 54.77900423463843538, + -98.68725830824647005, 51.02562354013957702, + -93.94726867835278483, 47.20919167069192923, + -89.59567144235911940, 43.36062114047545890, + -85.52649044264862255, 39.50382677214758331, + -81.65938939021744147, 35.65830698143167155, + -77.93201512996090230, 31.84082402428231973, + -74.29472888122153051, 28.06652943785655552, + -70.70698212428177953, 24.34974403770184992, + -67.13481958525987636, 20.70451849248449960, + -63.54916484650964748, 17.14505004500527363, + -59.92466443809041010, 13.68599977255221845, + -56.23894695811430466, 10.34273467530950796, + -52.47220691225792422, 7.13150506907926562, + -48.60705694133756083, 4.06955754229703981, + -44.62861191461713162, 1.17517574056815022, + -40.52477649322946007, -1.53236497867592791, + -36.28670514160293692, -4.03294710811188395, + -31.90939060170597941, -6.30588392466837799, + -27.39231472226267883, -8.33033151194296018, + -22.74006765897273752, -10.08584610023314454, + -17.96281473836326725, -11.55307649365679623, + -13.07647532339788832, -12.71455858250937965, + -8.10248720394762678, -13.55555303790300137, + -3.06707279862612925, -14.06484423228424063, + 1.99999999999999600, -19.59134287356426896, + 7.46002704385165138, -19.40755726556070115, + 12.88033504279662367, -18.85919797266866738, + 18.22391622121717347, -17.95503295463869264, + 23.45879403720244127, -16.70896594558295334, + 28.55966518490710015, -15.13909773195327269, + 33.50874372060166451, -13.26662598462660014, + 38.29585521437642370, -11.11472732454917889, + 42.91794868987490474, -8.70753794849322738, + 47.37824652323212149, -6.06930752874207258, + 51.68524592294901510, -3.22375917751487107, + 55.85174332801063457, -0.19365520038940884, + 59.89399913408759346, 2.99945241539763385, + 63.83111160507218784, 6.33531713186757095, + 67.68463403089357655, 9.79496963802120746, + 71.47845003929684538, 13.36059248585088000, + 75.23891693218338617, 17.01531594444593409, + 78.99529352424693229, 20.74294398926493699, + 82.78048521884274180, 24.52760876444716587, + 86.63216392910494790, 28.35333972343855180, + 90.59435351833845118, 32.20351875194761959, + 94.71961138434764393, 36.06017287638244539, + 99.07197748871266185, 39.90302877816822757, + 103.73088255981890882, 43.70821480745370025, + 108.79614728017998004, 47.44644408577498496, + 114.39390839554110357, 51.08045112569507751, + 120.68240638414135901, 54.56141374911145192, + 127.85428534979440940, 57.82417470821775396, + 136.12712761110682891, 60.78156430751474204, + 145.70578215046154469, 63.31961305621511116, + 156.69437932769909594, 65.29863962109097031, + 168.95645291666264143, 66.56908368553207822, + -178.00000000000002842, 67.00865712643570760, + -164.95645291666266985, 66.56908368553207822, + -152.69437932769912436, 65.29863962109097031, + -141.70578215046154469, 63.31961305621511116, + -132.12712761110685733, 60.78156430751474204, + -123.85428534979440940, 57.82417470821775396, + -116.68240638414138743, 54.56141374911145192, + -110.39390839554111778, 51.08045112569507751, + -104.79614728017998004, 47.44644408577498496, + -99.73088255981890882, 43.70821480745370025, + -95.07197748871266185, 39.90302877816822757, + -90.71961138434764393, 36.06017287638244539, + -86.59435351833845118, 32.20351875194761959, + -82.63216392910496211, 28.35333972343855180, + -78.78048521884275601, 24.52760876444716587, + -74.99529352424694650, 20.74294398926493699, + -71.23891693218340038, 17.01531594444593409, + -67.47845003929684538, 13.36059248585088000, + -63.68463403089357655, 9.79496963802120746, + -59.83111160507219495, 6.33531713186757095, + -55.89399913408760057, 2.99945241539763385, + -51.85174332801064168, -0.19365520038940884, + -47.68524592294902931, -3.22375917751487107, + -43.37824652323212860, -6.06930752874207258, + -38.91794868987490474, -8.70753794849322738, + -34.29585521437643081, -11.11472732454917889, + -29.50874372060167872, -13.26662598462660014, + -24.55966518490711081, -15.13909773195327269, + -19.45879403720245193, -16.70896594558295334, + -14.22391622121718058, -17.95503295463869264, + -8.88033504279663255, -18.85919797266866738, + -3.46002704385166071, -19.40755726556070115, + 1.99999999999999556, -25.43837119820565817, + 7.91994134356474966, -25.23910692117755517, + 13.78834011215970534, -24.64516330804185529, + 19.55772615655293478, -23.66774871950556403, + 25.18808138888132220, -22.32448212182583092, + 30.64904852681420522, -20.63801909702449677, + 35.92081946357622968, -18.63449057116782726, + 40.99386004100856695, -16.34197985921977292, + 45.86781613095838850, -13.78920372951385431, + 50.54999441522539882, -11.00448645615600007, + 55.05375641269628773, -8.01504773475606846, + 59.39706222812605318, -4.84657887125735343, + 63.60129716846514469, -1.52305829704237761, + 67.69043463465565935, 1.93324784086672352, + 71.69053915995505122, 5.50164833296652045, + 75.62959031828353318, 9.16278970353470612, + 79.53760403336076479, 12.89834287286321945, + 83.44703540885105042, 16.69062430422997068, + 87.39346098727328638, 20.52214967325533124, + 91.41655400532015108, 24.37510563102819106, + 95.56137924921046078, 28.23071203179445021, + 99.88003723653145016, 32.06843215731343832, + 104.43366587620558050, 35.86497163386139420, + 109.29473060869932510, 39.59298941537671368, + 114.54933935400606515, 43.21943287866469774, + 120.29889149413013172, 46.70342202473147353, + 126.65952710171551132, 49.99368843662634987, + 133.75638606304184464, 53.02581189527145256, + 141.70772458687929429, 55.72003625021812923, + 150.59283359977422379, 57.98141283097308474, + 160.40162010671977555, 59.70518607312001080, + 170.97884079028881388, 60.79043532425958318, + -178.00000000000002842, 61.16162880179432193, + -166.97884079028878546, 60.79043532425958318, + -156.40162010671977555, 59.70518607312001080, + -146.59283359977425221, 57.98141283097308474, + -137.70772458687926587, 55.72003625021812923, + -129.75638606304184464, 53.02581189527145256, + -122.65952710171549711, 49.99368843662634987, + -116.29889149413011751, 46.70342202473147353, + -110.54933935400606515, 43.21943287866469774, + -105.29473060869931089, 39.59298941537671368, + -100.43366587620559471, 35.86497163386139420, + -95.88003723653146437, 32.06843215731343832, + -91.56137924921046078, 28.23071203179445021, + -87.41655400532016529, 24.37510563102819106, + -83.39346098727328638, 20.52214967325533124, + -79.44703540885105042, 16.69062430422997068, + -75.53760403336077900, 12.89834287286321945, + -71.62959031828354739, 9.16278970353470612, + -67.69053915995506543, 5.50164833296652045, + -63.69043463465567356, 1.93324784086672352, + -59.60129716846515180, -1.52305829704237761, + -55.39706222812606740, -4.84657887125735343, + -51.05375641269629483, -8.01504773475606846, + -46.54999441522540593, -11.00448645615600007, + -41.86781613095839560, -13.78920372951385431, + -36.99386004100857406, -16.34197985921977292, + -31.92081946357624389, -18.63449057116782726, + -26.64904852681421943, -20.63801909702449677, + -21.18808138888132575, -22.32448212182583092, + -15.55772615655294011, -23.66774871950556403, + -9.78834011215971245, -24.64516330804185529, + -3.91994134356475854, -25.23910692117755517, + 1.99999999999999600, -31.84350531560286868, + 8.47781225763252877, -31.62546289221583962, + 14.88583393507996711, -30.97645080660140593, + 21.16079514785619509, -29.91127029421520689, + 27.25113130544622919, -28.45290874783366775, + 33.11998552133064067, -26.63042818683534207, + 38.74589265732009835, -24.47666503668304117, + 44.12160159745183563, -22.02610835172851722, + 49.25177543408178593, -19.31318879069118211, + 54.15029168417838434, -16.37106576006627279, + 58.83767345094592827, -13.23089104518995640, + 63.33895182482493169, -9.92146778444056032, + 67.68207179147748320, -6.46920580966549874, + 71.89683474738467339, -2.89828246890513030, + 76.01431282527251199, 0.76906207426874740, + 80.06665407661380129, 4.51214364124032485, + 84.08720437104130951, 8.31139512800513636, + 88.11088800598805904, 12.14788119395578647, + 92.17480572169650088, 16.00276430383186366, + 96.31902021078285259, 19.85670886149115333, + 100.58749981932022877, 23.68920100972959730, + 105.02917319698201482, 27.47775433026597725, + 109.69899804029304846, 31.19696754875727152, + 114.65884380836618561, 34.81740352705595143, + 119.97779813807069615, 38.30427835916992052, + 125.73118977168117283, 41.61600262291977259, + 131.99715883033749719, 44.70273256173389598, + 138.84908760812987794, 47.50530446205828383, + 146.34201331309026273, 49.95526128262041254, + 154.49218436749947614, 51.97706503648645082, + 163.25249258416440057, 53.49372597518600259, + 172.49285933765111167, 54.43639475342744305, + -178.00000000000002842, 54.75649468439711143, + -168.49285933765114009, 54.43639475342744305, + -159.25249258416440057, 53.49372597518600259, + -150.49218436749950456, 51.97706503648645082, + -142.34201331309029115, 49.95526128262041254, + -134.84908760812987794, 47.50530446205828383, + -127.99715883033752561, 44.70273256173389598, + -121.73118977168118704, 41.61600262291977259, + -115.97779813807071037, 38.30427835916992052, + -110.65884380836619982, 34.81740352705595143, + -105.69899804029306267, 31.19696754875727152, + -101.02917319698202903, 27.47775433026597725, + -96.58749981932022877, 23.68920100972959730, + -92.31902021078286680, 19.85670886149115333, + -88.17480572169651509, 16.00276430383186366, + -84.11088800598805904, 12.14788119395578647, + -80.08720437104132372, 8.31139512800513636, + -76.06665407661381550, 4.51214364124032485, + -72.01431282527251199, 0.76906207426874740, + -67.89683474738467339, -2.89828246890513030, + -63.68207179147749031, -6.46920580966549874, + -59.33895182482493880, -9.92146778444056032, + -54.83767345094592827, -13.23089104518995640, + -50.15029168417838434, -16.37106576006627279, + -45.25177543408179304, -19.31318879069118211, + -40.12160159745184274, -22.02610835172851722, + -34.74589265732009835, -24.47666503668304117, + -29.11998552133064422, -26.63042818683534207, + -23.25113130544624340, -28.45290874783366775, + -17.16079514785620930, -29.91127029421520689, + -10.88583393507997776, -30.97645080660140593, + -4.47781225763253765, -31.62546289221583962, + 1.99999999999999556, -38.87875527701606160, + 9.18708125479774296, -38.63683300066042392, + 16.27414365511956262, -37.91820250638208734, + 23.17259140539264095, -36.74332279213027164, + 29.81377244810832394, -35.14341202945125531, + 36.15298964123235947, -33.15698491696964822, + 42.16912769952225659, -30.82630445836842625, + 47.86123932751760179, -28.19436585459229860, + 53.24376548995357439, -25.30271165018388402, + 58.34173321358173325, -22.19009961204914561, + 63.18670666190918439, -18.89188184637555779, + 67.81377209114522486, -15.43989911427395079, + 72.25952713288313589, -11.86270778471484277, + 76.56090370117536281, -8.18599910867232694, + 80.75462189677602964, -4.43311662588922584, + 84.87709425893895343, -0.62561625366165385, + 88.96463936229967828, 3.21615784383777248, + 93.05390189983984328, 7.07248905242842518, + 97.18240447461120368, 10.92369066480776318, + 101.38917071735671982, 14.74948212031801020, + 105.71535755141637480, 18.52832281285148497, + 110.20481280215611264, 22.23668760419472790, + 114.90442707083992957, 25.84827445710250160, + 119.86406835852177721, 29.33315055785289260, + 125.13576891124590418, 32.65687547770472321, + 130.77168357131759535, 35.77969749489390949, + 136.82020109922683559, 38.65601141858346068, + 143.31958252044233859, 41.23439386179803279, + 150.28885177460347222, 43.45866798082070659, + 157.71667350566602295, 45.27051130755705799, + 165.55074086228117380, 46.61395503653132977, + 173.69217969866488716, 47.44157216781382402, + -178.00000000000002842, 47.72124472298391851, + -169.69217969866491558, 47.44157216781382402, + -161.55074086228117380, 46.61395503653132977, + -153.71667350566602295, 45.27051130755705799, + -146.28885177460344380, 43.45866798082070659, + -139.31958252044236701, 41.23439386179803279, + -132.82020109922686402, 38.65601141858346068, + -126.77168357131760956, 35.77969749489390949, + -121.13576891124588997, 32.65687547770472321, + -115.86406835852179142, 29.33315055785289260, + -110.90442707083994378, 25.84827445710250160, + -106.20481280215612685, 22.23668760419472790, + -101.71535755141638901, 18.52832281285148497, + -97.38917071735673403, 14.74948212031801020, + -93.18240447461121789, 10.92369066480776318, + -89.05390189983984328, 7.07248905242842518, + -84.96463936229969249, 3.21615784383777248, + -80.87709425893895343, -0.62561625366165385, + -76.75462189677602964, -4.43311662588922584, + -72.56090370117536281, -8.18599910867232694, + -68.25952713288315010, -11.86270778471484277, + -63.81377209114523907, -15.43989911427395079, + -59.18670666190919150, -18.89188184637555779, + -54.34173321358174746, -22.19009961204914561, + -49.24376548995358149, -25.30271165018388402, + -43.86123932751760890, -28.19436585459229860, + -38.16912769952225659, -30.82630445836842625, + -32.15298964123236658, -33.15698491696964822, + -25.81377244810833815, -35.14341202945125531, + -19.17259140539265516, -36.74332279213027164, + -12.27414365511956973, -37.91820250638208734, + -5.18708125479775184, -38.63683300066042392, + 1.99999999999999600, -46.61822072475269607, + 10.68839576675080139, -46.30624543028367412, + 19.18911080317679918, -45.38412606933155757, + 27.34546416576215222, -43.89055765093645078, + 35.05142021090807702, -41.88213387768103502, + 42.25559890256131013, -39.42480136235260346, + 48.95316063861176303, -36.58631299796493153, + 55.17233704527057370, -33.43100825355479344, + 60.96117255247643385, -30.01702215459737744, + 66.37723925505210332, -26.39536140379773599, + 71.48085419487965453, -22.61014977269736548, + 76.33122188826865795, -18.69947873486282575, + 80.98463901984250413, -14.69649678201448850, + 85.49400025392819202, -10.63053585607601548, + 89.90905356028720519, -6.52818370181667884, + 94.27704477012913742, -2.41427530523583922, + 98.64352777118510573, 1.68719008387413583, + 103.05319914949195947, 5.75218534812640314, + 107.55065416712271542, 9.75582109859903390, + 112.18096363521173942, 13.67152082594358298, + 116.98994385952536845, 17.47019805768939449, + 122.02393786261990272, 21.11944988981388605, + 127.32885205682140395, 24.58281725765701253, + 132.94811747553873715, 27.81921722445741807, + 138.91921485966395267, 30.78272820361294038, + 145.26850797118672176, 33.42299645292871446, + 152.00450116657361832, 35.68659928106228563, + 159.11038900941949237, 37.51968052984662449, + 166.53782868216146085, 38.87197455633482690, + 174.20476659278793363, 39.70189618399167131, + -178.00000000000002842, 39.98177927524729114, + -170.20476659278793363, 39.70189618399167131, + -162.53782868216148927, 38.87197455633482690, + -155.11038900941949237, 37.51968052984662449, + -148.00450116657361832, 35.68659928106228563, + -141.26850797118672176, 33.42299645292871446, + -134.91921485966395267, 30.78272820361294038, + -128.94811747553873715, 27.81921722445741807, + -123.32885205682140395, 24.58281725765701253, + -118.02393786261990272, 21.11944988981388605, + -112.98994385952538266, 17.47019805768939449, + -108.18096363521175363, 13.67152082594358298, + -103.55065416712271542, 9.75582109859903390, + -99.05319914949195947, 5.75218534812640314, + -94.64352777118511995, 1.68719008387413583, + -90.27704477012913742, -2.41427530523583922, + -85.90905356028720519, -6.52818370181667884, + -81.49400025392820623, -10.63053585607601548, + -76.98463901984250413, -14.69649678201448850, + -72.33122188826867216, -18.69947873486282575, + -67.48085419487966874, -22.61014977269736548, + -62.37723925505212463, -26.39536140379773599, + -56.96117255247644096, -30.01702215459737744, + -51.17233704527057370, -33.43100825355479344, + -44.95316063861176303, -36.58631299796493153, + -38.25559890256131723, -39.42480136235260346, + -31.05142021090808413, -41.88213387768103502, + -23.34546416576216643, -43.89055765093645078, + -15.18911080317680451, -45.38412606933155757, + -6.68839576675080849, -46.30624543028367412, + 1.99999999999999556, -55.13279625041899834, + 12.21451927262477000, -54.76594013073760436, + 22.09872704828332957, -53.68864395349499574, + 31.39769711559288012, -51.96422430511800883, + 39.96960367385942448, -49.68040408706254141, + 47.77902429867568657, -46.93247902115530223, + 54.86545369174167064, -43.81109166541020983, + 61.30916699148811233, -40.39598469592146301, + 67.20576633421555357, -36.75430974219705860, + 72.65107920217620574, -32.94148403257256064, + 77.73388667252179118, -29.00309226896920123, + 82.53343020155088539, -24.97699141478170048, + 87.11942634360052296, -20.89524495725207132, + 91.55320457280680557, -16.78577161643180204, + 95.88921878897831164, -12.67371220268469578, + 100.17656220612792595, -8.58256192749418645, + 104.46031614728221371, -4.53512373056416251, + 108.78265670493264849, -0.55433148808729282, + 113.18367588372132104, 3.33602075627005812, + 117.70187269530691765, 7.11062322287716331, + 122.37425008640165913, 10.74206713847307348, + 127.23592626204656142, 14.20035582641498806, + 132.31914580979304219, 17.45256126555711873, + 137.65157520973264127, 20.46271167815037728, + 143.25381666270027381, 23.19203115841211726, + 149.13620894054645305, 25.59967735271198208, + 155.29523174265207786, 27.64411687523928052, + 161.71017626763577368, 29.28521267182223298, + 168.34108626834759548, 30.48695101230148552, + 175.12910064316457692, 31.22051723140710777, + -178.00000000000002842, 31.46720374958099598, + -171.12910064316457692, 31.22051723140710777, + -164.34108626834762390, 30.48695101230148552, + -157.71017626763574526, 29.28521267182223298, + -151.29523174265207786, 27.64411687523928052, + -145.13620894054648147, 25.59967735271198208, + -139.25381666270027381, 23.19203115841211726, + -133.65157520973266969, 20.46271167815037728, + -128.31914580979304219, 17.45256126555711873, + -123.23592626204654721, 14.20035582641498806, + -118.37425008640165913, 10.74206713847307348, + -113.70187269530693186, 7.11062322287716331, + -109.18367588372132104, 3.33602075627005812, + -104.78265670493266271, -0.55433148808729282, + -100.46031614728222792, -4.53512373056416251, + -96.17656220612794016, -8.58256192749418645, + -91.88921878897831164, -12.67371220268469578, + -87.55320457280681978, -16.78577161643180204, + -83.11942634360052296, -20.89524495725207132, + -78.53343020155088539, -24.97699141478170048, + -73.73388667252179118, -29.00309226896920123, + -68.65107920217620574, -32.94148403257256064, + -63.20576633421556068, -36.75430974219705860, + -57.30916699148811233, -40.39598469592146301, + -50.86545369174167774, -43.81109166541020983, + -43.77902429867569367, -46.93247902115530223, + -35.96960367385943158, -49.68040408706254141, + -27.39769711559288723, -51.96422430511800883, + -18.09872704828334022, -53.68864395349499574, + -8.21451927262477888, -54.76594013073760436, + 1.99999999999999556, -64.48162218246979194, + 17.98037118830460201, -63.76341508305665684, + 32.62428016143745424, -61.72174914658255318, + 45.26325159838053480, -58.62403790832021144, + 55.90710322947827393, -54.76300160634040992, + 64.89227178252910733, -50.38082342875218700, + 72.61131289029847835, -45.65520975111467550, + 79.40370496549049051, -40.71192873614232610, + 85.53547443418635510, -35.64102404029981841, + 91.20945067766287195, -30.50961194071436822, + 96.58116190590757810, -25.37072818781791383, + 101.77265218881962028, -20.26925377022463337, + 106.88281945262124850, -15.24594064653214254, + 111.99465974304708027, -10.34025377357217934, + 117.18006743498143862, -5.59247248797413388, + 122.50268451884780063, -1.04529436283411115, + 128.01906809075350679, 3.25496397729521814, + 133.77826339847268855, 7.25759760165333567, + 139.81978239976487544, 10.90731557455581502, + 146.17005258660478262, 14.14485028302005709, + 152.83768595785250000, 16.90865838596959492, + 159.80845268931921055, 19.13794220572577487, + 167.04151372648448159, 20.77699151155475477, + 174.46891148411387462, 21.78042227826199806, + -178.00000000000002842, 22.11837781753019527, + -170.46891148411387462, 21.78042227826199806, + -163.04151372648448159, 20.77699151155475477, + -155.80845268931921055, 19.13794220572577487, + -148.83768595785252842, 16.90865838596959492, + -142.17005258660481104, 14.14485028302005709, + -135.81978239976487544, 10.90731557455581502, + -129.77826339847271697, 7.25759760165333567, + -124.01906809075352101, 3.25496397729521814, + -118.50268451884778642, -1.04529436283411115, + -113.18006743498142441, -5.59247248797413388, + -107.99465974304709448, -10.34025377357217934, + -102.88281945262124850, -15.24594064653214254, + -97.77265218881962028, -20.26925377022463337, + -92.58116190590757810, -25.37072818781791383, + -87.20945067766288616, -30.50961194071436822, + -81.53547443418635510, -35.64102404029981841, + -75.40370496549049051, -40.71192873614232610, + -68.61131289029849256, -45.65520975111467550, + -60.89227178252910733, -50.38082342875218700, + -51.90710322947828104, -54.76300160634040992, + -41.26325159838054901, -58.62403790832021144, + -28.62428016143746490, -61.72174914658255318, + -13.98037118830460557, -63.76341508305665684, + 1.99999999999999489, -74.69966643845165777, + 26.73698176730493614, -73.50791820369254026, + 46.45208209886830275, -70.36956689494368788, + 60.80977159628141493, -66.05660901455554779, + 71.45848324441054444, -61.11675773175377913, + 79.80759217874309286, -55.85321872580696834, + 86.74466514239207982, -50.43213273346431436, + 92.80540124701705906, -44.95206516381727369, + 98.32251211109890221, -39.47845092120540045, + 103.51336526710929320, -34.06038011063251503, + 108.52834753231333309, -28.73921107936183361, + 113.47768355265095863, -23.55333544230752452, + 118.44656525084906207, -18.54103902309049090, + 123.50369215326308847, -13.74234202233444968, + 128.70582948829564884, -9.20020460514958671, + 134.09971622507504208, -4.96122571165357140, + 139.72202213508867885, -1.07580930674149267, + 145.59779383610683112, 2.40232548413760538, + 151.73783261271663036, 5.41746050329454931, + 158.13565186637288207, 7.91416414131925894, + 164.76497871932286898, 9.84055665005913660, + 171.57900980622665088, 11.15231427016150256, + 178.51254101195692670, 11.81685015330946520, + -174.51254101195692670, 11.81685015330946520, + -167.57900980622667930, 11.15231427016150256, + -160.76497871932289740, 9.84055665005913660, + -154.13565186637291049, 7.91416414131925894, + -147.73783261271665879, 5.41746050329454931, + -141.59779383610683112, 2.40232548413760538, + -135.72202213508870727, -1.07580930674149267, + -130.09971622507501365, -4.96122571165357140, + -124.70582948829567727, -9.20020460514958671, + -119.50369215326308847, -13.74234202233444968, + -114.44656525084907628, -18.54103902309049090, + -109.47768355265095863, -23.55333544230752452, + -104.52834753231334730, -28.73921107936183361, + -99.51336526710929320, -34.06038011063251503, + -94.32251211109890221, -39.47845092120540045, + -88.80540124701705906, -44.95206516381727369, + -82.74466514239209403, -50.43213273346431436, + -75.80759217874310707, -55.85321872580696834, + -67.45848324441054444, -61.11675773175377913, + -56.80977159628143625, -66.05660901455554779, + -42.45208209886832407, -70.36956689494368788, + -22.73698176730495391, -73.50791820369254026, + 1.99999999999999600, -85.78166687108885924, + 61.87324357773108119, -82.33459525449173100, + 80.27768438369075454, -76.54083121758115738, + 89.42499621356618889, -70.41813086842684299, + 95.82057104813959825, -64.24948773689966686, + 101.11263541171206271, -58.11928872485961506, + 105.90317607822892398, -52.07335345715245722, + 110.47134737780878311, -46.14758359783259323, + 114.97138028465980142, -40.37601690822956613, + 119.50120511177952665, -34.79395188944583595, + 124.13018318901127657, -29.43951176908901957, + 128.91129483912590104, -24.35455549606792047, + 133.88641650799124250, -19.58517424494497305, + 139.08801440266989857, -15.18177122287804615, + 144.53869867413172301, -11.19861017149900739, + 150.24942513902280439, -7.69266569608215622, + 156.21700460768090579, -4.72161659803606781, + 162.42170502724644621, -2.34091526057735644, + 168.82591385569978115, -0.60006016949341945, + 175.37485237931247184, 0.46152536384328491, + -178.00000000000002842, 0.81833312891115950, + -171.37485237931247184, 0.46152536384328491, + -164.82591385569978115, -0.60006016949341945, + -158.42170502724641779, -2.34091526057735644, + -152.21700460768090579, -4.72161659803606781, + -146.24942513902283281, -7.69266569608215622, + -140.53869867413175143, -11.19861017149900739, + -135.08801440266989857, -15.18177122287804615, + -129.88641650799127092, -19.58517424494497305, + -124.91129483912591525, -24.35455549606792047, + -120.13018318901130499, -29.43951176908901957, + -115.50120511177955507, -34.79395188944583595, + -110.97138028465980142, -40.37601690822956613, + -106.47134737780881153, -46.14758359783259323, + -101.90317607822893820, -52.07335345715245722, + -97.11263541171207692, -58.11928872485961506, + -91.82057104813961246, -64.24948773689966686, + -85.42499621356618889, -70.41813086842684299, + -76.27768438369074033, -76.54083121758115738, + -57.87324357773110250, -82.33459525449173100, + -178.00000000000002842, -82.33591847955618448, + 143.22778135121401988, -79.54230031515010069, + 128.60898756414843547, -73.87349687490876704, + 124.32470998353319658, -67.47696540683871547, + 124.01228809564837263, -60.93070105520797597, + 125.59604777627561134, -54.43751136022503090, + 128.28024682035200499, -48.10755262562803125, + 131.71480673751509016, -42.02394287044850074, + 135.73150447900647464, -36.26259289105208694, + 140.24424010430539056, -30.89942669541638764, + 145.20455296082769792, -26.01294410398721979, + 150.57821873782722832, -21.68429473238734317, + 156.33085313434582986, -17.99542592709287092, + 162.41822389716838870, -15.02545489467462048, + 168.78020742076520833, -12.84550987892848184, + 175.33870321353637678, -11.51266748991093536, + -178.00000000000002842, -11.06408152044378745, + -171.33870321353637678, -11.51266748991093536, + -164.78020742076523675, -12.84550987892848184, + -158.41822389716841712, -15.02545489467462048, + -152.33085313434585828, -17.99542592709287092, + -146.57821873782722832, -21.68429473238734317, + -141.20455296082769792, -26.01294410398721979, + -136.24424010430539056, -30.89942669541638764, + -131.73150447900647464, -36.26259289105208694, + -127.71480673751507595, -42.02394287044850074, + -124.28024682035203341, -48.10755262562803125, + -121.59604777627559713, -54.43751136022503090, + -120.01228809564838684, -60.93070105520797597, + -120.32470998353322500, -67.47696540683871547, + -124.60898756414846389, -73.87349687490876704, + -139.22778135121404830, -79.54230031515010069, + -178.00000000000002842, -69.79343463141964321, + 167.63134400963940607, -68.62304968612301082, + 156.89871695437406629, -65.48334135458465255, + 150.54736374030309776, -61.10651485448561715, + 147.65287319536125210, -56.10790502157546911, + 147.17953110490091717, -50.89570090116846757, + 148.39568884343808008, -45.73876083396746139, + 150.83543074335267420, -40.82969132012996027, + 154.20166627754437627, -36.32066069380964279, + 158.29374271187359113, -32.34078423455542861, + 162.96127389198957758, -29.00296899656860816, + 168.07572003197756771, -26.40464312032804983, + 173.51374580520655400, -24.62495078480371546, + 179.14909640850049755, -23.72034636702055366, + -175.14909640850046912, -23.72034636702055366, + -169.51374580520655400, -24.62495078480371546, + -164.07572003197756771, -26.40464312032806049, + -158.96127389198957758, -29.00296899656861527, + -154.29374271187361956, -32.34078423455542861, + -150.20166627754440469, -36.32066069380964279, + -146.83543074335270262, -40.82969132012996027, + -144.39568884343808008, -45.73876083396746139, + -143.17953110490091717, -50.89570090116849599, + -143.65287319536128052, -56.10790502157547621, + -146.54736374030309776, -61.10651485448561715, + -152.89871695437406629, -65.48334135458465255, + -163.63134400963940607, -68.62304968612301082, + -178.00000000000002842, -56.85258712078380228, + 176.37525674045562596, -56.23764348052387874, + 171.72066876086481102, -54.50623527875517738, + 168.62326642581021474, -51.94637057143930292, + 167.22080918295719698, -48.91487419033276751, + 167.36621335583259906, -45.75617589753742465, + 168.79950790329058918, -42.76713193279059766, + 171.24101930549505823, -40.18968456371464271, + 174.42266827604018431, -38.21219158951615213, + 178.09065200901943626, -36.97055608025468132, + -178.00000000000002842, -36.54741287921620341, + -174.09065200901946469, -36.97055608025468132, + -170.42266827604018431, -38.21219158951615213, + -167.24101930549508666, -40.18968456371464271, + -164.79950790329061761, -42.76713193279059766, + -163.36621335583262749, -45.75617589753742465, + -163.22080918295719698, -48.91487419033276751, + -164.62326642581024316, -51.94637057143930292, + -167.72066876086483944, -54.50623527875517738, + -172.37525674045562596, -56.23764348052387874 }; }; namespace atlas { @@ -874,8 +1717,8 @@ CASE("t31c2.4") { for (int i = 0; i < grid.nx(j); i++, jglo++) { auto ll1 = PointLonLat(lonlat_arp_t32c24[2 * jglo + 0], lonlat_arp_t32c24[2 * jglo + 1]); auto ll2 = grid.lonlat(i, j); - EXPECT_APPROX_EQ(ll1.lon(), ll2.lon(), 1.e-10); - EXPECT_APPROX_EQ(ll1.lat(), ll2.lat(), 1.e-10); + EXPECT_APPROX_EQ(ll1.lon(), ll2.lon(), 1.e-9); + EXPECT_APPROX_EQ(ll1.lat(), ll2.lat(), 1.e-9); } } } diff --git a/src/tests/interpolation/CMakeLists.txt b/src/tests/interpolation/CMakeLists.txt index 29d220590..a5ac0e86a 100644 --- a/src/tests/interpolation/CMakeLists.txt +++ b/src/tests/interpolation/CMakeLists.txt @@ -75,6 +75,14 @@ ecbuild_add_executable( TARGET atlas_test_interpolation_structured2D NOINSTALL ) +ecbuild_add_test( TARGET atlas_test_interpolation_structured2D_regional + SOURCES test_interpolation_structured2D_regional.cc + LIBS atlas + MPI 2 + CONDITION eckit_HAVE_MPI + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + ecbuild_add_test( TARGET atlas_test_interpolation_non_linear SOURCES test_interpolation_non_linear.cc LIBS atlas @@ -104,7 +112,6 @@ ecbuild_add_test( TARGET atlas_test_interpolation_structured2D_to_unstructured ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) - ecbuild_add_test( TARGET atlas_test_interpolation_structured2D_to_points SOURCES test_interpolation_structured2D_to_points.cc LIBS atlas @@ -113,7 +120,6 @@ ecbuild_add_test( TARGET atlas_test_interpolation_structured2D_to_points ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) - ecbuild_add_test( TARGET atlas_test_interpolation_cubedsphere SOURCES test_interpolation_cubedsphere.cc LIBS atlas @@ -123,11 +129,19 @@ ecbuild_add_test( TARGET atlas_test_interpolation_cubedsphere ) ecbuild_add_test( TARGET atlas_test_interpolation_spherical_vector - OMP 4 SOURCES test_interpolation_spherical_vector.cc LIBS atlas - ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} + OMP 4 CONDITION atlas_HAVE_EIGEN + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + +ecbuild_add_test( TARGET atlas_test_interpolation_binning + SOURCES test_interpolation_binning.cc + LIBS atlas + MPI 6 + CONDITION eckit_HAVE_MPI AND MPI_SLOTS GREATER_EQUAL 6 + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) endif() diff --git a/src/tests/interpolation/test_interpolation_binning.cc b/src/tests/interpolation/test_interpolation_binning.cc new file mode 100644 index 000000000..bd35b20f9 --- /dev/null +++ b/src/tests/interpolation/test_interpolation_binning.cc @@ -0,0 +1,289 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + + +#include + +#include "atlas/array.h" +#include "atlas/field.h" +#include "atlas/field/FieldSet.h" +#include "atlas/functionspace/CubedSphereColumns.h" +#include "atlas/functionspace/NodeColumns.h" +#include "atlas/functionspace/StructuredColumns.h" +#include "atlas/grid.h" +#include "atlas/interpolation.h" +#include "atlas/mesh.h" +#include "atlas/meshgenerator/MeshGenerator.h" +#include "atlas/option/Options.h" +#include "atlas/output/Gmsh.h" +#include "atlas/runtime/Log.h" +#include "atlas/util/Config.h" +#include "atlas/util/CoordinateEnums.h" +#include "atlas/util/function/VortexRollup.h" + +#include "tests/AtlasTestEnvironment.h" + + +namespace atlas { +namespace test { + + +/// function to generate a synthetic field (vortex field) +Field getVortexField( + const functionspace::NodeColumns& csfs, + const std::string& vfield_name, + const size_t nb_levels) { + const auto lonlat = array::make_view(csfs.lonlat()); + + auto vfield = csfs.createField(option::name(vfield_name) | + option::levels(nb_levels)); + auto vfield_view = array::make_view(vfield); + double mean_field = 1.; + for (idx_t l=0; l < vfield.shape(1); ++l) { + for (idx_t i=0; i < vfield.shape(0); ++i) { + vfield_view(i, l) = mean_field + util::function::vortex_rollup( + lonlat(i, atlas::LON), lonlat(i, atlas::LAT), 1.); + } + } + + return vfield; +} + + +/// function to write a field set in a Gmsh file +void makeGmshOutput(const std::string& file_name, + const Mesh& mesh, + const FieldSet& fields) { + const auto& fs = fields[0].functionspace(); + + const auto config_gmsh = + util::Config("coordinates", "xyz") | + util::Config("ghost", true) | + util::Config("info", true); + const auto gmsh = output::Gmsh(file_name, config_gmsh); + gmsh.write(mesh); + gmsh.write(fields, fs); +} + + +/// function to carry out a dot product +double dotProd(const Field& field01, const Field& field02) { + double dprod{}; + + const auto field01_view = array::make_view(field01); + const auto field02_view = array::make_view(field02); + + for (idx_t l=0; l < field01_view.shape(1); ++l) { + for (idx_t i=0; i < field01_view.shape(0); ++i) { + dprod += field01_view(i, l) * field02_view(i, l); + } + } + eckit::mpi::comm().allReduceInPlace(dprod, eckit::mpi::Operation::SUM); + + return dprod; +} + + +//-- + + +/// test to carry out the rigridding from 'high' to 'low' resolution, +/// for a given type of grid (CS-LFR) +/// +/// after the regridding, the field set that has been generated is stored +/// in a Gmsh file (data visualization); +/// +CASE("rigridding from high to low resolution; grid type: CS-LFR") { + + // source grid (high res.) + auto csgrid_s = Grid("CS-LFR-100"); + auto csmesh_s = MeshGenerator("cubedsphere_dual").generate(csgrid_s); + auto csfs_s = functionspace::NodeColumns(csmesh_s); + + // target grid (low res.) + auto csgrid_t = Grid("CS-LFR-50"); + auto csmesh_t = MeshGenerator("cubedsphere_dual").generate(csgrid_t); + auto csfs_t = functionspace::NodeColumns(csmesh_t); + + size_t nb_levels = 1; + + auto field_01_s = getVortexField(csfs_s, "field_01_s", nb_levels); + + FieldSet fs_s; + fs_s.add(field_01_s); + + auto field_01_t = csfs_t.createField(option::name("field_01_t") | + option::levels(nb_levels)); + + FieldSet fs_t; + fs_t.add(field_01_t); + + + const auto scheme = util::Config("type", "binning") | + util::Config("scheme", option::type("cubedsphere-bilinear")); + + Interpolation regrid_high2low(scheme, csfs_s, csfs_t); + + // performing the regridding from high to low resolution + regrid_high2low.execute(fs_s, fs_t); + + fs_t["field_01_t"].haloExchange(); + + + //-- + + const std::string fname_s("regridding_h2l_cs_s.msh"); + + makeGmshOutput(fname_s, csmesh_s, fs_s["field_01_s"]); + + const std::string fname_t("regridding_h2l_cs_t.msh"); + + makeGmshOutput(fname_t, csmesh_t, fs_t["field_01_t"]); +} + + +/// test to carry out the rigridding from 'high' to 'low' resolution, +/// for a given type of grid (O) +/// +/// after the regridding, the field set that has been generated is stored +/// in a Gmsh file (data visualization); +/// +CASE("rigridding from high to low resolution; grid type: O") { + + // source grid (high res.) + auto ncgrid_s = Grid("O32"); + auto ncmesh_s = MeshGenerator("structured").generate(ncgrid_s); + auto ncfs_s = functionspace::StructuredColumns(ncgrid_s, option::halo(3)); + + // target grid (low res.) + auto ncgrid_t = Grid("O16"); + auto ncmesh_t = MeshGenerator("structured").generate(ncgrid_t); + auto ncfs_t = functionspace::StructuredColumns(ncgrid_t, option::halo(3)); + + size_t nb_levels = 1; + + auto field_01_s = getVortexField(ncfs_s, "field_01_s", nb_levels); + + FieldSet fs_s; + fs_s.add(field_01_s); + + auto field_01_t = ncfs_t.createField(option::name("field_01_t") | + option::levels(nb_levels)); + + FieldSet fs_t; + fs_t.add(field_01_t); + + + const auto scheme = util::Config("type", "binning") | + util::Config("scheme", option::type("structured-bilinear")); + + Interpolation regrid_high2low(scheme, ncfs_s, ncfs_t); + + // performing the regridding from high to low resolution + regrid_high2low.execute(fs_s, fs_t); + + fs_t["field_01_t"].haloExchange(); + + + //-- + + const std::string fname_s("regridding_h2l_nc_s.msh"); + + makeGmshOutput(fname_s, ncmesh_s, fs_s["field_01_s"]); + + const std::string fname_t("regridding_h2l_nc_t.msh"); + + makeGmshOutput(fname_t, ncmesh_t, fs_t["field_01_t"]); +} + + +/// test to carry out the 'dot-product' test for the rigridding from +/// 'high' to 'low' resolution, for a given type of grid (CS-LFR) +/// +CASE("dot-product test for the rigridding from high to low resolution; grid type: CS-LFR") { + + // source grid (high res.) + auto csgrid_s = Grid("CS-LFR-100"); + auto csmesh_s = MeshGenerator("cubedsphere_dual").generate(csgrid_s); + auto csfs_s = functionspace::NodeColumns(csmesh_s); + + // target grid (low res.) + auto csgrid_t = Grid("CS-LFR-50"); + auto csmesh_t = MeshGenerator("cubedsphere_dual").generate(csgrid_t); + auto csfs_t = functionspace::NodeColumns(csmesh_t); + + size_t nb_levels = 1; + + // source field + auto field_01_s = getVortexField(csfs_s, "field_01_s", nb_levels); + + FieldSet fs_s; + fs_s.add(field_01_s); + + // target field + auto field_01_t = csfs_t.createField(option::name("field_01_t") | + option::levels(nb_levels)); + + FieldSet fs_t; + fs_t.add(field_01_t); + + const auto scheme = util::Config("type", "binning") | + util::Config("scheme", option::type("cubedsphere-bilinear")) | + util::Config("adjoint", true); + + Interpolation regrid_high2low(scheme, csfs_s, csfs_t); + + // performing the regridding from high to low resolution + regrid_high2low.execute(fs_s, fs_t); + + + fs_t["field_01_t"].haloExchange(); + + // target field (adjoint) + auto field_01_ad_t = csfs_t.createField(option::name("field_01_ad_t") | + option::levels(nb_levels)); + array::make_view(field_01_ad_t).assign( + array::make_view(field_01_t)); + field_01_ad_t.adjointHaloExchange(); + + FieldSet fs_ad_t; + fs_ad_t.add(field_01_ad_t); + + // source field (adjoint) + auto field_01_ad_s = csfs_s.createField(option::name("field_01_ad_s") | + option::levels(nb_levels)); + array::make_view(field_01_ad_s).assign(0.); + + FieldSet fs_ad_s; + fs_ad_s.add(field_01_ad_s); + + // performing adjoint operation + regrid_high2low.execute_adjoint(fs_ad_s, fs_ad_t); + + + const auto t_dot_t = dotProd(fs_t["field_01_t"], fs_t["field_01_t"]); + const auto s_dot_ad_s = dotProd(fs_s["field_01_s"], fs_ad_s["field_01_ad_s"]); + + double scaled_diff = std::abs(t_dot_t - s_dot_ad_s)/std::abs(t_dot_t); + + // carrrying out a dot-product test ... + Log::info() << "\n- dot-product test:\n" + << "(Ax) . (Ax) = " << t_dot_t << "; " + << "x . (A^t A x) = " << s_dot_ad_s << "; " + << "scaled difference = " << scaled_diff << "\n" << std::endl; + + EXPECT(scaled_diff < 1e-12); +} + + +} // namespace test +} // namespace atlas + + +//-- + +int main(int argc, char** argv) { return atlas::test::run(argc, argv); } diff --git a/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc b/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc index 0ce3cd190..641005fb6 100644 --- a/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc +++ b/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc @@ -1,9 +1,5 @@ /* -<<<<<<< HEAD - * (C) Copyright 1996- ECMWF. -======= * (C) Copyright 2013 ECMWF. ->>>>>>> develop * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. diff --git a/src/tests/interpolation/test_interpolation_spherical_vector.cc b/src/tests/interpolation/test_interpolation_spherical_vector.cc index 4717c5a19..16dbdabf9 100644 --- a/src/tests/interpolation/test_interpolation_spherical_vector.cc +++ b/src/tests/interpolation/test_interpolation_spherical_vector.cc @@ -58,10 +58,10 @@ double vortexVertical(double lon, double lat) { void gmshOutput(const std::string& fileName, const FieldSet& fieldSet) { const auto functionSpace = fieldSet[0].functionspace(); - const auto structuredColums = functionspace::StructuredColumns(functionSpace); + const auto structuredColumns = functionspace::StructuredColumns(functionSpace); const auto nodeColumns = functionspace::NodeColumns(functionSpace); const auto mesh = - structuredColums ? Mesh(structuredColums.grid()) : nodeColumns.mesh(); + structuredColumns ? Mesh(structuredColumns.grid()) : nodeColumns.mesh(); const auto gmshConfig = Config("coordinates", "xyz") | Config("ghost", true) | Config("info", true); @@ -79,20 +79,35 @@ const auto generateNodeColums(const std::string& gridName, return functionspace::NodeColumns(mesh); } +// Helper function to create part-empty PointCloud +const auto generateEmptyPointCloud() { + const auto functionSpace = functionspace::PointCloud(std::vector{}); + return functionSpace; +} + // Helper struct to key different Functionspaces to strings struct FunctionSpaceFixtures { static const FunctionSpace& get(const std::string& fixture) { - static const auto functionSpaces = + static auto functionSpaces = std::map{ {"cubedsphere_mesh", generateNodeColums("CS-LFR-48", "cubedsphere_dual")}, {"gaussian_mesh", generateNodeColums("O48", "structured")}, {"structured_columns", functionspace::StructuredColumns(Grid("O48"), option::halo(1))}, + {"structured_columns_classic", + functionspace::StructuredColumns(Grid("F48"), option::halo(1))}, + {"structured_columns_classic_halo2", + functionspace::StructuredColumns(Grid("F48"), option::halo(2))}, + {"structured_columns_classic_highres_halo2", + functionspace::StructuredColumns(Grid("F96"), option::halo(2))}, + {"structured_columns_halo2", + functionspace::StructuredColumns(Grid("O48"), option::halo(2))}, {"structured_columns_lowres", functionspace::StructuredColumns(Grid("O24"), option::halo(1))}, {"structured_columns_hires", - functionspace::StructuredColumns(Grid("O96"), option::halo(1))}}; + functionspace::StructuredColumns(Grid("O96"), option::halo(1))}, + {"empty_point_cloud", generateEmptyPointCloud()}}; return functionSpaces.at(fixture); } }; @@ -109,15 +124,19 @@ struct FieldSpecFixtures { } }; -// Helper stcut to key different interpolation schemes to strings +// Helper struct to key different interpolation schemes to strings struct InterpSchemeFixtures { static const Config& get(const std::string& fixture) { static const auto cubedsphereBilinear = - option::type("cubedsphere-bilinear"); - static const auto finiteElement = option::type("finite-element"); - static const auto structuredLinear = - option::type("structured-linear2D") | option::halo(1); - + option::type("cubedsphere-bilinear") | Config("adjoint", true); + static const auto finiteElement = + option::type("finite-element") | Config("adjoint", true); + static const auto structuredLinear = option::type("structured-linear2D") | + option::halo(1) | + Config("adjoint", true); + static const auto structuredCubic = option::type("structured-bicubic") | + option::halo(2) | + Config("adjoint", true); static const auto sphericalVector = option::type("spherical-vector") | Config("adjoint", true); @@ -125,12 +144,15 @@ struct InterpSchemeFixtures { {"cubedsphere_bilinear", cubedsphereBilinear}, {"finite_element", finiteElement}, {"structured_linear", structuredLinear}, + {"structured_cubic", structuredCubic}, {"cubedsphere_bilinear_spherical", sphericalVector | Config("scheme", cubedsphereBilinear)}, {"finite_element_spherical", sphericalVector | Config("scheme", finiteElement)}, {"structured_linear_spherical", - sphericalVector | Config("scheme", structuredLinear)}}; + sphericalVector | Config("scheme", structuredLinear)}, + {"structured_cubic_spherical", + sphericalVector | Config("scheme", structuredCubic)}}; return interpSchemes.at(fixture); } }; @@ -218,40 +240,45 @@ void testInterpolation(const Config& config) { targetFunctionSpace.createField(errorFieldSpec))); errorView.assign(0.); - auto maxError = 0.; - ArrayForEach<0>::apply( - std::tie(targetLonLat, targetView, errorView), - [&](auto&& lonLat, auto&& targetColumn, auto&& errorColumn) { - const auto calcError = [&](auto&& targetElem, auto&& errorElem) { - auto trueValue = std::vector(targetElem.size()); - std::tie(trueValue[0], trueValue[1]) = - vortexHorizontal(lonLat(0), lonLat(1)); - if (targetElem.size() == 3) { - trueValue[2] = vortexVertical(lonLat(0), lonLat(1)); + if (config.has("tol")) { + auto maxError = 0.; + ArrayForEach<0>::apply( + std::tie(targetLonLat, targetView, errorView), + [&](auto&& lonLat, auto&& targetColumn, auto&& errorColumn) { + const auto calcError = [&](auto&& targetElem, auto&& errorElem) { + auto trueValue = std::vector(targetElem.size()); + std::tie(trueValue[0], trueValue[1]) = + vortexHorizontal(lonLat(0), lonLat(1)); + if (targetElem.size() == 3) { + trueValue[2] = vortexVertical(lonLat(0), lonLat(1)); + } + + auto errorSqrd = 0.; + for (auto k = 0; k < targetElem.size(); ++k) { + errorSqrd += (targetElem(k) - trueValue[k]) * + (targetElem(k) - trueValue[k]); + } + + errorElem = std::sqrt(errorSqrd); + maxError = std::max(maxError, static_cast(errorElem)); + }; + + if constexpr (Rank == 2) { + calcError(targetColumn, errorColumn); } - - auto errorSqrd = 0.; - for (auto k = 0; k < targetElem.size(); ++k) { - errorSqrd += - (targetElem(k) - trueValue[k]) * (targetElem(k) - trueValue[k]); + else if constexpr (Rank == 3) { + ArrayForEach<0>::apply(std::tie(targetColumn, errorColumn), + calcError); } + }); - errorElem = std::sqrt(errorSqrd); - maxError = std::max(maxError, static_cast(errorElem)); - }; - - if constexpr (Rank == 2) { - calcError(targetColumn, errorColumn); - } else if constexpr (Rank == 3) { - ArrayForEach<0>::apply(std::tie(targetColumn, errorColumn), - calcError); - } - }); - - EXPECT_APPROX_EQ(maxError, 0., config.getDouble("tol")); + EXPECT_APPROX_EQ(maxError, 0., config.getDouble("tol")); + } - gmshOutput(config.getString("file_id") + "_source.msh", sourceFieldSet); - gmshOutput(config.getString("file_id") + "_target.msh", targetFieldSet); + if (config.has("file_id")) { + gmshOutput(config.getString("file_id") + "_source.msh", sourceFieldSet); + gmshOutput(config.getString("file_id") + "_target.msh", targetFieldSet); + } // Adjoint test auto targetAdjoint = targetFunctionSpace.createField(fieldSpec); @@ -275,11 +302,15 @@ void testInterpolation(const Config& config) { constexpr auto tinyNum = 1e-13; const auto targetDotTarget = dotProduct(targetView, targetView); const auto sourceDotSourceAdjoint = dotProduct(sourceView, sourceAdjointView); - const auto dotProdRatio = targetDotTarget / sourceDotSourceAdjoint; - EXPECT_APPROX_EQ(dotProdRatio, 1., tinyNum); + + if (targetFunctionSpace.size() > 0) { + const auto dotProdRatio = targetDotTarget / sourceDotSourceAdjoint; + EXPECT_APPROX_EQ(dotProdRatio, 1., tinyNum); + } } -CASE("cubed sphere vector interpolation (3d-field, 2-vector)") { + +CASE("cubed sphere CS-LFR-48 vector interpolation (3d-field, 2-vector)") { const auto config = Config("source_fixture", "cubedsphere_mesh") .set("target_fixture", "gaussian_mesh") @@ -291,7 +322,7 @@ CASE("cubed sphere vector interpolation (3d-field, 2-vector)") { testInterpolation((config)); } -CASE("cubed sphere vector interpolation (3d-field, 3-vector)") { +CASE("cubed sphere CS-LFR-48 vector interpolation (3d-field, 3-vector)") { const auto config = Config("source_fixture", "cubedsphere_mesh") .set("target_fixture", "gaussian_mesh") @@ -303,6 +334,36 @@ CASE("cubed sphere vector interpolation (3d-field, 3-vector)") { testInterpolation((config)); } +CASE("cubed sphere CS-LFR-48 (spherical vector) to empty point cloud") { + const auto config = + Config("source_fixture", "cubedsphere_mesh") + .set("target_fixture", "empty_point_cloud") + .set("field_spec_fixture", "2vector") + .set("interp_fixture", "cubedsphere_bilinear_spherical"); + + testInterpolation((config)); +} + +CASE("cubed sphere CS-LFR-48 to empty point cloud") { + const auto config = + Config("source_fixture", "cubedsphere_mesh") + .set("target_fixture", "empty_point_cloud") + .set("field_spec_fixture", "2vector") + .set("interp_fixture", "cubedsphere_bilinear"); + + testInterpolation((config)); +} + + +CASE("finite element to empty point cloud") { + const auto config = Config("source_fixture", "gaussian_mesh") + .set("target_fixture", "cubedsphere_mesh") + .set("field_spec_fixture", "2vector") + .set("interp_fixture", "finite_element"); + + testInterpolation((config)); +} + CASE("finite element vector interpolation (2d-field, 2-vector)") { const auto config = Config("source_fixture", "gaussian_mesh") .set("target_fixture", "cubedsphere_mesh") @@ -314,7 +375,66 @@ CASE("finite element vector interpolation (2d-field, 2-vector)") { testInterpolation((config)); } -CASE("structured columns vector interpolation (2d-field, 2-vector)") { +CASE("structured columns F48 cubic vector spherical interpolation (3d-field, 2-vector)") { + const auto config = + Config("source_fixture", "structured_columns_classic_halo2") + .set("target_fixture", "cubedsphere_mesh") + .set("field_spec_fixture", "2vector") + .set("interp_fixture", "structured_cubic_spherical") + .set("file_id", "spherical_cubic_vector_classic_sc") + .set("tol", 0.0000082); + + testInterpolation((config)); +} + +CASE("structured columns F96 cubic vector spherical interpolation (2d-field, 2-vector)") { + const auto config = + Config("source_fixture", "structured_columns_classic_highres_halo2") + .set("target_fixture", "cubedsphere_mesh") + .set("field_spec_fixture", "2vector") + .set("interp_fixture", "structured_cubic_spherical") + .set("file_id", "spherical_2D_cubic_vector_highres_classic_sc") + .set("tol", 0.000000425); + + testInterpolation((config)); +} + +CASE("structured columns F96 cubic vector spherical interpolation (3d-field, 2-vector)") { + const auto config = + Config("source_fixture", "structured_columns_classic_highres_halo2") + .set("target_fixture", "cubedsphere_mesh") + .set("field_spec_fixture", "2vector") + .set("interp_fixture", "structured_cubic_spherical") + .set("file_id", "spherical_3D_cubic_vector_highres_classic_sc") + .set("tol", 0.00000085); + + testInterpolation((config)); +} + +CASE("structured columns O24 linear vector interpolation (2d-field, 2-vector)") { + const auto config = Config("source_fixture", "structured_columns_lowres") + .set("target_fixture", "gaussian_mesh") + .set("field_spec_fixture", "2vector") + .set("interp_fixture", "structured_linear_spherical") + .set("file_id", "spherical_vector_sc_lr") + .set("tol", 0.00056); + + testInterpolation((config)); +} + +CASE("structured columns O48 cubic vector spherical interpolation (3d-field, 2-vector)") { + const auto config = + Config("source_fixture", "structured_columns_halo2") + .set("target_fixture", "cubedsphere_mesh") + .set("field_spec_fixture", "2vector") + .set("interp_fixture", "structured_cubic_spherical") + .set("file_id", "spherical_cubic_vector_sc3") + .set("tol", 0.000007); + + testInterpolation((config)); +} + +CASE("structured columns O48 linear vector interpolation (2d-field, 2-vector)") { const auto config = Config("source_fixture", "structured_columns") .set("target_fixture", "cubedsphere_mesh") .set("field_spec_fixture", "2vector") @@ -325,18 +445,16 @@ CASE("structured columns vector interpolation (2d-field, 2-vector)") { testInterpolation((config)); } -CASE("structured columns vector interpolation (2d-field, 2-vector, low-res)") { - const auto config = Config("source_fixture", "structured_columns_lowres") - .set("target_fixture", "gaussian_mesh") +CASE("structured columns O48 to empty point cloud") { + const auto config = Config("source_fixture", "structured_columns") + .set("target_fixture", "empty_point_cloud") .set("field_spec_fixture", "2vector") - .set("interp_fixture", "structured_linear_spherical") - .set("file_id", "spherical_vector_sc_lr") - .set("tol", 0.00056); + .set("interp_fixture", "structured_linear"); testInterpolation((config)); } -CASE("structured columns vector interpolation (2d-field, 2-vector, hi-res)") { +CASE("structured columns O96 vector interpolation (2d-field, 2-vector, hi-res)") { const auto config = Config("source_fixture", "structured_columns_hires") .set("target_fixture", "gaussian_mesh") .set("field_spec_fixture", "2vector") diff --git a/src/tests/interpolation/test_interpolation_structured2D_regional.cc b/src/tests/interpolation/test_interpolation_structured2D_regional.cc new file mode 100644 index 000000000..8bba8692e --- /dev/null +++ b/src/tests/interpolation/test_interpolation_structured2D_regional.cc @@ -0,0 +1,217 @@ +/* + * (C) Copyright 2024 Meteorologisk Institutt + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "atlas/array.h" +#include "atlas/field.h" +#include "atlas/functionspace/StructuredColumns.h" +#include "atlas/grid.h" +#include "atlas/interpolation.h" +#include "atlas/option.h" +#include "atlas/util/Config.h" + +#include "tests/AtlasTestEnvironment.h" + +using atlas::functionspace::StructuredColumns; +using atlas::util::Config; + +namespace atlas { +namespace test { + +//----------------------------------------------------------------------------- + +static Config gridConfigs() { + Config gridConfigs; + + Config projectionConfig; + projectionConfig.set("type", "lambert_conformal_conic"); + projectionConfig.set("latitude0", 56.3); + projectionConfig.set("longitude0", 0.0); + + Config gridConfig; + gridConfig.set("type", "regional"); + std::vector lonlat = {9.9, 56.3}; + gridConfig.set("lonlat(centre)", lonlat); + gridConfig.set("projection", projectionConfig); + + const size_t sourceNx = 21; + const size_t sourceNy = 31; + const double sourceDx = 8.0e3; + const double sourceDy = 9.0e3; + + const size_t xFactor = 4; + const size_t yFactor = 3; + + const size_t targetNx = (sourceNx-1)*xFactor+1; + const size_t targetNy = (sourceNy-1)*yFactor+1; + const double targetDx = static_cast(sourceNx-1)/static_cast(targetNx-1)*sourceDx; + const double targetDy = static_cast(sourceNy-1)/static_cast(targetNy-1)*sourceDy; + + gridConfig.set("nx", sourceNx); + gridConfig.set("ny", sourceNy); + gridConfig.set("dx", sourceDx); + gridConfig.set("dy", sourceDy); + gridConfigs.set("source", gridConfig); + gridConfigs.set("source normalization", (sourceNx-1)*(sourceNy-1)); + + gridConfig.set("nx", targetNx); + gridConfig.set("ny", targetNy); + gridConfig.set("dx", targetDx); + gridConfig.set("dy", targetDy); + gridConfigs.set("target", gridConfig); + gridConfigs.set("target normalization", (targetNx-1)*(targetNy-1)); + + return gridConfigs; +} + +// Dot product +double dotProd(const Field& field01, const Field& field02) { + double dprod{}; + + const size_t ndim = field01.levels() > 0 ? 2 : 1; + if (ndim == 1) { + const auto field01_view = array::make_view(field01); + const auto field02_view = array::make_view(field02); + for (idx_t i=0; i < field01_view.shape(0); ++i) { + dprod += field01_view(i) * field02_view(i); + } + } else if (ndim == 2) { + const auto field01_view = array::make_view(field01); + const auto field02_view = array::make_view(field02); + for (idx_t l=0; l < field01_view.shape(1); ++l) { + for (idx_t i=0; i < field01_view.shape(0); ++i) { + dprod += field01_view(i, l) * field02_view(i, l); + } + } + } + eckit::mpi::comm().allReduceInPlace(dprod, eckit::mpi::Operation::SUM); + + return dprod; +} + + +CASE("test_interpolation_structured2D_regional_1d") { + Grid sourceGrid(gridConfigs().getSubConfiguration("source")); + Grid targetGrid(gridConfigs().getSubConfiguration("target")); + + StructuredColumns sourceFs(sourceGrid, option::halo(1)); + StructuredColumns targetFs(targetGrid, option::halo(1)); + + Interpolation interpolation(Config("type", "regional-linear-2d"), sourceFs, targetFs); + + auto sourceField = sourceFs.createField(Config("name", "source")); + auto targetField = targetFs.createField(Config("name", "target")); + + // Accuracy test + const auto sourceIView = array::make_view(sourceFs.index_i()); + const auto sourceJView = array::make_view(sourceFs.index_j()); + auto sourceView = array::make_view(sourceField); + const auto sourceGhostView = atlas::array::make_view(sourceFs.ghost()); + sourceView.assign(0.0); + for (idx_t i = 0; i < sourceFs.size(); ++i) { + if (sourceGhostView(i) == 0) { + sourceView(i) = static_cast((sourceIView(i)-1)*(sourceJView(i)-1)) + /static_cast(gridConfigs().getInt("source normalization")); + } + } + + interpolation.execute(sourceField, targetField); + + const auto targetIView = array::make_view(targetFs.index_i()); + const auto targetJView = array::make_view(targetFs.index_j()); + const auto targetView = array::make_view(targetField); + const auto targetGhostView = atlas::array::make_view(targetFs.ghost()); + const double tolerance = 1.e-12; + for (idx_t i = 0; i < targetFs.size(); ++i) { + if (targetGhostView(i) == 0) { + const double targetTest = static_cast((targetIView(i)-1)*(targetJView(i)-1)) + /static_cast(gridConfigs().getInt("target normalization")); + EXPECT_APPROX_EQ(targetView(i), targetTest, tolerance); + } + } + + // Adjoint test + auto targetAdjoint = targetFs.createField(); + array::make_view(targetAdjoint).assign(array::make_view(targetField)); + targetAdjoint.adjointHaloExchange(); + + auto sourceAdjoint = sourceFs.createField(); + array::make_view(sourceAdjoint).assign(0.); + interpolation.execute_adjoint(sourceAdjoint, targetAdjoint); + + const auto yDotY = dotProd(targetField, targetField); + const auto xDotXAdj = dotProd(sourceField, sourceAdjoint); + + EXPECT_APPROX_EQ(yDotY / xDotXAdj, 1., 1e-14); +} + + +CASE("test_interpolation_structured2D_regional_2d") { + Grid sourceGrid(gridConfigs().getSubConfiguration("source")); + Grid targetGrid(gridConfigs().getSubConfiguration("target")); + + const idx_t nlevs = 2; + StructuredColumns sourceFs(sourceGrid, option::halo(1) | option::levels(nlevs)); + StructuredColumns targetFs(targetGrid, option::halo(1) | option::levels(nlevs)); + + Interpolation interpolation(Config("type", "regional-linear-2d"), sourceFs, targetFs); + + auto sourceField = sourceFs.createField(Config("name", "source")); + auto targetField = targetFs.createField(Config("name", "target")); + + // Accuracy test + const auto sourceIView = array::make_view(sourceFs.index_i()); + const auto sourceJView = array::make_view(sourceFs.index_j()); + auto sourceView = array::make_view(sourceField); + const auto sourceGhostView = atlas::array::make_view(sourceFs.ghost()); + sourceView.assign(0.0); + for (idx_t i = 0; i < sourceFs.size(); ++i) { + if (sourceGhostView(i) == 0) { + for (idx_t k = 0; k < nlevs; ++k) { + sourceView(i, k) = static_cast((sourceIView(i)-1)*(sourceJView(i)-1)) + /static_cast(gridConfigs().getInt("source normalization")); + } + } + } + + interpolation.execute(sourceField, targetField); + + const auto targetIView = array::make_view(targetFs.index_i()); + const auto targetJView = array::make_view(targetFs.index_j()); + const auto targetView = array::make_view(targetField); + const auto targetGhostView = atlas::array::make_view(targetFs.ghost()); + const double tolerance = 1.e-12; + for (idx_t i = 0; i < targetFs.size(); ++i) { + if (targetGhostView(i) == 0) { + const double targetTest = static_cast((targetIView(i)-1)*(targetJView(i)-1)) + /static_cast(gridConfigs().getInt("target normalization")); + for (idx_t k = 0; k < nlevs; ++k) { + EXPECT_APPROX_EQ(targetView(i, k), targetTest, tolerance); + } + } + } + + // Adjoint test + auto targetAdjoint = targetFs.createField(); + array::make_view(targetAdjoint).assign(array::make_view(targetField)); + targetAdjoint.adjointHaloExchange(); + + auto sourceAdjoint = sourceFs.createField(); + array::make_view(sourceAdjoint).assign(0.); + interpolation.execute_adjoint(sourceAdjoint, targetAdjoint); + + const auto yDotY = dotProd(targetField, targetField); + const auto xDotXAdj = dotProd(sourceField, sourceAdjoint); + + EXPECT_APPROX_EQ(yDotY / xDotXAdj, 1., 1e-14); +} + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} diff --git a/src/tests/mesh/CMakeLists.txt b/src/tests/mesh/CMakeLists.txt index f9d7825e3..95aa8916b 100644 --- a/src/tests/mesh/CMakeLists.txt +++ b/src/tests/mesh/CMakeLists.txt @@ -175,9 +175,9 @@ endforeach() -atlas_add_cuda_test( +atlas_add_hic_test( TARGET atlas_test_connectivity_kernel - SOURCES test_connectivity_kernel.cu + SOURCES test_connectivity_kernel.hic LIBS atlas ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) diff --git a/src/tests/mesh/test_connectivity_kernel.cu b/src/tests/mesh/test_connectivity_kernel.hic similarity index 94% rename from src/tests/mesh/test_connectivity_kernel.cu rename to src/tests/mesh/test_connectivity_kernel.hic index f6e08e301..d897da28b 100644 --- a/src/tests/mesh/test_connectivity_kernel.cu +++ b/src/tests/mesh/test_connectivity_kernel.hic @@ -8,7 +8,7 @@ * does it submit to any jurisdiction. */ -#include +#include "hic/hic.h" #include "atlas/mesh/Connectivity.h" #include "tests/AtlasTestEnvironment.h" @@ -83,7 +83,7 @@ CASE( "test_block_connectivity" ) BlockConnectivity conn; bool* result; - cudaMallocManaged(&result, sizeof(bool)); + hicMallocManaged(&result, sizeof(bool)); *result = true; @@ -104,7 +104,7 @@ CASE( "test_block_connectivity" ) kernel_block<<<1,1>>>(conn, result); - cudaDeviceSynchronize(); + hicDeviceSynchronize(); EXPECT( *result == true ); @@ -127,12 +127,12 @@ CASE( "test_irregular_connectivity" ) EXPECT(conn(0,0) == 1 IN_FORTRAN); bool* result; - cudaMallocManaged(&result, sizeof(bool)); + hicMallocManaged(&result, sizeof(bool)); *result = true; kernel_irr<<<1,1>>>(conn, result); - cudaDeviceSynchronize(); + hicDeviceSynchronize(); EXPECT( *result == true ); @@ -155,12 +155,12 @@ CASE( "test_multiblock_connectivity" ) EXPECT(conn.block(0)(0,0) == 1 IN_FORTRAN); bool* result; - cudaMallocManaged(&result, sizeof(bool)); + hicMallocManaged(&result, sizeof(bool)); *result = true; kernel_multiblock<<<1,1>>>(conn, result); - cudaDeviceSynchronize(); + hicDeviceSynchronize(); EXPECT( *result == true ); diff --git a/src/tests/parallel/CMakeLists.txt b/src/tests/parallel/CMakeLists.txt index 7922b7af5..4824b9ce1 100644 --- a/src/tests/parallel/CMakeLists.txt +++ b/src/tests/parallel/CMakeLists.txt @@ -30,6 +30,17 @@ ecbuild_add_test( TARGET atlas_test_haloexchange ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_haloexchange_on_device + MPI 3 + CONDITION eckit_HAVE_MPI AND atlas_HAVE_GPU + SOURCES test_haloexchange.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_RUN_NGPUS=1 +) +if( TEST atlas_test_haloexchange_on_device ) + set_tests_properties ( atlas_test_haloexchange_on_device PROPERTIES LABELS "gpu") +endif() + ecbuild_add_test( TARGET atlas_test_gather MPI 3 CONDITION eckit_HAVE_MPI diff --git a/src/tests/parallel/test_haloexchange.cc b/src/tests/parallel/test_haloexchange.cc index 3bf2ccdd2..5009c4054 100644 --- a/src/tests/parallel/test_haloexchange.cc +++ b/src/tests/parallel/test_haloexchange.cc @@ -13,6 +13,8 @@ #include #include +#include "hic/hic.h" + #include "atlas/array.h" #include "atlas/array/ArrayView.h" #include "atlas/array/MakeView.h" @@ -105,7 +107,7 @@ struct validate { }; struct Fixture { - Fixture(bool on_device): on_device_(on_device) { + Fixture(bool _on_device = false): on_device(_on_device) { int nnodes_c[] = {5, 6, 7}; nb_nodes = vec(nnodes_c); N = nb_nodes[mpi::comm().rank()]; @@ -147,7 +149,7 @@ struct Fixture { std::vector gidx; int N; - bool on_device_; + bool on_device; }; //----------------------------------------------------------------------------- @@ -159,11 +161,15 @@ void test_rank0_arrview(Fixture& f) { arrv(j) = (size_t(f.part[j]) != mpi::comm().rank() ? 0 : f.gidx[j]); } - arr.syncHostDevice(); + if (f.on_device) { + arr.updateDevice(); + } - f.halo_exchange.execute(arr, f.on_device_); + f.halo_exchange.execute(arr, f.on_device); - arr.syncHostDevice(); + if( f.on_device ) { + arr.updateHost(); + } switch (mpi::comm().rank()) { case 0: { @@ -192,11 +198,15 @@ void test_rank1(Fixture& f) { arrv(j, 1) = (size_t(f.part[j]) != mpi::comm().rank() ? 0 : f.gidx[j] * 100); } - arr.syncHostDevice(); + if (f.on_device) { + arr.updateDevice(); + } - f.halo_exchange.execute(arr, f.on_device_); + f.halo_exchange.execute(arr, f.on_device); - arr.syncHostDevice(); + if (f.on_device) { + arr.updateHost(); + } switch (mpi::comm().rank()) { case 0: { @@ -226,8 +236,6 @@ void test_rank1_strided_v1(Fixture& f) { arrv_t(j, 1) = (size_t(f.part[j]) != mpi::comm().rank() ? 0 : f.gidx[j] * 100); } - arr_t.syncHostDevice(); - // create a wrap array where we fake the strides in a way that the second // dimension // (number of components) contains only one component but the associated @@ -240,19 +248,21 @@ void test_rank1_strided_v1(Fixture& f) { array::ArraySpec { array::make_shape(f.N, 1), #if ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA - array::make_strides(32, 1) - } + array::make_strides(32, 1) #else - array::make_strides(2, 1) - } + array::make_strides(2, 1) #endif - )); + })); - arr->syncHostDevice(); + if (f.on_device) { + arr->updateDevice(); + } - f.halo_exchange.execute(*arr, f.on_device_); + f.halo_exchange.execute(*arr, f.on_device); - arr->syncHostDevice(); + if (f.on_device) { + arr->updateHost(); + } switch (mpi::comm().rank()) { case 0: { @@ -282,8 +292,6 @@ void test_rank1_strided_v2(Fixture& f) { arrv_t(j, 1) = (size_t(f.part[j]) != mpi::comm().rank() ? 0 : f.gidx[j] * 100); } - arr_t.syncHostDevice(); - // create a wrap array where we fake the strides in a way that the second // dimension // (number of components) contains only one component but the associated @@ -295,9 +303,9 @@ void test_rank1_strided_v2(Fixture& f) { &(arrv_t(0, 1)), array::ArraySpec { array::make_shape(f.N, 1), #if ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA - array::make_strides(32, 1) + array::make_strides(32, 1) #else - array::make_strides(2, 1) + array::make_strides(2, 1) #endif })); @@ -333,11 +341,15 @@ void test_rank2(Fixture& f) { } } - arr.syncHostDevice(); + if (f.on_device) { + arr.updateDevice(); + } - f.halo_exchange.execute(arr, f.on_device_); + f.halo_exchange.execute(arr, f.on_device); - arr.syncHostDevice(); + if (f.on_device) { + arr.updateHost(); + } switch (mpi::comm().rank()) { case 0: { @@ -373,23 +385,26 @@ void test_rank2_l1(Fixture& f) { (size_t(f.part[p]) != mpi::comm().rank() ? 0 : f.gidx[p] * std::pow(10, i)); } } - arr_t.syncHostDevice(); std::unique_ptr arr(array::Array::wrap( arrv_t.data(), array::ArraySpec { array::make_shape(f.N, 1, 2), #if ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA - array::make_strides(96, 32, 1) + array::make_strides(96, 32, 1) #else - array::make_strides(6, 2, 1) + array::make_strides(6, 2, 1) #endif })); - arr_t.syncHostDevice(); + if (f.on_device) { + arr->updateDevice(); + } - f.halo_exchange.execute(*arr, false); + f.halo_exchange.execute(*arr, f.on_device); - arr_t.syncHostDevice(); + if (f.on_device) { + arr->updateHost(); + } switch (mpi::comm().rank()) { case 0: { @@ -426,7 +441,6 @@ void test_rank2_l1(Fixture& f) { } void test_rank2_l2_v2(Fixture& f) { -#if ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST // Test rank 2 halo-exchange array::ArrayT arr_t(f.N, 3, 2); array::ArrayView arrv_t = array::make_host_view(arr_t); @@ -443,13 +457,22 @@ void test_rank2_l2_v2(Fixture& f) { &arrv_t(0, 1, 1), array::ArraySpec { array::make_shape(f.N, 1, 1), #if ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA - array::make_strides(192, 32, 1) + array::make_strides(192, 32, 1) #else - array::make_strides(6, 2, 1) + array::make_strides(6, 2, 1) #endif })); - f.halo_exchange.execute(*arr, f.on_device_); + + if (f.on_device) { + arr->updateDevice(); + } + + f.halo_exchange.execute(*arr, f.on_device); + + if (f.on_device) { + arr->updateHost(); + } switch (mpi::comm().rank()) { case 0: { @@ -483,11 +506,9 @@ void test_rank2_l2_v2(Fixture& f) { break; } } -#endif } void test_rank2_v2(Fixture& f) { -#if ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST array::ArrayT arr_t(f.N, 3, 2); array::ArrayView arrv_t = array::make_view(arr_t); for (int p = 0; p < f.N; ++p) { @@ -503,13 +524,21 @@ void test_rank2_v2(Fixture& f) { &arrv_t(0, 0, 1), array::ArraySpec { array::make_shape(f.N, 3, 1), #if ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA - array::make_strides(192, 32, 2) + array::make_strides(192, 32, 2) #else - array::make_strides(6, 2, 2) + array::make_strides(6, 2, 2) #endif })); - f.halo_exchange.execute(*arr, f.on_device_); + if (f.on_device) { + arr->updateDevice(); + } + + f.halo_exchange.execute(*arr, f.on_device); + + if (f.on_device) { + arr->updateHost(); + } switch (mpi::comm().rank()) { case 0: { @@ -543,18 +572,21 @@ void test_rank2_v2(Fixture& f) { break; } } -#endif } void test_rank0_wrap(Fixture& f) { std::unique_ptr arr(array::Array::wrap(f.gidx.data(), array::make_shape(f.N))); array::ArrayView arrv = array::make_view(*arr); - arr->syncHostDevice(); + if (f.on_device) { + arr->updateDevice(); + } - f.halo_exchange.execute(*arr, f.on_device_); + f.halo_exchange.execute(*arr, f.on_device); - arr->syncHostDevice(); + if (f.on_device) { + arr->updateHost(); + } switch (mpi::comm().rank()) { case 0: { @@ -642,7 +674,6 @@ void test_rank2_paralleldim2(Fixture& f) { } void test_rank1_cinterface(Fixture& f) { -#if ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST array::ArrayT arr(f.N, 2); array::ArrayView arrv = array::make_host_view(arr); for (int j = 0; j < f.N; ++j) { @@ -650,8 +681,6 @@ void test_rank1_cinterface(Fixture& f) { arrv(j, 1) = (size_t(f.part[j]) != mpi::comm().rank() ? 0 : f.gidx[j] * 100); } - arr.syncHostDevice(); - int shapes[2] = {(int)arrv.shape(0), (int)arrv.shape(1)}; int strides[2] = {(int)arrv.stride(0), (int)arrv.stride(1)}; @@ -674,11 +703,10 @@ void test_rank1_cinterface(Fixture& f) { break; } } -#endif } CASE("test_haloexchange") { - Fixture f(false); + Fixture f; SECTION("test_rank0_arrview") { test_rank0_arrview(f); } @@ -701,20 +729,46 @@ CASE("test_haloexchange") { SECTION("test_rank1_paralleldim_1") { test_rank1_paralleldim1(f); } SECTION("test_rank2_paralleldim_2") { test_rank2_paralleldim2(f); } + SECTION("test_rank1_cinterface") { test_rank1_cinterface(f); } +} -#if ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA - f.on_device_ = true; +#if ATLAS_HAVE_GPU + +//----------------------------------------------------------------------------- + +static int devices() { + static int devices_ = [](){ + int n = 0; + auto err = hicGetDeviceCount(&n); + if (err != hicSuccess) { + n = 0; + static_cast(hicGetLastError()); + } + return n; + }(); + return devices_; +} + +CASE("test_haloexchange on device") { + if (devices() == 0) { + Log::warning() << "\"test_haloexchange on device skipped\": No devices available" << std::endl; + return; + } + + bool on_device = true; + Fixture f(on_device); SECTION("test_rank0_arrview") { test_rank0_arrview(f); } SECTION("test_rank1") { test_rank1(f); } SECTION("test_rank2") { test_rank2(f); } - SECTION("test_rank0_wrap") { test_rank0_wrap(f); } -#endif + SECTION("test_rank0_wrap") { test_rank0_wrap(f); } } +#endif + //----------------------------------------------------------------------------- diff --git a/src/tests/parallel/test_haloexchange_adjoint.cc b/src/tests/parallel/test_haloexchange_adjoint.cc index 816b68441..392bc3575 100644 --- a/src/tests/parallel/test_haloexchange_adjoint.cc +++ b/src/tests/parallel/test_haloexchange_adjoint.cc @@ -183,11 +183,15 @@ void test_rank0_arrview(Fixture& f) { } } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(arr, f.on_device_); - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } switch (mpi::comm().rank()) { case 0: { @@ -240,11 +244,15 @@ void test_rank0_arrview_adj_test(Fixture& f) { arrv(j) = arrv_init(j); } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute(arr, f.on_device_); - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } // sum1 POD sum1(0); @@ -252,11 +260,15 @@ void test_rank0_arrview_adj_test(Fixture& f) { sum1 += arrv(j) * arrv(j); } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(arr, f.on_device_); - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } // sum2 POD sum2(0); @@ -299,11 +311,15 @@ void test_rank1(Fixture& f) { } } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(arr, f.on_device_); - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } switch (mpi::comm().rank()) { case 0: { @@ -336,11 +352,15 @@ void test_rank1_adj_test(Fixture& f) { arrv(j, 1ul) = arrv_init(j, 1ul); } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute(arr, f.on_device_); - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } // sum1 POD sum1(0); @@ -350,11 +370,15 @@ void test_rank1_adj_test(Fixture& f) { } } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(arr, f.on_device_); - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } // sum2 POD sum2(0); @@ -399,8 +423,6 @@ void test_rank1_strided_v1(Fixture& f) { } } - arr_t.syncHostDevice(); - // create a wrap array where we fake the strides in a way that the second // dimension // (number of components) contains only one component but the associated @@ -460,9 +482,6 @@ void test_rank1_strided_v1_adj_test(Fixture& f) { arrv_t(j, 1ul) = arrv_init_t(j, 1ul); } - arr_init_t.syncHostDevice(); - arr_t.syncHostDevice(); - // create a wrap array where we fake the strides in a way that the second // dimension // (number of components) contains only one component but the associated @@ -483,11 +502,15 @@ void test_rank1_strided_v1_adj_test(Fixture& f) { #endif )); - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } f.halo_exchange_std->execute(*arr, f.on_device_); - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } // sum1 POD sum1(0); @@ -497,11 +520,15 @@ void test_rank1_strided_v1_adj_test(Fixture& f) { } } - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(*arr, f.on_device_); - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } // sum2 POD sum2(0); @@ -546,8 +573,6 @@ void test_rank1_strided_v2(Fixture& f) { } } - arr_t.syncHostDevice(); - // create a wrap array where we fake the strides in a way that the second // dimension // (number of components) contains only one component but the associated @@ -565,6 +590,10 @@ void test_rank1_strided_v2(Fixture& f) { #endif })); + if (f.on_device_) { + arr->syncHostDevice(); + } + f.halo_exchange_std->execute_adjoint(*arr, false); switch (mpi::comm().rank()) { @@ -599,9 +628,6 @@ void test_rank1_strided_v2_adj_test(Fixture& f) { arrv_t(j, 1ul) = arrv_init_t(j, 1ul); } - arr_init_t.syncHostDevice(); - arr_t.syncHostDevice(); - // create a wrap array where we fake the strides in a way that the second // dimension // (number of components) contains only one component but the associated @@ -619,11 +645,15 @@ void test_rank1_strided_v2_adj_test(Fixture& f) { #endif })); - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } f.halo_exchange_std->execute(*arr, f.on_device_); - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } // sum1 POD sum1(0); @@ -633,11 +663,15 @@ void test_rank1_strided_v2_adj_test(Fixture& f) { } } - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(*arr, f.on_device_); - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } // sum2 POD sum2(0); @@ -692,11 +726,15 @@ void test_rank2(Fixture& f) { } } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(arr, f.on_device_); - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } switch (mpi::comm().rank()) { case 0: { @@ -738,7 +776,9 @@ void test_rank2_adj_test(Fixture& f) { } } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute(arr, f.on_device_); @@ -752,11 +792,15 @@ void test_rank2_adj_test(Fixture& f) { } } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(arr, f.on_device_); - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } // sum2 POD sum2(0); @@ -810,8 +854,6 @@ void test_rank2_l1(Fixture& f) { } } - arr_t.syncHostDevice(); - std::unique_ptr arr(array::Array::wrap( arrv_t.data(), array::ArraySpec { array::make_shape(f.N, 1, 2), @@ -822,11 +864,8 @@ void test_rank2_l1(Fixture& f) { #endif })); - arr_t.syncHostDevice(); - f.halo_exchange_std->execute_adjoint(*arr, false); - arr_t.syncHostDevice(); switch (mpi::comm().rank()) { case 0: { @@ -877,7 +916,6 @@ void test_rank2_l1_adj_test(Fixture& f) { arrv_t(p, i, static_cast(1)) = arrv_init_t(p, i, static_cast(1)); } } - arr_t.syncHostDevice(); std::unique_ptr arr(array::Array::wrap( arrv_t.data(), array::ArraySpec { @@ -889,12 +927,8 @@ void test_rank2_l1_adj_test(Fixture& f) { #endif })); - arr_t.syncHostDevice(); - f.halo_exchange_std->execute(*arr, false); - arr_t.syncHostDevice(); - // sum1 POD sum1(0); for (std::size_t p = 0; p < static_cast(f.N); ++p) { @@ -905,12 +939,8 @@ void test_rank2_l1_adj_test(Fixture& f) { } } - arr->syncHostDevice(); - f.halo_exchange_std->execute_adjoint(*arr, false); - arr->syncHostDevice(); - // sum2 POD sum2(0); for (std::size_t p = 0; p < static_cast(f.N); ++p) { @@ -1153,7 +1183,8 @@ void test_rank2_v2(Fixture& f) { } void test_rank0_wrap(Fixture& f) { - std::unique_ptr arr(array::Array::wrap(f.gidx.data(), array::make_shape(f.N))); + std::vector existing = f.gidx; + std::unique_ptr arr(array::Array::wrap(existing.data(), array::make_shape(f.N))); array::ArrayView arrv = array::make_view(*arr); switch (mpi::comm().rank()) { case 0: { @@ -1179,11 +1210,15 @@ void test_rank0_wrap(Fixture& f) { } } - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(*arr, f.on_device_); - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } switch (mpi::comm().rank()) { case 0: { @@ -1202,6 +1237,7 @@ void test_rank0_wrap(Fixture& f) { break; } } + arr->deallocateDevice(); } void test_rank0_wrap_adj_test(Fixture& f) { @@ -1226,7 +1262,9 @@ void test_rank0_wrap_adj_test(Fixture& f) { arrv_init(j) = arrv(j); } - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } f.halo_exchange_std->execute(*arr, f.on_device_); @@ -1236,11 +1274,15 @@ void test_rank0_wrap_adj_test(Fixture& f) { sum1 += arrv(j) * arrv(j); } - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } f.halo_exchange_std->execute_adjoint(*arr, f.on_device_); - arr->syncHostDevice(); + if (f.on_device_) { + arr->syncHostDevice(); + } // sum2 POD sum2(0); @@ -1485,7 +1527,9 @@ void test_rank1_cinterface(Fixture& f) { } } - arr.syncHostDevice(); + if (f.on_device_) { + arr.syncHostDevice(); + } int shapes[2] = {(int)arrv.shape(0), (int)arrv.shape(1)}; int strides[2] = {(int)arrv.stride(0), (int)arrv.stride(1)}; diff --git a/src/tests/projection/fctest_projection.F90 b/src/tests/projection/fctest_projection.F90 index 0e2acf7dd..b7076b455 100644 --- a/src/tests/projection/fctest_projection.F90 +++ b/src/tests/projection/fctest_projection.F90 @@ -81,7 +81,7 @@ module projection_fixture call config%set("north_pole", [2.0_dp,46.7_dp] ) projection = atlas_Projection(config) FCTEST_CHECK_EQUAL( projection%type(), "rotated_schmidt" ) -FCTEST_CHECK_EQUAL( projection%hash(), "148d7ceb58250c0f48dc6b590941a341" ) +FCTEST_CHECK_EQUAL( projection%hash(), "3a39e0635b7d0a45f684696ca89825e6" ) call config%final() call projection%final() END_TEST @@ -91,7 +91,7 @@ module projection_fixture projection = atlas_RotatedSchmidtProjection( stretching_factor=2.0_dp, & north_pole=[2.0_dp,46.7_dp], rotation_angle=180.0_dp) FCTEST_CHECK_EQUAL( projection%type(), "rotated_schmidt" ) -FCTEST_CHECK_EQUAL( projection%hash(), "148d7ceb58250c0f48dc6b590941a341" ) +FCTEST_CHECK_EQUAL( projection%hash(), "3a39e0635b7d0a45f684696ca89825e6" ) call projection%final() END_TEST @@ -130,7 +130,7 @@ module projection_fixture call config%set("rotation_angle", 180.0_dp) projection = atlas_Projection(config) FCTEST_CHECK_EQUAL( projection%type(), "rotated_lonlat" ) -FCTEST_CHECK_EQUAL( projection%hash(), "2b6db0e1ccbe7c514dd726f408f92adb" ) +FCTEST_CHECK_EQUAL( projection%hash(), "79586cfbc8145cdef1a25d075a9ae07e" ) call config%final() call projection%final() END_TEST @@ -139,7 +139,7 @@ module projection_fixture type(atlas_Projection) :: projection projection = atlas_RotatedLonLatProjection([2.0_dp,46.7_dp],180._dp) FCTEST_CHECK_EQUAL( projection%type(), "rotated_lonlat" ) -FCTEST_CHECK_EQUAL( projection%hash(), "2b6db0e1ccbe7c514dd726f408f92adb" ) +FCTEST_CHECK_EQUAL( projection%hash(), "79586cfbc8145cdef1a25d075a9ae07e" ) call projection%final() END_TEST diff --git a/src/tests/projection/test_rotation.cc b/src/tests/projection/test_rotation.cc index 6a454bfc8..b45e3974e 100644 --- a/src/tests/projection/test_rotation.cc +++ b/src/tests/projection/test_rotation.cc @@ -35,7 +35,8 @@ bool equivalent(const PointLonLat& p1, const PointLonLat& p2) { auto f = [=](double lon) { return 10. + std::cos(lon * d2r); }; return is_approximately_equal(p1.lat(), p2.lat(), eps) && - (std::abs(p2.lat()) == 90. || is_approximately_equal(f(p1.lon()), f(p2.lon()), eps)); + (is_approximately_equal(std::abs(p2.lat()), 90., eps) || + is_approximately_equal(f(p1.lon()), f(p2.lon()), eps)); } #define EXPECT_EQUIVALENT(p1, p2) EXPECT(equivalent(p1, p2)) @@ -127,27 +128,27 @@ CASE("test_rotation_angle") { Rotation rot(Config("rotation_angle", 180.) | Config("north_pole", std::vector{2.0, 46.7})); const PointLonLat ref[] = { - {-178.00000, -46.70000}, {-178.00000, -16.70000}, {-178.00000, 13.30000}, {-178.00000, 43.30000}, - {-178.00000, 73.30000}, {2.00000, 76.70000}, {2.00000, 46.70000}, {-178.00000, -46.70000}, - {-162.62343, -19.46929}, {-152.02366, 8.65459}, {-139.57464, 36.43683}, {-113.10894, 61.43199}, - {-39.88245, 68.00825}, {2.00000, 46.70000}, {-178.00000, -46.70000}, {-148.83443, -27.31067}, - {-129.26346, -3.83700}, {-110.79116, 20.05422}, {-85.87917, 41.36507}, {-44.42496, 53.29508}, - {2.00000, 46.70000}, {-178.00000, -46.70000}, {-137.90794, -39.07002}, {-109.60146, -21.33906}, - {-88.00000, 0.00000}, {-66.39854, 21.33906}, {-38.09206, 39.07002}, {2.00000, 46.70000}, - {-178.00000, -46.70000}, {-131.57504, -53.29508}, {-90.12083, -41.36507}, {-65.20884, -20.05422}, - {-46.73654, 3.83700}, {-27.16557, 27.31067}, {2.00000, 46.70000}, {-178.00000, -46.70000}, - {-136.11755, -68.00825}, {-62.89106, -61.43199}, {-36.42536, -36.43683}, {-23.97634, -8.65459}, - {-13.37657, 19.46929}, {2.00000, 46.70000}, {-178.00000, -46.70000}, {-178.00000, -76.70000}, - {2.00000, -73.30000}, {2.00000, -43.30000}, {2.00000, -13.30000}, {2.00000, 16.70000}, - {2.00000, 46.70000}, {-178.00000, -46.70000}, {140.11755, -68.00825}, {66.89106, -61.43199}, - {40.42536, -36.43683}, {27.97634, -8.65459}, {17.37657, 19.46929}, {2.00000, 46.70000}, - {-178.00000, -46.70000}, {135.57504, -53.29508}, {94.12083, -41.36507}, {69.20884, -20.05422}, - {50.73654, 3.83700}, {31.16557, 27.31067}, {2.00000, 46.70000}, {-178.00000, -46.70000}, - {141.90794, -39.07002}, {113.60146, -21.33906}, {92.00000, 0.00000}, {70.39854, 21.33906}, - {42.09206, 39.07002}, {2.00000, 46.70000}, {-178.00000, -46.70000}, {152.83443, -27.31067}, - {133.26346, -3.83700}, {114.79116, 20.05422}, {89.87917, 41.36507}, {48.42496, 53.29508}, - {2.00000, 46.70000}, {-178.00000, -46.70000}, {166.62343, -19.46929}, {156.02366, 8.65459}, - {143.57464, 36.43683}, {117.10894, 61.43199}, {43.88245, 68.00825}, {2.00000, 46.70000}, + {-178.00000,-46.70000}, {-178.00000,-76.70000}, {2.00000,-73.30000}, {2.00000,-43.30000}, + {2.00000,-13.30000}, {2.00000,16.70000}, {2.00000,46.70000}, {-178.00000,-46.70000}, + {140.11755,-68.00825}, {66.89106,-61.43199}, {40.42536,-36.43683}, {27.97634,-8.65459}, + {17.37657,19.46929}, {2.00000,46.70000}, {-178.00000,-46.70000}, {135.57504,-53.29508}, + {94.12083,-41.36507}, {69.20884,-20.05422}, {50.73654,3.83700}, {31.16557,27.31067}, + {2.00000,46.70000}, {-178.00000,-46.70000}, {141.90794,-39.07002}, {113.60146,-21.33906}, + {92.00000,0.00000}, {70.39854,21.33906}, {42.09206,39.07002}, {2.00000,46.70000}, + {-178.00000,-46.70000}, {152.83443,-27.31067}, {133.26346,-3.83700}, {114.79116,20.05422}, + {89.87917,41.36507}, {48.42496,53.29508}, {2.00000,46.70000}, {-178.00000,-46.70000}, + {166.62343,-19.46929}, {156.02366,8.65459}, {143.57464,36.43683}, {117.10894,61.43199}, + {43.88245,68.00825}, {2.00000,46.70000}, {-178.00000,-46.70000}, {-178.00000,-16.70000}, + {-178.00000,13.30000}, {-178.00000,43.30000}, {-178.00000,73.30000}, {2.00000,76.70000}, + {2.00000,46.70000}, {-178.00000,-46.70000}, {-162.62343,-19.46929}, {-152.02366,8.65459}, + {-139.57464,36.43683}, {-113.10894,61.43199}, {-39.88245,68.00825}, {2.00000,46.70000}, + {-178.00000,-46.70000}, {-148.83443,-27.31067}, {-129.26346,-3.83700}, {-110.79116,20.05422}, + {-85.87917,41.36507}, {-44.42496,53.29508}, {2.00000,46.70000}, {-178.00000,-46.70000}, + {-137.90794,-39.07002}, {-109.60146,-21.33906}, {-88.00000,0.00000}, {-66.39854,21.33906}, + {-38.09206,39.07002}, {2.00000,46.70000}, {-178.00000,-46.70000}, {-131.57504,-53.29508}, + {-90.12083,-41.36507}, {-65.20884,-20.05422}, {-46.73654,3.83700}, {-27.16557,27.31067}, + {2.00000,46.70000}, {-178.00000,-46.70000}, {-136.11755,-68.00825}, {-62.89106,-61.43199}, + {-36.42536,-36.43683}, {-23.97634,-8.65459}, {-13.37657,19.46929}, {2.00000,46.70000}, }; @@ -200,14 +201,14 @@ CASE("test_rotation") { EXPECT_EQUIVALENT(magics.unrotate(r), p); p = {0., 0.}; - r = {-176., -50.}; + r = {4., 50.}; EXPECT_EQUIVALENT(rotation.rotate(p), r); EXPECT_EQUIVALENT(magics.rotate(p), r); EXPECT_EQUIVALENT(rotation.unrotate(r), p); EXPECT_EQUIVALENT(magics.unrotate(r), p); p = {-180., 45.}; - r = {-176., 85.}; + r = {-176., -5.}; EXPECT_EQUIVALENT(rotation.rotate(p), r); EXPECT_EQUIVALENT(magics.rotate(p), r); EXPECT_EQUIVALENT(rotation.unrotate(r), p); diff --git a/src/tests/util/CMakeLists.txt b/src/tests/util/CMakeLists.txt index 47d8b6def..7c7262ccf 100644 --- a/src/tests/util/CMakeLists.txt +++ b/src/tests/util/CMakeLists.txt @@ -94,3 +94,12 @@ ecbuild_add_test( TARGET atlas_test_unitsphere ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_pack_vector_fields + SOURCES test_pack_vector_fields.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + + + + diff --git a/src/tests/util/test_pack_vector_fields.cc b/src/tests/util/test_pack_vector_fields.cc new file mode 100644 index 000000000..3c6ea89df --- /dev/null +++ b/src/tests/util/test_pack_vector_fields.cc @@ -0,0 +1,235 @@ +/* + * (C) Crown Copyright 2024 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include +#include + +#include "atlas/array.h" +#include "atlas/field.h" +#include "atlas/functionspace.h" +#include "atlas/grid.h" +#include "atlas/option.h" +#include "atlas/util/Config.h" +#include "atlas/util/PackVectorFields.h" +#include "tests/AtlasTestEnvironment.h" + +namespace atlas { +namespace test { + +FieldSet setFields(const FunctionSpace& functionSpace, + const std::vector& fieldConfigs) { + auto fields = FieldSet{}; + + // Set unique values to all field elements. + auto value = 0; + for (const auto& fieldConfig : fieldConfigs) { + auto field = fields.add(functionSpace.createField(fieldConfig)); + for (auto arrayIdx = size_t{0}; arrayIdx < field.size(); arrayIdx++) { + field->data()[arrayIdx] = value++; + } + field.metadata().set("comment", "This field is made with love."); + auto vectorFieldName = std::string{}; + if (fieldConfig.get("vector_field_name", vectorFieldName)) { + field.metadata().set("vector_field_name", vectorFieldName); + } + } + return fields; +} + +FieldSet createOrderedTestFields() { + const auto grid = Grid("O16"); + const auto functionSpace = functionspace::StructuredColumns(grid); + + // Note: vector components 0 and 1 are contiguous in field set. + auto fieldConfigs = std::vector{}; + fieldConfigs.push_back(option::name("scalar") | option::levels(1) | + option::datatype(DataType::kind())); + fieldConfigs.push_back(option::name("vector_component_0") | + option::levels(1) | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + fieldConfigs.push_back(option::name("vector_component_1") | + option::levels(1) | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + + return setFields(functionSpace, fieldConfigs); +} + +FieldSet createUnorderedTestFields() { + auto fields = FieldSet{}; + + const auto grid = Grid("O16"); + const auto functionSpace = functionspace::StructuredColumns(grid); + + // Note: vector components 0 and 1 are not contiguous in field set. + auto fieldConfigs = std::vector{}; + fieldConfigs.push_back(option::name("vector_component_0") | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + fieldConfigs.push_back(option::name("scalar") | + option::datatype(DataType::kind())); + fieldConfigs.push_back(option::name("vector_component_1") | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + + return setFields(functionSpace, fieldConfigs); +} + +FieldSet createInconsistentRankFields() { + auto fields = FieldSet{}; + + const auto grid = Grid("O16"); + const auto functionSpace = functionspace::StructuredColumns(grid); + + // Note: vector components 0 and 1 have different ranks. + auto fieldConfigs = std::vector{}; + fieldConfigs.push_back(option::name("vector_component_0") | + option::levels(10) | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + fieldConfigs.push_back(option::name("scalar") | + option::datatype(DataType::kind())); + fieldConfigs.push_back(option::name("vector_component_1") | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + + return setFields(functionSpace, fieldConfigs); +} + +FieldSet createInconsistentDatatypeFields() { + auto fields = FieldSet{}; + + const auto grid = Grid("O16"); + const auto functionSpace = functionspace::StructuredColumns(grid); + + // Note: vector components 0 and 1 have different datatypes. + auto fieldConfigs = std::vector{}; + fieldConfigs.push_back(option::name("vector_component_0") | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + fieldConfigs.push_back(option::name("scalar") | + option::datatype(DataType::kind())); + fieldConfigs.push_back(option::name("vector_component_1") | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + + return setFields(functionSpace, fieldConfigs); +} + +FieldSet createInconsistentLevelsFields() { + auto fields = FieldSet{}; + + const auto grid = Grid("O16"); + const auto functionSpace = functionspace::StructuredColumns(grid); + + // Note: vector components 0 and 1 have different number of levels. + auto fieldConfigs = std::vector{}; + fieldConfigs.push_back(option::name("vector_component_0") | + option::levels(10) | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + fieldConfigs.push_back(option::name("scalar") | + option::datatype(DataType::kind())); + fieldConfigs.push_back(option::name("vector_component_1") | + option::levels(20) | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + + return setFields(functionSpace, fieldConfigs); +} + +FieldSet createInconsistentVariablesFields() { + auto fields = FieldSet{}; + + const auto grid = Grid("O16"); + const auto functionSpace = functionspace::StructuredColumns(grid); + + // Note: vector components 0 and 1 have different number of variables. + auto fieldConfigs = std::vector{}; + fieldConfigs.push_back(option::name("vector_component_0") | + option::datatype(DataType::kind()) | + option::variables(2) | + util::Config{"vector_field_name", "vector"}); + fieldConfigs.push_back(option::name("scalar") | + option::datatype(DataType::kind())); + fieldConfigs.push_back(option::name("vector_component_1") | + option::datatype(DataType::kind()) | + util::Config{"vector_field_name", "vector"}); + + return setFields(functionSpace, fieldConfigs); +} + +void checkTestFields(const FieldSet& fields) { + auto value = 0; + for (const auto& field : fields) { + for (auto arrayIdx = size_t{0}; arrayIdx < field.size(); arrayIdx++) { + EXPECT(field->data()[arrayIdx] == value++); + } + EXPECT(field.metadata().get("comment") == + "This field is made with love."); + } +} + +CASE("Basic pack and unpack") { + const auto fields = createOrderedTestFields(); + + const auto packedFields = util::pack_vector_fields(fields); + + EXPECT(!packedFields.has("vector_component_0")); + EXPECT(!packedFields.has("vector_component_1")); + EXPECT(packedFields.has("vector")); + EXPECT(packedFields.has("scalar")); + + const auto unpackedFields = util::unpack_vector_fields(packedFields); + + EXPECT(unpackedFields.has("vector_component_0")); + EXPECT(unpackedFields.has("vector_component_1")); + EXPECT(!unpackedFields.has("vector")); + EXPECT(unpackedFields.has("scalar")); + + checkTestFields(unpackedFields); +} + +CASE("unpack into existing field set") { + auto fields = createUnorderedTestFields(); + + const auto packedFields = util::pack_vector_fields(fields); + + EXPECT(!packedFields.has("vector_component_0")); + EXPECT(!packedFields.has("vector_component_1")); + EXPECT(packedFields.has("vector")); + EXPECT(packedFields.has("scalar")); + + // Need to unpack into existing field to guarantee field order is preserved. + array::make_view(fields["vector_component_0"]).assign(0.); + array::make_view(fields["vector_component_1"]).assign(0.); + util::unpack_vector_fields(packedFields, fields); + + EXPECT(fields.has("vector_component_0")); + EXPECT(fields.has("vector_component_1")); + EXPECT(!fields.has("vector")); + EXPECT(fields.has("scalar")); + + checkTestFields(fields); +} + +CASE("check that bad inputs throw") { + // Try to apply pack to inconsistent field sets. + EXPECT_THROWS(util::pack_vector_fields(createInconsistentRankFields())); + EXPECT_THROWS( + util::pack_vector_fields(createInconsistentDatatypeFields())); + EXPECT_THROWS( + util::pack_vector_fields(createInconsistentLevelsFields())); + EXPECT_THROWS( + util::pack_vector_fields(createInconsistentVariablesFields())); +} + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { return atlas::test::run(argc, argv); } diff --git a/tools/atlas-run b/tools/atlas-run index cd5d91252..39a062bd5 100755 --- a/tools/atlas-run +++ b/tools/atlas-run @@ -55,10 +55,13 @@ then elif command_exists srun ; then LAUNCH="srun ${ATLAS_RUN_MPI_ARGS}" if [ -n "${ATLAS_RUN_NGPUS}" ]; then - LAUNCH="${LAUNCH} --gres=gpu:${ATLAS_RUN_NGPUS}" + LAUNCH="${LAUNCH} --gpus-per-task=${ATLAS_RUN_NGPUS}" fi if [ -z "${ATLAS_RUN_NPROCS}" ]; then LAUNCH="${LAUNCH} -n 1" + if [ -n "${SLURM_GPUS}" ]; then + LAUNCH="${LAUNCH} --gpus-per-task=1" + fi else LAUNCH="${LAUNCH} -n ${ATLAS_RUN_NPROCS}" fi diff --git a/tools/c2f.py b/tools/c2f.py index cb30c421c..652b65935 100755 --- a/tools/c2f.py +++ b/tools/c2f.py @@ -7,9 +7,8 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. -#! /usr/bin/env python +#! /usr/bin/env python3 -from __future__ import print_function from argparse import ArgumentParser import re import time @@ -50,7 +49,7 @@ 'logical(c_bool)':'c_bool' } -function_signature = re.compile('''^ +function_signature = re.compile(r'''^ ( #1 Type (const\s+)? #2 Leading const ([^\*\&\s\[]+) #3 @@ -72,7 +71,7 @@ def __init__(self,msg): class Argument: - arg_signature = re.compile('''^ + arg_signature = re.compile(r'''^ ( #1 Type (const\s+)? #2 Leading const ([^\*\&\s\[]+) #3 @@ -180,9 +179,6 @@ def __init__(self,line): except KeyError: raise ParsingFailed("Could not parse return type for statement "+line) - def __nonzero__(self): # python 2 - return self.type != "void" - def __bool__(self): # python 3 return self.type != "void" @@ -304,9 +300,9 @@ def statements(self): with open(input,'r') as file: content = file.read() - externC = re.compile('^extern\s+"C"(\s*{)?') + externC = re.compile(r'^extern\s+"C"(\s*{)?') - regex_externC = [re.compile('^extern\s+"C"(\s*{)?'),re.compile('^{')] + regex_externC = [re.compile(r'^extern\s+"C"(\s*{)?'),re.compile('^{')] in_externC = [False,False] for line in content.splitlines():