From 802dc742f37f7497cbdb0b1350f049c6a502dc92 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 23 Jun 2020 20:34:47 +0100 Subject: [PATCH 001/161] travis: osx + openmpi workarounds --- tools/install-mpi.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/install-mpi.sh b/tools/install-mpi.sh index f019fa565..708e93585 100755 --- a/tools/install-mpi.sh +++ b/tools/install-mpi.sh @@ -24,6 +24,13 @@ case "$os" in openmpi) brew ls --versions openmpi || brew install openmpi echo "localhost slots=72" >> /usr/local/etc/openmpi-default-hostfile + # workaround for open-mpi/omp#7516 + echo "setting the mca gds to hash..." + echo "gds = hash" >> /usr/local/etc/pmix-mca-params.conf + + # workaround for open-mpi/ompi#5798 + echo "setting the mca btl_vader_backing_directory to /tmp..." + echo "btl_vader_backing_directory = /tmp" >> /usr/local/etc/openmpi-mca-params.conf ;; *) echo "Unknown MPI implementation: $MPI" From a3497bb2025aab5c3265cda406d09818d12a1db9 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 24 Jun 2020 17:21:45 +0100 Subject: [PATCH 002/161] ATLAS-302 Use posix_memalign for NativeDataStore --- src/atlas/array/native/NativeDataStore.h | 33 +++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/atlas/array/native/NativeDataStore.h b/src/atlas/array/native/NativeDataStore.h index 3d1408e33..f1ac9243d 100644 --- a/src/atlas/array/native/NativeDataStore.h +++ b/src/atlas/array/native/NativeDataStore.h @@ -10,11 +10,15 @@ #pragma once - #include // std::fill +#include // posix_memalign #include // std::numeric_limits::signaling_NaN +#include + #include "atlas/array/ArrayUtil.h" #include "atlas/library/config.h" +#include "atlas/runtime/Exception.h" +#include "eckit/log/Bytes.h" //------------------------------------------------------------------------------ @@ -45,9 +49,12 @@ void initialise( Value[], size_t ) {} template class DataStore : public ArrayDataStore { public: - DataStore( size_t size ) : data_store_( new Value[size] ), size_( size ) { initialise( data_store_, size_ ); } + DataStore( size_t size ) : size_( size ) { + alloc_aligned( data_store_, size_ ); + initialise( data_store_, size_ ); + } - virtual ~DataStore() override { delete[] data_store_; } + virtual ~DataStore() override { free_aligned( data_store_ ); } virtual void updateDevice() const override {} @@ -72,6 +79,26 @@ class DataStore : public ArrayDataStore { virtual void* voidDeviceData() override { return static_cast( data_store_ ); } private: + [[noreturn]] void throw_AllocationFailed( size_t bytes, const eckit::CodeLocation& loc ) { + std::ostringstream ss; + ss << "AllocationFailed: Could not allocate " << eckit::Bytes( bytes ); + throw_Exception( ss.str(), loc ); + } + + void alloc_aligned( Value*& ptr, size_t n ) { + const size_t alignment = 64 * sizeof( Value ); + size_t bytes = sizeof( Value ) * n; + int err = posix_memalign( (void**)&ptr, alignment, bytes ); + if ( err ) { + throw_AllocationFailed( bytes, Here() ); + } + } + + void free_aligned( Value*& ptr ) { + free( ptr ); + ptr = nullptr; + } + Value* data_store_; size_t size_; }; From a085aa7bd0881dfd32fb8327b4d666f17295fbb7 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 24 Jun 2020 17:25:57 +0100 Subject: [PATCH 003/161] ATLAS-302 Alignment for innermost field dimension --- src/atlas/array/Array.h | 8 +- src/atlas/array/ArraySpec.cc | 40 ++++---- src/atlas/array/ArraySpec.h | 6 +- src/atlas/array/gridtools/GridToolsArray.cc | 97 +++++++++++++++++-- .../array/gridtools/GridToolsArrayHelpers.h | 23 +++-- src/atlas/array/native/NativeArray.cc | 34 +++++-- src/atlas/field/Field.cc | 3 + src/atlas/field/Field.h | 3 + src/atlas/field/FieldCreatorArraySpec.cc | 27 +++--- src/atlas/field/FieldCreatorIFS.cc | 21 ++-- src/atlas/field/detail/FieldImpl.cc | 14 +++ src/atlas/field/detail/FieldImpl.h | 6 ++ .../functionspace/detail/StructuredColumns.cc | 13 ++- .../functionspace/detail/StructuredColumns.h | 2 + src/atlas/option/Options.cc | 4 + src/atlas/option/Options.h | 15 +++ src/tests/array/test_array.cc | 37 +++++++ .../functionspace/test_structuredcolumns.cc | 18 ++++ src/tests/grid/test_field.cc | 29 +++++- 19 files changed, 325 insertions(+), 75 deletions(-) diff --git a/src/atlas/array/Array.h b/src/atlas/array/Array.h index bd416e8ef..7385137d1 100644 --- a/src/atlas/array/Array.h +++ b/src/atlas/array/Array.h @@ -36,12 +36,15 @@ class ArrayT_impl; class Array : public util::Object { public: + Array() = default; virtual ~Array(); static Array* create( array::DataType, const ArrayShape& ); static Array* create( array::DataType, const ArrayShape&, const ArrayLayout& ); + static Array* create( array::DataType, ArraySpec&& ); + virtual size_t footprint() const = 0; template @@ -158,6 +161,7 @@ class Array : public util::Object { ArrayDataStore const* data_store() const { return data_store_.get(); } protected: + Array( ArraySpec&& spec ) : spec_( std::move( spec ) ) {} ArraySpec spec_; std::unique_ptr data_store_; @@ -178,10 +182,12 @@ class ArrayT : public Array { ArrayT( idx_t size0, idx_t size1, idx_t size2, idx_t size3 ); ArrayT( idx_t size0, idx_t size1, idx_t size2, idx_t size3, idx_t size4 ); - ArrayT( const ArraySpec& ); + ArrayT( ArraySpec&& ); ArrayT( const ArrayShape& ); + ArrayT( const ArrayShape&, const ArrayAlignment& ); + ArrayT( const ArrayShape&, const ArrayLayout& ); virtual void insert( idx_t idx1, idx_t size1 ); diff --git a/src/atlas/array/ArraySpec.cc b/src/atlas/array/ArraySpec.cc index b90708394..2fae44216 100644 --- a/src/atlas/array/ArraySpec.cc +++ b/src/atlas/array/ArraySpec.cc @@ -18,14 +18,14 @@ namespace atlas { namespace array { namespace { -idx_t compute_allocated_size( idx_t size, idx_t alignment ) { - idx_t div = size / alignment; - idx_t mod = size % alignment; - idx_t _allocated_size = div * alignment; +idx_t compute_aligned_size( idx_t size, idx_t alignment ) { + idx_t div = size / alignment; + idx_t mod = size % alignment; + idx_t _aligned_size = div * alignment; if ( mod > 0 ) { - _allocated_size += alignment; + _aligned_size += alignment; } - return _allocated_size; + return _aligned_size; } } // namespace @@ -33,25 +33,27 @@ ArraySpec::ArraySpec() : size_(), rank_(), allocated_size_(), contiguous_( true ArraySpec::ArraySpec( const ArrayShape& shape ) : ArraySpec( shape, ArrayAlignment() ) {} -ArraySpec::ArraySpec( const ArrayShape& shape, ArrayAlignment&& alignment ) { - if ( int( alignment ) > 1 ) { - ATLAS_NOTIMPLEMENTED; // innermost dimension needs to be padded - } +ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayAlignment& alignment ) { + ArrayShape aligned_shape = shape; + aligned_shape.back() = compute_aligned_size( aligned_shape.back(), alignment ); - rank_ = static_cast( shape.size() ); - size_ = 1; + rank_ = static_cast( shape.size() ); + size_ = 1; + allocated_size_ = 1; shape_.resize( rank_ ); strides_.resize( rank_ ); layout_.resize( rank_ ); for ( int j = rank_ - 1; j >= 0; --j ) { shape_[j] = shape[j]; - strides_[j] = size_; + strides_[j] = allocated_size_; layout_[j] = j; size_ *= shape_[j]; + allocated_size_ *= aligned_shape[j]; } - allocated_size_ = compute_allocated_size( size_, alignment ); - contiguous_ = true; + ATLAS_ASSERT( allocated_size_ == compute_aligned_size( shape_[0] * strides_[0], alignment ) ); + contiguous_ = ( size_ == allocated_size_ ); default_layout_ = true; + alignment_ = alignment; #ifdef ATLAS_HAVE_FORTRAN allocate_fortran_specs(); @@ -61,7 +63,7 @@ ArraySpec::ArraySpec( const ArrayShape& shape, ArrayAlignment&& alignment ) { ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayStrides& strides ) : ArraySpec( shape, strides, ArrayAlignment() ) {} -ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayStrides& strides, ArrayAlignment&& alignment ) { +ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayStrides& strides, const ArrayAlignment& alignment ) { ATLAS_ASSERT( shape.size() == strides.size(), "dimensions of shape and stride don't match" ); rank_ = static_cast( shape.size() ); @@ -75,7 +77,7 @@ ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayStrides& strides, Arra layout_[j] = j; size_ *= shape_[j]; } - allocated_size_ = compute_allocated_size( shape_[0] * strides_[0], alignment ); + allocated_size_ = compute_aligned_size( shape_[0] * strides_[0], alignment ); contiguous_ = ( size_ == allocated_size_ ); default_layout_ = true; @@ -88,7 +90,7 @@ ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayStrides& strides, cons ArraySpec( shape, strides, layout, ArrayAlignment() ) {} ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayStrides& strides, const ArrayLayout& layout, - ArrayAlignment&& alignment ) { + const ArrayAlignment& alignment ) { ATLAS_ASSERT( shape.size() == strides.size(), "dimensions of shape and stride don't match" ); rank_ = static_cast( shape.size() ); @@ -106,7 +108,7 @@ ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayStrides& strides, cons default_layout_ = false; } } - allocated_size_ = compute_allocated_size( shape_[layout_[0]] * strides_[layout_[0]], alignment ); + allocated_size_ = compute_aligned_size( shape_[layout_[0]] * strides_[layout_[0]], alignment ); contiguous_ = ( size_ == allocated_size_ ); #ifdef ATLAS_HAVE_FORTRAN diff --git a/src/atlas/array/ArraySpec.h b/src/atlas/array/ArraySpec.h index 863f5eea0..8c53949a6 100644 --- a/src/atlas/array/ArraySpec.h +++ b/src/atlas/array/ArraySpec.h @@ -43,9 +43,9 @@ class ArraySpec { ArraySpec( const ArrayShape& ); ArraySpec( const ArrayShape&, const ArrayStrides& ); ArraySpec( const ArrayShape&, const ArrayStrides&, const ArrayLayout& ); - ArraySpec( const ArrayShape&, ArrayAlignment&& ); - ArraySpec( const ArrayShape&, const ArrayStrides&, ArrayAlignment&& ); - ArraySpec( const ArrayShape&, const ArrayStrides&, const ArrayLayout&, ArrayAlignment&& ); + ArraySpec( const ArrayShape&, const ArrayAlignment& ); + ArraySpec( const ArrayShape&, const ArrayStrides&, const ArrayAlignment& ); + ArraySpec( const ArrayShape&, const ArrayStrides&, const ArrayLayout&, const ArrayAlignment& ); idx_t allocatedSize() const { return allocated_size_; } idx_t size() const { return size_; } idx_t rank() const { return rank_; } diff --git a/src/atlas/array/gridtools/GridToolsArray.cc b/src/atlas/array/gridtools/GridToolsArray.cc index ebeefb0a2..ab446903b 100644 --- a/src/atlas/array/gridtools/GridToolsArray.cc +++ b/src/atlas/array/gridtools/GridToolsArray.cc @@ -71,6 +71,47 @@ class ArrayT_impl { array_.spec_ = make_spec( gt_storage, dims... ); } + template > + void construct_aligned( UInts... dims ) { + static_assert( sizeof...( UInts ) > 0, "1" ); + auto gt_storage = create_gt_storage::type, + ::gridtools::alignment>( dims... ); + using data_store_t = typename std::remove_pointer::type; + array_.data_store_ = std::unique_ptr( new GridToolsDataStore( gt_storage ) ); + array_.spec_ = make_spec( gt_storage, dims... ); + } + + template > + void construct( ArrayAlignment alignment, UInts... dims ) { + static_assert( sizeof...( UInts ) > 0, "1" ); + switch ( int( alignment ) ) { + case 1: + construct( dims... ); + break; + case 2: + construct_aligned<2>( dims... ); + break; + case 4: + construct_aligned<4>( dims... ); + break; + case 8: + construct_aligned<8>( dims... ); + break; + case 16: + construct_aligned<16>( dims... ); + break; + case 32: + construct_aligned<32>( dims... ); + break; + case 64: + construct_aligned<64>( dims... ); + break; + default: + ATLAS_NOTIMPLEMENTED; + } + } + + void construct( const ArrayShape& shape ) { assert( shape.size() > 0 ); switch ( shape.size() ) { @@ -92,6 +133,28 @@ class ArrayT_impl { } } + void construct( const ArraySpec& spec ) { + auto& shape = spec.shape(); + assert( shape.size() > 0 ); + switch ( shape.size() ) { + case 1: + return construct( spec.alignment(), shape[0] ); + case 2: + return construct( spec.alignment(), shape[0], shape[1] ); + case 3: + return construct( spec.alignment(), shape[0], shape[1], shape[2] ); + case 4: + return construct( spec.alignment(), shape[0], shape[1], shape[2], shape[3] ); + case 5: + return construct( spec.alignment(), shape[0], shape[1], shape[2], shape[3], shape[4] ); + default: { + std::stringstream err; + err << "shape not recognized"; + throw_Exception( err.str(), Here() ); + } + } + } + void construct( const ArrayShape& shape, const ArrayLayout& layout ) { ATLAS_ASSERT( shape.size() > 0 ); ATLAS_ASSERT( shape.size() == layout.size() ); @@ -313,6 +376,26 @@ Array* Array::create( DataType datatype, const ArrayShape& shape, const ArrayLay } } +Array* Array::create( DataType datatype, ArraySpec&& spec ) { + switch ( datatype.kind() ) { + case DataType::KIND_REAL64: + return new ArrayT( std::move( spec ) ); + case DataType::KIND_REAL32: + return new ArrayT( std::move( spec ) ); + case DataType::KIND_INT32: + return new ArrayT( std::move( spec ) ); + case DataType::KIND_INT64: + return new ArrayT( std::move( spec ) ); + case DataType::KIND_UINT64: + return new ArrayT( std::move( spec ) ); + default: { + std::stringstream err; + err << "data kind " << datatype.kind() << " not recognised."; + throw_NotImplemented( err.str(), Here() ); + } + } +} + //------------------------------------------------------------------------------ Array::~Array() {} @@ -461,8 +544,7 @@ void ArrayT::resize( const ArrayShape& shape ) { template ArrayT::ArrayT( ArrayDataStore* ds, const ArraySpec& spec ) { data_store_ = std::unique_ptr( ds ); - - spec_ = spec; + spec_ = spec; } template @@ -492,16 +574,19 @@ ArrayT::ArrayT( const ArrayShape& shape ) { ArrayT_impl( *this ).construct( shape ); } +template +ArrayT::ArrayT( const ArrayShape& shape, const ArrayAlignment& alignment ) { + ArrayT_impl( *this ).construct( ArraySpec( shape, alignment ) ); +} + template ArrayT::ArrayT( const ArrayShape& shape, const ArrayLayout& layout ) { ArrayT_impl( *this ).construct( shape, layout ); } template -ArrayT::ArrayT( const ArraySpec& spec ) { - if ( not spec.contiguous() ) - ATLAS_NOTIMPLEMENTED; - ArrayT_impl( *this ).construct( spec.shape(), spec.layout() ); +ArrayT::ArrayT( ArraySpec&& spec ) { + ArrayT_impl( *this ).construct( spec ); } //------------------------------------------------------------------------------ diff --git a/src/atlas/array/gridtools/GridToolsArrayHelpers.h b/src/atlas/array/gridtools/GridToolsArrayHelpers.h index a0e5b1f9a..b30043c66 100644 --- a/src/atlas/array/gridtools/GridToolsArrayHelpers.h +++ b/src/atlas/array/gridtools/GridToolsArrayHelpers.h @@ -97,6 +97,7 @@ struct default_layout_t { using type = typename get_layout>::type; }; + template struct get_layout_map_component { // TODO: static_assert( ::gridtools::is_layout_map(), "Error: not a @@ -148,19 +149,20 @@ struct get_pack_size { using type = ::gridtools::static_uint; }; -template +template struct gt_storage_t { - using type = gridtools::storage_traits::data_store_t< - Value, gridtools::storage_traits::custom_layout_storage_info_t<0, LayoutMap, ::gridtools::zero_halo>>; + using type = + gridtools::storage_traits::data_store_t, Alignment>>; }; -template -typename gt_storage_t::type::value>::type* create_gt_storage( +template +typename gt_storage_t::type::value>::type* create_gt_storage( UInts... dims ) { static_assert( ( sizeof...( dims ) > 0 ), "Error: can not create storages without any dimension" ); - constexpr static unsigned int rank = get_pack_size::type::value; - typedef gridtools::storage_traits::custom_layout_storage_info_t<0, LayoutMap, ::gridtools::zero_halo> + typedef gridtools::storage_traits::custom_layout_storage_info_align_t<0, LayoutMap, ::gridtools::zero_halo, + Alignment> storage_info_ty; typedef gridtools::storage_traits::data_store_t data_store_t; @@ -175,6 +177,13 @@ typename gt_storage_t::type::value>::t return ds; } +template +typename gt_storage_t, get_pack_size::type::value>::type* +create_gt_storage( UInts... dims ) { + return create_gt_storage>( dims... ); +} + + template struct gt_wrap_storage_t { using type = gridtools::storage_traits::data_store_t>; diff --git a/src/atlas/array/native/NativeArray.cc b/src/atlas/array/native/NativeArray.cc index bd4b6f117..0a4d9d0db 100644 --- a/src/atlas/array/native/NativeArray.cc +++ b/src/atlas/array/native/NativeArray.cc @@ -82,6 +82,26 @@ Array* Array::create( DataType datatype, const ArrayShape& shape ) { } } +Array* Array::create( DataType datatype, ArraySpec&& spec ) { + switch ( datatype.kind() ) { + case DataType::KIND_REAL64: + return new ArrayT( std::move( spec ) ); + case DataType::KIND_REAL32: + return new ArrayT( std::move( spec ) ); + case DataType::KIND_INT32: + return new ArrayT( std::move( spec ) ); + case DataType::KIND_INT64: + return new ArrayT( std::move( spec ) ); + case DataType::KIND_UINT64: + return new ArrayT( std::move( spec ) ); + default: { + std::stringstream err; + err << "data kind " << datatype.kind() << " not recognised."; + throw_NotImplemented( err.str(), Here() ); + } + } +} + template ArrayT::ArrayT( ArrayDataStore* ds, const ArraySpec& spec ) { data_store_ = std::unique_ptr( ds ); @@ -125,6 +145,12 @@ ArrayT::ArrayT( const ArrayShape& shape ) { spec_ = ArraySpec( shape ); } +template +ArrayT::ArrayT( const ArrayShape& shape, const ArrayAlignment& alignment ) { + spec_ = ArraySpec( shape, alignment ); + data_store_ = std::unique_ptr( new native::DataStore( spec_.allocatedSize() ) ); +} + template ArrayT::ArrayT( const ArrayShape& shape, const ArrayLayout& layout ) { spec_ = ArraySpec( shape ); @@ -135,12 +161,8 @@ ArrayT::ArrayT( const ArrayShape& shape, const ArrayLayout& layout ) { } template -ArrayT::ArrayT( const ArraySpec& spec ) { - if ( not spec.contiguous() ) { - ATLAS_NOTIMPLEMENTED; - } - spec_ = spec; - data_store_ = std::unique_ptr( new native::DataStore( spec_.size() ) ); +ArrayT::ArrayT( ArraySpec&& spec ) : Array( std::move( spec ) ) { + data_store_ = std::unique_ptr( new native::DataStore( spec_.allocatedSize() ) ); } template diff --git a/src/atlas/field/Field.cc b/src/atlas/field/Field.cc index 352d90bfa..1ab3a905f 100644 --- a/src/atlas/field/Field.cc +++ b/src/atlas/field/Field.cc @@ -29,6 +29,9 @@ Field::Field( const eckit::Parametrisation& config ) : Handle( Implementation::c Field::Field( const std::string& name, array::DataType datatype, const array::ArrayShape& shape ) : Handle( Implementation::create( name, datatype, shape ) ) {} +Field::Field( const std::string& name, array::DataType datatype, array::ArraySpec&& spec ) : + Handle( Implementation::create( name, datatype, std::move( spec ) ) ) {} + Field::Field( const std::string& name, array::Array* array ) : Handle( Implementation::create( name, array ) ) {} template diff --git a/src/atlas/field/Field.h b/src/atlas/field/Field.h index f2efd1499..e9f9d3852 100644 --- a/src/atlas/field/Field.h +++ b/src/atlas/field/Field.h @@ -67,6 +67,9 @@ class Field : DOXYGEN_HIDE( public util::ObjectHandle ) { /// @brief Create field with given name, Datatype and ArrayShape Field( const std::string& name, array::DataType, const array::ArrayShape& = array::ArrayShape() ); + /// @brief Create field with given name, Datatype and ArraySpec + Field( const std::string& name, array::DataType, array::ArraySpec&& ); + /// @brief Create field with given name, and take ownership of given Array Field( const std::string& name, array::Array* ); diff --git a/src/atlas/field/FieldCreatorArraySpec.cc b/src/atlas/field/FieldCreatorArraySpec.cc index 83aa66b04..ab6e05fdc 100644 --- a/src/atlas/field/FieldCreatorArraySpec.cc +++ b/src/atlas/field/FieldCreatorArraySpec.cc @@ -38,26 +38,21 @@ FieldImpl* FieldCreatorArraySpec::createField( const eckit::Parametrisation& par else { s.assign( shape.begin(), shape.end() ); } - - array::DataType datatype = array::DataType::create(); - std::string datatype_str; - if ( params.get( "datatype", datatype_str ) ) { - datatype = array::DataType( datatype_str ); - } - else { - array::DataType::kind_t kind( array::DataType::kind() ); - params.get( "kind", kind ); - if ( !array::DataType::kind_valid( kind ) ) { - std::stringstream msg; - msg << "Could not create field. kind parameter unrecognized"; - throw_Exception( msg.str() ); - } - datatype = array::DataType( kind ); + array::DataType::kind_t kind = array::DataType::kind(); + params.get( "datatype", kind ); + if ( !array::DataType::kind_valid( kind ) ) { + std::stringstream msg; + msg << "Could not create field. kind parameter " << kind << " unrecognized"; + throw_Exception( msg.str() ); } + auto datatype = array::DataType( kind ); + + int alignment = 1; + params.get( "alignment", alignment ); std::string name; params.get( "name", name ); - return FieldImpl::create( name, datatype, array::ArrayShape( std::move( s ) ) ); + return FieldImpl::create( name, datatype, array::ArraySpec( std::move( s ), array::ArrayAlignment( alignment ) ) ); } namespace { diff --git a/src/atlas/field/FieldCreatorIFS.cc b/src/atlas/field/FieldCreatorIFS.cc index ce7e33a4a..c55796b74 100644 --- a/src/atlas/field/FieldCreatorIFS.cc +++ b/src/atlas/field/FieldCreatorIFS.cc @@ -39,21 +39,14 @@ FieldImpl* FieldCreatorIFS::createField( const eckit::Parametrisation& params ) params.get( "nlev", nlev ); params.get( "nvar", nvar ); - array::DataType datatype = array::DataType::create(); - std::string datatype_str; - if ( params.get( "datatype", datatype_str ) ) { - datatype = array::DataType( datatype_str ); - } - else { - array::DataType::kind_t kind( array::DataType::kind() ); - params.get( "kind", kind ); - if ( !array::DataType::kind_valid( kind ) ) { - std::stringstream msg; - msg << "Could not create field. kind parameter unrecognized"; - throw_Exception( msg.str() ); - } - datatype = array::DataType( kind ); + array::DataType::kind_t kind = array::DataType::kind(); + params.get( "datatype", kind ); + if ( !array::DataType::kind_valid( kind ) ) { + std::stringstream msg; + msg << "Could not create field. kind parameter " << kind << " unrecognized"; + throw_Exception( msg.str() ); } + auto datatype = array::DataType( kind ); nblk = std::ceil( static_cast( ngptot ) / static_cast( nproma ) ); diff --git a/src/atlas/field/detail/FieldImpl.cc b/src/atlas/field/detail/FieldImpl.cc index cf6acac9e..9b7e2b98a 100644 --- a/src/atlas/field/detail/FieldImpl.cc +++ b/src/atlas/field/detail/FieldImpl.cc @@ -43,6 +43,10 @@ FieldImpl* FieldImpl::create( const std::string& name, array::DataType datatype, return new FieldImpl( name, datatype, shape ); } +FieldImpl* FieldImpl::create( const std::string& name, array::DataType datatype, array::ArraySpec&& spec ) { + return new FieldImpl( name, datatype, std::move( spec ) ); +} + FieldImpl* FieldImpl::create( const std::string& name, array::Array* array ) { return new FieldImpl( name, array ); } @@ -58,6 +62,16 @@ FieldImpl::FieldImpl( const std::string& name, array::DataType datatype, const a set_variables( 0 ); } +FieldImpl::FieldImpl( const std::string& name, array::DataType datatype, array::ArraySpec&& spec ) : + functionspace_( new FunctionSpace() ) { + array_ = array::Array::create( datatype, std::move( spec ) ); + array_->attach(); + rename( name ); + set_levels( 0 ); + set_variables( 0 ); +} + + FieldImpl::FieldImpl( const std::string& name, array::Array* array ) : functionspace_( new FunctionSpace() ) { array_ = array; array_->attach(); diff --git a/src/atlas/field/detail/FieldImpl.h b/src/atlas/field/detail/FieldImpl.h index 9bdb57e09..a40015d45 100644 --- a/src/atlas/field/detail/FieldImpl.h +++ b/src/atlas/field/detail/FieldImpl.h @@ -45,6 +45,9 @@ class FieldImpl : public util::Object { static FieldImpl* create( const std::string& name, array::DataType, const array::ArrayShape& = array::ArrayShape() ); + /// @brief Create field with given name, Datatype and ArrayShape + static FieldImpl* create( const std::string& name, array::DataType, array::ArraySpec&& ); + /// @brief Create field with given name, Datatype of template and ArrayShape template static FieldImpl* create( const std::string& name, const array::ArrayShape& = array::ArrayShape() ); @@ -66,6 +69,9 @@ class FieldImpl : public util::Object { /// Allocate new Array internally FieldImpl( const std::string& name, array::DataType, const array::ArrayShape& ); + /// Allocate new Array internally + FieldImpl( const std::string& name, array::DataType, array::ArraySpec&& ); + /// Transfer ownership of Array FieldImpl( const std::string& name, array::Array* ); diff --git a/src/atlas/functionspace/detail/StructuredColumns.cc b/src/atlas/functionspace/detail/StructuredColumns.cc index 3e70875cc..0ef15c630 100644 --- a/src/atlas/functionspace/detail/StructuredColumns.cc +++ b/src/atlas/functionspace/detail/StructuredColumns.cc @@ -332,6 +332,17 @@ array::ArrayShape StructuredColumns::config_shape( const eckit::Configuration& c return shape; } + +array::ArrayAlignment StructuredColumns::config_alignment( const eckit::Configuration& config ) const { + int alignment( 1 ); + config.get( "alignment", alignment ); + return alignment; +} + +array::ArraySpec StructuredColumns::config_spec( const eckit::Configuration& config ) const { + return array::ArraySpec( config_shape( config ), config_alignment( config ) ); +} + size_t StructuredColumns::Map2to1::footprint() const { size_t size = sizeof( *this ); size += data_.size() * sizeof( decltype( data_ )::value_type ); @@ -503,7 +514,7 @@ StructuredColumns::~StructuredColumns() { // Create Field // ---------------------------------------------------------------------------- Field StructuredColumns::createField( const eckit::Configuration& options ) const { - Field field( config_name( options ), config_datatype( options ), config_shape( options ) ); + Field field( config_name( options ), config_datatype( options ), config_spec( options ) ); set_field_metadata( options, field ); return field; } diff --git a/src/atlas/functionspace/detail/StructuredColumns.h b/src/atlas/functionspace/detail/StructuredColumns.h index dd81fb1e0..0a5c87dde 100644 --- a/src/atlas/functionspace/detail/StructuredColumns.h +++ b/src/atlas/functionspace/detail/StructuredColumns.h @@ -176,6 +176,8 @@ class StructuredColumns : public FunctionSpaceImpl { std::string config_name( const eckit::Configuration& ) const; idx_t config_levels( const eckit::Configuration& ) const; array::ArrayShape config_shape( const eckit::Configuration& ) const; + array::ArrayAlignment config_alignment( const eckit::Configuration& ) const; + array::ArraySpec config_spec( const eckit::Configuration& ) const; void set_field_metadata( const eckit::Configuration&, Field& ) const; void check_bounds( idx_t i, idx_t j ) const { diff --git a/src/atlas/option/Options.cc b/src/atlas/option/Options.cc index d9b2472d3..4dbaa8f6a 100644 --- a/src/atlas/option/Options.cc +++ b/src/atlas/option/Options.cc @@ -77,6 +77,10 @@ pole_edges::pole_edges( bool _pole_edges ) { set( "pole_edges", _pole_edges ); } +alignment::alignment( int value ) { + set( "alignment", value ); +} + // ---------------------------------------------------------------------------- } // namespace option diff --git a/src/atlas/option/Options.h b/src/atlas/option/Options.h index 9ed0b3e04..60477c78b 100644 --- a/src/atlas/option/Options.h +++ b/src/atlas/option/Options.h @@ -79,6 +79,21 @@ class datatype : public util::Config { // ---------------------------------------------------------------------------- +class shape : public util::Config { +public: + template + shape( const std::initializer_list& list ) { + set( "shape", list ); + } +}; + +class alignment : public util::Config { +public: + alignment( int ); +}; + +// ---------------------------------------------------------------------------- + class halo : public util::Config { public: halo( size_t size ); diff --git a/src/tests/array/test_array.cc b/src/tests/array/test_array.cc index 9c1c5cc13..670422baf 100644 --- a/src/tests/array/test_array.cc +++ b/src/tests/array/test_array.cc @@ -588,6 +588,43 @@ CASE( "test_acc_map" ) { delete ds; } +CASE( "test_aligned_ArraySpec" ) { + auto spec = ArraySpec( make_shape( 10, 5, 3 ), ArrayAlignment( 4 ) ); + EXPECT_EQ( spec.shape()[0], 10 ); + EXPECT_EQ( spec.shape()[1], 5 ); + EXPECT_EQ( spec.shape()[2], 3 ); + EXPECT_EQ( spec.size(), 10 * 5 * 3 ); + EXPECT_EQ( spec.allocatedSize(), 10 * 5 * 4 ); + EXPECT_EQ( spec.contiguous(), false ); + EXPECT_EQ( spec.strides()[0], 5 * 4 ); + EXPECT_EQ( spec.strides()[1], 4 ); + EXPECT_EQ( spec.strides()[2], 1 ); +} + +CASE( "test_aligned_Array" ) { + auto shape = make_shape( 10, 5, 3 ); + auto alignment = ArrayAlignment( 4 ); + auto datatype = make_datatype(); + auto check_array = []( const Array& array ) { + EXPECT_EQ( array.shape()[0], 10 ); + EXPECT_EQ( array.shape()[1], 5 ); + EXPECT_EQ( array.shape()[2], 3 ); + EXPECT_EQ( array.size(), 10 * 5 * 3 ); + EXPECT_EQ( array.contiguous(), false ); + EXPECT_EQ( array.strides()[0], 5 * 4 ); + EXPECT_EQ( array.strides()[1], 4 ); + EXPECT_EQ( array.strides()[2], 1 ); + }; + SECTION( "ArrayT(shape,alignment)" ) { + ArrayT array{shape, alignment}; + check_array( array ); + } + SECTION( "Array::create(spec)" ) { + std::unique_ptr array{Array::create( datatype, ArraySpec( shape, alignment ) )}; + check_array( *array ); + } +} + //----------------------------------------------------------------------------- } // namespace test diff --git a/src/tests/functionspace/test_structuredcolumns.cc b/src/tests/functionspace/test_structuredcolumns.cc index d086de7d2..165153d96 100644 --- a/src/tests/functionspace/test_structuredcolumns.cc +++ b/src/tests/functionspace/test_structuredcolumns.cc @@ -428,6 +428,24 @@ CASE( "test_functionspace_StructuredColumns halo exchange adjoint test 1" ) { } +CASE( "create_aligned_field" ) { + std::string gridname = eckit::Resource( "--grid", "S20x3" ); + Grid grid( gridname ); + functionspace::StructuredColumns fs( grid, option::levels( 5 ) ); + Field field = fs.createField( option::variables( 3 ) | option::alignment( 4 ) ); + auto check_field = [&]( const Field& field ) { + EXPECT_EQ( field.shape()[0], fs.size() ); + EXPECT_EQ( field.shape()[1], 5 ); + EXPECT_EQ( field.shape()[2], 3 ); + EXPECT_EQ( field.size(), fs.size() * 5 * 3 ); + EXPECT_EQ( field.contiguous(), false ); + EXPECT_EQ( field.strides()[0], 5 * 4 ); + EXPECT_EQ( field.strides()[1], 4 ); + EXPECT_EQ( field.strides()[2], 1 ); + }; + check_field( field ); +} + //----------------------------------------------------------------------------- } // namespace test diff --git a/src/tests/grid/test_field.cc b/src/tests/grid/test_field.cc index 5f6941e56..94c33d98b 100644 --- a/src/tests/grid/test_field.cc +++ b/src/tests/grid/test_field.cc @@ -16,6 +16,7 @@ #include "atlas/field/FieldSet.h" #include "atlas/grid.h" #include "atlas/grid/Grid.h" +#include "atlas/option.h" #include "atlas/parallel/mpi/mpi.h" #include "atlas/runtime/Log.h" #include "atlas/util/ObjectHandle.h" @@ -47,7 +48,7 @@ class TakeArray { CASE( "test_fieldcreator" ) { Field field( util::Config( "creator", "ArraySpec" )( "shape", array::make_shape( 10, 2 ) )( - "datatype", array::DataType::real32().str() )( "name", "myfield" ) ); + "datatype", array::DataType::real32().kind() )( "name", "myfield" ) ); EXPECT( field.datatype() == array::DataType::real32() ); EXPECT( field.name() == "myfield" ); @@ -62,7 +63,7 @@ CASE( "test_fieldcreator" ) { util::Config ifs_parameters = util::Config( "creator", "IFS" )( "nlev", 137 )( "nproma", 10 )( "ngptot", g.size() ); Log::info() << "Creating IFS field " << std::endl; - Field ifs( util::Config( ifs_parameters )( "name", "myfield" )( "datatype", array::DataType::int32().str() )( + Field ifs( util::Config( ifs_parameters )( "name", "myfield" )( "datatype", array::DataType::int32().kind() )( "nvar", 8 ) ); ATLAS_DEBUG_VAR( ifs ); @@ -119,6 +120,30 @@ CASE( "test_wrap_rawdata_through_field" ) { Field field( "name", rawdata.data(), array::make_shape( 10, 2 ) ); } +CASE( "test_field_aligned" ) { + using namespace array; + auto check_field = []( const Field& field ) { + EXPECT_EQ( field.shape()[0], 10 ); + EXPECT_EQ( field.shape()[1], 5 ); + EXPECT_EQ( field.shape()[2], 3 ); + EXPECT_EQ( field.size(), 10 * 5 * 3 ); + EXPECT_EQ( field.contiguous(), false ); + EXPECT_EQ( field.strides()[0], 5 * 4 ); + EXPECT_EQ( field.strides()[1], 4 ); + EXPECT_EQ( field.strides()[2], 1 ); + }; + SECTION( "field(name,datatype,spec)" ) { + Field field( "name", make_datatype(), ArraySpec{make_shape( 10, 5, 3 ), ArrayAlignment( 4 )} ); + check_field( field ); + } + SECTION( "field(config)" ) { + Field field( util::Config( "creator", "ArraySpec" ) | // + option::datatypeT() | // + option::shape( {10, 5, 3} ) | // + option::alignment( 4 ) ); + check_field( field ); + } +} //----------------------------------------------------------------------------- From 229d35bcc43e6c203b785e9af6ee0ab04c5576f9 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 24 Jun 2020 17:51:49 +0100 Subject: [PATCH 004/161] ATLAS-302 Alignment for Fortran interface (and fix broken test atlas_fctest_field) --- src/atlas/field/FieldCreatorArraySpec.cc | 22 +++++++++++++------ src/atlas/field/FieldCreatorIFS.cc | 21 ++++++++++++------ src/atlas_f/field/atlas_Field_module.fypp | 13 +++++++---- .../atlas_FunctionSpace_module.F90 | 4 +++- src/tests/field/fctest_field.F90 | 17 ++++++++++++++ src/tests/grid/test_field.cc | 10 ++++----- 6 files changed, 63 insertions(+), 24 deletions(-) diff --git a/src/atlas/field/FieldCreatorArraySpec.cc b/src/atlas/field/FieldCreatorArraySpec.cc index ab6e05fdc..8d85499d7 100644 --- a/src/atlas/field/FieldCreatorArraySpec.cc +++ b/src/atlas/field/FieldCreatorArraySpec.cc @@ -38,14 +38,22 @@ FieldImpl* FieldCreatorArraySpec::createField( const eckit::Parametrisation& par else { s.assign( shape.begin(), shape.end() ); } - array::DataType::kind_t kind = array::DataType::kind(); - params.get( "datatype", kind ); - if ( !array::DataType::kind_valid( kind ) ) { - std::stringstream msg; - msg << "Could not create field. kind parameter " << kind << " unrecognized"; - throw_Exception( msg.str() ); + + array::DataType datatype = array::DataType::create(); + std::string datatype_str; + if ( params.get( "datatype", datatype_str ) ) { + datatype = array::DataType( datatype_str ); + } + else { + array::DataType::kind_t kind( array::DataType::kind() ); + params.get( "kind", kind ); + if ( !array::DataType::kind_valid( kind ) ) { + std::stringstream msg; + msg << "Could not create field. kind parameter unrecognized"; + throw_Exception( msg.str() ); + } + datatype = array::DataType( kind ); } - auto datatype = array::DataType( kind ); int alignment = 1; params.get( "alignment", alignment ); diff --git a/src/atlas/field/FieldCreatorIFS.cc b/src/atlas/field/FieldCreatorIFS.cc index c55796b74..ce7e33a4a 100644 --- a/src/atlas/field/FieldCreatorIFS.cc +++ b/src/atlas/field/FieldCreatorIFS.cc @@ -39,14 +39,21 @@ FieldImpl* FieldCreatorIFS::createField( const eckit::Parametrisation& params ) params.get( "nlev", nlev ); params.get( "nvar", nvar ); - array::DataType::kind_t kind = array::DataType::kind(); - params.get( "datatype", kind ); - if ( !array::DataType::kind_valid( kind ) ) { - std::stringstream msg; - msg << "Could not create field. kind parameter " << kind << " unrecognized"; - throw_Exception( msg.str() ); + array::DataType datatype = array::DataType::create(); + std::string datatype_str; + if ( params.get( "datatype", datatype_str ) ) { + datatype = array::DataType( datatype_str ); + } + else { + array::DataType::kind_t kind( array::DataType::kind() ); + params.get( "kind", kind ); + if ( !array::DataType::kind_valid( kind ) ) { + std::stringstream msg; + msg << "Could not create field. kind parameter unrecognized"; + throw_Exception( msg.str() ); + } + datatype = array::DataType( kind ); } - auto datatype = array::DataType( kind ); nblk = std::ceil( static_cast( ngptot ) / static_cast( nproma ) ); diff --git a/src/atlas_f/field/atlas_Field_module.fypp b/src/atlas_f/field/atlas_Field_module.fypp index dd505295c..6f859a5b2 100644 --- a/src/atlas_f/field/atlas_Field_module.fypp +++ b/src/atlas_f/field/atlas_Field_module.fypp @@ -61,6 +61,7 @@ contains procedure, public :: strides_array => Field__strides_array procedure, public :: strides_idx => Field__strides_idx generic :: strides => strides_array, strides_idx + generic :: stride => strides_idx procedure :: halo_exchange procedure :: dirty @@ -311,13 +312,14 @@ end function !------------------------------------------------------------------------------- #:for dtype, ftype, ctype in integer_types -function atlas_Field__create_name_kind_shape_${dtype}$(name,kind,shape) result(field) +function atlas_Field__create_name_kind_shape_${dtype}$(name,kind,shape,alignment) result(field) use atlas_field_c_binding use, intrinsic :: iso_c_binding, only : c_int, c_long type(atlas_Field) :: field character(len=*), intent(in) :: name integer(c_int), intent(in) :: kind ${ftype}$, intent(in) :: shape(:) + integer(c_int), intent(in), optional :: alignment type(atlas_Config) :: params @@ -325,8 +327,9 @@ function atlas_Field__create_name_kind_shape_${dtype}$(name,kind,shape) result(f call params%set("creator","ArraySpec") call params%set("shape",shape) call params%set("fortran",.True.) - call params%set("datatype",atlas_data_type(kind)) + call params%set("kind",kind) call params%set("name",name) + if( present(alignment) ) call params%set("alignment",alignment) field = atlas_Field__cptr( atlas__Field__create(params%CPTR_PGIBUG_B) ) call params%final() @@ -335,12 +338,13 @@ end function !------------------------------------------------------------------------------- -function atlas_Field__create_kind_shape_${dtype}$(kind,shape) result(field) +function atlas_Field__create_kind_shape_${dtype}$(kind,shape,alignment) result(field) use atlas_field_c_binding use, intrinsic :: iso_c_binding, only : c_int, c_long type(atlas_Field) :: field integer(c_int), intent(in) :: kind ${ftype}$, intent(in) :: shape(:) + integer(c_int), intent(in), optional :: alignment type(atlas_Config) :: params @@ -348,7 +352,8 @@ function atlas_Field__create_kind_shape_${dtype}$(kind,shape) result(field) call params%set("creator","ArraySpec") call params%set("shape",shape) call params%set("fortran",.True.) - call params%set("datatype",atlas_data_type(kind)) + call params%set("kind",kind) + if( present(alignment) ) call params%set("alignment",alignment) field = atlas_Field__cptr( atlas__Field__create(params%CPTR_PGIBUG_B) ) call params%final() diff --git a/src/atlas_f/functionspace/atlas_FunctionSpace_module.F90 b/src/atlas_f/functionspace/atlas_FunctionSpace_module.F90 index 2f1121d5a..93b0af2aa 100644 --- a/src/atlas_f/functionspace/atlas_FunctionSpace_module.F90 +++ b/src/atlas_f/functionspace/atlas_FunctionSpace_module.F90 @@ -101,7 +101,7 @@ function atlas_FunctionSpace__name(this) result(name) -function create_field_args(this,kind,name,levels,variables,type,global,owner) result(field) +function create_field_args(this,kind,name,levels,variables,type,alignment,global,owner) result(field) use atlas_functionspace_c_binding use, intrinsic :: iso_c_binding, only : c_int type(atlas_Field) :: field @@ -111,6 +111,7 @@ function create_field_args(this,kind,name,levels,variables,type,global,owner) re integer(c_int), intent(in), optional :: levels integer(c_int), intent(in), optional :: variables character(len=*), intent(in), optional :: type + integer(c_int), intent(in), optional :: alignment logical, intent(in), optional :: global integer(c_int), intent(in), optional :: owner @@ -124,6 +125,7 @@ function create_field_args(this,kind,name,levels,variables,type,global,owner) re if( present(levels) ) call options%set("levels",levels) if( present(variables) ) call options%set("variables",variables) if( present(type) ) call options%set("type",type) + if( present(alignment) ) call options%set("alignment",alignment) field = atlas_Field( atlas__FunctionSpace__create_field( this%CPTR_PGIBUG_A, options%CPTR_PGIBUG_B ) ) diff --git a/src/tests/field/fctest_field.F90 b/src/tests/field/fctest_field.F90 index b350d6900..ef915ce6e 100644 --- a/src/tests/field/fctest_field.F90 +++ b/src/tests/field/fctest_field.F90 @@ -165,6 +165,23 @@ module fcta_Field_fixture END_TEST +TEST( test_field_aligned ) + implicit none + type(atlas_Field) :: field + real(c_double), pointer :: view(:,:,:) + field = atlas_Field("field_2",atlas_real(c_double),(/3,5,10/),alignment=4) + call field%data(view) + FCTEST_CHECK_EQUAL( size(view,1) , 3 ) + FCTEST_CHECK_EQUAL( size(view,2) , 5 ) + FCTEST_CHECK_EQUAL( size(view,3) , 10 ) + FCTEST_CHECK_EQUAL( field%shape(1), 3 ) + FCTEST_CHECK_EQUAL( field%shape(2), 5 ) + FCTEST_CHECK_EQUAL( field%shape(3), 10 ) + FCTEST_CHECK_EQUAL( field%stride(1), 1 ) + FCTEST_CHECK_EQUAL( field%stride(2), 4 ) + FCTEST_CHECK_EQUAL( field%stride(3), 4*5 ) +END_TEST + ! ----------------------------------------------------------------------------- END_TESTSUITE diff --git a/src/tests/grid/test_field.cc b/src/tests/grid/test_field.cc index 94c33d98b..0294359ab 100644 --- a/src/tests/grid/test_field.cc +++ b/src/tests/grid/test_field.cc @@ -48,7 +48,7 @@ class TakeArray { CASE( "test_fieldcreator" ) { Field field( util::Config( "creator", "ArraySpec" )( "shape", array::make_shape( 10, 2 ) )( - "datatype", array::DataType::real32().kind() )( "name", "myfield" ) ); + "datatype", array::DataType::real32().str() )( "name", "myfield" ) ); EXPECT( field.datatype() == array::DataType::real32() ); EXPECT( field.name() == "myfield" ); @@ -63,7 +63,7 @@ CASE( "test_fieldcreator" ) { util::Config ifs_parameters = util::Config( "creator", "IFS" )( "nlev", 137 )( "nproma", 10 )( "ngptot", g.size() ); Log::info() << "Creating IFS field " << std::endl; - Field ifs( util::Config( ifs_parameters )( "name", "myfield" )( "datatype", array::DataType::int32().kind() )( + Field ifs( util::Config( ifs_parameters )( "name", "myfield" )( "datatype", array::DataType::int32().str() )( "nvar", 8 ) ); ATLAS_DEBUG_VAR( ifs ); @@ -137,9 +137,9 @@ CASE( "test_field_aligned" ) { check_field( field ); } SECTION( "field(config)" ) { - Field field( util::Config( "creator", "ArraySpec" ) | // - option::datatypeT() | // - option::shape( {10, 5, 3} ) | // + Field field( util::Config( "creator", "ArraySpec" ) | // + util::Config( "datatype", make_datatype().str() ) | // + option::shape( {10, 5, 3} ) | // option::alignment( 4 ) ); check_field( field ); } From a6264dfbadccdaeb03d21f24b559eaa3ec50c566 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 24 Jun 2020 19:41:17 +0000 Subject: [PATCH 005/161] ATLAS-302 Fix test_array as GridTools CUDA backend now no longer aligns to 32 but to 1 by default (better to control ourselves) --- src/tests/array/test_array.cc | 75 +++++++++++------------------------ 1 file changed, 23 insertions(+), 52 deletions(-) diff --git a/src/tests/array/test_array.cc b/src/tests/array/test_array.cc index 670422baf..e32d4e972 100644 --- a/src/tests/array/test_array.cc +++ b/src/tests/array/test_array.cc @@ -19,18 +19,6 @@ #include "atlas/array/gridtools/GridToolsMakeView.h" #endif - -#ifdef ATLAS_HAVE_GRIDTOOLS_STORAGE -#if ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA -#define PADDED 1 -#else -#define PADDED 0 -#endif -#else -#define PADDED 0 -#endif -#define NOT_PADDED 1 - PADDED - using namespace atlas::array; namespace atlas { @@ -137,7 +125,7 @@ CASE( "test_array_shape" ) { EXPECT_EQ( ds->rank(), 2 ); EXPECT_EQ( ds->stride( 0 ), gt_hv.storage_info().stride<0>() ); EXPECT_EQ( ds->stride( 1 ), gt_hv.storage_info().stride<1>() ); - EXPECT( ds->contiguous() == NOT_PADDED ); + EXPECT( ds->contiguous() ); delete ds; } #endif @@ -153,11 +141,9 @@ CASE( "test_spec" ) { EXPECT( ds->spec().shapef()[1] == 5 ); EXPECT( ds->spec().shapef()[2] == 4 ); - if ( NOT_PADDED ) { - EXPECT_EQ( ds->spec().strides()[0], 6 * 5 ); - EXPECT_EQ( ds->spec().strides()[1], 6 ); - EXPECT_EQ( ds->spec().strides()[2], 1 ); - } + EXPECT_EQ( ds->spec().strides()[0], 6 * 5 ); + EXPECT_EQ( ds->spec().strides()[1], 6 ); + EXPECT_EQ( ds->spec().strides()[2], 1 ); EXPECT( ds->spec().hasDefaultLayout() == true ); delete ds; @@ -173,12 +159,10 @@ CASE( "test_spec_layout" ) { EXPECT( ds->spec().shapef()[0] == 6 ); EXPECT( ds->spec().shapef()[1] == 5 ); EXPECT( ds->spec().shapef()[2] == 4 ); - if ( NOT_PADDED ) { - EXPECT( ds->spec().strides()[0] == 6 * 5 ); - EXPECT( ds->spec().strides()[1] == 6 ); - EXPECT( ds->spec().strides()[2] == 1 ); - EXPECT( ds->spec().size() == ds->spec().allocatedSize() ); - } + EXPECT( ds->spec().strides()[0] == 6 * 5 ); + EXPECT( ds->spec().strides()[1] == 6 ); + EXPECT( ds->spec().strides()[2] == 1 ); + EXPECT( ds->spec().size() == ds->spec().allocatedSize() ); EXPECT( ds->spec().hasDefaultLayout() == true ); EXPECT( ds->spec().layout()[0] == 0 ); EXPECT( ds->spec().layout()[1] == 1 ); @@ -198,11 +182,9 @@ CASE( "test_spec_layout_rev" ) { EXPECT( ds->spec().shapef()[0] == 4 ); EXPECT( ds->spec().shapef()[1] == 5 ); EXPECT( ds->spec().shapef()[2] == 6 ); - if ( NOT_PADDED ) { - EXPECT( ds->spec().strides()[0] == 1 ); - EXPECT( ds->spec().strides()[1] == 4 ); - EXPECT( ds->spec().strides()[2] == 4 * 5 ); - } + EXPECT( ds->spec().strides()[0] == 1 ); + EXPECT( ds->spec().strides()[1] == 4 ); + EXPECT( ds->spec().strides()[2] == 4 * 5 ); EXPECT( ds->spec().hasDefaultLayout() == false ); EXPECT( ds->spec().layout()[0] == 2 ); EXPECT( ds->spec().layout()[1] == 1 ); @@ -252,11 +234,8 @@ CASE( "test_copy_gt_ctr" ) { EXPECT( hv2( 1, 1 ) == 7 ); auto dims = hv.data_view().storage_info().total_lengths(); - ATLAS_DEBUG_VAR( dims[0] ); - ATLAS_DEBUG_VAR( dims[1] ); - EXPECT( dims[0] == 3 ); - if ( NOT_PADDED ) - EXPECT( dims[1] == 2 ); + EXPECT_EQ( dims[0], 3 ); + EXPECT_EQ( dims[1], 2 ); delete ds; } #endif @@ -482,11 +461,9 @@ CASE( "test_ArrayT" ) { ArrayT ds( 2, 3, 4 ); EXPECT( ds.size() == 2 * 3 * 4 ); - if ( NOT_PADDED ) { - EXPECT( ds.stride( 0 ) == 3 * 4 ); - EXPECT( ds.stride( 1 ) == 4 ); - EXPECT( ds.stride( 2 ) == 1 ); - } + EXPECT( ds.stride( 0 ) == 3 * 4 ); + EXPECT( ds.stride( 1 ) == 4 ); + EXPECT( ds.stride( 2 ) == 1 ); EXPECT( ds.shape( 0 ) == 2 ); EXPECT( ds.shape( 1 ) == 3 ); EXPECT( ds.shape( 2 ) == 4 ); @@ -496,11 +473,9 @@ CASE( "test_ArrayT" ) { ArrayT ds( make_shape( 2, 3, 4 ) ); EXPECT( ds.size() == 2 * 3 * 4 ); - if ( NOT_PADDED ) { - EXPECT( ds.stride( 0 ) == 3 * 4 ); - EXPECT( ds.stride( 1 ) == 4 ); - EXPECT( ds.stride( 2 ) == 1 ); - } + EXPECT( ds.stride( 0 ) == 3 * 4 ); + EXPECT( ds.stride( 1 ) == 4 ); + EXPECT( ds.stride( 2 ) == 1 ); EXPECT( ds.shape( 0 ) == 2 ); EXPECT( ds.shape( 1 ) == 3 ); EXPECT( ds.shape( 2 ) == 4 ); @@ -510,11 +485,9 @@ CASE( "test_ArrayT" ) { ArrayT ds( ArraySpec( make_shape( 2, 3, 4 ) ) ); EXPECT( ds.size() == 2 * 3 * 4 ); - if ( NOT_PADDED ) { - EXPECT( ds.stride( 0 ) == 3 * 4 ); - EXPECT( ds.stride( 1 ) == 4 ); - EXPECT( ds.stride( 2 ) == 1 ); - } + EXPECT( ds.stride( 0 ) == 3 * 4 ); + EXPECT( ds.stride( 1 ) == 4 ); + EXPECT( ds.stride( 2 ) == 1 ); EXPECT( ds.shape( 0 ) == 2 ); EXPECT( ds.shape( 1 ) == 3 ); EXPECT( ds.shape( 2 ) == 4 ); @@ -544,9 +517,7 @@ CASE( "test_valid" ) { CASE( "test_wrap" ) { array::ArrayT arr_t( 3, 2 ); EXPECT( arr_t.shape( 0 ) == 3 ); - if ( NOT_PADDED ) { - EXPECT( arr_t.stride( 0 ) == 2 ); - } + EXPECT( arr_t.stride( 0 ) == 2 ); EXPECT( arr_t.shape( 1 ) == 2 ); EXPECT( arr_t.stride( 1 ) == 1 ); EXPECT( arr_t.rank() == 2 ); From 62098090bdb5f3439bf0d30a320b0d35a8bbe1d4 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 1 Jul 2020 14:34:58 +0000 Subject: [PATCH 006/161] Restore compatibility with eckit 1.4.0 --- src/atlas/library/Library.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index d2fe5f9f2..45ca15886 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -129,7 +129,7 @@ void load_library( const eckit::PathName& library_dir, const std::string library if ( !eckit::system::Library::exists( library_name ) ) { std::ostringstream ss; ss << "Shared library " << library_path << " loaded but Library " << library_name << " not registered"; - throw eckit::UnexpectedState( ss.str(), Here() ); + throw eckit::UnexpectedState( ss.str() ); } } From 61cfea076f6278da8886499e4205f8cb5a3d05b5 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 1 Jul 2020 13:15:08 +0000 Subject: [PATCH 007/161] Fix reporting on traces where iterations may differ --- src/atlas/runtime/trace/CallStack.h | 4 ++ src/atlas/runtime/trace/Timings.cc | 97 +++++++++++++++++++++++++++-- src/tests/runtime/test_trace.cc | 52 ++++++++++++++++ 3 files changed, 148 insertions(+), 5 deletions(-) diff --git a/src/atlas/runtime/trace/CallStack.h b/src/atlas/runtime/trace/CallStack.h index 06142adc3..a5a9e104e 100644 --- a/src/atlas/runtime/trace/CallStack.h +++ b/src/atlas/runtime/trace/CallStack.h @@ -44,6 +44,10 @@ class CallStack { operator bool() const { return not stack_.empty(); } +public: + CallStack() = default; + CallStack( const CallStack& other ) : stack_( other.stack_ ) {} + private: std::list stack_; mutable size_t hash_{0}; diff --git a/src/atlas/runtime/trace/Timings.cc b/src/atlas/runtime/trace/Timings.cc index 61410da59..aec243558 100644 --- a/src/atlas/runtime/trace/Timings.cc +++ b/src/atlas/runtime/trace/Timings.cc @@ -22,6 +22,7 @@ #include "eckit/filesystem/PathName.h" #include "atlas/parallel/mpi/mpi.h" +#include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "atlas/runtime/trace/CallStack.h" #include "atlas/runtime/trace/CodeLocation.h" @@ -66,6 +67,80 @@ class TimingsRegistry { private: std::string filter_filepath( const std::string& filepath ) const; + + friend class Tree; + friend class Node; +}; + +struct Node { + Node() : index( -1 ) {} + Node( size_t _index ) : index( _index ) { + size_t _nest = TimingsRegistry::instance().nest_[index]; + + + const auto& this_stack = TimingsRegistry::instance().stack_[index]; + auto this_stack_hash = TimingsRegistry::instance().stack_[index].hash(); + auto is_child = [&]( size_t i ) -> bool { + CallStack child_stack = TimingsRegistry::instance().stack_[i]; + child_stack.pop_front(); + auto child_stack_hash = child_stack.hash(); + return child_stack_hash == this_stack_hash; + }; + + + for ( size_t i = index + 1; i < TimingsRegistry::instance().size(); ++i ) { + if ( TimingsRegistry::instance().nest_[i] == _nest + 1 ) { + if ( is_child( i ) ) { + children.emplace_back( new Node( i ) ); + } + } + } + } + std::vector> children; + std::unique_ptr parent; + long index; + void print( std::ostream& out ) { + if ( index >= 0 ) { + size_t _nest = nest(); + for ( size_t i = 1; i < _nest; ++i ) { + out << " "; + } + out << TimingsRegistry::instance().titles_[index] << std::endl; + } + for ( auto& child : children ) { + child->print( out ); + } + } + size_t nest() const { return TimingsRegistry::instance().nest_[index]; } + void order( std::vector& order ) const { + if ( index >= 0 ) { + order.emplace_back( index ); + } + for ( auto& child : children ) { + child->order( order ); + } + } +}; +struct Tree { + Node root; + Tree() { + for ( size_t j = 0; j < TimingsRegistry::instance().size(); ++j ) { + auto& nest = TimingsRegistry::instance().nest_[j]; + if ( nest == 1 ) { + auto& children = root.children; + children.emplace_back( new Node( j ) ); + } + } + } + void print( std::ostream& out ) { root.print( out ); } + std::vector order() const { + std::vector order; + order.reserve( TimingsRegistry::instance().size() ); + root.order( order ); + ATLAS_ASSERT( order.size() == TimingsRegistry::instance().size(), + "Likely a atlas_Trace has not finalised properly" ); + return order; + } }; size_t TimingsRegistry::add( const CodeLocation& loc, const CallStack& stack, const std::string& title, @@ -139,6 +214,8 @@ void TimingsRegistry::report( std::ostream& out, const eckit::Configuration& con std::vector excluded_labels_vector = config.getStringVector( "exclude", std::vector() ); std::vector include_back; + auto order = Tree().order(); + for ( auto& label : excluded_labels_vector ) { size_t found = label.find( "/*" ); if ( found != std::string::npos ) { @@ -175,7 +252,8 @@ void TimingsRegistry::report( std::ostream& out, const eckit::Configuration& con std::vector excluded_nest_stored( size() ); long excluded_nest = size(); - for ( size_t j = 0; j < size(); ++j ) { + for ( size_t jj = 0; jj < size(); ++jj ) { + size_t j = order[jj]; if ( nest_[j] > excluded_nest ) { excluded_timers.insert( j ); } @@ -261,11 +339,19 @@ void TimingsRegistry::report( std::ostream& out, const eckit::Configuration& con std::vector prefix_( size() ); if ( indent ) { std::vector active( max_nest, false ); - for ( long k = long( size() ) - 1; k >= 0; --k ) { + for ( long kk = long( size() ) - 1; kk >= 0; --kk ) { + long k = order[kk]; const auto& nest = nest_[k]; const CallStack& this_stack = stack_[k]; - const CallStack& next_stack = ( k == long( size() ) - 1 ) ? this_stack : stack_[k + 1]; + const CallStack* next_stack_ptr; + if ( kk == size() - 1 ) { + next_stack_ptr = &this_stack; + } + else { + next_stack_ptr = &stack_[order[kk + 1]]; + } + const CallStack& next_stack = *next_stack_ptr; auto this_it = this_stack.rbegin(); auto next_it = next_stack.rbegin(); @@ -308,7 +394,8 @@ void TimingsRegistry::report( std::ostream& out, const eckit::Configuration& con } } - for ( size_t j = 0; j < size(); ++j ) { + for ( size_t i = 0; i < size(); ++i ) { + size_t j = order[i]; auto& tot = tot_timings_[j]; auto& max = max_timings_[j]; auto& min = std::min( max, min_timings_[j] ); @@ -387,7 +474,7 @@ std::string Timings::report() { } std::string Timings::report( const Configuration& config ) { - std::stringstream out; + std::ostringstream out; TimingsRegistry::instance().report( out, config ); return out.str(); } diff --git a/src/tests/runtime/test_trace.cc b/src/tests/runtime/test_trace.cc index 9abd0805a..622314aed 100644 --- a/src/tests/runtime/test_trace.cc +++ b/src/tests/runtime/test_trace.cc @@ -69,6 +69,58 @@ CASE( "test barrier" ) { EXPECT( runtime::trace::Barriers::state() == Library::instance().traceBarriers() ); } +// -------------------------------------------------------------------------- + + +void haloexchange() { + ATLAS_TRACE(); +} +void wind_const() { + ATLAS_TRACE(); +} +void wind_next() { + ATLAS_TRACE(); + haloexchange(); +} +static int count = 0; +void dp_meth() { + ATLAS_TRACE(); + if ( count == 0 ) { + wind_const(); + } + else { + wind_next(); + } + ++count; +} +void tracer_interpolation() { + ATLAS_TRACE(); + haloexchange(); +} +void execute_sladv() { + ATLAS_TRACE(); + dp_meth(); + tracer_interpolation(); +} + +CASE( "test report" ) { + SECTION( "1" ) { + for ( int i = 0; i < 3; ++i ) { + execute_sladv(); + } + } + SECTION( "2" ) { + count = 0; + for ( int i = 0; i < 3; ++i ) { + execute_sladv(); + } + } + Log::info() << atlas::Trace::report() << std::endl; +} + +// -------------------------------------------------------------------------- + + } // namespace test } // namespace atlas From b3e4646527961a22eb9b87b1f1fc0b656c493bfe Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 1 Jul 2020 13:19:56 +0000 Subject: [PATCH 008/161] Trace Field allocations and Field memory with ATLAS_TRACE=1 --- src/atlas/array/native/NativeDataStore.h | 50 ++++++++++++++++++++++-- src/atlas/field/FieldCreatorArraySpec.cc | 11 +++++- src/atlas/field/detail/FieldImpl.cc | 3 ++ src/atlas/field/detail/FieldImpl.h | 5 +++ 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/src/atlas/array/native/NativeDataStore.h b/src/atlas/array/native/NativeDataStore.h index f1ac9243d..3f6472274 100644 --- a/src/atlas/array/native/NativeDataStore.h +++ b/src/atlas/array/native/NativeDataStore.h @@ -11,13 +11,15 @@ #pragma once #include // std::fill -#include // posix_memalign -#include // std::numeric_limits::signaling_NaN +#include +#include // posix_memalign +#include // std::numeric_limits::signaling_NaN #include #include "atlas/array/ArrayUtil.h" #include "atlas/library/config.h" #include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" #include "eckit/log/Bytes.h" //------------------------------------------------------------------------------ @@ -26,6 +28,43 @@ namespace atlas { namespace array { namespace native { +struct MemoryHighWatermark { + std::atomic bytes_{0}; + std::atomic high_{0}; + void print( std::ostream& out ) const { out << eckit::Bytes( double( bytes_ ) ); } + friend std::ostream& operator<<( std::ostream& out, const MemoryHighWatermark& v ) { + v.print( out ); + return out; + } + MemoryHighWatermark& operator+=( const size_t& bytes ) { + bytes_ += bytes; + update_maximum(); + Log::trace() << "Memory: " << eckit::Bytes( double( bytes_ ) ) << "\t( +" << eckit::Bytes( double( bytes ) ) + << " \t| high watermark " << eckit::Bytes( double( high_ ) ) << "\t)" << std::endl; + return *this; + } + MemoryHighWatermark& operator-=( const size_t& bytes ) { + bytes_ -= bytes; + Log::trace() << "Memory: " << eckit::Bytes( double( bytes_ ) ) << "\t( -" << eckit::Bytes( double( bytes ) ) + << " \t| high watermark " << eckit::Bytes( double( high_ ) ) << "\t)" << std::endl; + return *this; + } + +private: + MemoryHighWatermark() = default; + void update_maximum() noexcept { + size_t prev_value = high_; + while ( prev_value < bytes_ && !high_.compare_exchange_weak( prev_value, bytes_ ) ) { + } + } + +public: + static MemoryHighWatermark& instance() { + static MemoryHighWatermark _instance; + return _instance; + } +}; + template static constexpr Value invalid_value() { return std::numeric_limits::has_signaling_NaN @@ -88,7 +127,9 @@ class DataStore : public ArrayDataStore { void alloc_aligned( Value*& ptr, size_t n ) { const size_t alignment = 64 * sizeof( Value ); size_t bytes = sizeof( Value ) * n; - int err = posix_memalign( (void**)&ptr, alignment, bytes ); + MemoryHighWatermark::instance() += bytes; + + int err = posix_memalign( (void**)&ptr, alignment, bytes ); if ( err ) { throw_AllocationFailed( bytes, Here() ); } @@ -97,8 +138,11 @@ class DataStore : public ArrayDataStore { void free_aligned( Value*& ptr ) { free( ptr ); ptr = nullptr; + MemoryHighWatermark::instance() -= footprint(); } + size_t footprint() const { return sizeof( Value ) * size_; } + Value* data_store_; size_t size_; }; diff --git a/src/atlas/field/FieldCreatorArraySpec.cc b/src/atlas/field/FieldCreatorArraySpec.cc index 8d85499d7..505786c42 100644 --- a/src/atlas/field/FieldCreatorArraySpec.cc +++ b/src/atlas/field/FieldCreatorArraySpec.cc @@ -18,6 +18,7 @@ #include "atlas/array/DataType.h" #include "atlas/field/detail/FieldImpl.h" #include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" namespace atlas { namespace field { @@ -60,7 +61,15 @@ FieldImpl* FieldCreatorArraySpec::createField( const eckit::Parametrisation& par std::string name; params.get( "name", name ); - return FieldImpl::create( name, datatype, array::ArraySpec( std::move( s ), array::ArrayAlignment( alignment ) ) ); + Log::trace() << "Create field " << name << "\t shape=["; + for ( size_t i = 0; i < s.size(); ++i ) { + Log::trace() << s[i] << ( i < s.size() - 1 ? "," : "" ); + } + Log::trace() << "]" << std::endl; + auto field = + FieldImpl::create( name, datatype, array::ArraySpec( std::move( s ), array::ArrayAlignment( alignment ) ) ); + field->callbackOnDestruction( [field]() { Log::trace() << "Destroy field " << field->name() << std::endl; } ); + return field; } namespace { diff --git a/src/atlas/field/detail/FieldImpl.cc b/src/atlas/field/detail/FieldImpl.cc index 9b7e2b98a..a194f46af 100644 --- a/src/atlas/field/detail/FieldImpl.cc +++ b/src/atlas/field/detail/FieldImpl.cc @@ -83,6 +83,9 @@ FieldImpl::FieldImpl( const std::string& name, array::Array* array ) : functions FieldImpl::~FieldImpl() { array_->detach(); if ( array_->owners() == 0 ) { + for ( auto& f : callback_on_destruction_ ) { + f(); + } delete array_; } delete functionspace_; diff --git a/src/atlas/field/detail/FieldImpl.h b/src/atlas/field/detail/FieldImpl.h index a40015d45..4ed8ce39d 100644 --- a/src/atlas/field/detail/FieldImpl.h +++ b/src/atlas/field/detail/FieldImpl.h @@ -13,6 +13,7 @@ #pragma once +#include #include #include @@ -198,6 +199,9 @@ class FieldImpl : public util::Object { void haloExchange( bool on_device = false ) const; void adjointHaloExchange( bool on_device = false ) const; + + void callbackOnDestruction( std::function&& f ) { callback_on_destruction_.emplace_back( std::move( f ) ); } + private: // methods void print( std::ostream& os, bool dump = false ) const; @@ -206,6 +210,7 @@ class FieldImpl : public util::Object { util::Metadata metadata_; array::Array* array_; FunctionSpace* functionspace_; + std::vector> callback_on_destruction_; }; //---------------------------------------------------------------------------------------------------------------------- From 15913165dd27cdb134638d4adc0ceb32f8146f5a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 1 Jul 2020 13:40:17 +0000 Subject: [PATCH 009/161] ATLAS-303 Allow 64bit idx_t --- CMakeLists.txt | 4 +- src/atlas/array/ArraySpec.cc | 14 +-- src/atlas/array/ArraySpec.h | 8 +- src/atlas/field/FieldSet.h | 15 +++- src/atlas/functionspace/Spectral.h | 4 +- src/atlas/grid/detail/grid/Structured.cc | 2 +- .../interpolation/method/fe/FiniteElement.cc | 6 +- .../structured/StructuredInterpolation2D.tcc | 2 +- .../structured/StructuredInterpolation3D.tcc | 4 +- src/atlas/mesh/Elements.cc | 4 +- src/atlas/mesh/actions/BuildHalo.cc | 32 +++---- .../actions/ReorderReverseCuthillMckee.cc | 24 ++--- .../detail/StructuredMeshGenerator.cc | 2 +- src/atlas/trans/local/TransLocal.cc | 6 +- src/atlas/trans/local/TransLocal.h | 2 +- src/atlas/util/Allocate.cc | 8 +- src/atlas/util/Allocate.h | 20 ++--- src/atlas_f/grid/atlas_Grid_module.F90 | 4 +- .../mesh/atlas_Connectivity_module.F90 | 4 +- src/atlas_f/util/atlas_KDTree_module.F90 | 62 +++++++------ src/atlas_f/util/atlas_allocate_module.F90 | 90 ++++++++++++++----- src/tests/array/test_array.cc | 2 +- src/tests/functionspace/test_stencil.cc | 2 +- src/tests/grid/fctest_grids.F90 | 2 +- src/tests/mesh/test_mesh_reorder.cc | 6 +- src/tests/numerics/test_fvm_nabla.cc | 8 +- src/tests/util/fctest_kdtree.F90 | 5 +- 27 files changed, 204 insertions(+), 138 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a32af1d43..87b073fff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,8 +24,8 @@ find_package( ecbuild 3.1.0 REQUIRED ) project( atlas LANGUAGES CXX ) -set( ATLAS_BITS_GLOBAL 64 ) # bits used to define a global index (atlas::gidx_t) -set( ATLAS_BITS_LOCAL 32 ) # bits used to define a local index (atlas::idx_t) +set( ATLAS_BITS_GLOBAL 64 CACHE STRING "bits used to define a global index (atlas::gidx_t)" ) +set( ATLAS_BITS_LOCAL 32 CACHE STRING "bits used to define a local index (atlas::idx_t)" ) ################################################################################ # Required packages diff --git a/src/atlas/array/ArraySpec.cc b/src/atlas/array/ArraySpec.cc index 2fae44216..34ecee15d 100644 --- a/src/atlas/array/ArraySpec.cc +++ b/src/atlas/array/ArraySpec.cc @@ -18,10 +18,10 @@ namespace atlas { namespace array { namespace { -idx_t compute_aligned_size( idx_t size, idx_t alignment ) { - idx_t div = size / alignment; - idx_t mod = size % alignment; - idx_t _aligned_size = div * alignment; +size_t compute_aligned_size( size_t size, size_t alignment ) { + size_t div = size / alignment; + size_t mod = size % alignment; + size_t _aligned_size = div * alignment; if ( mod > 0 ) { _aligned_size += alignment; } @@ -47,10 +47,10 @@ ArraySpec::ArraySpec( const ArrayShape& shape, const ArrayAlignment& alignment ) shape_[j] = shape[j]; strides_[j] = allocated_size_; layout_[j] = j; - size_ *= shape_[j]; - allocated_size_ *= aligned_shape[j]; + size_ *= size_t( shape_[j] ); + allocated_size_ *= size_t( aligned_shape[j] ); } - ATLAS_ASSERT( allocated_size_ == compute_aligned_size( shape_[0] * strides_[0], alignment ) ); + ATLAS_ASSERT( allocated_size_ == compute_aligned_size( size_t( shape_[0] ) * size_t( strides_[0] ), alignment ) ); contiguous_ = ( size_ == allocated_size_ ); default_layout_ = true; alignment_ = alignment; diff --git a/src/atlas/array/ArraySpec.h b/src/atlas/array/ArraySpec.h index 8c53949a6..bd7318ee8 100644 --- a/src/atlas/array/ArraySpec.h +++ b/src/atlas/array/ArraySpec.h @@ -26,9 +26,9 @@ namespace array { class ArraySpec { private: - idx_t size_; + size_t size_; idx_t rank_; - idx_t allocated_size_; + size_t allocated_size_; ArrayShape shape_; ArrayStrides strides_; ArrayLayout layout_; @@ -46,8 +46,8 @@ class ArraySpec { ArraySpec( const ArrayShape&, const ArrayAlignment& ); ArraySpec( const ArrayShape&, const ArrayStrides&, const ArrayAlignment& ); ArraySpec( const ArrayShape&, const ArrayStrides&, const ArrayLayout&, const ArrayAlignment& ); - idx_t allocatedSize() const { return allocated_size_; } - idx_t size() const { return size_; } + size_t allocatedSize() const { return allocated_size_; } + size_t size() const { return size_; } idx_t rank() const { return rank_; } const ArrayShape& shape() const { return shape_; } const ArrayAlignment& alignment() const { return alignment_; } diff --git a/src/atlas/field/FieldSet.h b/src/atlas/field/FieldSet.h index 80292f960..9192647b8 100644 --- a/src/atlas/field/FieldSet.h +++ b/src/atlas/field/FieldSet.h @@ -15,6 +15,7 @@ #pragma once +#include #include #include #include @@ -54,8 +55,11 @@ class FieldSetImpl : public util::Object { const std::string& name() const { return name_; } std::string& name() { return name_; } - const Field& operator[]( const idx_t& i ) const { return field( i ); } - Field& operator[]( const idx_t& i ) { return field( i ); } + const Field& operator[]( const std::int32_t& i ) const { return field( i ); } + Field& operator[]( const std::int32_t& i ) { return field( i ); } + + const Field& operator[]( const std::int64_t& i ) const { return field( i ); } + Field& operator[]( const std::int64_t& i ) { return field( i ); } const Field& operator[]( const std::string& name ) const { return field( name ); } Field& operator[]( const std::string& name ) { return field( name ); } @@ -138,8 +142,11 @@ class FieldSet : DOXYGEN_HIDE( public util::ObjectHandle ) const std::string& name() const { return get()->name(); } std::string& name() { return get()->name(); } - const Field& operator[]( const idx_t& i ) const { return get()->operator[]( i ); } - Field& operator[]( const idx_t& i ) { return get()->operator[]( i ); } + const Field& operator[]( const std::int32_t& i ) const { return get()->operator[]( i ); } + Field& operator[]( const std::int32_t& i ) { return get()->operator[]( i ); } + + const Field& operator[]( const std::int64_t& i ) const { return get()->operator[]( i ); } + Field& operator[]( const std::int64_t& i ) { return get()->operator[]( i ); } const Field& operator[]( const std::string& name ) const { return get()->operator[]( name ); } Field& operator[]( const std::string& name ) { return get()->operator[]( name ); } diff --git a/src/atlas/functionspace/Spectral.h b/src/atlas/functionspace/Spectral.h index b4d6e7940..1d89a55ca 100644 --- a/src/atlas/functionspace/Spectral.h +++ b/src/atlas/functionspace/Spectral.h @@ -127,7 +127,7 @@ class Spectral : public functionspace::FunctionSpaceImpl { } } else { - const int nb_zonal_wavenumbers{zonal_wavenumbers.size()}; + const int nb_zonal_wavenumbers{static_cast( zonal_wavenumbers.size() )}; atlas_omp_parallel_for( int jm = 0; jm < nb_zonal_wavenumbers; ++jm ) { const int m = zonal_wavenumbers( jm ); for ( int n = m; n <= truncation; ++n ) { @@ -153,7 +153,7 @@ class Spectral : public functionspace::FunctionSpaceImpl { } } else { - const int nb_zonal_wavenumbers{zonal_wavenumbers.size()}; + const int nb_zonal_wavenumbers{static_cast( zonal_wavenumbers.size() )}; atlas_omp_parallel_for( int jm = 0; jm < nb_zonal_wavenumbers; ++jm ) { const int m = zonal_wavenumbers( jm ); for ( int n = m; n <= truncation; ++n ) { diff --git a/src/atlas/grid/detail/grid/Structured.cc b/src/atlas/grid/detail/grid/Structured.cc index d17d7f7f5..3ea526720 100644 --- a/src/atlas/grid/detail/grid/Structured.cc +++ b/src/atlas/grid/detail/grid/Structured.cc @@ -561,7 +561,7 @@ void Structured::crop( const Domain& dom ) { cropped_xmin[jcropped] = normalise( x( i_xmin, j ) ); cropped_xmax[jcropped] = normalise( x( i_xmax, j ) ); - cropped_nx[jcropped] = std::max( 1, n ); + cropped_nx[jcropped] = std::max( 1, n ); } } } diff --git a/src/atlas/interpolation/method/fe/FiniteElement.cc b/src/atlas/interpolation/method/fe/FiniteElement.cc index d2d28639e..5752782c1 100644 --- a/src/atlas/interpolation/method/fe/FiniteElement.cc +++ b/src/atlas/interpolation/method/fe/FiniteElement.cc @@ -136,7 +136,7 @@ void FiniteElement::print( std::ostream& out ) const { auto stencil_points_loc = array::make_view( field_stencil_points_loc ); auto stencil_weights_loc = array::make_view( field_stencil_weigths_loc ); - auto stencil_size_loc = array::make_view( field_stencil_size_loc ); + auto stencil_size_loc = array::make_view( field_stencil_size_loc ); stencil_size_loc.assign( 0 ); for ( Matrix::const_iterator it = matrix_.begin(); it != matrix_.end(); ++it ) { @@ -154,12 +154,12 @@ void FiniteElement::print( std::ostream& out ) const { tgt.createField( option::variables( Stencil::max_stencil_size ) | option::global( 0 ) ); auto field_stencil_weights_glb = tgt.createField( option::variables( Stencil::max_stencil_size ) | option::global( 0 ) ); - auto field_stencil_size_glb = tgt.createField( option::global( 0 ) ); + auto field_stencil_size_glb = tgt.createField( option::global( 0 ) ); auto stencil_points_glb = array::make_view( field_stencil_points_glb ); auto stencil_weights_glb = array::make_view( field_stencil_weights_glb ); - auto stencil_size_glb = array::make_view( field_stencil_size_glb ); + auto stencil_size_glb = array::make_view( field_stencil_size_glb ); tgt.gather().gather( stencil_size_loc, stencil_size_glb ); tgt.gather().gather( stencil_points_loc, stencil_points_glb ); diff --git a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc index 5ed82331c..49ca6a631 100644 --- a/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc +++ b/src/atlas/interpolation/method/structured/StructuredInterpolation2D.tcc @@ -64,7 +64,7 @@ void StructuredInterpolation2D::do_setup( const Grid& source, const Grid ATLAS_ASSERT( StructuredGrid( source ) ); FunctionSpace source_fs = - functionspace::StructuredColumns( source, option::halo( std::max( kernel_->stencil_halo(), 1 ) ) ); + functionspace::StructuredColumns( source, option::halo( std::max( kernel_->stencil_halo(), 1 ) ) ); // guarantee "1" halo for pole treatment! FunctionSpace target_fs = functionspace::PointCloud( target ); diff --git a/src/atlas/interpolation/method/structured/StructuredInterpolation3D.tcc b/src/atlas/interpolation/method/structured/StructuredInterpolation3D.tcc index d27e29ba2..f3a5f7416 100644 --- a/src/atlas/interpolation/method/structured/StructuredInterpolation3D.tcc +++ b/src/atlas/interpolation/method/structured/StructuredInterpolation3D.tcc @@ -149,8 +149,6 @@ void StructuredInterpolation3D::do_execute( const FieldSet& src_fields, return; } - ATLAS_TRACE( "StructuredInterpolation<" + Kernel::className() + ">::do_execute()" ); - const idx_t N = src_fields.size(); ATLAS_ASSERT( N == tgt_fields.size() ); @@ -159,6 +157,8 @@ void StructuredInterpolation3D::do_execute( const FieldSet& src_fields, haloExchange( src_fields ); + ATLAS_TRACE( "StructuredInterpolation<" + Kernel::className() + ">::do_execute()" ); + array::DataType datatype = src_fields[0].datatype(); int rank = src_fields[0].rank(); diff --git a/src/atlas/mesh/Elements.cc b/src/atlas/mesh/Elements.cc index 663aa196d..32d170c76 100644 --- a/src/atlas/mesh/Elements.cc +++ b/src/atlas/mesh/Elements.cc @@ -155,13 +155,13 @@ idx_t Elements::add( const idx_t nb_elements ) { template <> array::LocalIndexView Elements::indexview( Field& field ) const { - auto local_view = array::make_host_view( field ).slice( array::Range{begin(), begin() + size()} ); + auto local_view = array::make_host_view( field ).slice( array::Range{begin(), begin() + size()} ); return array::LocalIndexView( local_view.data(), local_view.shape(), local_view.strides() ); } template <> array::LocalIndexView Elements::indexview( const Field& field ) const { - auto local_view = array::make_host_view( field ).slice( array::Range{begin(), begin() + size()} ); + auto local_view = array::make_host_view( field ).slice( array::Range{begin(), begin() + size()} ); return array::LocalIndexView( local_view.data(), local_view.shape(), local_view.strides() ); } diff --git a/src/atlas/mesh/actions/BuildHalo.cc b/src/atlas/mesh/actions/BuildHalo.cc index b18406580..d4e92b162 100644 --- a/src/atlas/mesh/actions/BuildHalo.cc +++ b/src/atlas/mesh/actions/BuildHalo.cc @@ -77,8 +77,8 @@ void make_nodes_global_index_human_readable( const mesh::actions::BuildHalo& bui // and could receive different gidx for different tasks // unused // int mypart = mpi::rank(); - int nparts = mpi::size(); - idx_t root = 0; + int nparts = mpi::size(); + size_t root = 0; array::ArrayView nodes_glb_idx = array::make_view( nodes.global_index() ); // nodes_glb_idx.dump( Log::info() ); @@ -105,7 +105,7 @@ void make_nodes_global_index_human_readable( const mesh::actions::BuildHalo& bui } std::vector glb_idx( points_to_edit.size() ); - idx_t nb_nodes = static_cast( glb_idx.size() ); + int nb_nodes = static_cast( glb_idx.size() ); for ( idx_t i = 0; i < nb_nodes; ++i ) { glb_idx[i] = nodes_glb_idx( points_to_edit[i] ); } @@ -135,7 +135,7 @@ void make_nodes_global_index_human_readable( const mesh::actions::BuildHalo& bui std::vector recvdispls( mpi::size() ); ATLAS_TRACE_MPI( GATHER ) { mpi::comm().gather( nb_nodes, recvcounts, root ); } - int glb_nb_nodes = std::accumulate( recvcounts.begin(), recvcounts.end(), 0 ); + idx_t glb_nb_nodes = std::accumulate( recvcounts.begin(), recvcounts.end(), 0 ); recvdispls[0] = 0; for ( int jpart = 1; jpart < nparts; ++jpart ) { // start at 1 @@ -225,8 +225,8 @@ void make_cells_global_index_human_readable( const mesh::actions::BuildHalo& bui } std::vector glb_idx( cells_to_edit.size() ); - const idx_t nb_cells = static_cast( glb_idx.size() ); - for ( idx_t i = 0; i < nb_cells; ++i ) { + const int nb_cells = static_cast( glb_idx.size() ); + for ( int i = 0; i < nb_cells; ++i ) { glb_idx[i] = cells_glb_idx( cells_to_edit[i] ); } @@ -333,10 +333,10 @@ void build_lookup_node2elem( const Mesh& mesh, Node2Elem& node2elem ) { } } -void accumulate_partition_bdry_nodes_old( Mesh& mesh, std::vector& bdry_nodes ) { +void accumulate_partition_bdry_nodes_old( Mesh& mesh, std::vector& bdry_nodes ) { ATLAS_TRACE(); - std::set bdry_nodes_set; + std::set bdry_nodes_set; std::vector facet_nodes; std::vector connectivity_facet_to_elem; @@ -364,10 +364,10 @@ void accumulate_partition_bdry_nodes_old( Mesh& mesh, std::vector& bdry_nod } } } - bdry_nodes = std::vector( bdry_nodes_set.begin(), bdry_nodes_set.end() ); + bdry_nodes = std::vector( bdry_nodes_set.begin(), bdry_nodes_set.end() ); } -void accumulate_partition_bdry_nodes( Mesh& mesh, idx_t halo, std::vector& bdry_nodes ) { +void accumulate_partition_bdry_nodes( Mesh& mesh, idx_t halo, std::vector& bdry_nodes ) { #ifndef ATLAS_103 /* deprecated */ accumulate_partition_bdry_nodes_old( mesh, bdry_nodes ); @@ -384,10 +384,10 @@ void accumulate_partition_bdry_nodes( Mesh& mesh, idx_t halo, std::vector& } template -std::vector filter_nodes( std::vector nodes, const Predicate& predicate ) { - std::vector filtered; +std::vector filter_nodes( std::vector nodes, const Predicate& predicate ) { + std::vector filtered; filtered.reserve( nodes.size() ); - for ( int inode : nodes ) { + for ( idx_t inode : nodes ) { if ( predicate( inode ) ) { filtered.push_back( inode ); } @@ -623,7 +623,7 @@ class BuildHaloHelper { array::ArrayView elem_flags; array::ArrayView elem_glb_idx; - std::vector bdry_nodes; + std::vector bdry_nodes; Node2Elem node_to_elem; Uid2Node uid2node; UniqueLonLat compute_uid; @@ -1125,8 +1125,8 @@ void increase_halo_interior( BuildHaloHelper& helper ) { // 1) Find boundary nodes of this partition: accumulate_partition_bdry_nodes( helper.mesh, helper.halosize, helper.bdry_nodes ); - const std::vector& bdry_nodes = helper.bdry_nodes; - const idx_t nb_bdry_nodes = static_cast( bdry_nodes.size() ); + const std::vector& bdry_nodes = helper.bdry_nodes; + const idx_t nb_bdry_nodes = static_cast( bdry_nodes.size() ); // 2) Communicate uid of these boundary nodes to other partitions diff --git a/src/atlas/mesh/actions/ReorderReverseCuthillMckee.cc b/src/atlas/mesh/actions/ReorderReverseCuthillMckee.cc index 3b6916b05..6c19ee34d 100644 --- a/src/atlas/mesh/actions/ReorderReverseCuthillMckee.cc +++ b/src/atlas/mesh/actions/ReorderReverseCuthillMckee.cc @@ -56,14 +56,14 @@ class CuthillMckee { } // Cuthill-Mckee algorithm - std::vector order() { + std::vector order() { auto degrees = computeDegrees( sparse_ ); - std::queue Q; - std::vector R; + std::queue Q; + std::vector R; R.reserve( degrees.size() ); - std::vector sorted_row; + std::vector sorted_row; sorted_row.reserve( sparse_.cols() ); NotVisited_t not_visited; @@ -87,7 +87,7 @@ class CuthillMckee { // Simple Breadth First Search while ( !Q.empty() ) { - int row = Q.front(); + idx_t row = Q.front(); sorted_row.clear(); @@ -102,7 +102,7 @@ class CuthillMckee { } std::sort( sorted_row.begin(), sorted_row.end(), - [&]( int i, int j ) { return degrees[i] - degrees[j]; } ); + [&]( idx_t i, idx_t j ) { return degrees[i] - degrees[j]; } ); for ( size_t i = 0; i < sorted_row.size(); i++ ) { Q.emplace( sorted_row[i] ); @@ -125,11 +125,11 @@ class ReverseCuthillMckee { ReverseCuthillMckee( eckit::linalg::SparseMatrix&& m ) : cuthill_mckee_( std::move( m ) ) {} // Reverse Cuthill-Mckee algorithm - std::vector order() { - std::vector cuthill = cuthill_mckee_.order(); + std::vector order() { + std::vector cuthill = cuthill_mckee_.order(); - int size = static_cast( cuthill.size() ); - int n = size; + idx_t size = static_cast( cuthill.size() ); + idx_t n = size; if ( n % 2 == 0 ) { n -= 1; @@ -137,8 +137,8 @@ class ReverseCuthillMckee { n = n / 2; - for ( int i = 0; i <= n; i++ ) { - int j = cuthill[size - 1 - i]; + for ( idx_t i = 0; i <= n; i++ ) { + idx_t j = cuthill[size - 1 - i]; cuthill[size - 1 - i] = cuthill[i]; cuthill[i] = j; } diff --git a/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc b/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc index 0b626aad4..c2cdc6901 100644 --- a/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc +++ b/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc @@ -751,7 +751,7 @@ We need to connect to next region int nb_region_nodes = 0; for ( int jlat = region.north; jlat <= region.south; ++jlat ) { n = offset.at( jlat ); - region.lat_begin.at( jlat ) = std::max( 0, region.lat_begin.at( jlat ) ); + region.lat_begin.at( jlat ) = std::max( 0, region.lat_begin.at( jlat ) ); for ( idx_t jlon = 0; jlon < rg.nx( jlat ); ++jlon ) { if ( distribution.partition( n ) == mypart ) { region.lat_begin.at( jlat ) = std::min( region.lat_begin.at( jlat ), jlon ); diff --git a/src/atlas/trans/local/TransLocal.cc b/src/atlas/trans/local/TransLocal.cc index 1f344a1c4..3ce0834e0 100644 --- a/src/atlas/trans/local/TransLocal.cc +++ b/src/atlas/trans/local/TransLocal.cc @@ -292,9 +292,9 @@ TransLocal::TransLocal( const Cache& cache, const Grid& grid, const Domain& doma double fft_threshold = 0.0; // fraction of latitudes of the full grid down to which FFT is used. // This threshold needs to be adjusted depending on the dgemm and FFT performance of the machine // on which this code is running! - int nlats = 0; - int nlonsMax = 0; - int neqtr = 0; + idx_t nlats = 0; + idx_t nlonsMax = 0; + idx_t neqtr = 0; useFFT_ = TransParameters( config ).fft(); unstruct_precomp_ = ( config.has( "precompute" ) ? precompute_ : false ); no_symmetry_ = false; diff --git a/src/atlas/trans/local/TransLocal.h b/src/atlas/trans/local/TransLocal.h index ca1458c9c..14089166e 100644 --- a/src/atlas/trans/local/TransLocal.h +++ b/src/atlas/trans/local/TransLocal.h @@ -184,7 +184,7 @@ class TransLocal : public trans::TransImpl { std::vector jlonMin_; idx_t jlatMin_; idx_t jlatMinLeg_; - idx_t nlonsMaxGlobal_; + int nlonsMaxGlobal_; std::vector nlonsGlobal_; std::vector nlat0_; idx_t nlatsGlobal_; diff --git a/src/atlas/util/Allocate.cc b/src/atlas/util/Allocate.cc index 8ce6008d0..40a67a769 100644 --- a/src/atlas/util/Allocate.cc +++ b/src/atlas/util/Allocate.cc @@ -78,16 +78,16 @@ void deallocate_host( void* ptr ) { //------------------------------------------------------------------------------ extern "C" { -void atlas__allocate_managedmem_double( double*& a, int N ) { +void atlas__allocate_managedmem_double( double*& a, size_t N ) { allocate_managedmem( a, N ); } -void atlas__allocate_managedmem_float( float*& a, int N ) { +void atlas__allocate_managedmem_float( float*& a, size_t N ) { allocate_managedmem( a, N ); } -void atlas__allocate_managedmem_int( int*& a, int N ) { +void atlas__allocate_managedmem_int( int*& a, size_t N ) { allocate_managedmem( a, N ); } -void atlas__allocate_managedmem_long( long*& a, int N ) { +void atlas__allocate_managedmem_long( long*& a, size_t N ) { allocate_managedmem( a, N ); } void atlas__deallocate_managedmem( void*& a ) { diff --git a/src/atlas/util/Allocate.h b/src/atlas/util/Allocate.h index 50258876d..76d871089 100644 --- a/src/atlas/util/Allocate.h +++ b/src/atlas/util/Allocate.h @@ -32,9 +32,9 @@ void deallocate_host( void* ptr ); } // namespace detail template -void allocate_managedmem( T*& data, idx_t N ) { +void allocate_managedmem( T*& data, size_t N ) { if ( N != 0 ) { - detail::allocate_cudamanaged( reinterpret_cast( &data ), static_cast( N ) * sizeof( T ) ); + detail::allocate_cudamanaged( reinterpret_cast( &data ), N * sizeof( T ) ); } } @@ -47,9 +47,9 @@ void delete_managedmem( T*& data ) { } template -void allocate_devicemem( T*& data, idx_t N ) { +void allocate_devicemem( T*& data, size_t N ) { if ( N != 0 ) { - detail::allocate_cuda( reinterpret_cast( &data ), static_cast( N ) * sizeof( T ) ); + detail::allocate_cuda( reinterpret_cast( &data ), N * sizeof( T ) ); } } @@ -62,9 +62,9 @@ void delete_devicemem( T*& data ) { } template -void allocate_hostmem( T*& data, idx_t N ) { +void allocate_hostmem( T*& data, size_t N ) { if ( N != 0 ) { - detail::allocate_host( reinterpret_cast( &data ), static_cast( N ) * sizeof( T ) ); + detail::allocate_host( reinterpret_cast( &data ), N * sizeof( T ) ); } } @@ -80,10 +80,10 @@ void delete_hostmem( T*& data ) { //------------------------------------------------------------------------------ extern "C" { -void atlas__allocate_managedmem_double( double*& a, idx_t N ); -void atlas__allocate_managedmem_float( float*& a, idx_t N ); -void atlas__allocate_managedmem_int( int*& a, idx_t N ); -void atlas__allocate_managedmem_long( long*& a, idx_t N ); +void atlas__allocate_managedmem_double( double*& a, size_t N ); +void atlas__allocate_managedmem_float( float*& a, size_t N ); +void atlas__allocate_managedmem_int( int*& a, size_t N ); +void atlas__allocate_managedmem_long( long*& a, size_t N ); void atlas__deallocate_managedmem( void*& a ); } diff --git a/src/atlas_f/grid/atlas_Grid_module.F90 b/src/atlas_f/grid/atlas_Grid_module.F90 index f62276fbe..aa9939bda 100644 --- a/src/atlas_f/grid/atlas_Grid_module.F90 +++ b/src/atlas_f/grid/atlas_Grid_module.F90 @@ -669,7 +669,7 @@ function Structured__ij_int32(this, gidx) result(ij) use atlas_grid_Structured_c_binding integer(c_int), intent (in) :: gidx class(atlas_StructuredGrid), intent(in) :: this - integer(c_int) :: ij (2) + integer(ATLAS_KIND_IDX) :: ij (2) call atlas__grid__Structured__index2ij(this%CPTR_PGIBUG_A, c_gidx(gidx), ij(1), ij(2)) ij = ij + 1 end function @@ -679,7 +679,7 @@ function Structured__ij_int64(this, gidx) result(ij) use atlas_grid_Structured_c_binding integer(c_long), intent (in) :: gidx class(atlas_StructuredGrid), intent(in) :: this - integer(c_int) :: ij (2) + integer(ATLAS_KIND_IDX) :: ij (2) call atlas__grid__Structured__index2ij(this%CPTR_PGIBUG_A, c_gidx(gidx), ij(1), ij(2)) ij = ij + 1 end function diff --git a/src/atlas_f/mesh/atlas_Connectivity_module.F90 b/src/atlas_f/mesh/atlas_Connectivity_module.F90 index 955c1e2ac..11702a2ab 100644 --- a/src/atlas_f/mesh/atlas_Connectivity_module.F90 +++ b/src/atlas_f/mesh/atlas_Connectivity_module.F90 @@ -245,8 +245,8 @@ pure function access_value(this,c,r) result(val) end function pure function access_rows(this) result(val) - use, intrinsic :: iso_c_binding, only : c_int - integer(c_int) :: val + use atlas_kinds_module + integer(ATLAS_KIND_IDX) :: val class(atlas_ConnectivityAccess), intent(in) :: this val = this%rows_ end function diff --git a/src/atlas_f/util/atlas_KDTree_module.F90 b/src/atlas_f/util/atlas_KDTree_module.F90 index 327d8f622..f29402cd8 100644 --- a/src/atlas_f/util/atlas_KDTree_module.F90 +++ b/src/atlas_f/util/atlas_KDTree_module.F90 @@ -110,27 +110,30 @@ end subroutine atlas_IndexKDTree__delete subroutine IndexKDTree__reserve(this, size) use atlas_KDTree_c_binding use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this - integer(c_int), intent(in) :: size + integer(ATLAS_KIND_IDX), intent(in) :: size call atlas__IndexKDTree__reserve(this%CPTR_PGIBUG_A, size) end subroutine IndexKDTree__reserve subroutine IndexKDTree__insert_separate_coords(this, lon, lat, index) use atlas_KDTree_c_binding use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this real(c_double), intent(in) :: lon real(c_double), intent(in) :: lat - integer(c_int), intent(in) :: index + integer(ATLAS_KIND_IDX), intent(in) :: index call atlas__IndexKDTree__insert(this%CPTR_PGIBUG_A, lon, lat, index) end subroutine IndexKDTree__insert_separate_coords subroutine IndexKDTree__insert_vectorized_coords(this, lonlat, index) use atlas_KDTree_c_binding use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this real(c_double), intent(in) :: lonlat(2) - integer(c_int), intent(in) :: index + integer(ATLAS_KIND_IDX), intent(in) :: index call atlas__IndexKDTree__insert(this%CPTR_PGIBUG_A, lonlat(1), lonlat(2), index) end subroutine IndexKDTree__insert_vectorized_coords @@ -141,16 +144,18 @@ subroutine IndexKDTree__build_only(this) call atlas__IndexKDTree__build(this%CPTR_PGIBUG_A) end subroutine IndexKDTree__build_only -subroutine IndexKDTree__build_list_separate_coords(this, k, lons, lats, indices) +subroutine IndexKDTree__build_list_separate_coords(this, n, lons, lats, indices) use atlas_KDTree_c_binding use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this - integer(c_int), intent(in) :: k - real(c_double), intent(in) :: lons(k) - real(c_double), intent(in) :: lats(k) - integer(c_int), intent(in), optional :: indices(k) - integer(c_int) :: i, index - do i = 1, k + integer(ATLAS_KIND_IDX), intent(in) :: n + real(c_double), intent(in) :: lons(n) + real(c_double), intent(in) :: lats(n) + integer(ATLAS_KIND_IDX), intent(in), optional :: indices(n) + integer(ATLAS_KIND_IDX) :: i + integer(ATLAS_KIND_IDX) :: index + do i = 1, n if (present(indices)) then index = indices(i) else @@ -161,15 +166,16 @@ subroutine IndexKDTree__build_list_separate_coords(this, k, lons, lats, indices) call this%build() end subroutine IndexKDTree__build_list_separate_coords -subroutine IndexKDTree__build_list_vectorized_coords(this, k, lonlats, indices) +subroutine IndexKDTree__build_list_vectorized_coords(this, n, lonlats, indices) use atlas_KDTree_c_binding use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this - integer(c_int), intent(in) :: k - real(c_double), intent(in) :: lonlats(k,2) - integer(c_int), intent(in), optional :: indices(k) - integer(c_int) :: i, index - do i = 1, k + integer(ATLAS_KIND_IDX), intent(in) :: n + real(c_double), intent(in) :: lonlats(n,2) + integer(ATLAS_KIND_IDX), intent(in), optional :: indices(n) + integer(ATLAS_KIND_IDX) :: i, index + do i = 1, n if (present(indices)) then index = indices(i) else @@ -184,11 +190,12 @@ subroutine IndexKDTree__closestPoints_separate_coords(this, plon, plat, k, indic use atlas_KDTree_c_binding use, intrinsic :: iso_c_binding, only : c_f_pointer use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this real(c_double), intent(in) :: plon real(c_double), intent(in) :: plat integer(c_int), intent(in) :: k - integer(c_int), intent(out) :: indices(k) + integer(ATLAS_KIND_IDX), intent(out) :: indices(k) real(c_double), intent(out), optional :: distances(k) real(c_double), intent(out), optional :: lons(k) real(c_double), intent(out), optional :: lats(k) @@ -196,7 +203,7 @@ subroutine IndexKDTree__closestPoints_separate_coords(this, plon, plat, k, indic type(c_ptr) :: distances_cptr type(c_ptr) :: lons_cptr type(c_ptr) :: lats_cptr - integer(c_int), pointer :: indices_fptr(:) + integer(ATLAS_KIND_IDX), pointer :: indices_fptr(:) real(c_double), pointer :: distances_fptr(:) real(c_double), pointer :: lons_fptr(:) real(c_double), pointer :: lats_fptr(:) @@ -228,17 +235,18 @@ subroutine IndexKDTree__closestPoints_vectorized_coords(this, point, k, indices, use atlas_KDTree_c_binding use, intrinsic :: iso_c_binding, only : c_f_pointer use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this real(c_double), intent(in) :: point(2) integer(c_int), intent(in) :: k - integer(c_int), intent(out) :: indices(k) + integer(ATLAS_KIND_IDX), intent(out) :: indices(k) real(c_double), intent(out), optional :: distances(k) real(c_double), intent(out), optional :: lonlats(k,2) type(c_ptr) :: indices_cptr type(c_ptr) :: distances_cptr type(c_ptr) :: lons_cptr type(c_ptr) :: lats_cptr - integer(c_int), pointer :: indices_fptr(:) + integer(ATLAS_KIND_IDX), pointer :: indices_fptr(:) real(c_double), pointer :: distances_fptr(:) real(c_double), pointer :: lons_fptr(:) real(c_double), pointer :: lats_fptr(:) @@ -267,10 +275,11 @@ end subroutine IndexKDTree__closestPoints_vectorized_coords subroutine IndexKDTree__closestPoint_separate_coords(this, plon, plat, index, distance, lon, lat) use atlas_KDTree_c_binding use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this real(c_double), intent(in) :: plon real(c_double), intent(in) :: plat - integer(c_int), intent(out) :: index + integer(ATLAS_KIND_IDX), intent(out) :: index real(c_double), intent(out), optional :: distance real(c_double), intent(out), optional :: lon real(c_double), intent(out), optional :: lat @@ -284,9 +293,10 @@ end subroutine IndexKDTree__closestPoint_separate_coords subroutine IndexKDTree__closestPoint_vectorized_coords(this, point, index, distance, lonlat) use atlas_KDTree_c_binding use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this real(c_double), intent(in) :: point(2) - integer(c_int), intent(out) :: index + integer(ATLAS_KIND_IDX), intent(out) :: index real(c_double), intent(out), optional :: distance real(c_double), intent(out), optional :: lonlat(2) real(c_double) :: distance_tmp, lon_tmp, lat_tmp @@ -302,12 +312,13 @@ subroutine IndexKDTree__closestPointsWithinRadius_separate_coords(this, plon, pl use atlas_KDTree_c_binding use, intrinsic :: iso_c_binding, only : c_f_pointer use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this real(c_double), intent(in) :: plon real(c_double), intent(in) :: plat real(c_double), intent(in) :: radius integer(c_int), intent(out) :: k - integer(c_int), allocatable, intent(inout), optional :: indices(:) + integer(ATLAS_KIND_IDX), allocatable, intent(inout), optional :: indices(:) real(c_double), allocatable, intent(inout), optional :: distances(:) real(c_double), allocatable, intent(inout), optional :: lons(:) real(c_double), allocatable, intent(inout), optional :: lats(:) @@ -367,11 +378,12 @@ subroutine IndexKDTree__closestPointsWithinRadius_vectorized_coords(this, point, use atlas_KDTree_c_binding use, intrinsic :: iso_c_binding, only : c_f_pointer use fckit_c_interop_module + use atlas_kinds_module class(atlas_IndexKDTree), intent(in) :: this real(c_double), intent(in) :: point(2) real(c_double), intent(in) :: radius integer(c_int), intent(out) :: k - integer(c_int), allocatable, intent(inout), optional :: indices(:) + integer(ATLAS_KIND_IDX), allocatable, intent(inout), optional :: indices(:) real(c_double), allocatable, intent(inout), optional :: distances(:) real(c_double), allocatable, intent(inout), optional :: lonlats(:,:) integer(c_size_t) :: k_tmp @@ -381,7 +393,7 @@ subroutine IndexKDTree__closestPointsWithinRadius_vectorized_coords(this, point, type(c_ptr) :: distances_cptr real(c_double), pointer :: lons_fptr(:) real(c_double), pointer :: lats_fptr(:) - integer(c_int), pointer :: indices_fptr(:) + integer(ATLAS_KIND_IDX), pointer :: indices_fptr(:) real(c_double), pointer :: distances_fptr(:) call atlas__IndexKDTree__closestPointsWithinRadius(this%CPTR_PGIBUG_A, point(1), point(2), radius, & & k_tmp, lons_cptr, lats_cptr, indices_cptr, distances_cptr) diff --git a/src/atlas_f/util/atlas_allocate_module.F90 b/src/atlas_f/util/atlas_allocate_module.F90 index 0937d98ed..3cc1d3a7a 100644 --- a/src/atlas_f/util/atlas_allocate_module.F90 +++ b/src/atlas_f/util/atlas_allocate_module.F90 @@ -21,11 +21,13 @@ module atlas_allocate_module module procedure atlas_allocate_managedmem_real64_r1 module procedure atlas_allocate_managedmem_real32_r1 module procedure atlas_allocate_managedmem_int32_r1 - module procedure atlas_allocate_managedmem_int64_r1 + module procedure atlas_allocate_managedmem_int64_r1_int32 + module procedure atlas_allocate_managedmem_int64_r1_int64 module procedure atlas_allocate_managedmem_real64_r2 module procedure atlas_allocate_managedmem_real32_r2 module procedure atlas_allocate_managedmem_int32_r2 - module procedure atlas_allocate_managedmem_int64_r2 + module procedure atlas_allocate_managedmem_int64_r2_int32 + module procedure atlas_allocate_managedmem_int64_r2_int64 end interface interface atlas_deallocate_managedmem @@ -49,8 +51,10 @@ subroutine atlas_allocate_managedmem_real64_r1( A, dims ) real(c_double), pointer :: a(:) integer(c_int) :: dims(:) type(c_ptr) :: value_cptr - if( dims(1) > 0 ) then - call atlas__allocate_managedmem_double( value_cptr, dims(1) ) + integer(c_size_t) :: size + size = dims(1) + if( size > 0 ) then + call atlas__allocate_managedmem_double( value_cptr, size ) call c_f_pointer(value_cptr,a,dims) endif end subroutine @@ -61,8 +65,10 @@ subroutine atlas_allocate_managedmem_real32_r1( A, dims ) real(c_float), pointer :: a(:) integer(c_int) :: dims(:) type(c_ptr) :: value_cptr - if( dims(1) > 0 ) then - call atlas__allocate_managedmem_float( value_cptr, dims(1) ) + integer(c_size_t) :: size + size = dims(1) + if( size > 0 ) then + call atlas__allocate_managedmem_float( value_cptr, size ) call c_f_pointer(value_cptr,a,dims) endif end subroutine @@ -73,20 +79,38 @@ subroutine atlas_allocate_managedmem_int32_r1( A, dims ) integer(c_int), pointer :: a(:) integer(c_int) :: dims(:) type(c_ptr) :: value_cptr - if( dims(1) > 0 ) then - call atlas__allocate_managedmem_int( value_cptr, dims(1) ) + integer(c_size_t) :: size + size = dims(1) + if( size > 0 ) then + call atlas__allocate_managedmem_int( value_cptr, size ) call c_f_pointer(value_cptr,a,dims) endif end subroutine -subroutine atlas_allocate_managedmem_int64_r1( A, dims ) +subroutine atlas_allocate_managedmem_int64_r1_int32( A, dims ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding integer(c_long), pointer :: a(:) - integer(c_int) :: dims(:) + integer(c_int32_t) :: dims(:) + type(c_ptr) :: value_cptr + integer(c_size_t) :: size + size = dims(1) + if( size > 0 ) then + call atlas__allocate_managedmem_long( value_cptr, size ) + call c_f_pointer(value_cptr,a,dims) + endif +end subroutine + +subroutine atlas_allocate_managedmem_int64_r1_int64( A, dims ) + use, intrinsic :: iso_c_binding + use atlas_allocate_c_binding + integer(c_long), pointer :: a(:) + integer(c_int64_t) :: dims(:) type(c_ptr) :: value_cptr - if( dims(1) > 0 ) then - call atlas__allocate_managedmem_long( value_cptr, dims(1) ) + integer(c_size_t) :: size + size = dims(1) + if( size > 0 ) then + call atlas__allocate_managedmem_long( value_cptr, size ) call c_f_pointer(value_cptr,a,dims) endif end subroutine @@ -97,20 +121,38 @@ subroutine atlas_allocate_managedmem_int32_r2( A, dims ) integer(c_int), pointer :: a(:,:) integer(c_int) :: dims(:) type(c_ptr) :: value_cptr - if( dims(1)*dims(2) > 0 ) then - call atlas__allocate_managedmem_int( value_cptr, dims(1)*dims(2) ) + integer(c_size_t) :: size + size = dims(1)*dims(2) + if( size > 0 ) then + call atlas__allocate_managedmem_int( value_cptr, size ) call c_f_pointer(value_cptr,a,dims) endif end subroutine -subroutine atlas_allocate_managedmem_int64_r2( A, dims ) +subroutine atlas_allocate_managedmem_int64_r2_int32( A, dims ) use, intrinsic :: iso_c_binding use atlas_allocate_c_binding integer(c_long), pointer :: a(:,:) - integer(c_int) :: dims(:) + integer(c_int32_t) :: dims(:) + type(c_ptr) :: value_cptr + integer(c_size_t) :: size + size = dims(1)*dims(2) + if( size > 0 ) then + call atlas__allocate_managedmem_long( value_cptr, size ) + call c_f_pointer(value_cptr,a,dims) + endif +end subroutine + +subroutine atlas_allocate_managedmem_int64_r2_int64( A, dims ) + use, intrinsic :: iso_c_binding + use atlas_allocate_c_binding + integer(c_long), pointer :: a(:,:) + integer(c_int64_t) :: dims(:) type(c_ptr) :: value_cptr - if( dims(1)*dims(2) > 0 ) then - call atlas__allocate_managedmem_long( value_cptr, dims(1)*dims(2) ) + integer(c_size_t) :: size + size = dims(1)*dims(2) + if( size > 0 ) then + call atlas__allocate_managedmem_long( value_cptr, size ) call c_f_pointer(value_cptr,a,dims) endif end subroutine @@ -121,8 +163,10 @@ subroutine atlas_allocate_managedmem_real64_r2( A, dims ) real(c_double), pointer :: a(:,:) integer(c_int) :: dims(:) type(c_ptr) :: value_cptr - if( dims(1)*dims(2) > 0 ) then - call atlas__allocate_managedmem_double( value_cptr, dims(1)*dims(2) ) + integer(c_size_t) :: size + size = dims(1)*dims(2) + if( size > 0 ) then + call atlas__allocate_managedmem_double( value_cptr, size ) call c_f_pointer(value_cptr,a,dims) endif end subroutine @@ -133,8 +177,10 @@ subroutine atlas_allocate_managedmem_real32_r2( A, dims ) real(c_float), pointer :: a(:,:) integer(c_int) :: dims(:) type(c_ptr) :: value_cptr - if( dims(1)*dims(2) > 0 ) then - call atlas__allocate_managedmem_float( value_cptr, dims(1)*dims(2) ) + integer(c_size_t) :: size + size = dims(1)*dims(2) + if( size > 0 ) then + call atlas__allocate_managedmem_float( value_cptr, size ) call c_f_pointer(value_cptr,a,dims) endif end subroutine diff --git a/src/tests/array/test_array.cc b/src/tests/array/test_array.cc index e32d4e972..016648087 100644 --- a/src/tests/array/test_array.cc +++ b/src/tests/array/test_array.cc @@ -125,7 +125,7 @@ CASE( "test_array_shape" ) { EXPECT_EQ( ds->rank(), 2 ); EXPECT_EQ( ds->stride( 0 ), gt_hv.storage_info().stride<0>() ); EXPECT_EQ( ds->stride( 1 ), gt_hv.storage_info().stride<1>() ); - EXPECT( ds->contiguous() ); + EXPECT( ds->contiguous() ); delete ds; } #endif diff --git a/src/tests/functionspace/test_stencil.cc b/src/tests/functionspace/test_stencil.cc index ebf9a39e6..7eef40eab 100644 --- a/src/tests/functionspace/test_stencil.cc +++ b/src/tests/functionspace/test_stencil.cc @@ -195,7 +195,7 @@ CASE( "test vertical stencil" ) { const double eps = 1.e-14; for ( idx_t k = 0; k < nlev; ++k ) { - idx_t k_expected = std::max( 0, std::min( nlev - 1, k ) ); + idx_t k_expected = std::max( 0, std::min( nlev - 1, k ) ); EXPECT( compute_lower( zcoord[k] ) == k_expected ); EXPECT( compute_lower( zcoord[k] - eps ) == k_expected ); EXPECT( compute_lower( zcoord[k] + eps ) == k_expected ); diff --git a/src/tests/grid/fctest_grids.F90 b/src/tests/grid/fctest_grids.F90 index a1c61d257..ca8666763 100644 --- a/src/tests/grid/fctest_grids.F90 +++ b/src/tests/grid/fctest_grids.F90 @@ -31,7 +31,7 @@ use atlas_module implicit none type(atlas_StructuredGrid) :: grid - integer :: i, j, i1, j1 + integer(ATLAS_KIND_IDX) :: i, j, i1, j1 integer(ATLAS_KIND_IDX) :: jglo, jglo1 grid = atlas_StructuredGrid ("N16") diff --git a/src/tests/mesh/test_mesh_reorder.cc b/src/tests/mesh/test_mesh_reorder.cc index 6c8b6738d..b2b8cf54e 100644 --- a/src/tests/mesh/test_mesh_reorder.cc +++ b/src/tests/mesh/test_mesh_reorder.cc @@ -251,7 +251,7 @@ Mesh get_mesh() { } } -void test_reordering( const util::Config& reorder_config, const std::vector& expected = {} ) { +void test_reordering( const util::Config& reorder_config, const std::vector& expected = {} ) { std::string type = reorder_config.getString( "type" ); // clang-format off std::string title = type == "none" ? "Default order" : @@ -318,7 +318,7 @@ void test_reordering( const util::Config& reorder_config, const std::vector CASE( "test_hilbert_reordering" ) { auto reorder_config = option::type( "hilbert" ) | util::Config( "recursion", 30 ); - std::vector expected; + std::vector expected; if ( grid_name() == "O16" && mpi::comm().size() == 1 ) { // clang-format off expected = {0,20,1,21,45,74,73,44,72,104,105,141,140,180,224,225,181,182,226,227,142,106,107,143,183,228,184,229,230,185,145,144,108,76,47,75,46,22,2,23,3,77,48,78,49,24,4,25,5,26,51,80,79,50,111,148,112,113,150,149,190,191,236,235,234,189,233,232,187,188,147,110,146,109,186,231,279,280,332,331,388,448,449,450,389,390,451,452,391,334,333,281,282,335,336,283,284,285,338,337,394,395,455,456,454,453,393,392,517,586,587,518,519,520,521,589,590,588,661,662,663,739,740,738,737,660,659,735,736,734,733,656,657,658,585,516,515,584,583,514,513,581,582,654,655,732,731,730,729,653,652,728,727,725,726,649,650,651,578,577,509,510,511,579,580,512,447,446,386,387,278,277,330,329,276,328,384,385,445,444,443,442,382,383,327,275,274,326,325,273,272,324,380,381,441,440,504,505,572,573,574,506,507,508,576,575,647,648,724,723,722,646,645,644,720,721,800,801,802,882,881,880,957,956,1028,1029,1030,958,959,1031,1032,960,884,883,803,804,805,806,885,886,887,807,808,810,809,889,888,963,965,964,1036,1035,1034,962,961,1033,1101,1100,1164,1165,1102,1103,1104,1167,1166,1226,1227,1282,1281,1225,1224,1280,1279,1278,1222,1223,1163,1099,1098,1162,1161,1097,1096,1160,1220,1221,1277,1276,1328,1376,1377,1329,1330,1331,1378,1422,1462,1461,1421,1420,1460,1496,1497,1528,1556,1580,1557,1581,1529,1498,1499,1530,1558,1582,1559,1531,1500,1464,1425,1424,1463,1423,1380,1379,1332,1333,1381,1334,1382,1335,1336,1337,1384,1383,1427,1466,1426,1465,1501,1532,1583,1560,1533,1502,1503,1534,1584,1561,1585,1562,1535,1504,1469,1430,1429,1468,1467,1428,1385,1338,1339,1386,1387,1340,1289,1288,1233,1234,1175,1112,1111,1110,1174,1173,1109,1108,1172,1231,1232,1287,1286,1285,1230,1229,1284,1283,1228,1169,1168,1105,1106,1107,1170,1171,1040,969,968,1039,1038,1037,966,967,891,890,811,812,813,892,893,894,815,814,816,817,896,895,970,1041,1042,971,972,1043,1044,1045,974,973,899,898,897,818,819,820,821,822,901,900,975,1046,1047,976,977,1048,1049,978,903,902,823,824,825,826,904,905,906,827,828,830,829,908,907,981,983,982,1053,1052,1051,980,979,1050,1117,1116,1179,1180,1118,1119,1120,1182,1181,1240,1241,1295,1294,1239,1238,1293,1292,1291,1236,1237,1178,1115,1114,1177,1176,1113,1235,1290,1388,1341,1342,1343,1389,1432,1471,1470,1431,1505,1563,1586,1536,1506,1507,1537,1564,1587,1565,1538,1508,1473,1435,1434,1472,1433,1391,1390,1344,1345,1392,1346,1393,1347,1348,1349,1395,1394,1437,1475,1436,1474,1509,1539,1588,1566,1540,1510,1511,1541,1589,1567,1439,1477,1476,1438,1396,1350,1351,1397,1301,1247,1127,1126,1189,1188,1125,1124,1187,1245,1246,1300,1299,1298,1244,1243,1297,1296,1242,1184,1183,1121,1122,1123,1185,1186,1057,987,986,1056,1055,1054,984,985,910,909,831,832,833,911,912,913,835,834,836,837,915,914,988,1058,1059,989,990,1060,1061,991,917,916,838,839,759,681,680,758,757,755,756,678,679,605,604,534,535,536,606,607,537,471,470,409,297,351,350,296,295,349,407,408,469,468,467,466,405,406,348,294,347,346,293,292,345,403,404,465,464,530,599,600,601,531,532,533,603,602,675,676,677,754,753,752,674,673,750,751,749,748,671,672,598,529,528,597,596,527,526,595,668,669,670,747,746,745,744,667,666,743,742,741,664,665,592,591,522,523,524,593,594,525,459,460,398,397,458,457,396,339,286,287,340,341,288,289,343,342,399,461,462,400,401,463,402,344,290,291,242,241,196,155,117,154,116,153,194,195,240,239,193,238,237,192,151,114,152,115,82,52,81,27,6,28,53,83,54,84,29,7,8,30,56,86,85,55,118,156,197,243,198,244,245,199,157,119,120,158,246,200,201,247,159,121,87,57,31,9,10,32,11,33,59,90,89,58,88,122,123,161,160,202,248,249,203,204,250,251,162,124,125,163,205,252,206,253,254,207,165,164,126,92,61,91,60,34,12,35,13,93,62,94,63,36,14,37,15,38,65,96,95,64,129,168,130,131,170,169,212,213,260,259,258,211,257,256,209,210,167,128,166,127,208,255,305,306,360,359,418,480,481,482,419,420,483,484,421,362,361,307,308,363,364,309,310,311,366,365,424,425,487,488,486,485,423,422,551,622,623,552,553,554,555,625,626,624,699,700,701,779,780,778,777,698,697,775,776,774,773,694,695,696,621,550,549,620,619,548,547,617,618,692,693,772,771,770,769,691,690,768,767,765,766,687,688,689,614,613,543,544,545,615,616,546,479,478,416,417,304,303,358,357,302,356,414,415,477,476,475,474,412,413,355,301,300,354,353,299,298,352,410,411,473,472,538,539,608,609,610,540,541,542,612,611,685,686,764,763,762,684,683,682,760,761,840,841,842,920,919,918,993,992,1062,1063,1064,994,995,1065,1066,996,922,921,843,844,845,846,923,924,925,847,848,850,849,927,926,999,1001,1000,1070,1069,1068,998,997,1067,1133,1132,1194,1195,1134,1135,1136,1197,1196,1254,1255,1308,1307,1253,1252,1306,1305,1304,1250,1251,1193,1131,1130,1192,1191,1129,1128,1190,1248,1249,1303,1302,1352,1398,1399,1353,1354,1355,1400,1442,1480,1479,1441,1440,1478,1512,1513,1542,1568,1590,1569,1591,1543,1514,1515,1544,1570,1592,1571,1545,1516,1482,1445,1444,1481,1443,1402,1401,1356,1357,1403,1358,1404,1359,1360,1361,1406,1405,1447,1484,1446,1483,1517,1546,1593,1572,1547,1518,1519,1548,1594,1573,1595,1574,1549,1520,1487,1450,1449,1486,1485,1448,1407,1362,1363,1408,1409,1364,1315,1314,1261,1262,1205,1144,1143,1142,1204,1203,1141,1140,1202,1259,1260,1313,1312,1311,1258,1257,1310,1309,1256,1199,1198,1137,1138,1139,1200,1201,1074,1005,1004,1073,1072,1071,1002,1003,929,928,851,852,853,930,931,932,855,854,856,857,934,933,1006,1075,1076,1007,1008,1077,1078,1079,1010,1009,937,936,935,858,859,860,861,862,939,938,1011,1080,1081,1012,1013,1082,1083,1014,941,940,863,864,865,866,942,943,944,867,868,870,869,946,945,1017,1019,1018,1087,1086,1085,1016,1015,1084,1149,1148,1209,1210,1150,1151,1152,1212,1211,1268,1269,1321,1320,1267,1266,1319,1318,1317,1264,1265,1208,1147,1146,1207,1206,1145,1263,1316,1410,1365,1366,1367,1411,1452,1489,1488,1451,1521,1575,1596,1550,1522,1523,1551,1576,1597,1577,1552,1524,1491,1455,1454,1490,1453,1413,1412,1368,1369,1414,1370,1415,1371,1372,1373,1417,1416,1457,1493,1456,1492,1525,1553,1598,1578,1554,1526,1527,1555,1599,1579,1459,1495,1494,1458,1418,1374,1375,1419,1327,1275,1159,1158,1219,1218,1157,1156,1217,1273,1274,1326,1325,1324,1272,1271,1323,1322,1270,1214,1213,1153,1154,1155,1215,1216,1091,1023,1022,1090,1089,1088,1020,1021,948,947,871,872,873,949,950,951,875,874,876,877,953,952,1024,1092,1093,1025,1026,1094,1095,1027,955,954,878,879,799,719,718,798,797,795,796,716,717,641,640,568,569,570,642,643,571,503,502,439,323,379,378,322,321,377,437,438,501,500,499,498,435,436,376,320,375,374,319,318,373,433,434,497,496,564,635,636,637,565,566,567,639,638,713,714,715,794,793,792,712,711,790,791,789,788,709,710,634,563,562,633,632,561,560,631,706,707,708,787,786,785,784,705,704,783,782,781,702,703,628,627,556,557,558,629,630,559,491,492,428,427,490,489,426,367,312,313,368,369,314,315,371,370,429,493,494,430,431,495,432,372,316,317,266,265,218,175,135,174,134,173,216,217,264,263,215,262,261,214,171,132,172,133,98,66,97,39,16,40,67,99,68,100,41,17,18,42,70,102,101,69,136,176,219,267,220,268,269,221,177,137,138,178,270,222,223,271,179,139,103,71,43,19,1600,1601,1602,1603,1604,1605,1606,1607,1608,1609,1610,1611,1612,1613,1614,1615,1616,1617,1618,1619,1620,1621,1622,1623,1624,1625,1626,1627,1628,1629,1630,1631}; @@ -338,7 +338,7 @@ CASE( "test_hilbert_reordering" ) { CASE( "test_reverse_cuthill_mckee_reordering" ) { auto reorder_config = option::type( "reverse_cuthill_mckee" ); - std::vector expected; + std::vector expected; if ( grid_name() == "O16" && mpi::comm().size() == 1 ) { // clang-format off expected = {1579,1599,1555,1554,1578,1598,1527,1526,1525,1553,1577,1597,1495,1494,1493,1492,1524,1552,1576,1596,1459,1458,1457,1456,1455,1491,1523,1551,1575,1595,1419,1418,1417,1416,1415,1414,1454,1490,1522,1550,1574,1573,1594,1375,1374,1373,1372,1371,1370,1369,1413,1453,1489,1521,1549,1548,1547,1572,1593,1327,1326,1325,1324,1323,1322,1321,1320,1368,1412,1452,1488,1520,1519,1518,1517,1546,1571,1592,1275,1274,1273,1272,1271,1270,1269,1268,1267,1319,1367,1411,1451,1487,1486,1485,1484,1483,1516,1545,1570,1591,1219,1218,1217,1216,1215,1214,1213,1212,1211,1210,1266,1318,1366,1410,1450,1449,1448,1447,1446,1445,1482,1515,1544,1569,1590,1159,1095,1158,1157,1156,1155,1154,1153,1152,1151,1150,1149,1209,1148,1265,1317,1365,1409,1408,1407,1406,1405,1404,1589,1403,1444,1481,1514,1543,1568,1567,1027,955,799,879,1094,1026,1093,1092,1091,1090,1089,1088,1087,1086,1085,1084,1083,1208,1147,1264,1316,1364,1363,1362,1361,1360,1359,1358,1566,1588,1357,1402,1443,1480,1513,1542,1541,1540,878,954,719,798,1082,1025,953,1024,1023,1022,1021,1020,1019,1018,1017,1016,1015,1014,1013,1207,1146,1263,1315,1314,1313,1312,1311,1310,1309,1308,1539,1565,1587,1307,1356,1401,1442,1479,1512,1511,1510,1509,877,797,643,718,1081,1012,876,952,951,950,949,948,947,946,945,944,943,942,941,940,939,1206,1145,1262,1261,1260,1259,1258,1257,1256,1255,1254,1508,1538,1564,1586,1253,1306,1355,1400,1441,1478,1477,1476,1475,1474,796,717,571,642,1080,1011,938,874,875,872,873,870,871,868,869,866,867,864,865,862,863,861,1205,1144,1204,1203,1202,1201,1200,1199,1198,1197,1196,1473,1507,1537,1563,1585,1584,1583,1582,1580,1581,1195,1252,1305,1354,1399,1440,1439,1438,1437,1436,1435,795,716,641,503,570,1079,1010,860,937,794,793,792,791,790,789,788,787,786,785,784,783,782,780,781,1143,1078,1142,1141,1140,1139,1138,1137,1136,1135,1134,1133,1434,1472,1506,1536,1562,1561,1560,1559,1558,1556,1557,1194,1132,1251,1304,1353,1398,1397,1396,1395,1394,1393,1392,715,640,569,439,502,1009,936,859,779,714,713,712,711,710,709,708,707,706,705,704,703,702,701,1077,1008,1076,1075,1074,1073,1072,1071,1070,1069,1068,1067,1066,1391,1433,1471,1505,1535,1534,1533,1532,1531,1530,1528,1529,1193,1131,1250,1303,1352,1351,1350,1349,1348,1347,1346,1345,639,568,501,379,438,858,935,778,700,638,637,636,635,634,633,632,631,630,629,628,627,626,1065,1007,934,1006,1005,1004,1003,1002,1001,1000,999,998,997,996,995,1344,1390,1432,1470,1504,1503,1502,1501,1500,1499,1498,1496,1497,1192,1130,1249,1302,1301,1300,1299,1298,1297,1296,1295,1294,567,500,437,323,378,857,777,699,625,566,565,564,563,562,561,560,559,558,557,556,555,1064,994,856,933,932,931,930,929,928,927,926,925,924,923,922,921,920,1293,1343,1389,1431,1469,1468,1467,1466,1465,1464,1463,1462,1460,1461,1191,1129,1248,1247,1246,1245,1244,1243,1242,1241,1240,1239,499,436,377,271,322,776,698,624,554,498,497,496,495,494,493,492,491,490,489,488,1063,993,919,854,855,852,853,850,851,848,849,846,847,844,845,842,843,841,1238,1292,1342,1388,1430,1429,1428,1427,1426,1425,1424,1423,1422,1420,1421,1190,1128,1189,1188,1187,1186,1185,1184,1183,1182,1181,1180,435,376,321,223,270,775,697,623,553,487,434,433,432,431,430,429,428,427,426,425,1062,992,840,918,774,773,772,771,770,769,768,767,766,765,764,763,762,760,761,1179,1237,1291,1341,1387,1386,1385,1384,1383,1382,1381,1380,1379,1378,1376,1377,1127,1061,1126,1125,1124,1123,1122,1121,1120,1119,1118,1117,1116,375,320,269,179,222,696,622,552,486,424,374,373,372,371,370,369,368,367,366,991,917,839,759,695,694,693,692,691,690,689,688,687,686,685,684,683,682,1178,1236,1290,1340,1339,1338,1337,1336,1335,1334,1333,1332,1331,1330,1328,1329,1115,1060,990,1059,1058,1057,1056,1055,1054,1053,1052,1051,1050,1049,319,268,221,139,178,621,551,485,423,365,318,317,316,315,314,313,312,311,838,916,758,681,620,619,618,617,616,615,614,613,612,611,610,609,608,1177,1235,1289,1288,1287,1286,1285,1284,1283,1282,1281,1280,1279,1278,1276,1277,1048,1114,989,915,988,987,986,985,984,983,982,981,980,979,978,977,267,220,177,103,138,550,484,422,364,310,266,265,264,263,262,261,260,837,757,680,607,549,548,547,546,545,544,543,542,541,540,539,538,1176,1234,1233,1232,1231,1230,1229,1228,1227,1226,1225,1224,1223,1222,1220,1221,1047,976,1113,836,914,913,912,911,910,909,908,907,906,905,904,903,902,901,219,176,137,71,102,483,421,363,309,259,218,217,216,215,214,213,756,679,606,537,482,481,480,479,478,477,476,475,474,473,472,1175,1174,1173,1172,1171,1170,1169,1168,1167,1166,1165,1164,1163,1162,1160,1161,1046,975,1112,900,834,835,832,833,830,831,828,829,826,827,824,825,822,823,821,175,136,101,43,70,420,362,308,258,212,174,173,172,171,170,755,678,605,536,471,419,418,417,416,415,414,413,412,411,410,1111,1110,1109,1108,1107,1106,1105,1104,1103,1102,1101,1100,1099,1098,1096,1097,1045,974,1044,820,899,754,753,752,751,750,749,748,747,746,745,744,743,742,740,741,19,135,100,69,42,361,307,257,211,169,134,133,132,131,677,604,535,470,409,360,359,358,357,356,355,354,353,352,1043,1042,1041,1040,1039,1038,1037,1036,1035,1034,1033,1032,1031,1030,1028,1029,973,898,972,819,739,676,675,674,673,672,671,670,669,668,667,666,665,664,663,18,99,68,41,306,256,210,168,130,98,97,96,603,534,469,408,351,305,304,303,302,301,300,299,298,971,970,969,968,967,966,965,964,963,962,961,960,956,959,958,957,818,897,896,738,662,602,601,600,599,598,597,596,595,594,593,592,591,590,17,67,40,255,209,167,129,95,66,65,533,468,407,350,297,254,253,252,251,250,249,248,895,894,893,892,891,890,889,888,887,886,885,884,880,881,883,882,816,817,737,661,589,532,531,530,529,528,527,526,525,524,523,522,521,800,16,39,208,166,128,94,64,38,467,406,349,296,247,207,206,205,204,203,202,814,815,812,813,810,811,808,809,806,807,804,805,801,802,803,736,660,588,520,466,465,464,463,462,461,460,459,458,457,456,720,15,165,127,93,63,37,405,348,295,246,201,164,163,162,161,160,735,734,733,732,731,730,729,728,727,726,725,724,723,721,722,659,587,519,455,404,403,402,401,400,399,398,397,396,395,644,14,126,92,62,36,347,294,245,200,159,125,124,123,122,658,657,656,655,654,653,652,651,650,649,648,647,646,645,586,518,454,394,346,345,344,343,342,341,340,339,338,572,13,91,61,35,293,244,199,158,121,90,89,88,585,584,583,582,581,580,579,578,577,576,575,574,573,517,453,393,337,292,291,290,289,288,287,286,285,504,12,60,34,243,198,157,120,87,59,58,516,515,514,513,512,511,510,509,508,507,506,505,452,392,336,284,242,241,240,239,238,237,236,440,11,33,197,156,119,86,57,32,451,450,449,448,447,446,445,444,443,442,441,391,335,283,235,196,195,194,193,192,191,380,10,155,118,85,56,31,390,389,388,387,386,385,384,383,382,381,334,282,234,190,154,153,152,151,150,324,9,117,84,55,30,333,332,331,330,329,328,327,326,325,281,233,189,149,116,115,114,113,272,8,83,54,29,280,279,278,277,276,275,274,273,232,188,148,112,82,81,80,224,7,53,28,231,230,229,228,227,226,225,187,147,111,79,52,51,180,6,27,186,185,184,183,182,181,146,110,78,50,26,140,5,145,144,143,142,141,109,77,49,25,104,4,108,107,106,105,76,48,24,72,3,75,74,73,47,23,44,2,46,45,22,20,1,21,0,1631,1630,1629,1628,1627,1626,1625,1624,1623,1622,1621,1620,1619,1618,1617,1616,1615,1614,1613,1612,1611,1610,1609,1608,1607,1606,1605,1604,1603,1602,1601,1600}; diff --git a/src/tests/numerics/test_fvm_nabla.cc b/src/tests/numerics/test_fvm_nabla.cc index b26154074..0524dfca7 100644 --- a/src/tests/numerics/test_fvm_nabla.cc +++ b/src/tests/numerics/test_fvm_nabla.cc @@ -228,7 +228,7 @@ CASE( "test_grad" ) { } double min, max, mean; - int N; + idx_t N; fvm.node_columns().minimum( fields["xder"], min ); fvm.node_columns().maximum( fields["xder"], max ); @@ -291,7 +291,7 @@ CASE( "test_div" ) { } double min, max, mean; - int N; + idx_t N; fvm.node_columns().minimum( fields["div"], min ); fvm.node_columns().maximum( fields["div"], max ); fvm.node_columns().mean( fields["div"], mean, N ); @@ -368,7 +368,7 @@ CASE( "test_curl" ) { } double min, max, mean; - int N; + idx_t N; // Vorticity! fvm.node_columns().minimum( fields["vor"], min ); @@ -406,7 +406,7 @@ CASE( "test_lapl" ) { } double min, max, mean; - int N; + idx_t N; fvm.node_columns().minimum( fields["lapl"], min ); fvm.node_columns().maximum( fields["lapl"], max ); fvm.node_columns().mean( fields["lapl"], mean, N ); diff --git a/src/tests/util/fctest_kdtree.F90 b/src/tests/util/fctest_kdtree.F90 index 980d3a8fd..acbb8537c 100644 --- a/src/tests/util/fctest_kdtree.F90 +++ b/src/tests/util/fctest_kdtree.F90 @@ -47,8 +47,9 @@ end module fcta_KDTree_fixture type(atlas_Geometry) :: geometry type(atlas_IndexKDTree) :: kdtree type(atlas_StructuredGrid) :: grid - integer(c_int) :: n, i, ix, iy, k, kk - integer(c_int), allocatable :: tree_indices(:), indices(:), indices_rad(:), result_indices(:) + integer(ATLAS_KIND_IDX) :: n, i, ix, iy + integer(c_int) :: k, kk + integer(ATLAS_KIND_IDX), allocatable :: tree_indices(:), indices(:), indices_rad(:), result_indices(:) real(c_double) :: lonlat(2), xyz(3), plonlat(2), pxyz(3) real(c_double), allocatable :: tree_lonlats(:,:), tree_distances(:) real(c_double), allocatable :: lonlats(:,:), distances(:) From 321b83485b7c5c6290e6bdae88533dc549157222 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 1 Jul 2020 14:28:48 +0000 Subject: [PATCH 010/161] Fix warnings --- src/atlas/array/LocalView.cc | 78 +++---------------- .../distribution/DistributionFunction.h | 4 +- .../partitioner/EqualRegionsPartitioner.h | 1 + .../MatchingMeshPartitionerLonLatPolygon.h | 2 + src/atlas/parallel/HaloExchange.h | 3 - 5 files changed, 16 insertions(+), 72 deletions(-) diff --git a/src/atlas/array/LocalView.cc b/src/atlas/array/LocalView.cc index 45991ca03..f4a2882d7 100644 --- a/src/atlas/array/LocalView.cc +++ b/src/atlas/array/LocalView.cc @@ -110,24 +110,22 @@ namespace array { template LocalView make_view( const TYPE data[], const ArrayShape& ); \ template LocalView make_view( const TYPE data[], const ArrayShape& ); \ \ - template LocalView make_view( TYPE data[], size_t ); \ - template LocalView make_view( TYPE data[], size_t ); \ - template LocalView make_view( const TYPE data[], size_t ); \ - template LocalView make_view( const TYPE data[], size_t ); \ - \ - \ template class LocalView; \ template class LocalView; \ template void LocalView::assign( const TYPE& ); +#define EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK1( TYPE ) \ + template LocalView make_view( TYPE data[], size_t ); \ + template LocalView make_view( TYPE data[], size_t ); \ + template LocalView make_view( const TYPE data[], size_t ); \ + template LocalView make_view( const TYPE data[], size_t ); + #define EXPLICIT_TEMPLATE_INSTANTIATION( RANK ) \ EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK( int, RANK ) \ EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK( long, RANK ) \ EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK( float, RANK ) \ EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK( double, RANK ) -//template LocalView make_view( int data[], const ArrayShape& ); -//template LocalView make_view( int data[], size_t ); // For each RANK in [1..9] EXPLICIT_TEMPLATE_INSTANTIATION( 1 ) @@ -139,67 +137,13 @@ EXPLICIT_TEMPLATE_INSTANTIATION( 6 ) EXPLICIT_TEMPLATE_INSTANTIATION( 7 ) EXPLICIT_TEMPLATE_INSTANTIATION( 8 ) EXPLICIT_TEMPLATE_INSTANTIATION( 9 ) +EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK1( int ) +EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK1( long ) +EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK1( float ) +EXPLICIT_TEMPLATE_INSTANTIATION_TYPE_RANK1( double ) -#undef EXPLICIT_TEMPLATE_INSTANTIATION +#undef EXPLICIT_TEMPLATE_INSTANTIATION -//#define EXPLICIT_TEMPLATE_INSTANTIATION( Rank ) \ -// template class LocalView; \ -// template class LocalView; \ -// template class LocalView; \ -// template class LocalView; \ -// template class LocalView; \ -// template class LocalView; \ -// template class LocalView; \ -// template class LocalView; \ -// template class LocalView; \ -// template class LocalView; \ -// template LocalView make_view( const int data[], \ -// const ArrayShape& ); \ -// template LocalView make_view( const int data[], \ -// const ArrayShape& ); \ -// template LocalView make_view( const long data[], \ -// const ArrayShape& ); \ -// template LocalView make_view( const long data[], \ -// const ArrayShape& ); \ -// template LocalView make_view( const float data[], \ -// const ArrayShape& ); \ -// template LocalView make_view( const float data[], \ -// const ArrayShape& ); \ -// template LocalView make_view( const double data[], \ -// const ArrayShape& ); \ -// template LocalView make_view( \ -// const double data[], const ArrayShape& ); \ -// \ -// template LocalView make_view( const int data[], \ -// size_t ); \ -// template LocalView make_view( const int data[], \ -// size_t ); \ -// template LocalView make_view( const long data[], \ -// size_t ); \ -// template LocalView make_view( const long data[], \ -// size_t ); \ -// template LocalView make_view( const float data[], \ -// size_t ); \ -// template LocalView make_view( const float data[], \ -// size_t ); \ -// template LocalView make_view( const double data[], \ -// size_t ); \ -// template LocalView make_view( \ -// const double data[], size_t ); - - -//// For each NDims in [1..9] -//EXPLICIT_TEMPLATE_INSTANTIATION( 1 ) -//EXPLICIT_TEMPLATE_INSTANTIATION( 2 ) -//EXPLICIT_TEMPLATE_INSTANTIATION( 3 ) -//EXPLICIT_TEMPLATE_INSTANTIATION( 4 ) -//EXPLICIT_TEMPLATE_INSTANTIATION( 5 ) -//EXPLICIT_TEMPLATE_INSTANTIATION( 6 ) -//EXPLICIT_TEMPLATE_INSTANTIATION( 7 ) -//EXPLICIT_TEMPLATE_INSTANTIATION( 8 ) -//EXPLICIT_TEMPLATE_INSTANTIATION( 9 ) - -//#undef EXPLICIT_TEMPLATE_INSTANTIATION } // namespace array } // namespace atlas diff --git a/src/atlas/grid/detail/distribution/DistributionFunction.h b/src/atlas/grid/detail/distribution/DistributionFunction.h index 8e414aa96..6f22c73f8 100644 --- a/src/atlas/grid/detail/distribution/DistributionFunction.h +++ b/src/atlas/grid/detail/distribution/DistributionFunction.h @@ -60,8 +60,8 @@ class DistributionFunctionT : public DistributionFunction { } ATLAS_ALWAYS_INLINE void partition( gidx_t begin, gidx_t end, int partitions[] ) const override { - const Derived& derived = *static_cast(this); - size_t i = 0; + const Derived& derived = *static_cast( this ); + size_t i = 0; for ( gidx_t n = begin; n < end; ++n, ++i ) { partitions[i] = derived.function( n ); } diff --git a/src/atlas/grid/detail/partitioner/EqualRegionsPartitioner.h b/src/atlas/grid/detail/partitioner/EqualRegionsPartitioner.h index c95e06579..2ecea2056 100644 --- a/src/atlas/grid/detail/partitioner/EqualRegionsPartitioner.h +++ b/src/atlas/grid/detail/partitioner/EqualRegionsPartitioner.h @@ -86,6 +86,7 @@ class EqualRegionsPartitioner : public Partitioner { int nb_bands() const { return bands_.size(); } int nb_regions( int band ) const { return sectors_[band]; } + using Partitioner::partition; virtual void partition( const Grid&, int part[] ) const; virtual std::string type() const { return "equal_regions"; } diff --git a/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerLonLatPolygon.h b/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerLonLatPolygon.h index 53e4419b1..01e1306b3 100644 --- a/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerLonLatPolygon.h +++ b/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerLonLatPolygon.h @@ -28,6 +28,8 @@ class MatchingMeshPartitionerLonLatPolygon : public MatchingMeshPartitioner { MatchingMeshPartitioner( nb_partitions ) {} MatchingMeshPartitionerLonLatPolygon( const Mesh& mesh ) : MatchingMeshPartitioner( mesh ) {} + using Partitioner::partition; + /** * @brief Partition a grid, using the same partitions from a pre-partitioned * mesh. diff --git a/src/atlas/parallel/HaloExchange.h b/src/atlas/parallel/HaloExchange.h index e883cf868..fee7d27ec 100644 --- a/src/atlas/parallel/HaloExchange.h +++ b/src/atlas/parallel/HaloExchange.h @@ -54,9 +54,6 @@ class HaloExchange : public util::Object { void setup( const int part[], const idx_t remote_idx[], const int base, idx_t size, idx_t halo_begin ); - // template - // void execute( DATA_TYPE field[], idx_t nb_vars ) const; - template void execute( array::Array& field, bool on_device = false ) const; From 2c76ea45298b0aa09cb6032d851ba4b254d437ff Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 15 Jul 2020 09:34:57 +0000 Subject: [PATCH 011/161] ATLAS-304 Fix use of INIT_SNAN feature --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 87b073fff..6c2251258 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,7 @@ include( features/PROJ ) include( features/SANDBOX ) include( features/CLANG_TIDY ) include( features/INCLUDE_WHAT_YOU_USE ) +include( features/INIT_SNAN ) include( features/DOCS ) ################################################################################ From 7ae1c51285fc00bbe8dd2efca78c54dc679b2ad9 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 15 Jul 2020 09:38:52 +0000 Subject: [PATCH 012/161] ATLAS_ASSERT macro converts to const char* to string --- src/atlas/runtime/Exception.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/atlas/runtime/Exception.h b/src/atlas/runtime/Exception.h index b6c581ca9..8e1b325a8 100644 --- a/src/atlas/runtime/Exception.h +++ b/src/atlas/runtime/Exception.h @@ -41,7 +41,8 @@ inline void Assert( bool success, const char* code, const char* file, int line, throw_AssertionFailed( code, eckit::CodeLocation( file, line, func ) ); } } -inline void Assert( bool success, const char* code, const char* msg, const char* file, int line, const char* func ) { +inline void Assert( bool success, const char* code, const std::string& msg, const char* file, int line, + const char* func ) { if ( not success ) { throw_AssertionFailed( code, msg, eckit::CodeLocation( file, line, func ) ); } @@ -54,7 +55,8 @@ inline void Assert( bool success, const char* code, const char* msg, const char* #define ATLAS_NOTIMPLEMENTED ::atlas::throw_NotImplemented( Here() ) #define ATLAS_ASSERT_NOMSG( code ) ::atlas::detail::Assert( bool( code ), #code, __FILE__, __LINE__, __func__ ) -#define ATLAS_ASSERT_MSG( code, msg ) ::atlas::detail::Assert( bool( code ), #code, msg, __FILE__, __LINE__, __func__ ) +#define ATLAS_ASSERT_MSG( code, msg ) \ + ::atlas::detail::Assert( bool( code ), #code, std::string( msg ), __FILE__, __LINE__, __func__ ) #define ATLAS_ASSERT( ... ) __ATLAS_SPLICE( __ATLAS_ASSERT_, __ATLAS_NARG( __VA_ARGS__ ) )( __VA_ARGS__ ) #define __ATLAS_ASSERT_1 ATLAS_ASSERT_NOMSG From 4cc81584c2439837931f9cdd4093569476e2bdce Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 15 Jul 2020 12:15:12 +0000 Subject: [PATCH 013/161] ATLAS-305 Support floating point exceptions and signal handling --- cmake/features/INIT_SNAN.cmake | 20 ++ src/atlas/CMakeLists.txt | 2 + src/atlas/library/FloatingPointExceptions.cc | 288 +++++++++++++++++++ src/atlas/library/FloatingPointExceptions.h | 64 +++++ src/atlas/library/Library.cc | 8 +- src/atlas/library/defines.h.in | 2 + src/atlas/runtime/AtlasTool.cc | 41 +-- src/tests/AtlasTestEnvironment.h | 9 + src/tests/trans/CMakeLists.txt | 4 + 9 files changed, 400 insertions(+), 38 deletions(-) create mode 100644 src/atlas/library/FloatingPointExceptions.cc create mode 100644 src/atlas/library/FloatingPointExceptions.h diff --git a/cmake/features/INIT_SNAN.cmake b/cmake/features/INIT_SNAN.cmake index 493c63c21..c366fe91f 100644 --- a/cmake/features/INIT_SNAN.cmake +++ b/cmake/features/INIT_SNAN.cmake @@ -17,3 +17,23 @@ if( ${CMAKE_BUILD_TYPE} MATCHES "Debug" ) endif() endif() + +cmake_push_check_state(RESET) +include(CheckSymbolExists) + set(CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE) + if(UNIX) + set(CMAKE_REQUIRED_LIBRARIES m) + endif() + check_symbol_exists(feenableexcept "fenv.h" atlas_HAVE_FEENABLEEXCEPT) + check_symbol_exists(fedisableexcept "fenv.h" atlas_HAVE_FEDISABLEEXCEPT) + if( atlas_HAVE_FEENABLEEXCEPT ) + set( atlas_HAVE_FEENABLEEXCEPT 1 ) + else() + set( atlas_HAVE_FEENABLEEXCEPT 0 ) + endif() + if( atlas_HAVE_FEDISABLEEXCEPT ) + set( atlas_HAVE_FEDISABLEEXCEPT 1 ) + else() + set( atlas_HAVE_FEDISABLEEXCEPT 0 ) + endif() +cmake_pop_check_state() diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 5a3359929..b34828152 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -27,6 +27,8 @@ install( FILES list( APPEND atlas_srcs library.h library/config.h +library/FloatingPointExceptions.h +library/FloatingPointExceptions.cc library/Library.h library/Library.cc library/Plugin.h diff --git a/src/atlas/library/FloatingPointExceptions.cc b/src/atlas/library/FloatingPointExceptions.cc new file mode 100644 index 000000000..31068bde1 --- /dev/null +++ b/src/atlas/library/FloatingPointExceptions.cc @@ -0,0 +1,288 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "atlas/library/FloatingPointExceptions.h" + +#include +#include +#include + +#include "eckit/config/LibEcKit.h" +#include "eckit/config/Resource.h" +#include "eckit/utils/StringTools.h" + +#include "atlas/library/config.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" +#include "atlas/runtime/Trace.h" + +static int atlas_feenableexcept( int excepts ) { +#if ATLAS_HAVE_FEENABLEEXCEPT + return ::feenableexcept( excepts ); +#else + return 0; +#endif +} + +#ifdef UNUSED +static int atlas_fedisableexcept( int excepts ) { +#if ATLAS_HAVE_FEDISABLEEXCEPT + return ::fedisableexcept( excepts ); +#else + return 0; +#endif +} +#endif + +namespace atlas { +namespace library { + +static std::map str_to_except = {{"FE_INVALID", FE_INVALID}, {"FE_INEXACT", FE_INEXACT}, + {"FE_DIVBYZERO", FE_DIVBYZERO}, {"FE_OVERFLOW", FE_OVERFLOW}, + {"FE_UNDERFLOW", FE_UNDERFLOW}, {"FE_ALL_EXCEPT", FE_ALL_EXCEPT}}; +static std::map except_to_str = {{FE_INVALID, "FE_INVALID"}, {FE_INEXACT, "FE_INEXACT"}, + {FE_DIVBYZERO, "FE_DIVBYZERO"}, {FE_OVERFLOW, "FE_OVERFLOW"}, + {FE_UNDERFLOW, "FE_UNDERFLOW"}, {FE_ALL_EXCEPT, "FE_ALL_EXCEPT"}}; + +static std::map str_to_signal = {{"SIGINT", SIGINT}, {"SIGILL", SIGILL}, {"SIGABRT", SIGABRT}, + {"SIGFPE", SIGFPE}, {"SIGKILL", SIGKILL}, {"SIGSEGV", SIGSEGV}, + {"SIGTERM", SIGTERM}}; +static std::map signal_to_str = {{SIGINT, "SIGINT"}, {SIGILL, "SIGILL"}, {SIGABRT, "SIGABRT"}, + {SIGFPE, "SIGFPE"}, {SIGKILL, "SIGKILL"}, {SIGSEGV, "SIGSEGV"}, + {SIGTERM, "SIGTERM"}}; + +// ------------------------------------------------------------------------------------ + +class Signal { + // Not sure if this should be made public (in header file) just yet + +public: + Signal(); + + Signal( int signum ); + + Signal( int signum, signal_action_t ); + + Signal( int signum, signal_handler_t signal_handler ); + + operator int() const { return signum_; } + int signum() const { return signum_; } + + const std::string& code() const { return signal_to_str[signum_]; } + + std::string str() const { return str_; } + const signal_handler_t& handler() const { return signal_action_.sa_handler; } + const struct sigaction* action() const { return &signal_action_; } + +private: + friend std::ostream& operator<<( std::ostream&, const Signal& ); + + int signum_; + std::string str_; + struct sigaction signal_action_; +}; + +// ------------------------------------------------------------------------------------ + +class Signals { + // Not sure if this should be made public (in header file) just yet + +private: + Signals() {} + +public: + static Signals& instance(); + void setSignalHandlers(); + void setSignalHandler( const Signal& ); + void restoreSignalHandler( int signum ); + void restoreAllSignalHandlers(); + const Signal& signal( int signum ) const; + +private: + typedef std::map registered_signals_t; + registered_signals_t registered_signals_; +}; + +// ------------------------------------------------------------------------------------ + + +// ------------------------------------------------------------------------------------ + +[[noreturn]] void atlas_signal_handler( int signum, siginfo_t* si, void* /*unused*/ ) { + Signal signal = Signals::instance().signal( signum ); + + std::string signal_code; + if ( signum == SIGFPE ) { + switch ( si->si_code ) { + case FPE_FLTDIV: + signal_code = " [FE_DIVBYZERO]"; + break; + case FPE_FLTINV: + signal_code = " [FE_INVALID]"; + break; + case FPE_FLTOVF: + signal_code = " [FE_OVERFLOW]"; + break; + case FPE_FLTUND: + signal_code = " [FE_UNDERFLOW]"; + break; + case FPE_FLTRES: + signal_code = " [FE_INEXACT]"; + break; + } + } + + std::ostream& out = Log::error(); + out << "\n" + << "=========================================\n" + << signal << signal_code << " (signal intercepted by atlas)\n"; + out << "-----------------------------------------\n" + << "BACKTRACE\n" + << "-----------------------------------------\n" + << backtrace() << "\n" + << "=========================================\n" + << std::endl; + + eckit::LibEcKit::instance().abort(); + + // Just in case we end up here, which normally we shouldn't. + std::_Exit( EXIT_FAILURE ); +} + +//---------------------------------------------------------------------------------------------------------------------- + + +Signals& Signals::instance() { + static Signals signals; + return signals; +} + +void Signals::restoreSignalHandler( int signum ) { + if ( registered_signals_.find( signum ) != registered_signals_.end() ) { + Log::debug() << "\n"; + std::signal( signum, SIG_DFL ); + Log::debug() << "Atlas restored default signal handler for signal " << std::setw( 7 ) << std::left + << registered_signals_[signum].code() << " [" << registered_signals_[signum] << "]\n"; + Log::debug() << std::endl; + registered_signals_.erase( signum ); + } +} + +void Signals::restoreAllSignalHandlers() { + Log::debug() << "\n"; + for ( registered_signals_t::const_iterator it = registered_signals_.begin(); it != registered_signals_.end(); + ++it ) { + std::signal( it->first, SIG_DFL ); + Log::debug() << "Atlas restored default signal handler for signal " << std::setw( 7 ) << std::left + << it->second.code() << " [" << it->second.str() << "]\n"; + } + Log::debug() << std::endl; + registered_signals_.clear(); +} + +const Signal& Signals::signal( int signum ) const { + return registered_signals_.at( signum ); +} + +std::ostream& operator<<( std::ostream& out, const Signal& signal ) { + out << signal.str(); + return out; +} + +void Signals::setSignalHandlers() { + Log::debug() << "\n"; + setSignalHandler( SIGINT ); + setSignalHandler( SIGILL ); + setSignalHandler( SIGABRT ); + setSignalHandler( SIGFPE ); + setSignalHandler( SIGKILL ); + setSignalHandler( SIGSEGV ); + setSignalHandler( SIGTERM ); +} + +void Signals::setSignalHandler( const Signal& signal ) { + registered_signals_[signal] = signal; + sigaction( signal, signal.action(), nullptr ); + Log::debug() << "Atlas registered signal handler for signal " << std::setw( 7 ) << std::left << signal.code() + << " [" << signal << "]" << std::endl; +} + + +Signal::Signal() : signum_( 0 ), str_() { + signal_action_.sa_handler = SIG_DFL; +} + +Signal::Signal( int signum ) : Signal( signum, atlas_signal_handler ) {} + +Signal::Signal( int signum, signal_handler_t signal_handler ) : signum_( signum ), str_( strsignal( signum ) ) { + memset( &signal_action_, 0, sizeof( signal_action_ ) ); + sigemptyset( &signal_action_.sa_mask ); + signal_action_.sa_handler = signal_handler; + signal_action_.sa_flags = 0; +} + +Signal::Signal( int signum, signal_action_t signal_action ) : signum_( signum ), str_( strsignal( signum ) ) { + memset( &signal_action_, 0, sizeof( signal_action_ ) ); + sigemptyset( &signal_action_.sa_mask ); + signal_action_.sa_sigaction = signal_action; + signal_action_.sa_flags = SA_SIGINFO; +} + + +void enable_floating_point_exceptions() { + std::vector floating_point_exceptions = + eckit::Resource>( "atlasFPE;$ATLAS_FPE", {"false"} ); + { + bool _enable = false; + int _excepts = 0; + auto enable = [&]( int except ) { + _excepts |= except; + _enable = true; + Log::debug() << "Atlas enabled floating point exception " << except_to_str[except] << std::endl; + }; + bool skip_map = false; + if ( floating_point_exceptions.size() == 1 ) { + std::string s = eckit::StringTools::lower( floating_point_exceptions[0] ); + if ( s == "no" || s == "off" || s == "false" || s == "0" ) { + _enable = false; + skip_map = true; + } + else if ( s == "yes" || s == "on" || s == "true" || s == "1" ) { + enable( FE_INVALID ); + enable( FE_DIVBYZERO ); + enable( FE_OVERFLOW ); + skip_map = true; + } + } + if ( not skip_map ) { + for ( auto& s : floating_point_exceptions ) { + if ( str_to_except.find( s ) == str_to_except.end() ) { + throw eckit::UserError( + s + " is not a valid floating point exception code. " + "Valid codes: [FE_INVALID,FE_INEXACT,FE_DIVBYZERO,FE_OVERFLOW,FE_ALL_EXCEPT]", + Here() ); + } + enable( str_to_except[s] ); + } + } + if ( _enable ) { + atlas_feenableexcept( _excepts ); + } + } +} + +void enable_atlas_signal_handler() { + if ( eckit::Resource( "atlasSignalHandler;$ATLAS_SIGNAL_HANDLER", false ) ) { + Signals::instance().setSignalHandlers(); + } +} + +} // namespace library +} // namespace atlas diff --git a/src/atlas/library/FloatingPointExceptions.h b/src/atlas/library/FloatingPointExceptions.h new file mode 100644 index 000000000..87c97d604 --- /dev/null +++ b/src/atlas/library/FloatingPointExceptions.h @@ -0,0 +1,64 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include +#include + +namespace atlas { +namespace library { +// ------------------------------------------------------------------------------------ + +/* @brief Enable floating point exceptions + * + * An environment variable is responsible to enable: + * - ATLAS_FPE=0 --> Default. do not enable anything in Atlas (other libraries or programs may still do so) + * - ATLAS_FPE=1 --> FE_INVALID, FE_DIVBYZERO, FE_OVERFLOW + * - ATLAS_FPE=VALUE1,VALUE2,... --> Specified codes only. Valid codes: + * FE_INVALID, FE_DIVBYZERO, FE_OVERFLOW, FE_UNDERFLOW, FE_INEXACT, FE_ALL_EXCEPT + * + * @note This function is called automatically within Library::initialize() and should + * not be called directly + */ +void enable_floating_point_exceptions(); + +// ------------------------------------------------------------------------------------ + +/* @brief Enable atlas signal handler for all signals + * + * An environment variable is responsible to enable: + * - ATLAS_SIGNAL_HANDLER=0 --> Default. Atlas will not set any signal handlers + * - ATLAS_SIGNAL_HANDLER=1 --> Enable atlas_signal_handler instead of the default for signals: + * + * Enabled signals: + * - SIGABRT + * - SIGFPE + * - SIGILL + * - SIGINT + * - SIGSEGV + * - SIGTERM + * - SIGKILL + * + * @note This function is called automatically within Library::initialize() and should + * not be called directly + */ +void enable_atlas_signal_handler(); + +// ------------------------------------------------------------------------------------ + +typedef void ( *signal_handler_t )( int ); +typedef void ( *signal_action_t )( int, ::siginfo_t*, void* ); + +[[noreturn]] void atlas_signal_handler( int signum, ::siginfo_t* si, void* unused ); + +// ------------------------------------------------------------------------------------ + +} // namespace library +} // namespace atlas diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index 45ca15886..b0a821ae0 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -8,11 +8,14 @@ * nor does it submit to any jurisdiction. */ +#include "atlas/library/Library.h" + #include #include #include // for dynamic loading (should be delegated to eckit) +#include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/eckit.h" #include "eckit/filesystem/LocalPathName.h" @@ -25,7 +28,7 @@ #include "eckit/types/Types.h" #include "eckit/utils/Translator.h" -#include "atlas/library/Library.h" +#include "atlas/library/FloatingPointExceptions.h" #include "atlas/library/Plugin.h" #include "atlas/library/config.h" #include "atlas/library/git_sha1.h" @@ -234,7 +237,6 @@ void Library::initialise( const eckit::Parametrisation& config ) { warning_channel_.reset(); } - std::vector plugin_names = eckit::Resource>( "atlasPlugins;$ATLAS_PLUGINS", {} ); @@ -286,6 +288,8 @@ void Library::initialise( const eckit::Parametrisation& config ) { } } + library::enable_floating_point_exceptions(); + library::enable_atlas_signal_handler(); // Summary if ( getEnv( "ATLAS_LOG_RANK", 0 ) == int( mpi::rank() ) ) { diff --git a/src/atlas/library/defines.h.in b/src/atlas/library/defines.h.in index 6ce6200f1..c130458dc 100644 --- a/src/atlas/library/defines.h.in +++ b/src/atlas/library/defines.h.in @@ -30,6 +30,8 @@ #define ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST @ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST@ #define ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA @ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA@ #define ATLAS_HAVE_TRANS @atlas_HAVE_TRANS@ +#define ATLAS_HAVE_FEENABLEEXCEPT @atlas_HAVE_FEENABLEEXCEPT@ +#define ATLAS_HAVE_FEDISABLEEXCEPT @atlas_HAVE_FEDISABLEEXCEPT@ #ifdef __CUDACC__ #define ATLAS_HOST_DEVICE __host__ __device__ diff --git a/src/atlas/runtime/AtlasTool.cc b/src/atlas/runtime/AtlasTool.cc index 7d8f14fb4..9e0383ca9 100644 --- a/src/atlas/runtime/AtlasTool.cc +++ b/src/atlas/runtime/AtlasTool.cc @@ -102,16 +102,15 @@ std::string getEnv( const std::string& env, const std::string& default_value ) { } void setEnv( const std::string& env, bool value ) { - ::setenv( env.c_str(), eckit::Translator()( value ).c_str(), 0 ); + constexpr int DO_NOT_REPLACE_IF_EXISTS = 0; + ::setenv( env.c_str(), eckit::Translator()( value ).c_str(), DO_NOT_REPLACE_IF_EXISTS ); } + } // namespace namespace atlas { -static std::string exception_what; -static eckit::CodeLocation exception_location; -static std::string exception_callstack; static bool use_logfile; static std::string logfile_name; static std::string workdir; @@ -147,9 +146,6 @@ static std::string workdir; << exception.callStack() << "\n" << "=========================================\n" << std::endl; - exception_what = exception.what(); - exception_location = exception.location(); - exception_callstack = exception.callStack(); } catch ( const eckit::Exception& exception ) { out << "\n" @@ -169,9 +165,6 @@ static std::string workdir; << exception.callStack() << "\n" << "=========================================\n" << std::endl; - exception_what = exception.what(); - exception_location = exception.location(); - exception_callstack = backtrace(); } catch ( const std::exception& exception ) { out << "\n" @@ -185,9 +178,6 @@ static std::string workdir; << backtrace() << "\n" << "=========================================\n" << std::endl; - exception_what = exception.what(); - exception_location = eckit::CodeLocation(); - exception_callstack = backtrace(); } catch ( ... ) { out << "\n" @@ -198,9 +188,6 @@ static std::string workdir; << "-----------------------------------------\n" << backtrace() << "\n" << "=========================================" << std::endl; - exception_what = "Uncaught exception"; - exception_location = eckit::CodeLocation(); - exception_callstack = backtrace(); } } @@ -269,6 +256,8 @@ atlas::AtlasTool::AtlasTool( int argc, char** argv ) : eckit::Tool( argc, argv ) std::set_terminate( &atlas_terminate ); setEnv( "ECKIT_EXCEPTION_IS_SILENT", true ); setEnv( "ECKIT_ASSERT_FAILED_IS_SILENT", true ); + setEnv( "ATLAS_FPE", true ); + setEnv( "ATLAS_SIGNAL_HANDLER", true ); add_option( new SimpleOption( "help", "Print this help" ) ); add_option( new SimpleOption( "debug", "Debug level" ) ); taskID( eckit::mpi::comm( "world" ).rank() ); @@ -303,26 +292,6 @@ int atlas::AtlasTool::start() { catch ( ... ) { atlas_terminate(); } - // catch ( eckit::Exception& e ) { - // status = 1; - // Log::error() << "** " << e.what() << e.location() << std::endl; - // Log::error() << "** Backtrace:\n" << e.callStack() << '\n'; - // Log::error() << "** Exception caught in " << Here() << " terminates " << name() << std::endl; - // } - // catch ( std::exception& e ) { - // status = 1; - // Log::error() << "** " << e.what() << " caught in " << Here() << '\n'; - // Log::error() << "** Exception terminates " << name() << std::endl; - // } - // catch ( ... ) { - // status = 1; - // Log::error() << "** Exception caught in " << Here() << '\n'; - // Log::error() << "** Exception terminates " << name() << std::endl; - // } - // if ( status ) { - // Log::error() << std::flush; - // eckit::LibEcKit::instance().abort(); - // } } void atlas::AtlasTool::run() {} diff --git a/src/tests/AtlasTestEnvironment.h b/src/tests/AtlasTestEnvironment.h index 1bf237023..71e0c7e7f 100644 --- a/src/tests/AtlasTestEnvironment.h +++ b/src/tests/AtlasTestEnvironment.h @@ -295,6 +295,11 @@ int getEnv( const std::string& env, int default_value ) { return default_value; } +void setEnv( const std::string& env, bool value ) { + constexpr int DO_NOT_REPLACE_IF_EXISTS = 0; + ::setenv( env.c_str(), eckit::Translator()( value ).c_str(), DO_NOT_REPLACE_IF_EXISTS ); +} + } // namespace struct AtlasTestEnvironment { @@ -346,6 +351,10 @@ struct AtlasTestEnvironment { Log::error() << "Calling MPI_Abort" << std::endl; eckit::mpi::comm().abort(); } ); + + setEnv( "ATLAS_FPE", true ); + setEnv( "ATLAS_SIGNAL_HANDLER", true ); + atlas::initialize(); eckit::mpi::comm().barrier(); } diff --git a/src/tests/trans/CMakeLists.txt b/src/tests/trans/CMakeLists.txt index 630fde943..eabe73a0a 100644 --- a/src/tests/trans/CMakeLists.txt +++ b/src/tests/trans/CMakeLists.txt @@ -43,6 +43,8 @@ ecbuild_add_test( TARGET atlas_test_trans CONDITION atlas_HAVE_TRANS AND eckit_HAVE_MPI AND transi_HAVE_MPI LIBS atlas transi ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} + # There seems to be a issue with vd2uv raising FE_INVALID, only on specific arch (bamboo-leap42-gnu63) + ATLAS_FPE=FE_OVERFLOW,FE_DIVBYZERO ) ecbuild_add_test( TARGET atlas_test_trans_serial @@ -50,6 +52,8 @@ ecbuild_add_test( TARGET atlas_test_trans_serial CONDITION atlas_HAVE_TRANS LIBS atlas transi ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} + # There seems to be a issue with vd2uv raising FE_INVALID, only on specific arch (bamboo-leap42-gnu63) + ATLAS_FPE=FE_OVERFLOW,FE_DIVBYZERO ) ecbuild_add_test( TARGET atlas_test_trans_invtrans_grad From 8fbb3a6c4ba1bdd49c3484002790327923251faa Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 29 Jul 2020 13:09:56 +0100 Subject: [PATCH 014/161] Prevent overly agressive optimization by clang 9, resulting in FE_DIVBYZERO (in branch prediction?) --- src/atlas/trans/local/LegendrePolynomials.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/atlas/trans/local/LegendrePolynomials.cc b/src/atlas/trans/local/LegendrePolynomials.cc index 8e2a986c7..ea7e81e6d 100644 --- a/src/atlas/trans/local/LegendrePolynomials.cc +++ b/src/atlas/trans/local/LegendrePolynomials.cc @@ -55,9 +55,9 @@ void compute_legendre_polynomials_lat( const int trc, // truncation (in) // -------------------- // 1. First two columns // -------------------- - double zdlx1 = ( M_PI_2 - lat ); // theta - double zdlx = std::cos( zdlx1 ); // cos(theta) - double zdlsita = std::sqrt( 1. - zdlx * zdlx ); // sin(theta) (this is how trans library does it) + double zdlx1 = ( M_PI_2 - lat ); // theta + double zdlx = std::cos( zdlx1 ); // cos(theta) + volatile double zdlsita = std::sqrt( 1. - zdlx * zdlx ); // sin(theta) (this is how trans library does it) legpol[idxmn( 0, 0 )] = 1.; double vsin[trc + 1], vcos[trc + 1]; From da5731fb5525ad22c14dc632d919eae0d2d1c5c5 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 29 Jul 2020 15:07:12 +0100 Subject: [PATCH 015/161] Workaround Cray 8.6 compiler bug --- src/atlas/library/FloatingPointExceptions.cc | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/atlas/library/FloatingPointExceptions.cc b/src/atlas/library/FloatingPointExceptions.cc index 31068bde1..158d91140 100644 --- a/src/atlas/library/FloatingPointExceptions.cc +++ b/src/atlas/library/FloatingPointExceptions.cc @@ -17,6 +17,7 @@ #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/utils/StringTools.h" +#include "eckit/utils/Translator.h" #include "atlas/library/config.h" #include "atlas/runtime/Exception.h" @@ -202,7 +203,6 @@ void Signals::setSignalHandlers() { setSignalHandler( SIGILL ); setSignalHandler( SIGABRT ); setSignalHandler( SIGFPE ); - setSignalHandler( SIGKILL ); setSignalHandler( SIGSEGV ); setSignalHandler( SIGTERM ); } @@ -235,10 +235,20 @@ Signal::Signal( int signum, signal_action_t signal_action ) : signum_( signum ), signal_action_.sa_flags = SA_SIGINFO; } - void enable_floating_point_exceptions() { - std::vector floating_point_exceptions = - eckit::Resource>( "atlasFPE;$ATLAS_FPE", {"false"} ); + // Following line gives runtime errors with Cray 8.6 due to compiler bug (but works with Cray 8.5 and Cray 8.7) + // std::vector floating_point_exceptions = eckit::Resource>( "atlasFPE;$ATLAS_FPE", {"false"} ); + // Instead, manually access environment + std::vector floating_point_exceptions{"false"}; + if ( ::getenv( "ATLAS_FPE" ) ) { + std::string env( ::getenv( "ATLAS_FPE" ) ); + std::vector tmp = eckit::Translator>()( env ); + floating_point_exceptions = tmp; + // Above trick with "tmp" is what avoids the Cray 8.6 compiler bug + } + else { + floating_point_exceptions = eckit::Resource>( "atlasFPE", {"false"} ); + } { bool _enable = false; int _excepts = 0; From b6747bd7776c6f0dec449517866ef13749f930c4 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 30 Jul 2020 10:47:16 +0000 Subject: [PATCH 016/161] Restore default signal handler within atlas_signal_handler --- src/atlas/library/FloatingPointExceptions.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/atlas/library/FloatingPointExceptions.cc b/src/atlas/library/FloatingPointExceptions.cc index 158d91140..764639900 100644 --- a/src/atlas/library/FloatingPointExceptions.cc +++ b/src/atlas/library/FloatingPointExceptions.cc @@ -151,6 +151,7 @@ class Signals { << "=========================================\n" << std::endl; + Signals::instance().restoreSignalHandler(signum); eckit::LibEcKit::instance().abort(); // Just in case we end up here, which normally we shouldn't. From 22557383e1ab9893d5dc38948088a787b1b20eb8 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 30 Jul 2020 10:48:48 +0000 Subject: [PATCH 017/161] Fix FE_INVALID and FE_DIVBYZERO for Cray 8.7 Release build due to too aggressive optimization, or compiler bugs... --- src/CMakeLists.txt | 9 +++++ src/atlas/grid/Vertical.cc | 33 ++++++++++--------- .../grid/detail/spacing/LinearSpacing.cc | 10 ++++-- .../kernels/CubicHorizontalKernel.h | 4 +++ .../structured/kernels/CubicVerticalKernel.h | 5 +++ .../kernels/QuasiCubicHorizontalKernel.h | 5 +++ src/atlas/library/defines.h.in | 3 ++ .../CubicInterpolationPrototype.h | 18 ++++++---- src/tests/mesh/test_rgg.cc | 4 +-- 9 files changed, 65 insertions(+), 26 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 950d9a2bc..368560508 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -60,6 +60,15 @@ else() set( atlas_HAVE_GRIDTOOLS_STORAGE 0 ) endif() +set( atlas_BUILD_TYPE_RELEASE 0 ) +set( atlas_BUILD_TYPE_DEBUG 0 ) +if( CMAKE_BUILD_TYPE MATCHES "Release" ) + set( atlas_BUILD_TYPE_RELEASE 1 ) +endif() +if( CMAKE_BUILD_TYPE MATCHES "Debug" ) + set( atlas_BUILD_TYPE_DEBUG 1 ) +endif() + add_subdirectory( atlas_acc_support ) add_subdirectory( atlas ) diff --git a/src/atlas/grid/Vertical.cc b/src/atlas/grid/Vertical.cc index 912033d35..45b184577 100644 --- a/src/atlas/grid/Vertical.cc +++ b/src/atlas/grid/Vertical.cc @@ -22,21 +22,24 @@ namespace { std::vector linspace( double start, double end, idx_t N, bool endpoint ) { std::vector x_; - x_.resize( N ); - - double step; - if ( endpoint && N > 1 ) { - step = ( end - start ) / double( N - 1 ); - } - else if ( N > 0 ) { - step = ( end - start ) / double( N ); - } - else { - step = 0.; - } - - for ( idx_t i = 0; i < N; ++i ) { - x_[i] = start + i * step; + if( N > 0 ) { + volatile double _N = N; // volatile keyword prevents agressive optimization by Cray compiler + x_.resize( N ); + + double step; + if ( endpoint && N > 1 ) { + step = ( end - start ) / (_N - 1 ); + } + else if ( N > 0 ) { + step = ( end - start ) / _N; + } + else { + step = 0.; + } + + for ( idx_t i = 0; i < N; ++i ) { + x_[i] = start + i * step; + } } return x_; } diff --git a/src/atlas/grid/detail/spacing/LinearSpacing.cc b/src/atlas/grid/detail/spacing/LinearSpacing.cc index 8409356cd..9f00f059a 100644 --- a/src/atlas/grid/detail/spacing/LinearSpacing.cc +++ b/src/atlas/grid/detail/spacing/LinearSpacing.cc @@ -84,11 +84,15 @@ void LinearSpacing::setup( double start, double end, long N, bool endpoint ) { x_.resize( N ); double step; - if ( endpoint && N > 1 ) { - step = ( end - start ) / double( N - 1 ); + volatile double _N = N; // volatile keyword prevents agressive optimization by Cray compiler + if( start == end ) { + step = 0.; + } + else if ( endpoint && N > 1 ) { + step = ( end - start ) / ( _N - 1 ); } else { - step = ( end - start ) / double( N ); + step = ( end - start ) / _N; } for ( long i = 0; i < N; ++i ) { diff --git a/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h b/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h index 097c4d5f1..d04f0e01d 100644 --- a/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h @@ -119,6 +119,10 @@ class CubicHorizontalKernel { auto& weights_j = weights.weights_j; weights_j[0] = ( dl2 * dl3 * dl4 ) / dcl1; +#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE + // prevents FE_INVALID somehow (tested with Cray 8.7) + ATLAS_ASSERT( !std::isnan(weights_j[0]) ); +#endif weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; diff --git a/src/atlas/interpolation/method/structured/kernels/CubicVerticalKernel.h b/src/atlas/interpolation/method/structured/kernels/CubicVerticalKernel.h index a741b9204..de5755479 100644 --- a/src/atlas/interpolation/method/structured/kernels/CubicVerticalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/CubicVerticalKernel.h @@ -11,6 +11,7 @@ #pragma once #include +#include "atlas/library/config.h" #include "atlas/grid/Stencil.h" #include "atlas/grid/StencilComputer.h" #include "atlas/grid/Vertical.h" @@ -91,6 +92,10 @@ class CubicVerticalKernel { double d3 = z - zvec[3]; w[0] = ( d1 * d2 * d3 ) / dc0; +#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE + // prevents FE_INVALID somehow (tested with Cray 8.7) + ATLAS_ASSERT( ! std::isnan(w[0])); +#endif w[1] = ( d0 * d2 * d3 ) / dc1; w[2] = ( d0 * d1 * d3 ) / dc2; w[3] = 1. - w[0] - w[1] - w[2]; diff --git a/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h b/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h index 8773e4bea..9292fcdd0 100644 --- a/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h @@ -17,6 +17,7 @@ #include "eckit/linalg/Triplet.h" +#include "atlas/library/config.h" #include "atlas/array/ArrayView.h" #include "atlas/functionspace/StructuredColumns.h" #include "atlas/grid/Stencil.h" @@ -134,6 +135,10 @@ class QuasiCubicHorizontalKernel { auto& weights_j = weights.weights_j; weights_j[0] = ( dl2 * dl3 * dl4 ) / dcl1; +#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE + // prevents FE_INVALID somehow (tested with Cray 8.7) + ATLAS_ASSERT( ! std::isnan(weights_j[0]) ); +#endif weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; diff --git a/src/atlas/library/defines.h.in b/src/atlas/library/defines.h.in index c130458dc..b66f7a79b 100644 --- a/src/atlas/library/defines.h.in +++ b/src/atlas/library/defines.h.in @@ -32,6 +32,8 @@ #define ATLAS_HAVE_TRANS @atlas_HAVE_TRANS@ #define ATLAS_HAVE_FEENABLEEXCEPT @atlas_HAVE_FEENABLEEXCEPT@ #define ATLAS_HAVE_FEDISABLEEXCEPT @atlas_HAVE_FEDISABLEEXCEPT@ +#define ATLAS_BUILD_TYPE_DEBUG @atlas_BUILD_TYPE_DEBUG@ +#define ATLAS_BUILD_TYPE_RELEASE @atlas_BUILD_TYPE_RELEASE@ #ifdef __CUDACC__ #define ATLAS_HOST_DEVICE __host__ __device__ @@ -53,4 +55,5 @@ #define ATLAS_ALWAYS_INLINE inline #endif + #endif diff --git a/src/tests/interpolation/CubicInterpolationPrototype.h b/src/tests/interpolation/CubicInterpolationPrototype.h index 5a0e62e3b..a61255cb5 100644 --- a/src/tests/interpolation/CubicInterpolationPrototype.h +++ b/src/tests/interpolation/CubicInterpolationPrototype.h @@ -8,6 +8,9 @@ * nor does it submit to any jurisdiction. */ +#include "eckit/linalg/SparseMatrix.h" +#include "eckit/types/Types.h" + #include "atlas/array/ArrayView.h" #include "atlas/array/MakeView.h" #include "atlas/functionspace/StructuredColumns.h" @@ -15,11 +18,7 @@ #include "atlas/grid/Stencil.h" #include "atlas/grid/StencilComputer.h" #include "atlas/grid/Vertical.h" -#include "eckit/linalg/SparseMatrix.h" - - -#include "atlas/runtime/Log.h" -#include "eckit/types/Types.h" +#include "atlas/runtime/Exception.h" using namespace eckit; using namespace atlas::functionspace; @@ -60,7 +59,6 @@ class CubicVerticalInterpolation { template void compute_weights( const double z, const stencil_t& stencil, weights_t& weights ) const { auto& w = weights.weights_k; - std::array zvec; for ( idx_t k = 0; k < 4; ++k ) { zvec[k] = vertical_( stencil.k( k ) ); @@ -140,6 +138,10 @@ class CubicVerticalInterpolation { double d3 = z - zvec[3]; w[0] = ( d1 * d2 * d3 ) / dc0; +#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE + // prevents FE_INVALID somehow (tested with Cray 8.7) + ATLAS_ASSERT( ! std::isnan(w[0]) ); +#endif w[1] = ( d0 * d2 * d3 ) / dc1; w[2] = ( d0 * d1 * d3 ) / dc2; w[3] = 1. - w[0] - w[1] - w[2]; @@ -254,6 +256,10 @@ class CubicHorizontalInterpolation { auto& weights_j = weights.weights_j; weights_j[0] = ( dl2 * dl3 * dl4 ) / dcl1; +#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE + // prevents FE_INVALID somehow (tested with Cray 8.7) + ATLAS_ASSERT( !std::isnan(weights_j[0]) ); +#endif weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; diff --git a/src/tests/mesh/test_rgg.cc b/src/tests/mesh/test_rgg.cc index e34d46eae..67a2b7e2c 100644 --- a/src/tests/mesh/test_rgg.cc +++ b/src/tests/mesh/test_rgg.cc @@ -406,9 +406,9 @@ ASSERT(0); Log::error() << "node " << gid << " is not owned by anyone" << std::endl; } } - EXPECT( nb_owned == grid.size() ); + EXPECT_EQ( nb_owned, grid.size() ); - EXPECT( eckit::types::is_approximately_equal( area, check_area, 1e-10 ) ); + EXPECT_APPROX_EQ( area, check_area, 1e-9 ); } CASE( "test_meshgen_ghost_at_end" ) { From 1a5f29fa2b5fbb38c3f4683887b8fbcc9fc85dcb Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 30 Jul 2020 12:08:24 +0100 Subject: [PATCH 018/161] Cosmetic changes from cppcheck and clang-format --- src/apps/atlas-meshgen.cc | 2 +- src/atlas/grid/StencilComputer.cc | 3 +-- src/atlas/grid/Vertical.cc | 6 ++--- src/atlas/grid/Vertical.h | 7 ++---- src/atlas/grid/detail/grid/Structured.cc | 14 +++++++---- .../partitioner/CheckerboardPartitioner.cc | 23 ++++--------------- .../partitioner/CheckerboardPartitioner.h | 4 ++-- .../grid/detail/spacing/LinearSpacing.cc | 4 ++-- .../kernels/CubicHorizontalKernel.h | 10 ++++---- .../structured/kernels/CubicVerticalKernel.h | 6 ++--- .../kernels/QuasiCubicHorizontalKernel.h | 12 +++++----- src/atlas/library/FloatingPointExceptions.cc | 2 +- src/atlas/library/Library.cc | 4 ++-- .../CubicInterpolationPrototype.h | 14 +++++------ 14 files changed, 49 insertions(+), 62 deletions(-) diff --git a/src/apps/atlas-meshgen.cc b/src/apps/atlas-meshgen.cc index cbd66c00f..c056e9d7e 100644 --- a/src/apps/atlas-meshgen.cc +++ b/src/apps/atlas-meshgen.cc @@ -126,7 +126,7 @@ Meshgen2Gmsh::Meshgen2Gmsh( int argc, char** argv ) : AtlasTool( argc, argv ) { //----------------------------------------------------------------------------- -std::string get_arg( const AtlasTool::Args& args, const std::string& flag, const std::string default_value = "" ) { +std::string get_arg( const AtlasTool::Args& args, const std::string& flag, const std::string& default_value = "" ) { for ( int i = 0; i < args.count() - 1; ++i ) { if ( args( i ) == flag ) { return args( i + 1 ); diff --git a/src/atlas/grid/StencilComputer.cc b/src/atlas/grid/StencilComputer.cc index 0e967dc68..4a67056ba 100644 --- a/src/atlas/grid/StencilComputer.cc +++ b/src/atlas/grid/StencilComputer.cc @@ -15,8 +15,7 @@ namespace atlas { namespace grid { -ComputeLower::ComputeLower( const Vertical& z ) { - nlev_ = z.size(); +ComputeLower::ComputeLower( const Vertical& z ) : nlev_{z.size()} { z_.resize( nlev_ ); double dz = std::numeric_limits::max(); constexpr double tol = 1.e-12; diff --git a/src/atlas/grid/Vertical.cc b/src/atlas/grid/Vertical.cc index 45b184577..eb26acc8a 100644 --- a/src/atlas/grid/Vertical.cc +++ b/src/atlas/grid/Vertical.cc @@ -22,13 +22,13 @@ namespace { std::vector linspace( double start, double end, idx_t N, bool endpoint ) { std::vector x_; - if( N > 0 ) { - volatile double _N = N; // volatile keyword prevents agressive optimization by Cray compiler + if ( N > 0 ) { + volatile double _N = N; // volatile keyword prevents agressive optimization by Cray compiler x_.resize( N ); double step; if ( endpoint && N > 1 ) { - step = ( end - start ) / (_N - 1 ); + step = ( end - start ) / ( _N - 1 ); } else if ( N > 0 ) { step = ( end - start ) / _N; diff --git a/src/atlas/grid/Vertical.h b/src/atlas/grid/Vertical.h index daff12986..e2f7a7405 100644 --- a/src/atlas/grid/Vertical.h +++ b/src/atlas/grid/Vertical.h @@ -60,9 +60,9 @@ class Vertical { friend std::ostream& operator<<( std::ostream& os, const Vertical& v ); private: + idx_t size_; idx_t k_begin_; idx_t k_end_; - idx_t size_; std::vector z_; double min_; double max_; @@ -80,10 +80,7 @@ Vertical::Vertical( idx_t levels, const vector_t& z, const Interval& interval, c //--------------------------------------------------------------------------------------------------------------------- template -Vertical::Vertical( idx_t levels, const vector_t& z, const util::Config& ) { - size_ = levels; - k_begin_ = 0; - k_end_ = size_; +Vertical::Vertical( idx_t levels, const vector_t& z, const util::Config& ) : size_{levels}, k_begin_{0}, k_end_{size_} { assert( size_ == static_cast( z.size() ) ); z_.resize( size_ ); for ( idx_t k = 0; k < size_; ++k ) { diff --git a/src/atlas/grid/detail/grid/Structured.cc b/src/atlas/grid/detail/grid/Structured.cc index 3ea526720..69bb9da0d 100644 --- a/src/atlas/grid/detail/grid/Structured.cc +++ b/src/atlas/grid/detail/grid/Structured.cc @@ -294,11 +294,15 @@ Structured::XSpace::Implementation::Implementation( const Spacing& spacing, idx_ } Structured::XSpace::Implementation::Implementation( const std::vector& spacings ) : - ny_( spacings.size() ), nx_( ny_ ), xmin_( ny_ ), xmax_( ny_ ), dx_( ny_ ) { - nxmax_ = 0; - nxmin_ = std::numeric_limits::max(); - min_ = std::numeric_limits::max(); - max_ = -std::numeric_limits::max(); + ny_( spacings.size() ), + nxmax_( 0 ), + nxmin_( std::numeric_limits::max() ), + nx_( ny_ ), + xmin_( ny_ ), + xmax_( ny_ ), + dx_( ny_ ), + min_( std::numeric_limits::max() ), + max_( -std::numeric_limits::max() ) { for ( idx_t j = 0; j < ny_; ++j ) { const spacing::LinearSpacing& linspace = dynamic_cast( *spacings[j].get() ); diff --git a/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.cc b/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.cc index 31d98bc2e..0d6fac93f 100644 --- a/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.cc +++ b/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.cc @@ -31,31 +31,18 @@ namespace grid { namespace detail { namespace partitioner { -CheckerboardPartitioner::CheckerboardPartitioner() : Partitioner() { - nbands_ = 0; // to be computed later - checkerboard_ = true; // default -} +CheckerboardPartitioner::CheckerboardPartitioner() : Partitioner() {} -CheckerboardPartitioner::CheckerboardPartitioner( int N ) : Partitioner( N ) { - nbands_ = 0; // to be computed later - checkerboard_ = true; // default -} +CheckerboardPartitioner::CheckerboardPartitioner( int N ) : Partitioner( N ) {} CheckerboardPartitioner::CheckerboardPartitioner( int N, const eckit::Parametrisation& config ) : Partitioner( N ) { - nbands_ = 0; // to be computed later config.get( "bands", nbands_ ); - checkerboard_ = true; // default } -CheckerboardPartitioner::CheckerboardPartitioner( int N, int nbands ) : Partitioner( N ) { - nbands_ = nbands; - checkerboard_ = true; // default -} +CheckerboardPartitioner::CheckerboardPartitioner( int N, int nbands ) : Partitioner( N ), nbands_{nbands} {} -CheckerboardPartitioner::CheckerboardPartitioner( int N, int nbands, bool checkerboard ) : Partitioner( N ) { - nbands_ = nbands; - checkerboard_ = checkerboard; -} +CheckerboardPartitioner::CheckerboardPartitioner( int N, int nbands, bool checkerboard ) : + Partitioner( N ), nbands_{nbands} {} CheckerboardPartitioner::Checkerboard CheckerboardPartitioner::checkerboard( const Grid& grid ) const { // grid dimensions diff --git a/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.h b/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.h index 567355602..9d1eda305 100644 --- a/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.h +++ b/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.h @@ -57,8 +57,8 @@ class CheckerboardPartitioner : public Partitioner { void check() const; private: - idx_t nbands_; // number of bands from configuration - bool checkerboard_; // exact (true) or approximate (false) checkerboard + idx_t nbands_ = 0; // number of bands from configuration + bool checkerboard_ = true; // exact (true) or approximate (false) checkerboard }; } // namespace partitioner diff --git a/src/atlas/grid/detail/spacing/LinearSpacing.cc b/src/atlas/grid/detail/spacing/LinearSpacing.cc index 9f00f059a..7174ef2fc 100644 --- a/src/atlas/grid/detail/spacing/LinearSpacing.cc +++ b/src/atlas/grid/detail/spacing/LinearSpacing.cc @@ -84,8 +84,8 @@ void LinearSpacing::setup( double start, double end, long N, bool endpoint ) { x_.resize( N ); double step; - volatile double _N = N; // volatile keyword prevents agressive optimization by Cray compiler - if( start == end ) { + volatile double _N = N; // volatile keyword prevents agressive optimization by Cray compiler + if ( start == end ) { step = 0.; } else if ( endpoint && N > 1 ) { diff --git a/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h b/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h index d04f0e01d..90293058b 100644 --- a/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/CubicHorizontalKernel.h @@ -119,13 +119,13 @@ class CubicHorizontalKernel { auto& weights_j = weights.weights_j; weights_j[0] = ( dl2 * dl3 * dl4 ) / dcl1; -#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE +#if defined( _CRAYC ) && ATLAS_BUILD_TYPE_RELEASE // prevents FE_INVALID somehow (tested with Cray 8.7) - ATLAS_ASSERT( !std::isnan(weights_j[0]) ); + ATLAS_ASSERT( !std::isnan( weights_j[0] ) ); #endif - weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; - weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; - weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; + weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; + weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; + weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; } template diff --git a/src/atlas/interpolation/method/structured/kernels/CubicVerticalKernel.h b/src/atlas/interpolation/method/structured/kernels/CubicVerticalKernel.h index de5755479..625b7fe51 100644 --- a/src/atlas/interpolation/method/structured/kernels/CubicVerticalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/CubicVerticalKernel.h @@ -11,10 +11,10 @@ #pragma once #include -#include "atlas/library/config.h" #include "atlas/grid/Stencil.h" #include "atlas/grid/StencilComputer.h" #include "atlas/grid/Vertical.h" +#include "atlas/library/config.h" #include "atlas/util/Point.h" namespace atlas { @@ -92,9 +92,9 @@ class CubicVerticalKernel { double d3 = z - zvec[3]; w[0] = ( d1 * d2 * d3 ) / dc0; -#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE +#if defined( _CRAYC ) && ATLAS_BUILD_TYPE_RELEASE // prevents FE_INVALID somehow (tested with Cray 8.7) - ATLAS_ASSERT( ! std::isnan(w[0])); + ATLAS_ASSERT( !std::isnan( w[0] ) ); #endif w[1] = ( d0 * d2 * d3 ) / dc1; w[2] = ( d0 * d1 * d3 ) / dc2; diff --git a/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h b/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h index 9292fcdd0..d34e3c435 100644 --- a/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h +++ b/src/atlas/interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h @@ -17,11 +17,11 @@ #include "eckit/linalg/Triplet.h" -#include "atlas/library/config.h" #include "atlas/array/ArrayView.h" #include "atlas/functionspace/StructuredColumns.h" #include "atlas/grid/Stencil.h" #include "atlas/grid/StencilComputer.h" +#include "atlas/library/config.h" #include "atlas/runtime/Exception.h" #include "atlas/util/CoordinateEnums.h" #include "atlas/util/Point.h" @@ -135,13 +135,13 @@ class QuasiCubicHorizontalKernel { auto& weights_j = weights.weights_j; weights_j[0] = ( dl2 * dl3 * dl4 ) / dcl1; -#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE +#if defined( _CRAYC ) && ATLAS_BUILD_TYPE_RELEASE // prevents FE_INVALID somehow (tested with Cray 8.7) - ATLAS_ASSERT( ! std::isnan(weights_j[0]) ); + ATLAS_ASSERT( !std::isnan( weights_j[0] ) ); #endif - weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; - weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; - weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; + weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; + weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; + weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; } template diff --git a/src/atlas/library/FloatingPointExceptions.cc b/src/atlas/library/FloatingPointExceptions.cc index 764639900..9202103dd 100644 --- a/src/atlas/library/FloatingPointExceptions.cc +++ b/src/atlas/library/FloatingPointExceptions.cc @@ -151,7 +151,7 @@ class Signals { << "=========================================\n" << std::endl; - Signals::instance().restoreSignalHandler(signum); + Signals::instance().restoreSignalHandler( signum ); eckit::LibEcKit::instance().abort(); // Just in case we end up here, which normally we shouldn't. diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index b0a821ae0..a0bb74fad 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -91,7 +91,7 @@ int getEnv( const std::string& env, int default_value ) { } -bool library_exists( const eckit::PathName& library_dir, const std::string library_name, +bool library_exists( const eckit::PathName& library_dir, const std::string& library_name, eckit::PathName& library_path ) { // WARNING: Mostly copy-paste from eckit develop before 1.13 release std::vector library_file_names; @@ -107,7 +107,7 @@ bool library_exists( const eckit::PathName& library_dir, const std::string libra } -void load_library( const eckit::PathName& library_dir, const std::string library_name ) { +void load_library( const eckit::PathName& library_dir, const std::string& library_name ) { // WARNING: Mostly copy-paste from eckit develop before 1.13 release std::vector library_file_names; library_file_names.push_back( "lib" + library_name + ".so" ); diff --git a/src/tests/interpolation/CubicInterpolationPrototype.h b/src/tests/interpolation/CubicInterpolationPrototype.h index a61255cb5..f849386bc 100644 --- a/src/tests/interpolation/CubicInterpolationPrototype.h +++ b/src/tests/interpolation/CubicInterpolationPrototype.h @@ -138,9 +138,9 @@ class CubicVerticalInterpolation { double d3 = z - zvec[3]; w[0] = ( d1 * d2 * d3 ) / dc0; -#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE +#if defined( _CRAYC ) && ATLAS_BUILD_TYPE_RELEASE // prevents FE_INVALID somehow (tested with Cray 8.7) - ATLAS_ASSERT( ! std::isnan(w[0]) ); + ATLAS_ASSERT( !std::isnan( w[0] ) ); #endif w[1] = ( d0 * d2 * d3 ) / dc1; w[2] = ( d0 * d1 * d3 ) / dc2; @@ -256,13 +256,13 @@ class CubicHorizontalInterpolation { auto& weights_j = weights.weights_j; weights_j[0] = ( dl2 * dl3 * dl4 ) / dcl1; -#if defined(_CRAYC) && ATLAS_BUILD_TYPE_RELEASE +#if defined( _CRAYC ) && ATLAS_BUILD_TYPE_RELEASE // prevents FE_INVALID somehow (tested with Cray 8.7) - ATLAS_ASSERT( !std::isnan(weights_j[0]) ); + ATLAS_ASSERT( !std::isnan( weights_j[0] ) ); #endif - weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; - weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; - weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; + weights_j[1] = ( dl1 * dl3 * dl4 ) / dcl2; + weights_j[2] = ( dl1 * dl2 * dl4 ) / dcl3; + weights_j[3] = 1. - weights_j[0] - weights_j[1] - weights_j[2]; } template From e8f84ae7c4cfc059c4019356ce0774a3e01e1e33 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 30 Jul 2020 12:46:11 +0100 Subject: [PATCH 019/161] Print source and build dir with 'atlas --info' --- src/atlas/library/Library.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index a0bb74fad..181b7c22c 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -409,6 +409,8 @@ void Library::Information::print( std::ostream& out ) const { out << " Build:" << std::endl; out << " build type : " << ATLAS_BUILD_TYPE << '\n' << " timestamp : " << ATLAS_BUILD_TIMESTAMP << '\n' + << " source dir : " << ATLAS_DEVELOPER_SRC_DIR << '\n' + << " build dir : " << ATLAS_DEVELOPER_BIN_DIR << '\n' << " op. system : " << ATLAS_OS_NAME << " (" << ATLAS_OS_STR << ")" << '\n' << " processor : " << ATLAS_SYS_PROCESSOR << std::endl << " c compiler : " << ATLAS_C_COMPILER_ID << " " << ATLAS_C_COMPILER_VERSION << '\n' @@ -470,7 +472,7 @@ void Library::Information::print( std::ostream& out ) const { } out << " \n Dependencies: \n"; - + out << " ecbuild version (" << ECBUILD_VERSION << ")" << '\n'; if ( Library::exists( "eckit" ) ) { out << " " << str( Library::lookup( "eckit" ) ) << '\n'; } From c9a4947438078d318366affccbd4fea38522300b Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Jul 2020 12:21:38 +0100 Subject: [PATCH 020/161] ATLAS-307 Fix division by zero on very fast interpolations (pe. unit tests) when timer elapsed() returns 0. on calculating rate --- .../interpolation/method/knn/KNearestNeighbours.cc | 12 ++++++++---- .../interpolation/method/knn/NearestNeighbour.cc | 13 ++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc index 781d21e4c..8924fdd7c 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc @@ -10,13 +10,14 @@ #include "atlas/interpolation/method/knn/KNearestNeighbours.h" +#include + #include "eckit/log/Plural.h" -#include "eckit/log/Timer.h" +#include "eckit/types/FloatCompare.h" #include "atlas/array.h" #include "atlas/functionspace/NodeColumns.h" -#include "atlas/grid/Grid.h" -#include "atlas/grid/StructuredGrid.h" +#include "atlas/grid.h" #include "atlas/interpolation/method/MethodFactory.h" #include "atlas/mesh/Nodes.h" #include "atlas/mesh/actions/BuildXYZField.h" @@ -92,7 +93,10 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp for ( size_t ip = 0; ip < out_npts; ++ip ) { if ( ip && ( ip % 1000 == 0 ) ) { - double rate = ip / timer.elapsed(); + auto elapsed = timer.elapsed(); + auto rate = eckit::types::is_approximately_equal( elapsed, 0. ) + ? std::numeric_limits::infinity() + : ( ip / elapsed ); Log::debug() << eckit::BigNum( ip ) << " (at " << rate << " points/s)..." << std::endl; } diff --git a/src/atlas/interpolation/method/knn/NearestNeighbour.cc b/src/atlas/interpolation/method/knn/NearestNeighbour.cc index 95fed2591..cb8f6159d 100644 --- a/src/atlas/interpolation/method/knn/NearestNeighbour.cc +++ b/src/atlas/interpolation/method/knn/NearestNeighbour.cc @@ -5,16 +5,20 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. and Interpolation + * nor does it submit to any jurisdiction. */ +#include "atlas/interpolation/method/knn/NearestNeighbour.h" + +#include + #include "eckit/log/Plural.h" +#include "eckit/types/FloatCompare.h" #include "atlas/array.h" #include "atlas/functionspace/NodeColumns.h" #include "atlas/grid.h" #include "atlas/interpolation/method/MethodFactory.h" -#include "atlas/interpolation/method/knn/NearestNeighbour.h" #include "atlas/mesh/Nodes.h" #include "atlas/mesh/actions/BuildXYZField.h" #include "atlas/meshgenerator.h" @@ -80,7 +84,10 @@ void NearestNeighbour::do_setup( const FunctionSpace& source, const FunctionSpac Trace timer( Here(), "atlas::interpolation::method::NearestNeighbour::do_setup()" ); for ( size_t ip = 0; ip < out_npts; ++ip ) { if ( ip && ( ip % 1000 == 0 ) ) { - double rate = ip / timer.elapsed(); + auto elapsed = timer.elapsed(); + auto rate = eckit::types::is_approximately_equal( elapsed, 0. ) + ? std::numeric_limits::infinity() + : ( ip / elapsed ); Log::debug() << eckit::BigNum( ip ) << " (at " << rate << " points/s)..." << std::endl; } From adb019d92e1d4fc2a523196e406b76e986859baf Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 28 Aug 2020 23:17:45 +0100 Subject: [PATCH 021/161] Fix typo causing broken MathchingMeshPartitionerSphericalPolygon --- .../partitioner/MatchingMeshPartitionerSphericalPolygon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerSphericalPolygon.cc b/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerSphericalPolygon.cc index 1b2882475..11aa71a21 100644 --- a/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerSphericalPolygon.cc +++ b/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerSphericalPolygon.cc @@ -48,7 +48,7 @@ void MatchingMeshPartitionerSphericalPolygon::partition( const Grid& grid, int p bool includesNorthPole = ( mpi_rank == 0 ); bool includesSouthPole = ( mpi_rank == mpi_size - 1 ); - if ( not prePartitionedMesh_.projection() ) { + if ( prePartitionedMesh_.projection() ) { ATLAS_NOTIMPLEMENTED; } const util::SphericalPolygon poly{prePartitionedMesh_.polygon( 0 )}; From 86ddde3abd69d060c35b39b51f45f9743cd930fe Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 00:49:03 +0100 Subject: [PATCH 022/161] ATLAS-269, MIR-465: grid-box average interpolation --- src/atlas/CMakeLists.txt | 6 + src/atlas/functionspace.h | 1 + src/atlas/functionspace/Points.cc | 118 +++++++++++ src/atlas/functionspace/Points.h | 141 +++++++++++++ .../interpolation/method/knn/GridBoxMethod.cc | 194 ++++++++++++++++++ .../interpolation/method/knn/GridBoxMethod.h | 55 +++++ .../method/knn/KNearestNeighboursBase.cc | 30 ++- .../method/knn/KNearestNeighboursBase.h | 2 + src/atlas/util/GridBox.cc | 124 +++++++++++ src/atlas/util/GridBox.h | 117 +++++++++++ 10 files changed, 786 insertions(+), 2 deletions(-) create mode 100644 src/atlas/functionspace/Points.cc create mode 100644 src/atlas/functionspace/Points.h create mode 100644 src/atlas/interpolation/method/knn/GridBoxMethod.cc create mode 100644 src/atlas/interpolation/method/knn/GridBoxMethod.h create mode 100644 src/atlas/util/GridBox.cc create mode 100644 src/atlas/util/GridBox.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index b34828152..928d5bfa7 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -404,6 +404,8 @@ functionspace/Spectral.h functionspace/Spectral.cc functionspace/PointCloud.h functionspace/PointCloud.cc +functionspace/Points.h +functionspace/Points.cc functionspace/detail/FunctionSpaceImpl.h functionspace/detail/FunctionSpaceImpl.cc functionspace/detail/FunctionSpaceInterface.h @@ -497,6 +499,8 @@ interpolation/method/Ray.cc interpolation/method/Ray.h interpolation/method/fe/FiniteElement.cc interpolation/method/fe/FiniteElement.h +interpolation/method/knn/GridBoxMethod.cc +interpolation/method/knn/GridBoxMethod.h interpolation/method/knn/KNearestNeighbours.cc interpolation/method/knn/KNearestNeighbours.h interpolation/method/knn/KNearestNeighboursBase.cc @@ -615,6 +619,8 @@ util/KDTree.cc util/KDTree.h util/PolygonXY.cc util/PolygonXY.h +util/GridBox.cc +util/GridBox.h util/Metadata.cc util/Metadata.h util/Point.cc diff --git a/src/atlas/functionspace.h b/src/atlas/functionspace.h index 01de76bc5..907824424 100644 --- a/src/atlas/functionspace.h +++ b/src/atlas/functionspace.h @@ -17,5 +17,6 @@ #include "atlas/functionspace/FunctionSpace.h" #include "atlas/functionspace/NodeColumns.h" #include "atlas/functionspace/PointCloud.h" +#include "atlas/functionspace/Points.h" #include "atlas/functionspace/Spectral.h" #include "atlas/functionspace/StructuredColumns.h" diff --git a/src/atlas/functionspace/Points.cc b/src/atlas/functionspace/Points.cc new file mode 100644 index 000000000..a44c0536c --- /dev/null +++ b/src/atlas/functionspace/Points.cc @@ -0,0 +1,118 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +#include "atlas/functionspace/Points.h" +#include "atlas/array.h" +#include "atlas/grid/Grid.h" +#include "atlas/grid/Iterator.h" +#include "atlas/option/Options.h" +#include "atlas/runtime/Exception.h" +#include "atlas/util/CoordinateEnums.h" +#include "atlas/util/Earth.h" + + +namespace atlas { +namespace functionspace { +namespace detail { + + +Points::Points( const Grid& grid ) : + lonlat_( "lonlat", array::make_datatype(), array::make_shape( grid.size(), 2 ) ), + xyz_( "xyz", array::make_datatype(), array::make_shape( grid.size(), 3 ) ) { + auto lonlat = array::make_view( lonlat_ ); + auto xyz = array::make_view( xyz_ ); + + PointXYZ q; + idx_t j = 0; + for ( auto p : grid.lonlat() ) { + util::Earth::convertSphericalToCartesian( p, q ); + + lonlat( j, LON ) = p.lon(); + lonlat( j, LAT ) = p.lat(); + + xyz( j, XX ) = q.x(); + xyz( j, YY ) = q.y(); + xyz( j, ZZ ) = q.z(); + + ++j; + } +} + + +Points::~Points() = default; + + +Field Points::createField( const Field& other, const eckit::Configuration& config ) const { + return createField( option::datatype( other.datatype() ) | config ); +} + + +Field Points::createField( const eckit::Configuration& ) const { + ATLAS_NOTIMPLEMENTED; +} + + +idx_t Points::size() const { + return lonlat_.shape( 0 ); +} + + +size_t Points::footprint() const { + return sizeof( *this ); +} + + +std::string Points::distribution() const { + return std::string( "serial" ); +} + + +std::string Points::type() const { + return "Points"; +} + + +const Field& Points::ghost() const { + if ( not ghost_ ) { + ghost_ = Field( "ghost", array::make_datatype(), array::make_shape( size() ) ); + array::make_view( ghost_ ).assign( 0 ); + } + return ghost_; +} + + +template <> +const PointXYZ Points::IteratorT::operator*() const { + return {view_( n_, XX ), view_( n_, YY ), view_( ZZ )}; +} + + +template <> +const PointLonLat Points::IteratorT::operator*() const { + return {view_( n_, LON ), view_( n_, LAT )}; +} + + +} // namespace detail + + +Points::Points( const Grid& grid ) : + FunctionSpace( new detail::Points( grid ) ), + functionspace_( dynamic_cast( get() ) ) {} + + +Points::Points( const FunctionSpace& functionspace ) : + FunctionSpace( functionspace ), + functionspace_( dynamic_cast( get() ) ) {} + + +} // namespace functionspace +} // namespace atlas diff --git a/src/atlas/functionspace/Points.h b/src/atlas/functionspace/Points.h new file mode 100644 index 000000000..c5d61e847 --- /dev/null +++ b/src/atlas/functionspace/Points.h @@ -0,0 +1,141 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +#pragma once + +#include "atlas/array/ArrayView.h" +#include "atlas/field/Field.h" +#include "atlas/functionspace/FunctionSpace.h" +#include "atlas/functionspace/detail/FunctionSpaceImpl.h" +#include "atlas/runtime/Exception.h" +#include "atlas/util/Point.h" + + +namespace atlas { +class Grid; +} + + +namespace atlas { +namespace functionspace { + + +namespace detail { + +class Points final : public FunctionSpaceImpl { +public: + Points( const Grid& ); + ~Points() override; + + using FunctionSpaceImpl::createField; + Field createField( const Field&, const eckit::Configuration& ) const override; + Field createField( const eckit::Configuration& ) const override; + + idx_t size() const override; + size_t footprint() const override; + std::string distribution() const override; + std::string type() const override; + + const Field& lonlat() const { return lonlat_; } + const Field& xyz() const { return xyz_; } + const Field& ghost() const; + + + template + struct IteratorT { + IteratorT( const Field& field, bool begin = true ) : + field_( field ), + view_( array::make_view( field_ ) ), + m_( field_.shape( 0 ) ), + n_( begin ? 0 : m_ ) {} + + bool next( PointXYZ& p ) { + if ( n_ < m_ ) { + p = operator*(); + ++n_; + return true; + } + return false; + } + + const IteratorT& operator++() { + ++n_; + return *this; + } + + const T operator*() const { ATLAS_NOTIMPLEMENTED; } + + bool operator==( const IteratorT& other ) const { return n_ == other.n_; } + bool operator!=( const IteratorT& other ) const { return n_ != other.n_; } + + private: + const Field& field_; + const array::ArrayView view_; + idx_t m_; + idx_t n_; + }; + + + template + struct IterateT { + using iterator = IteratorT; + using const_iterator = iterator; + + IterateT( const Field& field ) : field_( field ) {} + iterator begin() const { return {field_}; } + iterator end() const { return {field_, false}; } + + private: + const Field& field_; + }; + + + struct Iterate { + Iterate( const Points& fs ) : fs_( fs ) {} + IterateT xyz() const { return {fs_.xyz()}; } + IterateT lonlat() const { return {fs_.lonlat()}; } + + private: + const Points& fs_; + }; + + Iterate iterate() const { return Iterate( *this ); } + +private: + Field lonlat_; + Field xyz_; + mutable Field ghost_; +}; + + +} // namespace detail + + +class Points : public FunctionSpace { +public: + Points( const Grid& ); + Points( const FunctionSpace& ); + + operator bool() const { return functionspace_ != nullptr; } + + const Field& lonlat() const { return functionspace_->lonlat(); } + const Field& xyz() const { return functionspace_->xyz(); } + const Field& ghost() const { return functionspace_->ghost(); } + + detail::Points::Iterate iterate() const { return functionspace_->iterate(); } + +private: + const detail::Points* functionspace_; +}; + + +} // namespace functionspace +} // namespace atlas diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc new file mode 100644 index 000000000..83ca9c0a9 --- /dev/null +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -0,0 +1,194 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. and Interpolation + */ + + +#include "atlas/interpolation/method/knn/GridBoxMethod.h" + +#include +#include + +#include "eckit/log/Plural.h" +#include "eckit/log/ProgressTimer.h" +#include "eckit/types/FloatCompare.h" + +#include "atlas/functionspace/Points.h" +#include "atlas/grid.h" +#include "atlas/interpolation/method/MethodFactory.h" +#include "atlas/parallel/mpi/mpi.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" +#include "atlas/runtime/Trace.h" +#include "atlas/util/GridBox.h" + + +namespace atlas { +namespace interpolation { +namespace method { + + +MethodBuilder __builder( "grid-box-average" ); + + +void GridBoxMethod::setup( const Grid& source, const Grid& target ) { + if ( mpi::size() > 1 ) { + ATLAS_NOTIMPLEMENTED; + } + + ATLAS_ASSERT_MSG( !source.projection() && !target.projection(), + "GridBoxMethod: rotated/projected grids not supported" ); + + sourceGrid_ = source; + targetGrid_ = target; + + setup( functionspace::Points( source ), functionspace::Points( target ) ); +} + + +GridBoxMethod::~GridBoxMethod() = default; + + +void GridBoxMethod::print( std::ostream& out ) const { + out << "GridBoxMethod[]"; +} + + +void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& target ) { + if ( mpi::size() > 1 ) { + ATLAS_NOTIMPLEMENTED; + } + + ATLAS_ASSERT( sourceGrid_ ); + ATLAS_ASSERT( targetGrid_ ); + source_ = source; + target_ = target; + + functionspace::Points src = source; + functionspace::Points tgt = target; + ATLAS_ASSERT( src ); + ATLAS_ASSERT( tgt ); + + auto src_npts = size_t( src.size() ); + auto tgt_npts = size_t( tgt.size() ); + Log::debug() << "GridBoxMethod: intersect " << eckit::BigNum( tgt_npts ) << " from " + << eckit::Plural( tgt_npts, "grid box" ) << std::endl; + + + // build point-search tree + { + Trace timer( Here(), "GridBoxMethod: build point search tree" ); + buildPointSearchTree( src ); + ATLAS_ASSERT( pTree_ != nullptr ); + } + + + // helper structures + size_t nbFailures = 0; + std::forward_list failures; + + std::vector weights_triplets; + std::vector triplets; + + + // set grid boxes + const util::GridBoxes sourceBoxes( sourceGrid_ ); + const util::GridBoxes targetBoxes( targetGrid_ ); + const auto R = sourceBoxes.getLongestGridBoxDiagonal() + targetBoxes.getLongestGridBoxDiagonal(); + + + // intersect grid boxes + { + eckit::ProgressTimer progress( "Projecting", tgt_npts, "point", double( 5. ), Log::info() ); + Trace timer( Here(), "GridBoxMethod: intersect grid boxes" ); + + size_t i = 0; + for ( auto p : tgt.iterate().xyz() ) { + if ( ++progress ) { + Log::info() << "GridBoxMethod: " << *pTree_ << std::endl; + } + + // lookup + auto closest = pTree_->findInSphere( p, R ); + ASSERT( !closest.empty() ); + + + // calculate grid box intersections + triplets.clear(); + triplets.reserve( closest.size() ); + + auto& box = targetBoxes.at( i ); + double area = box.area(); + ASSERT( area > 0. ); + + double sumSmallAreas = 0.; + bool areaMatch = false; + for ( auto& c : closest ) { + auto j = c.payload(); + auto smallBox = sourceBoxes.at( j ); + + if ( box.intersects( smallBox ) ) { + double smallArea = smallBox.area(); + ASSERT( smallArea > 0. ); + + triplets.emplace_back( i, j, smallArea / area ); + sumSmallAreas += smallArea; + + if ( ( areaMatch = eckit::types::is_approximately_equal( area, sumSmallAreas, 1. /*m^2*/ ) ) ) { + break; + } + } + } + + + // insert the interpolant weights into the global (sparse) interpolant matrix + if ( areaMatch ) { + std::copy( triplets.begin(), triplets.end(), std::back_inserter( weights_triplets ) ); + } + else { + ++nbFailures; + failures.push_front( i ); + } + + + ++i; + } + } + + + // statistics + Log::debug() << "Intersected " << eckit::Plural( weights_triplets.size(), "grid box" ) << std::endl; + + if ( nbFailures > 0 ) { + Log::warning() << "Failed to intersect points: "; + size_t count = 0; + auto sep = ""; + for ( const auto& f : failures ) { + Log::warning() << sep << f; + sep = ", "; + if ( ++count > 10 ) { + Log::warning() << "..."; + break; + } + } + Log::warning() << std::endl; + } + + + // fill sparse matrix + { + Trace timer( Here(), "GridBoxMethod: build sparse matrix" ); + Matrix A( tgt_npts, src_npts, weights_triplets ); + matrix_.swap( A ); + } +} + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h new file mode 100644 index 000000000..a9f0861cf --- /dev/null +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -0,0 +1,55 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +#pragma once + +#include "atlas/interpolation/method/knn/KNearestNeighboursBase.h" + +#include "atlas/functionspace.h" +#include "atlas/grid.h" + + +namespace atlas { +namespace interpolation { +namespace method { + + +class GridBoxMethod : public KNearestNeighboursBase { +public: + GridBoxMethod( const Config& config ) : KNearestNeighboursBase( config ) {} + virtual ~GridBoxMethod() override; + + virtual void print( std::ostream& ) const override; + +protected: + /** + * @brief Create an interpolant sparse matrix relating two functionspaces, using grid-box average method + * @param source functionspace containing source points + * @param target functionspace containing target points + */ + virtual void setup( const FunctionSpace& source, const FunctionSpace& target ) override; + + virtual void setup( const Grid& source, const Grid& target ) override; + + virtual const FunctionSpace& source() const override { return source_; } + virtual const FunctionSpace& target() const override { return target_; } + +private: + FunctionSpace source_; + FunctionSpace target_; + Grid sourceGrid_; + Grid targetGrid_; +}; + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc index f85a65a2c..a02fb407b 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc @@ -23,8 +23,7 @@ namespace interpolation { namespace method { void KNearestNeighboursBase::buildPointSearchTree( Mesh& meshSource ) { - using namespace atlas; - eckit::TraceTimer tim( "atlas::interpolation::method::KNearestNeighboursBase::setup()" ); + eckit::TraceTimer tim( "KNearestNeighboursBase::setup()" ); // generate 3D point coordinates mesh::actions::BuildXYZField( "xyz" )( meshSource ); @@ -52,6 +51,33 @@ void KNearestNeighboursBase::buildPointSearchTree( Mesh& meshSource ) { } } +void KNearestNeighboursBase::buildPointSearchTree(const functionspace::Points& points ) { + eckit::TraceTimer tim( "KNearestNeighboursBase::buildPointSearchTree()" ); + + // build point-search tree + pTree_.reset( new PointIndex3 ); + + static bool fastBuildKDTrees = eckit::Resource( "$ATLAS_FAST_BUILD_KDTREES", true ); + + if ( fastBuildKDTrees ) { + std::vector pidx; + pidx.reserve( points.size() ); + + idx_t ip = 0; + for ( auto p : points.iterate().xyz() ) { + pidx.emplace_back( p, ip++ ); + } + + pTree_->build( pidx.begin(), pidx.end() ); + } + else { + idx_t ip = 0; + for ( auto p : points.iterate().xyz() ) { + pTree_->insert( PointIndex3::Value( p, ip++ ) ); + } + } +} + } // namespace method } // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h index 0e92070ff..0265fdd94 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h @@ -14,6 +14,7 @@ #include "atlas/interpolation/method/Method.h" #include "atlas/interpolation/method/PointIndex3.h" +#include "atlas/functionspace/Points.h" namespace atlas { namespace interpolation { @@ -26,6 +27,7 @@ class KNearestNeighboursBase : public Method { protected: void buildPointSearchTree( Mesh& meshSource ); + void buildPointSearchTree( const functionspace::Points& ); std::unique_ptr pTree_; }; diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc new file mode 100644 index 000000000..996a987bf --- /dev/null +++ b/src/atlas/util/GridBox.cc @@ -0,0 +1,124 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/util/GridBox.h" + +#include +#include + +#include "eckit/types/FloatCompare.h" + +#include "atlas/runtime/Exception.h" +#include "atlas/util/Earth.h" +#include "atlas/util/Point.h" +#include "atlas/grid.h" + + +static constexpr double GLOBE = 360.; +static constexpr double NORTH_POLE = 90.; +static constexpr double SOUTH_POLE = -90.; + +double normalise( double lon, double minimum ) { + while ( lon < minimum ) { + lon += GLOBE; + } + while ( lon >= minimum + GLOBE ) { + lon -= GLOBE; + } + return lon; +} + + +namespace atlas { +namespace util { + + +GridBox::GridBox( double north, double west, double south, double east ) : + north_( north ), + west_( west ), + south_( south ), + east_( east ) { + ATLAS_ASSERT( SOUTH_POLE <= south_ && south_ <= north_ && north_ <= NORTH_POLE ); + ATLAS_ASSERT( west_ <= east_ && east_ <= west_ + GLOBE ); +} + + +double GridBox::area() const { + return util::Earth::area( {west_, north_}, {east_, south_} ); +} + + +double GridBox::diagonal() const { + return util::Earth::distance( {west_, north_}, {east_, south_} ); +} + + +bool GridBox::intersects( GridBox& other ) const { + double n = std::min( north_, other.north_ ); + double s = std::max( south_, other.south_ ); + + if ( !eckit::types::is_strictly_greater( n, s ) ) { + return false; + } + + auto intersect = []( const GridBox& a, const GridBox& b, double& w, double& e ) { + double ref = normalise( b.west_, a.west_ ); + double w_ = std::max( a.west_, ref ); + double e_ = std::min( a.east_, normalise( b.east_, ref ) ); + + if ( eckit::types::is_strictly_greater( e_, w_ ) ) { + w = w_; + e = e_; + return true; + } + return false; + }; + + double w = std::min( west_, other.west_ ); + double e = w; + + if ( west_ <= other.west_ ? intersect( *this, other, w, e ) || intersect( other, *this, w, e ) + : intersect( other, *this, w, e ) || intersect( *this, other, w, e ) ) { + ATLAS_ASSERT( w <= e ); + other = {n, w, s, e}; + return true; + } + return false; +} + + +void GridBox::print( std::ostream& out ) const { + out << "GridBox[north=" << north_ << ",west=" << west_ << ",south=" << south_ << ",east=" << east_ << "]"; +} + + +GridBoxes::GridBoxes(const Grid& grid) { + + //std::vector(grid.gridBoxes()) + + ATLAS_ASSERT(idx_t(size()) == grid.size()); +} + + +double GridBoxes::getLongestGridBoxDiagonal() const { + double R = 0.; + for (auto& box : *this) { + R = std::max(R, box.diagonal()); + } + + ATLAS_ASSERT(R > 0.); + return R; +} + + +} // namespace util +} // namespace atlas diff --git a/src/atlas/util/GridBox.h b/src/atlas/util/GridBox.h new file mode 100644 index 000000000..9d904be88 --- /dev/null +++ b/src/atlas/util/GridBox.h @@ -0,0 +1,117 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include +#include + + +namespace atlas { +class Grid; +} + + +namespace atlas { +namespace util { + + +class GridBox { +public: + // -- Exceptions + // None + + // -- Constructors + + GridBox( double north, double west, double south, double east ); + + // -- Destructor + // None + + // -- Convertors + // None + + // -- Operators + // None + + // -- Methods + + double area() const; + double diagonal() const; + bool intersects( GridBox& ) const; + + // -- Overridden methods + // None + + // -- Class members + // None + + // -- Class methods + // None + +protected: + // -- Members + // None + + // -- Methods + // None + + // -- Overridden methods + // None + + // -- Class members + // None + + // -- Class methods + // None + + // -- Friends + // None + +private: + // -- Members + + double north_; + double west_; + double south_; + double east_; + + // -- Methods + + void print( std::ostream& ) const; + + // -- Overridden methods + // None + + // -- Class members + // None + + // -- Class methods + // None + + // -- Friends + + friend std::ostream& operator<<( std::ostream& s, const GridBox& p ) { + p.print( s ); + return s; + } +}; + + +struct GridBoxes : std::vector { + GridBoxes( const Grid& ); + double getLongestGridBoxDiagonal() const; +}; + + +} // namespace util +} // namespace atlas From 5438107fc720c33cfbba1573a3c97d48e4171b1c Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 11:36:30 +0100 Subject: [PATCH 023/161] ATLAS-269, MIR-465: grid-box average interpolation --- src/atlas/util/GridBox.cc | 56 ++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index 996a987bf..690481fd4 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -16,11 +16,12 @@ #include #include "eckit/types/FloatCompare.h" +#include "eckit/types/Fraction.h" +#include "atlas/grid.h" #include "atlas/runtime/Exception.h" #include "atlas/util/Earth.h" #include "atlas/util/Point.h" -#include "atlas/grid.h" static constexpr double GLOBE = 360.; @@ -101,21 +102,62 @@ void GridBox::print( std::ostream& out ) const { } -GridBoxes::GridBoxes(const Grid& grid) { +GridBoxes::GridBoxes( const Grid& grid ) { + StructuredGrid structured( grid ); + if ( !structured || !structured.domain().global() || grid.projection() ) { + throw_NotImplemented( "GridBoxes only support structured, unprojected/unrotated global grids", Here() ); + } + + + // Calculate grid-box parallels (latitude midpoints) + auto& y = structured.yspace(); + + std::vector lat; + lat.reserve( y.size() + 1 ); - //std::vector(grid.gridBoxes()) + lat.push_back( 90. ); + for ( auto b = y.begin(), a = b++; b != y.end(); a = b++ ) { + lat.push_back( ( *b + *a ) / 2. ); + } + lat.push_back( -90. ); + + + // Calculate grid-box meridians (longitude midpoints) + auto& x = structured.xspace(); + ATLAS_ASSERT( x.nx().size() == x.dx().size() ); + ATLAS_ASSERT( x.nx().size() == x.xmin().size() ); + + clear(); + reserve( grid.size() ); + for ( size_t j = 0; j < x.nx().size(); ++j ) { + eckit::Fraction dx( x.dx()[j] ); + eckit::Fraction xmin( x.xmin()[j] ); + auto n = ( xmin / dx ).integralPart(); + if ( n * dx < xmin ) { + n += 1; + } + + eckit::Fraction lon1 = ( n * dx ) - ( dx / 2 ); + for ( idx_t i = 0; i < x.nx()[j]; ++i ) { + double lon0 = lon1; + lon1 += dx; + emplace_back( GridBox( lat[j], lon0, lat[j + 1], lon1 ) ); + } + } - ATLAS_ASSERT(idx_t(size()) == grid.size()); + ATLAS_ASSERT( idx_t( size() ) == grid.size() ); } double GridBoxes::getLongestGridBoxDiagonal() const { + ATLAS_ASSERT( !empty() ); + double R = 0.; - for (auto& box : *this) { - R = std::max(R, box.diagonal()); + for ( auto& box : *this ) { + R = std::max( R, box.diagonal() ); } - ATLAS_ASSERT(R > 0.); + ATLAS_ASSERT( R > 0. ); return R; } From 7fbd16aa8717f274874dddfd04cd4655135a200e Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 15:31:37 +0100 Subject: [PATCH 024/161] ATLAS-269, MIR-465: grid-box average interpolation matrix-free --- src/atlas/functionspace/Points.cc | 2 +- .../interpolation/method/knn/GridBoxMethod.cc | 81 +++++++++++-------- .../interpolation/method/knn/GridBoxMethod.h | 15 +++- src/atlas/util/GridBox.h | 1 + 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/src/atlas/functionspace/Points.cc b/src/atlas/functionspace/Points.cc index a44c0536c..2d14cff07 100644 --- a/src/atlas/functionspace/Points.cc +++ b/src/atlas/functionspace/Points.cc @@ -10,11 +10,11 @@ #include "atlas/functionspace/Points.h" + #include "atlas/array.h" #include "atlas/grid/Grid.h" #include "atlas/grid/Iterator.h" #include "atlas/option/Options.h" -#include "atlas/runtime/Exception.h" #include "atlas/util/CoordinateEnums.h" #include "atlas/util/Earth.h" diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 83ca9c0a9..dd3c92722 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -25,7 +25,6 @@ #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "atlas/runtime/Trace.h" -#include "atlas/util/GridBox.h" namespace atlas { @@ -36,10 +35,21 @@ namespace method { MethodBuilder __builder( "grid-box-average" ); +GridBoxMethod::GridBoxMethod( const Method::Config& config ) : KNearestNeighboursBase( config ), matrixFree_( false ) { + config.get( "matrix_free", matrixFree_ ); +} + + +GridBoxMethod::~GridBoxMethod() = default; + + +void GridBoxMethod::print( std::ostream& out ) const { + out << "GridBoxMethod[]"; +} + + void GridBoxMethod::setup( const Grid& source, const Grid& target ) { - if ( mpi::size() > 1 ) { - ATLAS_NOTIMPLEMENTED; - } + ATLAS_TRACE( "GridBoxMethod::setup()" ); ATLAS_ASSERT_MSG( !source.projection() && !target.projection(), "GridBoxMethod: rotated/projected grids not supported" ); @@ -51,15 +61,9 @@ void GridBoxMethod::setup( const Grid& source, const Grid& target ) { } -GridBoxMethod::~GridBoxMethod() = default; - - -void GridBoxMethod::print( std::ostream& out ) const { - out << "GridBoxMethod[]"; -} - - void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& target ) { + ATLAS_TRACE( "GridBoxMethod::setup()" ); + if ( mpi::size() > 1 ) { ATLAS_NOTIMPLEMENTED; } @@ -74,18 +78,12 @@ void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& tar ATLAS_ASSERT( src ); ATLAS_ASSERT( tgt ); - auto src_npts = size_t( src.size() ); - auto tgt_npts = size_t( tgt.size() ); - Log::debug() << "GridBoxMethod: intersect " << eckit::BigNum( tgt_npts ) << " from " - << eckit::Plural( tgt_npts, "grid box" ) << std::endl; - + buildPointSearchTree( src ); + ATLAS_ASSERT( pTree_ != nullptr ); - // build point-search tree - { - Trace timer( Here(), "GridBoxMethod: build point search tree" ); - buildPointSearchTree( src ); - ATLAS_ASSERT( pTree_ != nullptr ); - } + Trace timer( Here(), "GridBoxMethod: build grid boxes" ); + sourceBoxes_ = util::GridBoxes( sourceGrid_ ); + targetBoxes_ = util::GridBoxes( targetGrid_ ); // helper structures @@ -96,16 +94,12 @@ void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& tar std::vector triplets; - // set grid boxes - const util::GridBoxes sourceBoxes( sourceGrid_ ); - const util::GridBoxes targetBoxes( targetGrid_ ); - const auto R = sourceBoxes.getLongestGridBoxDiagonal() + targetBoxes.getLongestGridBoxDiagonal(); - - // intersect grid boxes { - eckit::ProgressTimer progress( "Projecting", tgt_npts, "point", double( 5. ), Log::info() ); - Trace timer( Here(), "GridBoxMethod: intersect grid boxes" ); + eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ), Log::info() ); + Trace timer( Here(), "GridBoxMethod: intersecting grid boxes" ); + + const auto R = sourceBoxes_.getLongestGridBoxDiagonal() + targetBoxes_.getLongestGridBoxDiagonal(); size_t i = 0; for ( auto p : tgt.iterate().xyz() ) { @@ -122,7 +116,7 @@ void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& tar triplets.clear(); triplets.reserve( closest.size() ); - auto& box = targetBoxes.at( i ); + auto& box = targetBoxes_.at( i ); double area = box.area(); ASSERT( area > 0. ); @@ -130,7 +124,7 @@ void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& tar bool areaMatch = false; for ( auto& c : closest ) { auto j = c.payload(); - auto smallBox = sourceBoxes.at( j ); + auto smallBox = sourceBoxes_.at( j ); if ( box.intersects( smallBox ) ) { double smallArea = smallBox.area(); @@ -183,12 +177,31 @@ void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& tar // fill sparse matrix { Trace timer( Here(), "GridBoxMethod: build sparse matrix" ); - Matrix A( tgt_npts, src_npts, weights_triplets ); + Matrix A( targetBoxes_.size(), sourceBoxes_.size(), weights_triplets ); matrix_.swap( A ); } } +void GridBoxMethod::execute( const FieldSet& source, FieldSet& target ) const { + ATLAS_TRACE( "GridBoxMethod::execute()" ); + + const idx_t N = source.size(); + ATLAS_ASSERT( N == target.size() ); + + for ( idx_t i = 0; i < N; ++i ) { + Log::debug() << "GridBoxMethod::execute on field " << ( i + 1 ) << '/' << N << "..." << std::endl; + execute( source[i], target[i] ); + } +} + + +void GridBoxMethod::execute( const Field& source, Field& target ) const { + Log::info() << std::endl; + ATLAS_NOTIMPLEMENTED; +} + + } // namespace method } // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index a9f0861cf..80203411f 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -15,6 +15,7 @@ #include "atlas/functionspace.h" #include "atlas/grid.h" +#include "atlas/util/GridBox.h" namespace atlas { @@ -24,29 +25,37 @@ namespace method { class GridBoxMethod : public KNearestNeighboursBase { public: - GridBoxMethod( const Config& config ) : KNearestNeighboursBase( config ) {} + GridBoxMethod( const Config& ); virtual ~GridBoxMethod() override; +protected: virtual void print( std::ostream& ) const override; -protected: /** * @brief Create an interpolant sparse matrix relating two functionspaces, using grid-box average method * @param source functionspace containing source points * @param target functionspace containing target points */ virtual void setup( const FunctionSpace& source, const FunctionSpace& target ) override; - virtual void setup( const Grid& source, const Grid& target ) override; + virtual void execute( const FieldSet& source, FieldSet& target ) const override; + virtual void execute( const Field& source, Field& target ) const override; + virtual const FunctionSpace& source() const override { return source_; } virtual const FunctionSpace& target() const override { return target_; } private: FunctionSpace source_; FunctionSpace target_; + Grid sourceGrid_; Grid targetGrid_; + + util::GridBoxes sourceBoxes_; + util::GridBoxes targetBoxes_; + + bool matrixFree_; }; diff --git a/src/atlas/util/GridBox.h b/src/atlas/util/GridBox.h index 9d904be88..19f20dca9 100644 --- a/src/atlas/util/GridBox.h +++ b/src/atlas/util/GridBox.h @@ -109,6 +109,7 @@ class GridBox { struct GridBoxes : std::vector { GridBoxes( const Grid& ); + using vector::vector; double getLongestGridBoxDiagonal() const; }; From 82b65d96d3c3f2f2be2c5d128682f8733e359e96 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 18:44:43 +0100 Subject: [PATCH 025/161] ATLAS-269, MIR-465: grid-box average interpolation matrix-free --- .../interpolation/method/knn/GridBoxMethod.cc | 127 ++++++++---------- .../interpolation/method/knn/GridBoxMethod.h | 2 + 2 files changed, 60 insertions(+), 69 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index dd3c92722..467625392 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -48,40 +48,66 @@ void GridBoxMethod::print( std::ostream& out ) const { } -void GridBoxMethod::setup( const Grid& source, const Grid& target ) { - ATLAS_TRACE( "GridBoxMethod::setup()" ); +void GridBoxMethod::setup( const FunctionSpace& /*source*/, const FunctionSpace& /*target*/ ) { + ATLAS_NOTIMPLEMENTED; +} - ATLAS_ASSERT_MSG( !source.projection() && !target.projection(), - "GridBoxMethod: rotated/projected grids not supported" ); - sourceGrid_ = source; - targetGrid_ = target; +bool GridBoxMethod::intersect( size_t i, const util::GridBox& box, const PointIndex3::NodeList& closest, + std::vector& triplets ) { + ASSERT( !closest.empty() ); + + triplets.clear(); + triplets.reserve( closest.size() ); + + double area = box.area(); + ASSERT( area > 0. ); + + double sumSmallAreas = 0.; + for ( auto& c : closest ) { + auto j = c.payload(); + auto smallBox = sourceBoxes_.at( j ); - setup( functionspace::Points( source ), functionspace::Points( target ) ); + if ( box.intersects( smallBox ) ) { + double smallArea = smallBox.area(); + ASSERT( smallArea > 0. ); + + triplets.emplace_back( i, j, smallArea / area ); + sumSmallAreas += smallArea; + + if ( eckit::types::is_approximately_equal( area, sumSmallAreas, 1. /*m^2*/ ) ) { + return true; + } + } + } + + triplets.clear(); + return false; } -void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& target ) { +void GridBoxMethod::setup( const Grid& source, const Grid& target ) { ATLAS_TRACE( "GridBoxMethod::setup()" ); if ( mpi::size() > 1 ) { ATLAS_NOTIMPLEMENTED; } - ATLAS_ASSERT( sourceGrid_ ); - ATLAS_ASSERT( targetGrid_ ); - source_ = source; - target_ = target; + ATLAS_ASSERT( source ); + ATLAS_ASSERT( target ); + sourceGrid_ = source; + targetGrid_ = target; functionspace::Points src = source; functionspace::Points tgt = target; ATLAS_ASSERT( src ); ATLAS_ASSERT( tgt ); + source_ = src; + target_ = tgt; buildPointSearchTree( src ); ATLAS_ASSERT( pTree_ != nullptr ); - Trace timer( Here(), "GridBoxMethod: build grid boxes" ); sourceBoxes_ = util::GridBoxes( sourceGrid_ ); targetBoxes_ = util::GridBoxes( targetGrid_ ); @@ -94,11 +120,11 @@ void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& tar std::vector triplets; - // intersect grid boxes + // intersect grid boxes and insert the interpolant weights into the global (sparse) interpolant matrix { - eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ), Log::info() ); - Trace timer( Here(), "GridBoxMethod: intersecting grid boxes" ); + ATLAS_TRACE( "GridBoxMethod::setup: intersecting grid boxes" ); + eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ), Log::info() ); const auto R = sourceBoxes_.getLongestGridBoxDiagonal() + targetBoxes_.getLongestGridBoxDiagonal(); size_t i = 0; @@ -107,41 +133,8 @@ void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& tar Log::info() << "GridBoxMethod: " << *pTree_ << std::endl; } - // lookup - auto closest = pTree_->findInSphere( p, R ); - ASSERT( !closest.empty() ); - - - // calculate grid box intersections - triplets.clear(); - triplets.reserve( closest.size() ); - - auto& box = targetBoxes_.at( i ); - double area = box.area(); - ASSERT( area > 0. ); - - double sumSmallAreas = 0.; - bool areaMatch = false; - for ( auto& c : closest ) { - auto j = c.payload(); - auto smallBox = sourceBoxes_.at( j ); - - if ( box.intersects( smallBox ) ) { - double smallArea = smallBox.area(); - ASSERT( smallArea > 0. ); - - triplets.emplace_back( i, j, smallArea / area ); - sumSmallAreas += smallArea; - - if ( ( areaMatch = eckit::types::is_approximately_equal( area, sumSmallAreas, 1. /*m^2*/ ) ) ) { - break; - } - } - } - - - // insert the interpolant weights into the global (sparse) interpolant matrix - if ( areaMatch ) { + if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, R ), triplets ) ) { + ATLAS_ASSERT( !triplets.empty() ); std::copy( triplets.begin(), triplets.end(), std::back_inserter( weights_triplets ) ); } else { @@ -149,34 +142,30 @@ void GridBoxMethod::setup( const FunctionSpace& source, const FunctionSpace& tar failures.push_front( i ); } - ++i; } - } - - // statistics - Log::debug() << "Intersected " << eckit::Plural( weights_triplets.size(), "grid box" ) << std::endl; - - if ( nbFailures > 0 ) { - Log::warning() << "Failed to intersect points: "; - size_t count = 0; - auto sep = ""; - for ( const auto& f : failures ) { - Log::warning() << sep << f; - sep = ", "; - if ( ++count > 10 ) { - Log::warning() << "..."; - break; + if ( nbFailures > 0 ) { + Log::warning() << "Failed to intersect grid boxes: "; + size_t count = 0; + auto sep = ""; + for ( const auto& f : failures ) { + Log::warning() << sep << f; + sep = ", "; + if ( ++count > 10 ) { + Log::warning() << "... (" << nbFailures << " total)"; + break; + } } + Log::warning() << std::endl; + throw_Exception( "Failed to intersect grid boxes" ); } - Log::warning() << std::endl; } // fill sparse matrix { - Trace timer( Here(), "GridBoxMethod: build sparse matrix" ); + ATLAS_TRACE( "GridBoxMethod::setup: build sparse matrix" ); Matrix A( targetBoxes_.size(), sourceBoxes_.size(), weights_triplets ); matrix_.swap( A ); } diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index 80203411f..af0ac3392 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -45,6 +45,8 @@ class GridBoxMethod : public KNearestNeighboursBase { virtual const FunctionSpace& source() const override { return source_; } virtual const FunctionSpace& target() const override { return target_; } + bool intersect( size_t i, const util::GridBox& iBox, const PointIndex3::NodeList&, std::vector& ); + private: FunctionSpace source_; FunctionSpace target_; From 8372d41d2416eb784f3942ce4db04f063813d0b4 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 20:40:29 +0100 Subject: [PATCH 026/161] ATLAS-269, MIR-465: grid-box average interpolation matrix-free --- src/atlas/functionspace/Points.cc | 11 ++ src/atlas/functionspace/Points.h | 3 + .../interpolation/method/knn/GridBoxMethod.cc | 142 ++++++++++++------ .../interpolation/method/knn/GridBoxMethod.h | 10 +- 4 files changed, 115 insertions(+), 51 deletions(-) diff --git a/src/atlas/functionspace/Points.cc b/src/atlas/functionspace/Points.cc index 2d14cff07..3b6fc93e3 100644 --- a/src/atlas/functionspace/Points.cc +++ b/src/atlas/functionspace/Points.cc @@ -15,6 +15,7 @@ #include "atlas/grid/Grid.h" #include "atlas/grid/Iterator.h" #include "atlas/option/Options.h" +#include "atlas/runtime/Log.h" #include "atlas/util/CoordinateEnums.h" #include "atlas/util/Earth.h" @@ -60,6 +61,16 @@ Field Points::createField( const eckit::Configuration& ) const { } +void Points::haloExchange( const FieldSet&, bool ) const { + Log::debug() << "Points::haloExchange: ignored" << std::endl; +} + + +void Points::haloExchange( const Field&, bool ) const { + Log::debug() << "Points::haloExchange: ignored" << std::endl; +} + + idx_t Points::size() const { return lonlat_.shape( 0 ); } diff --git a/src/atlas/functionspace/Points.h b/src/atlas/functionspace/Points.h index c5d61e847..34d44b004 100644 --- a/src/atlas/functionspace/Points.h +++ b/src/atlas/functionspace/Points.h @@ -39,6 +39,9 @@ class Points final : public FunctionSpaceImpl { Field createField( const Field&, const eckit::Configuration& ) const override; Field createField( const eckit::Configuration& ) const override; + void haloExchange( const FieldSet&, bool /*on_device*/ = false ) const override; + void haloExchange( const Field&, bool /*on_device*/ = false ) const override; + idx_t size() const override; size_t footprint() const override; std::string distribution() const override; diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 467625392..fa4e12b0c 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -11,7 +11,7 @@ #include "atlas/interpolation/method/knn/GridBoxMethod.h" -#include +#include #include #include "eckit/log/Plural.h" @@ -35,8 +35,26 @@ namespace method { MethodBuilder __builder( "grid-box-average" ); -GridBoxMethod::GridBoxMethod( const Method::Config& config ) : KNearestNeighboursBase( config ), matrixFree_( false ) { - config.get( "matrix_free", matrixFree_ ); +void give_up( const std::forward_list& failures ) { + Log::warning() << "Failed to intersect grid boxes: "; + + size_t count = 0; + auto sep = ""; + for ( const auto& f : failures ) { + if ( count++ < 10 ) { + Log::warning() << sep << f; + sep = ", "; + } + } + Log::warning() << "... (" << eckit::Plural( count, "total failure" ) << std::endl; + + throw_Exception( "Failed to intersect grid boxes" ); +} + + +GridBoxMethod::GridBoxMethod( const Method::Config& config ) : KNearestNeighboursBase( config ) { + config.get( "matrix_free", matrixFree_ = false ); + config.get( "fail_early", failEarly_ = true ); } @@ -54,7 +72,7 @@ void GridBoxMethod::setup( const FunctionSpace& /*source*/, const FunctionSpace& bool GridBoxMethod::intersect( size_t i, const util::GridBox& box, const PointIndex3::NodeList& closest, - std::vector& triplets ) { + std::vector& triplets ) const { ASSERT( !closest.empty() ); triplets.clear(); @@ -81,6 +99,12 @@ bool GridBoxMethod::intersect( size_t i, const util::GridBox& box, const PointIn } } + if ( failEarly_ ) { + Log::error() << "Failed to intersect grid box " << i << ", " << box << std::endl; + throw_Exception( "Failed to intersect grid box" ); + } + + failures_.push_front( i ); triplets.clear(); return false; } @@ -111,83 +135,103 @@ void GridBoxMethod::setup( const Grid& source, const Grid& target ) { sourceBoxes_ = util::GridBoxes( sourceGrid_ ); targetBoxes_ = util::GridBoxes( targetGrid_ ); + searchRadius_ = sourceBoxes_.getLongestGridBoxDiagonal() + targetBoxes_.getLongestGridBoxDiagonal(); + failures_.clear(); - // helper structures - size_t nbFailures = 0; - std::forward_list failures; - - std::vector weights_triplets; - std::vector triplets; + if ( matrixFree_ ) { + Matrix A; + matrix_.swap( A ); + return; + } + std::vector allTriplets; - // intersect grid boxes and insert the interpolant weights into the global (sparse) interpolant matrix { ATLAS_TRACE( "GridBoxMethod::setup: intersecting grid boxes" ); - eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ), Log::info() ); - const auto R = sourceBoxes_.getLongestGridBoxDiagonal() + targetBoxes_.getLongestGridBoxDiagonal(); + eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ) ); + std::vector triplets; size_t i = 0; for ( auto p : tgt.iterate().xyz() ) { if ( ++progress ) { Log::info() << "GridBoxMethod: " << *pTree_ << std::endl; } - if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, R ), triplets ) ) { - ATLAS_ASSERT( !triplets.empty() ); - std::copy( triplets.begin(), triplets.end(), std::back_inserter( weights_triplets ) ); - } - else { - ++nbFailures; - failures.push_front( i ); + if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { + std::copy( triplets.begin(), triplets.end(), std::back_inserter( allTriplets ) ); } ++i; } - if ( nbFailures > 0 ) { - Log::warning() << "Failed to intersect grid boxes: "; - size_t count = 0; - auto sep = ""; - for ( const auto& f : failures ) { - Log::warning() << sep << f; - sep = ", "; - if ( ++count > 10 ) { - Log::warning() << "... (" << nbFailures << " total)"; - break; - } - } - Log::warning() << std::endl; - throw_Exception( "Failed to intersect grid boxes" ); + if ( !failures_.empty() ) { + give_up( failures_ ); } } - - // fill sparse matrix { - ATLAS_TRACE( "GridBoxMethod::setup: build sparse matrix" ); - Matrix A( targetBoxes_.size(), sourceBoxes_.size(), weights_triplets ); + ATLAS_TRACE( "GridBoxMethod::setup: build interpolant matrix" ); + Matrix A( targetBoxes_.size(), sourceBoxes_.size(), allTriplets ); matrix_.swap( A ); } } void GridBoxMethod::execute( const FieldSet& source, FieldSet& target ) const { - ATLAS_TRACE( "GridBoxMethod::execute()" ); + if ( matrixFree_ ) { + ATLAS_TRACE( "GridBoxMethod::execute()" ); - const idx_t N = source.size(); - ATLAS_ASSERT( N == target.size() ); - for ( idx_t i = 0; i < N; ++i ) { - Log::debug() << "GridBoxMethod::execute on field " << ( i + 1 ) << '/' << N << "..." << std::endl; - execute( source[i], target[i] ); - } -} + // ensure setup() + functionspace::Points tgt = target_; + ATLAS_ASSERT( tgt ); + ATLAS_ASSERT( searchRadius_ > 0. ); + ATLAS_ASSERT( !sourceBoxes_.empty() ); + ATLAS_ASSERT( !targetBoxes_.empty() ); -void GridBoxMethod::execute( const Field& source, Field& target ) const { - Log::info() << std::endl; - ATLAS_NOTIMPLEMENTED; + // set arrays + ATLAS_ASSERT( source.size() == 1 && source.field( 0 ).rank() == 1 ); + ATLAS_ASSERT( target.size() == 1 && target.field( 0 ).rank() == 1 ); + + auto xarray = atlas::array::make_view( source.field( 0 ) ); + auto yarray = atlas::array::make_view( target.field( 0 ) ); + ATLAS_ASSERT( xarray.size() == idx_t( sourceBoxes_.size() ) ); + ATLAS_ASSERT( yarray.size() == idx_t( targetBoxes_.size() ) ); + + yarray.assign( 0. ); + failures_.clear(); + + + // interpolate + eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ) ); + + std::vector triplets; + size_t i = 0; + for ( auto p : tgt.iterate().xyz() ) { + if ( ++progress ) { + Log::info() << "GridBoxMethod: " << *pTree_ << std::endl; + } + + if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { + auto& y = yarray[i /*t.col()*/]; + for ( auto& t : triplets ) { + y += xarray[t.col()] * t.value(); + } + } + + ++i; + } + + if ( !failures_.empty() ) { + give_up( failures_ ); + } + + return; + } + + Method::execute( source, target ); } diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index af0ac3392..29e2f502b 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -13,6 +13,8 @@ #include "atlas/interpolation/method/knn/KNearestNeighboursBase.h" +#include + #include "atlas/functionspace.h" #include "atlas/grid.h" #include "atlas/util/GridBox.h" @@ -40,12 +42,11 @@ class GridBoxMethod : public KNearestNeighboursBase { virtual void setup( const Grid& source, const Grid& target ) override; virtual void execute( const FieldSet& source, FieldSet& target ) const override; - virtual void execute( const Field& source, Field& target ) const override; virtual const FunctionSpace& source() const override { return source_; } virtual const FunctionSpace& target() const override { return target_; } - bool intersect( size_t i, const util::GridBox& iBox, const PointIndex3::NodeList&, std::vector& ); + bool intersect( size_t i, const util::GridBox& iBox, const PointIndex3::NodeList&, std::vector& ) const; private: FunctionSpace source_; @@ -57,7 +58,12 @@ class GridBoxMethod : public KNearestNeighboursBase { util::GridBoxes sourceBoxes_; util::GridBoxes targetBoxes_; + double searchRadius_; + + mutable std::forward_list failures_; + bool matrixFree_; + bool failEarly_; }; From 7549f2030f0be2c5c8005add3dd690b9854dfdde Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 20:50:53 +0100 Subject: [PATCH 027/161] ATLAS-269, MIR-465: avoid Intel compiler problem --- src/atlas/util/GridBox.cc | 3 +++ src/atlas/util/GridBox.h | 1 + 2 files changed, 4 insertions(+) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index 690481fd4..002e48349 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -149,6 +149,9 @@ GridBoxes::GridBoxes( const Grid& grid ) { } +GridBoxes::GridBoxes() = default; + + double GridBoxes::getLongestGridBoxDiagonal() const { ATLAS_ASSERT( !empty() ); diff --git a/src/atlas/util/GridBox.h b/src/atlas/util/GridBox.h index 19f20dca9..8ba383c4f 100644 --- a/src/atlas/util/GridBox.h +++ b/src/atlas/util/GridBox.h @@ -109,6 +109,7 @@ class GridBox { struct GridBoxes : std::vector { GridBoxes( const Grid& ); + GridBoxes(); using vector::vector; double getLongestGridBoxDiagonal() const; }; From d1d6d7daf9939d58b96e53a4bf46451f6eb2619b Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 20:57:30 +0100 Subject: [PATCH 028/161] ATLAS-269, MIR-465: cleanup --- src/atlas/util/GridBox.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index 002e48349..150f66d7e 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -115,11 +115,11 @@ GridBoxes::GridBoxes( const Grid& grid ) { std::vector lat; lat.reserve( y.size() + 1 ); - lat.push_back( 90. ); + lat.push_back( NORTH_POLE ); for ( auto b = y.begin(), a = b++; b != y.end(); a = b++ ) { lat.push_back( ( *b + *a ) / 2. ); } - lat.push_back( -90. ); + lat.push_back( SOUTH_POLE ); // Calculate grid-box meridians (longitude midpoints) From d75c8a4d4c02f6b7345848591801d9c25fe47fac Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 22:54:54 +0100 Subject: [PATCH 029/161] ATLAS-269, MIR-465: cleanup --- src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc index a02fb407b..e2dade3f7 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc @@ -23,7 +23,7 @@ namespace interpolation { namespace method { void KNearestNeighboursBase::buildPointSearchTree( Mesh& meshSource ) { - eckit::TraceTimer tim( "KNearestNeighboursBase::setup()" ); + eckit::TraceTimer tim( "KNearestNeighboursBase::buildPointSearchTree()" ); // generate 3D point coordinates mesh::actions::BuildXYZField( "xyz" )( meshSource ); From 1447c431902e89e0c6dba9f3caffc898cfc1af35 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 3 Apr 2020 23:41:36 +0100 Subject: [PATCH 030/161] ATLAS-269, MIR-465: cleanup --- src/atlas/interpolation/method/knn/GridBoxMethod.cc | 13 ++++++------- src/atlas/interpolation/method/knn/GridBoxMethod.h | 4 ---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index fa4e12b0c..64a90c9ac 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -18,7 +18,6 @@ #include "eckit/log/ProgressTimer.h" #include "eckit/types/FloatCompare.h" -#include "atlas/functionspace/Points.h" #include "atlas/grid.h" #include "atlas/interpolation/method/MethodFactory.h" #include "atlas/parallel/mpi/mpi.h" @@ -119,11 +118,9 @@ void GridBoxMethod::setup( const Grid& source, const Grid& target ) { ATLAS_ASSERT( source ); ATLAS_ASSERT( target ); - sourceGrid_ = source; - targetGrid_ = target; - functionspace::Points src = source; - functionspace::Points tgt = target; + functionspace::Points src( source ); + functionspace::Points tgt( target ); ATLAS_ASSERT( src ); ATLAS_ASSERT( tgt ); source_ = src; @@ -132,8 +129,8 @@ void GridBoxMethod::setup( const Grid& source, const Grid& target ) { buildPointSearchTree( src ); ATLAS_ASSERT( pTree_ != nullptr ); - sourceBoxes_ = util::GridBoxes( sourceGrid_ ); - targetBoxes_ = util::GridBoxes( targetGrid_ ); + sourceBoxes_ = util::GridBoxes( source ); + targetBoxes_ = util::GridBoxes( target ); searchRadius_ = sourceBoxes_.getLongestGridBoxDiagonal() + targetBoxes_.getLongestGridBoxDiagonal(); failures_.clear(); @@ -186,6 +183,8 @@ void GridBoxMethod::execute( const FieldSet& source, FieldSet& target ) const { // ensure setup() functionspace::Points tgt = target_; ATLAS_ASSERT( tgt ); + + ATLAS_ASSERT( pTree_ != nullptr ); ATLAS_ASSERT( searchRadius_ > 0. ); ATLAS_ASSERT( !sourceBoxes_.empty() ); ATLAS_ASSERT( !targetBoxes_.empty() ); diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index 29e2f502b..d627d1c5d 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -16,7 +16,6 @@ #include #include "atlas/functionspace.h" -#include "atlas/grid.h" #include "atlas/util/GridBox.h" @@ -52,9 +51,6 @@ class GridBoxMethod : public KNearestNeighboursBase { FunctionSpace source_; FunctionSpace target_; - Grid sourceGrid_; - Grid targetGrid_; - util::GridBoxes sourceBoxes_; util::GridBoxes targetBoxes_; From 2018c16679fb1c9079b8eb65117d6dbec24f97de Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 17 Apr 2020 05:50:16 +0100 Subject: [PATCH 031/161] ATLAS-269, MIR-465: API changes --- src/atlas/interpolation/method/knn/GridBoxMethod.cc | 6 +++--- src/atlas/interpolation/method/knn/GridBoxMethod.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 64a90c9ac..6296f1cfc 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -65,7 +65,7 @@ void GridBoxMethod::print( std::ostream& out ) const { } -void GridBoxMethod::setup( const FunctionSpace& /*source*/, const FunctionSpace& /*target*/ ) { +void GridBoxMethod::do_setup( const FunctionSpace& /*source*/, const FunctionSpace& /*target*/ ) { ATLAS_NOTIMPLEMENTED; } @@ -109,7 +109,7 @@ bool GridBoxMethod::intersect( size_t i, const util::GridBox& box, const PointIn } -void GridBoxMethod::setup( const Grid& source, const Grid& target ) { +void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { ATLAS_TRACE( "GridBoxMethod::setup()" ); if ( mpi::size() > 1 ) { @@ -175,7 +175,7 @@ void GridBoxMethod::setup( const Grid& source, const Grid& target ) { } -void GridBoxMethod::execute( const FieldSet& source, FieldSet& target ) const { +void GridBoxMethod::do_execute( const FieldSet& source, FieldSet& target ) const { if ( matrixFree_ ) { ATLAS_TRACE( "GridBoxMethod::execute()" ); diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index d627d1c5d..2eb3bd1f0 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -37,10 +37,10 @@ class GridBoxMethod : public KNearestNeighboursBase { * @param source functionspace containing source points * @param target functionspace containing target points */ - virtual void setup( const FunctionSpace& source, const FunctionSpace& target ) override; - virtual void setup( const Grid& source, const Grid& target ) override; + virtual void do_setup( const FunctionSpace& source, const FunctionSpace& target ) override; + virtual void do_setup( const Grid& source, const Grid& target ) override; - virtual void execute( const FieldSet& source, FieldSet& target ) const override; + virtual void do_execute( const FieldSet& source, FieldSet& target ) const override; virtual const FunctionSpace& source() const override { return source_; } virtual const FunctionSpace& target() const override { return target_; } From bb7deb8fc240fab3d872c079618dbab9dc4ee579 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Mon, 20 Apr 2020 17:57:17 +0100 Subject: [PATCH 032/161] ATLAS-269, MIR-465: API changes --- src/atlas/functionspace/Points.cc | 2 +- src/atlas/functionspace/Points.h | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/atlas/functionspace/Points.cc b/src/atlas/functionspace/Points.cc index 3b6fc93e3..bfc842dda 100644 --- a/src/atlas/functionspace/Points.cc +++ b/src/atlas/functionspace/Points.cc @@ -91,7 +91,7 @@ std::string Points::type() const { } -const Field& Points::ghost() const { +atlas::Field Points::ghost() const { if ( not ghost_ ) { ghost_ = Field( "ghost", array::make_datatype(), array::make_shape( size() ) ); array::make_view( ghost_ ).assign( 0 ); diff --git a/src/atlas/functionspace/Points.h b/src/atlas/functionspace/Points.h index 34d44b004..e8afc8180 100644 --- a/src/atlas/functionspace/Points.h +++ b/src/atlas/functionspace/Points.h @@ -47,9 +47,10 @@ class Points final : public FunctionSpaceImpl { std::string distribution() const override; std::string type() const override; - const Field& lonlat() const { return lonlat_; } - const Field& xyz() const { return xyz_; } - const Field& ghost() const; + atlas::Field lonlat() const override { return lonlat_; } + atlas::Field ghost() const override; + + const atlas::Field xyz() const { return xyz_; } template @@ -129,9 +130,9 @@ class Points : public FunctionSpace { operator bool() const { return functionspace_ != nullptr; } - const Field& lonlat() const { return functionspace_->lonlat(); } - const Field& xyz() const { return functionspace_->xyz(); } - const Field& ghost() const { return functionspace_->ghost(); } + Field lonlat() const { return functionspace_->lonlat(); } + Field xyz() const { return functionspace_->xyz(); } + Field ghost() const { return functionspace_->ghost(); } detail::Points::Iterate iterate() const { return functionspace_->iterate(); } From 7c2ead0ed220a684c13509502ad8773e9336d20a Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Apr 2020 10:59:32 +0100 Subject: [PATCH 033/161] ATLAS-269, MIR-465: Points functionspace lower memory overhead, calculating xyz on-demand --- src/atlas/functionspace/Points.cc | 17 +++++----------- src/atlas/functionspace/Points.h | 32 ++++++++++++++----------------- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/src/atlas/functionspace/Points.cc b/src/atlas/functionspace/Points.cc index bfc842dda..8d2ba9109 100644 --- a/src/atlas/functionspace/Points.cc +++ b/src/atlas/functionspace/Points.cc @@ -26,23 +26,13 @@ namespace detail { Points::Points( const Grid& grid ) : - lonlat_( "lonlat", array::make_datatype(), array::make_shape( grid.size(), 2 ) ), - xyz_( "xyz", array::make_datatype(), array::make_shape( grid.size(), 3 ) ) { + lonlat_( "lonlat", array::make_datatype(), array::make_shape( grid.size(), 2 ) ) { auto lonlat = array::make_view( lonlat_ ); - auto xyz = array::make_view( xyz_ ); - PointXYZ q; idx_t j = 0; for ( auto p : grid.lonlat() ) { - util::Earth::convertSphericalToCartesian( p, q ); - lonlat( j, LON ) = p.lon(); lonlat( j, LAT ) = p.lat(); - - xyz( j, XX ) = q.x(); - xyz( j, YY ) = q.y(); - xyz( j, ZZ ) = q.z(); - ++j; } } @@ -102,7 +92,10 @@ atlas::Field Points::ghost() const { template <> const PointXYZ Points::IteratorT::operator*() const { - return {view_( n_, XX ), view_( n_, YY ), view_( ZZ )}; + PointXYZ q; + Point2 p = {view_( n_, LON ), view_( n_, LAT )}; + util::Earth::convertSphericalToCartesian( p, q ); + return q; } diff --git a/src/atlas/functionspace/Points.h b/src/atlas/functionspace/Points.h index e8afc8180..e16a50bf1 100644 --- a/src/atlas/functionspace/Points.h +++ b/src/atlas/functionspace/Points.h @@ -50,18 +50,16 @@ class Points final : public FunctionSpaceImpl { atlas::Field lonlat() const override { return lonlat_; } atlas::Field ghost() const override; - const atlas::Field xyz() const { return xyz_; } - template struct IteratorT { - IteratorT( const Field& field, bool begin = true ) : - field_( field ), - view_( array::make_view( field_ ) ), - m_( field_.shape( 0 ) ), + IteratorT( const Field& field, bool begin ) : + lonlat_( field ), + view_( array::make_view( lonlat_ ) ), + m_( lonlat_.shape( 0 ) ), n_( begin ? 0 : m_ ) {} - bool next( PointXYZ& p ) { + bool next( T& p ) { if ( n_ < m_ ) { p = operator*(); ++n_; @@ -81,7 +79,7 @@ class Points final : public FunctionSpaceImpl { bool operator!=( const IteratorT& other ) const { return n_ != other.n_; } private: - const Field& field_; + const Field lonlat_; const array::ArrayView view_; idx_t m_; idx_t n_; @@ -93,29 +91,28 @@ class Points final : public FunctionSpaceImpl { using iterator = IteratorT; using const_iterator = iterator; - IterateT( const Field& field ) : field_( field ) {} - iterator begin() const { return {field_}; } - iterator end() const { return {field_, false}; } + IterateT( const Field& lonlat ) : lonlat_( lonlat ) {} + iterator begin() const { return {lonlat_, true}; } + iterator end() const { return {lonlat_, false}; } private: - const Field& field_; + const Field lonlat_; }; struct Iterate { - Iterate( const Points& fs ) : fs_( fs ) {} - IterateT xyz() const { return {fs_.xyz()}; } - IterateT lonlat() const { return {fs_.lonlat()}; } + Iterate( const Points& points ) : lonlat_( points.lonlat() ) {} + IterateT xyz() const { return {lonlat_}; } + IterateT lonlat() const { return {lonlat_}; } private: - const Points& fs_; + const Field lonlat_; }; Iterate iterate() const { return Iterate( *this ); } private: Field lonlat_; - Field xyz_; mutable Field ghost_; }; @@ -131,7 +128,6 @@ class Points : public FunctionSpace { operator bool() const { return functionspace_ != nullptr; } Field lonlat() const { return functionspace_->lonlat(); } - Field xyz() const { return functionspace_->xyz(); } Field ghost() const { return functionspace_->ghost(); } detail::Points::Iterate iterate() const { return functionspace_->iterate(); } From 076fb28c2b2ef2df9fc5fba998fcc3565e9d18ec Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Apr 2020 19:11:03 +0100 Subject: [PATCH 034/161] ATLAS-269, MIR-465: Points functionspace const correctness --- src/atlas/functionspace/Points.cc | 16 ++++++++++++++-- src/atlas/functionspace/Points.h | 5 ++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/atlas/functionspace/Points.cc b/src/atlas/functionspace/Points.cc index 8d2ba9109..81fb4b546 100644 --- a/src/atlas/functionspace/Points.cc +++ b/src/atlas/functionspace/Points.cc @@ -11,10 +11,13 @@ #include "atlas/functionspace/Points.h" +// #include + #include "atlas/array.h" #include "atlas/grid/Grid.h" #include "atlas/grid/Iterator.h" #include "atlas/option/Options.h" +#include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "atlas/util/CoordinateEnums.h" #include "atlas/util/Earth.h" @@ -33,6 +36,9 @@ Points::Points( const Grid& grid ) : for ( auto p : grid.lonlat() ) { lonlat( j, LON ) = p.lon(); lonlat( j, LAT ) = p.lat(); + + // shouldn't be necessary + // lonlat( j, LAT ) = std::min(90., std::max(-90., p.lat())); ++j; } } @@ -90,8 +96,14 @@ atlas::Field Points::ghost() const { } +template +T Points::IteratorT::operator*() const { + ATLAS_NOTIMPLEMENTED; +} + + template <> -const PointXYZ Points::IteratorT::operator*() const { +PointXYZ Points::IteratorT::operator*() const { PointXYZ q; Point2 p = {view_( n_, LON ), view_( n_, LAT )}; util::Earth::convertSphericalToCartesian( p, q ); @@ -100,7 +112,7 @@ const PointXYZ Points::IteratorT::operator*() const { template <> -const PointLonLat Points::IteratorT::operator*() const { +PointLonLat Points::IteratorT::operator*() const { return {view_( n_, LON ), view_( n_, LAT )}; } diff --git a/src/atlas/functionspace/Points.h b/src/atlas/functionspace/Points.h index e16a50bf1..3b5714385 100644 --- a/src/atlas/functionspace/Points.h +++ b/src/atlas/functionspace/Points.h @@ -15,7 +15,6 @@ #include "atlas/field/Field.h" #include "atlas/functionspace/FunctionSpace.h" #include "atlas/functionspace/detail/FunctionSpaceImpl.h" -#include "atlas/runtime/Exception.h" #include "atlas/util/Point.h" @@ -73,7 +72,7 @@ class Points final : public FunctionSpaceImpl { return *this; } - const T operator*() const { ATLAS_NOTIMPLEMENTED; } + T operator*() const; bool operator==( const IteratorT& other ) const { return n_ == other.n_; } bool operator!=( const IteratorT& other ) const { return n_ != other.n_; } @@ -81,7 +80,7 @@ class Points final : public FunctionSpaceImpl { private: const Field lonlat_; const array::ArrayView view_; - idx_t m_; + const idx_t m_; idx_t n_; }; From 37eeff66eccee2f29347679c1b21ee88bd8c6841 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 23 Apr 2020 00:22:31 +0100 Subject: [PATCH 035/161] ATLAS-269, MIR-465: work in progress --- src/atlas/interpolation/method/knn/GridBoxMethod.cc | 4 +--- src/atlas/util/GridBox.cc | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 6296f1cfc..4cb8e0f01 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -209,9 +209,7 @@ void GridBoxMethod::do_execute( const FieldSet& source, FieldSet& target ) const std::vector triplets; size_t i = 0; for ( auto p : tgt.iterate().xyz() ) { - if ( ++progress ) { - Log::info() << "GridBoxMethod: " << *pTree_ << std::endl; - } + ++progress; if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { auto& y = yarray[i /*t.col()*/]; diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index 150f66d7e..ca9cc3666 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -104,7 +104,7 @@ void GridBox::print( std::ostream& out ) const { GridBoxes::GridBoxes( const Grid& grid ) { StructuredGrid structured( grid ); - if ( !structured || !structured.domain().global() || grid.projection() ) { + if ( !structured || /*!structured.domain().global() ||*/ grid.projection() ) { throw_NotImplemented( "GridBoxes only support structured, unprojected/unrotated global grids", Here() ); } From 2d4211f636fbfa652442d4b69f8c237eac506f32 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 23 Apr 2020 15:12:49 +0100 Subject: [PATCH 036/161] ATLAS-269, MIR-465: work in progress --- src/atlas/interpolation/method/Method.cc | 2 +- .../interpolation/method/knn/GridBoxMethod.cc | 75 +++++++++---------- .../interpolation/method/knn/GridBoxMethod.h | 2 +- 3 files changed, 39 insertions(+), 40 deletions(-) diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index ebdd1f7e7..c32b7464f 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -198,7 +198,7 @@ void Method::do_execute( const FieldSet& fieldsSource, FieldSet& fieldsTarget ) for ( idx_t i = 0; i < fieldsSource.size(); ++i ) { Log::debug() << "Method::do_execute() on field " << ( i + 1 ) << '/' << N << "..." << std::endl; - Method::do_execute( fieldsSource[i], fieldsTarget[i] ); + do_execute( fieldsSource[i], fieldsTarget[i] ); } } diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 4cb8e0f01..6b0c5b169 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -175,60 +175,59 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { } -void GridBoxMethod::do_execute( const FieldSet& source, FieldSet& target ) const { - if ( matrixFree_ ) { - ATLAS_TRACE( "GridBoxMethod::execute()" ); - +void GridBoxMethod::do_execute( const Field& source, Field& target ) const { + if ( !matrixFree_ ) { + Method::execute( source, target ); + return; + } - // ensure setup() - functionspace::Points tgt = target_; - ATLAS_ASSERT( tgt ); + ATLAS_TRACE( "GridBoxMethod::execute()" ); - ATLAS_ASSERT( pTree_ != nullptr ); - ATLAS_ASSERT( searchRadius_ > 0. ); - ATLAS_ASSERT( !sourceBoxes_.empty() ); - ATLAS_ASSERT( !targetBoxes_.empty() ); + // ensure setup() + functionspace::Points tgt = target_; + ATLAS_ASSERT( tgt ); - // set arrays - ATLAS_ASSERT( source.size() == 1 && source.field( 0 ).rank() == 1 ); - ATLAS_ASSERT( target.size() == 1 && target.field( 0 ).rank() == 1 ); + ATLAS_ASSERT( pTree_ != nullptr ); + ATLAS_ASSERT( searchRadius_ > 0. ); + ATLAS_ASSERT( !sourceBoxes_.empty() ); + ATLAS_ASSERT( !targetBoxes_.empty() ); - auto xarray = atlas::array::make_view( source.field( 0 ) ); - auto yarray = atlas::array::make_view( target.field( 0 ) ); - ATLAS_ASSERT( xarray.size() == idx_t( sourceBoxes_.size() ) ); - ATLAS_ASSERT( yarray.size() == idx_t( targetBoxes_.size() ) ); - yarray.assign( 0. ); - failures_.clear(); + // set arrays + ATLAS_ASSERT( source.rank() == 1 ); + ATLAS_ASSERT( target.rank() == 1 ); + auto xarray = atlas::array::make_view( source ); + auto yarray = atlas::array::make_view( target ); + ATLAS_ASSERT( xarray.size() == idx_t( sourceBoxes_.size() ) ); + ATLAS_ASSERT( yarray.size() == idx_t( targetBoxes_.size() ) ); - // interpolate - eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ) ); + yarray.assign( 0. ); + failures_.clear(); - std::vector triplets; - size_t i = 0; - for ( auto p : tgt.iterate().xyz() ) { - ++progress; - if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { - auto& y = yarray[i /*t.col()*/]; - for ( auto& t : triplets ) { - y += xarray[t.col()] * t.value(); - } - } + // interpolate + eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ) ); - ++i; - } + std::vector triplets; + size_t i = 0; + for ( auto p : tgt.iterate().xyz() ) { + ++progress; - if ( !failures_.empty() ) { - give_up( failures_ ); + if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { + auto& y = yarray[i]; + for ( auto& t : triplets ) { + y += xarray[t.col()] * t.value(); + } } - return; + ++i; } - Method::execute( source, target ); + if ( !failures_.empty() ) { + give_up( failures_ ); + } } diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index 2eb3bd1f0..c420bb476 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -40,7 +40,7 @@ class GridBoxMethod : public KNearestNeighboursBase { virtual void do_setup( const FunctionSpace& source, const FunctionSpace& target ) override; virtual void do_setup( const Grid& source, const Grid& target ) override; - virtual void do_execute( const FieldSet& source, FieldSet& target ) const override; + virtual void do_execute( const Field& source, Field& target ) const override; virtual const FunctionSpace& source() const override { return source_; } virtual const FunctionSpace& target() const override { return target_; } From a5cfb62aba1a49fd7cd89e2824f82c994c48e37b Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Apr 2020 09:32:06 +0100 Subject: [PATCH 037/161] ATLAS-269, MIR-465: bug fix (infinite loop) --- src/atlas/interpolation/method/knn/GridBoxMethod.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 6b0c5b169..698096236 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -177,7 +177,7 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { void GridBoxMethod::do_execute( const Field& source, Field& target ) const { if ( !matrixFree_ ) { - Method::execute( source, target ); + Method::do_execute( source, target ); return; } From edfa20287324538123f2b1284fd3dd30e726fcae Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Apr 2020 09:32:20 +0100 Subject: [PATCH 038/161] ATLAS-269, MIR-465: unit test --- src/tests/interpolation/CMakeLists.txt | 6 + .../test_interpolation_grid_box_average.cc | 110 ++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 src/tests/interpolation/test_interpolation_grid_box_average.cc diff --git a/src/tests/interpolation/CMakeLists.txt b/src/tests/interpolation/CMakeLists.txt index 187af2b9b..0df8ae423 100644 --- a/src/tests/interpolation/CMakeLists.txt +++ b/src/tests/interpolation/CMakeLists.txt @@ -19,6 +19,12 @@ ecbuild_add_test( TARGET atlas_test_interpolation_finite_element ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_interpolation_grid_box_average + SOURCES test_interpolation_grid_box_average.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + ecbuild_add_test( TARGET atlas_test_interpolation_cubic_prototype SOURCES test_interpolation_cubic_prototype.cc CubicInterpolationPrototype.h LIBS atlas diff --git a/src/tests/interpolation/test_interpolation_grid_box_average.cc b/src/tests/interpolation/test_interpolation_grid_box_average.cc new file mode 100644 index 000000000..7321eab99 --- /dev/null +++ b/src/tests/interpolation/test_interpolation_grid_box_average.cc @@ -0,0 +1,110 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +#include + +#include "eckit/types/FloatCompare.h" + +#include "atlas/array.h" +#include "atlas/field.h" +#include "atlas/grid.h" +#include "atlas/interpolation.h" +#include "atlas/option.h" +#include "atlas/util/Config.h" +#include "atlas/util/GridBox.h" + +#include "tests/AtlasTestEnvironment.h" + + +namespace atlas { +namespace test { + + +double integral( const Grid& grid, const Field& field ) { + auto values = array::make_view( field ); + + auto boxes = util::GridBoxes( grid ); + ATLAS_ASSERT( boxes.size() == size_t( field.shape( 0 ) ) ); + + double i = 0.; + for ( size_t c = 0; c < boxes.size(); c++ ) { + i += boxes[c].area() * values( c ); + } + return i; +} + + +CASE( "test_interpolation_grid_box_average" ) { + using eckit::types::is_approximately_equal; + + Grid gridA( "O32" ); + Grid gridB( "O64" ); + Grid gridC( "O32" ); + + + // the integral of field = 1 is Earth's surface area + auto ref = util::Earth::area(); + Log::info() << "Earth's surface = " << ref << " m2 (reference)" << std::endl; + static double interpolation_tolerance = 1e3; // (Earth's surface area is large) + + Field fieldA( "A", array::make_datatype(), array::make_shape( gridA.size() ) ); + array::make_view( fieldA ).assign( 1. ); + + + SECTION( "matrix-free" ) { + Field fieldB( "B", array::make_datatype(), array::make_shape( gridB.size() ) ); + Field fieldC( "C", array::make_datatype(), array::make_shape( gridC.size() ) ); + + auto config = option::type( "grid-box-average" ); + config.set( "matrix_free", true ); + + Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); + Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); + + double integrals[]{integral( gridA, fieldA ), integral( gridB, fieldB ), integral( gridC, fieldC )}; + Log::info() << "Integral A: " << integrals[0] << "\n" + << "Integral B: " << integrals[1] << "\n" + << "Integral C: " << integrals[2] << std::endl; + + EXPECT( is_approximately_equal( integrals[0], ref, interpolation_tolerance ) ); + EXPECT( is_approximately_equal( integrals[1], ref, interpolation_tolerance ) ); + EXPECT( is_approximately_equal( integrals[2], ref, interpolation_tolerance ) ); + } + + SECTION( "matrix-based" ) { + Field fieldB( "B", array::make_datatype(), array::make_shape( gridB.size() ) ); + Field fieldC( "C", array::make_datatype(), array::make_shape( gridC.size() ) ); + + auto config = option::type( "grid-box-average" ); + config.set( "matrix_free", false ); + + Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); + Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); + + double integrals[]{integral( gridA, fieldA ), integral( gridB, fieldB ), integral( gridC, fieldC )}; + Log::info() << "Integral A: " << integrals[0] << "\n" + << "Integral B: " << integrals[1] << "\n" + << "Integral C: " << integrals[2] << std::endl; + + EXPECT( is_approximately_equal( integrals[0], ref, interpolation_tolerance ) ); + EXPECT( is_approximately_equal( integrals[1], ref, interpolation_tolerance ) ); + EXPECT( is_approximately_equal( integrals[2], ref, interpolation_tolerance ) ); + } +} + + +} // namespace test +} // namespace atlas + + +int main( int argc, char** argv ) { + return atlas::test::run( argc, argv ); +} From bcc6931e7f2ec3761e8c52ec78010bf57a5e5999 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Apr 2020 14:34:25 +0100 Subject: [PATCH 039/161] ATLAS-269, MIR-465: unit test refactor --- .../test_interpolation_grid_box_average.cc | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_grid_box_average.cc b/src/tests/interpolation/test_interpolation_grid_box_average.cc index 7321eab99..80174ec0b 100644 --- a/src/tests/interpolation/test_interpolation_grid_box_average.cc +++ b/src/tests/interpolation/test_interpolation_grid_box_average.cc @@ -9,8 +9,6 @@ */ -#include - #include "eckit/types/FloatCompare.h" #include "atlas/array.h" @@ -43,60 +41,58 @@ double integral( const Grid& grid, const Field& field ) { CASE( "test_interpolation_grid_box_average" ) { - using eckit::types::is_approximately_equal; - Grid gridA( "O32" ); Grid gridB( "O64" ); Grid gridC( "O32" ); - // the integral of field = 1 is Earth's surface area - auto ref = util::Earth::area(); - Log::info() << "Earth's surface = " << ref << " m2 (reference)" << std::endl; - static double interpolation_tolerance = 1e3; // (Earth's surface area is large) + // the integral of a constant field = 1 is Earth's surface area + struct reference_t { + reference_t( double value, double tolerance ) : value_( value ), tolerance_( tolerance ) { + Log::info() << "Reference: " << value_ << ", abs. tolerance: +-" << tolerance_ << std::endl; + } + + void check( std::string label, double value ) { + Log::info() << label << value << std::endl; + EXPECT( eckit::types::is_approximately_equal( value, value_, tolerance_ ) ); + } + + const double value_; + const double tolerance_; + } reference( util::Earth::area(), 1.e2 ); Field fieldA( "A", array::make_datatype(), array::make_shape( gridA.size() ) ); array::make_view( fieldA ).assign( 1. ); + reference.check( "Integral A: ", integral( gridA, fieldA ) ); - SECTION( "matrix-free" ) { + SECTION( "matrix-based" ) { Field fieldB( "B", array::make_datatype(), array::make_shape( gridB.size() ) ); Field fieldC( "C", array::make_datatype(), array::make_shape( gridC.size() ) ); auto config = option::type( "grid-box-average" ); - config.set( "matrix_free", true ); + config.set( "matrix_free", false ); Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); - Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); - - double integrals[]{integral( gridA, fieldA ), integral( gridB, fieldB ), integral( gridC, fieldC )}; - Log::info() << "Integral A: " << integrals[0] << "\n" - << "Integral B: " << integrals[1] << "\n" - << "Integral C: " << integrals[2] << std::endl; + reference.check( "Integral B: ", integral( gridB, fieldB ) ); - EXPECT( is_approximately_equal( integrals[0], ref, interpolation_tolerance ) ); - EXPECT( is_approximately_equal( integrals[1], ref, interpolation_tolerance ) ); - EXPECT( is_approximately_equal( integrals[2], ref, interpolation_tolerance ) ); + Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); + reference.check( "Integral C: ", integral( gridC, fieldC ) ); } - SECTION( "matrix-based" ) { + + SECTION( "matrix-free" ) { Field fieldB( "B", array::make_datatype(), array::make_shape( gridB.size() ) ); Field fieldC( "C", array::make_datatype(), array::make_shape( gridC.size() ) ); auto config = option::type( "grid-box-average" ); - config.set( "matrix_free", false ); + config.set( "matrix_free", true ); Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); - Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); + reference.check( "Integral B: ", integral( gridB, fieldB ) ); - double integrals[]{integral( gridA, fieldA ), integral( gridB, fieldB ), integral( gridC, fieldC )}; - Log::info() << "Integral A: " << integrals[0] << "\n" - << "Integral B: " << integrals[1] << "\n" - << "Integral C: " << integrals[2] << std::endl; - - EXPECT( is_approximately_equal( integrals[0], ref, interpolation_tolerance ) ); - EXPECT( is_approximately_equal( integrals[1], ref, interpolation_tolerance ) ); - EXPECT( is_approximately_equal( integrals[2], ref, interpolation_tolerance ) ); + Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); + reference.check( "Integral C: ", integral( gridC, fieldC ) ); } } From 22ce298e926ea15dff9247f21478d3abd1821bf3 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Apr 2020 15:52:56 +0100 Subject: [PATCH 040/161] ATLAS-269, MIR-465: work in progress (partial revert) --- src/atlas/interpolation/method/Method.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index c32b7464f..ebdd1f7e7 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -198,7 +198,7 @@ void Method::do_execute( const FieldSet& fieldsSource, FieldSet& fieldsTarget ) for ( idx_t i = 0; i < fieldsSource.size(); ++i ) { Log::debug() << "Method::do_execute() on field " << ( i + 1 ) << '/' << N << "..." << std::endl; - do_execute( fieldsSource[i], fieldsTarget[i] ); + Method::do_execute( fieldsSource[i], fieldsTarget[i] ); } } From 85a4047fa60995d6a4f34bcb910a9f075c556ca6 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Apr 2020 18:29:23 +0100 Subject: [PATCH 041/161] ATLAS-269, MIR-465: testing FieldSet interpolation --- .../interpolation/method/knn/GridBoxMethod.cc | 21 +++- .../interpolation/method/knn/GridBoxMethod.h | 1 + .../test_interpolation_grid_box_average.cc | 108 +++++++++++++++--- 3 files changed, 114 insertions(+), 16 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 698096236..b72a0a0cb 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -175,14 +175,31 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { } +void GridBoxMethod::do_execute( const FieldSet& source, FieldSet& target ) const { + ATLAS_ASSERT( source.size() == target.size() ); + + // Matrix-based interpolation is handled by base (Method) class + // TODO: exploit sparse/dense matrix multiplication + for ( idx_t i = 0; i < source.size(); ++i ) { + if ( matrixFree_ ) { + GridBoxMethod::do_execute( source[i], target[i] ); + } + else { + Method::do_execute( source[i], target[i] ); + } + } +} + + void GridBoxMethod::do_execute( const Field& source, Field& target ) const { + ATLAS_TRACE( "atlas::interpolation::method::GridBoxMethod::do_execute()" ); + + // Matrix-based interpolation is handled by base (Method) class if ( !matrixFree_ ) { Method::do_execute( source, target ); return; } - ATLAS_TRACE( "GridBoxMethod::execute()" ); - // ensure setup() functionspace::Points tgt = target_; diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index c420bb476..49b2d7a21 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -40,6 +40,7 @@ class GridBoxMethod : public KNearestNeighboursBase { virtual void do_setup( const FunctionSpace& source, const FunctionSpace& target ) override; virtual void do_setup( const Grid& source, const Grid& target ) override; + virtual void do_execute( const FieldSet& source, FieldSet& target ) const override; virtual void do_execute( const Field& source, Field& target ) const override; virtual const FunctionSpace& source() const override { return source_; } diff --git a/src/tests/interpolation/test_interpolation_grid_box_average.cc b/src/tests/interpolation/test_interpolation_grid_box_average.cc index 80174ec0b..7c01c3578 100644 --- a/src/tests/interpolation/test_interpolation_grid_box_average.cc +++ b/src/tests/interpolation/test_interpolation_grid_box_average.cc @@ -8,6 +8,7 @@ * nor does it submit to any jurisdiction. */ +#include #include "eckit/types/FloatCompare.h" @@ -40,13 +41,45 @@ double integral( const Grid& grid, const Field& field ) { } +void reset_fields( Field& f1, Field& f2, Field& f3 ) { + array::make_view( f1 ).assign( 1. ); + array::make_view( f2 ).assign( 0. ); + array::make_view( f3 ).assign( 0. ); +} + + +void reset_fieldsets( FieldSet& f1, FieldSet& f2, FieldSet& f3 ) { + ATLAS_ASSERT( f1.size() == f2.size() ); + ATLAS_ASSERT( f1.size() == f3.size() ); + + double value = 1.; // or your money back :-) + for ( idx_t i = 0; i < f1.size(); ++i, value += 1. ) { + array::make_view( f1[i] ).assign( value ); + array::make_view( f2[i] ).assign( 0. ); + array::make_view( f3[i] ).assign( 0. ); + } +} + + +FieldSet create_fieldset( std::string name, idx_t size, size_t number ) { + ATLAS_ASSERT( 1 <= number ); + + FieldSet set; + for ( size_t i = 1; i <= number; ++i ) { + set.add( Field( name + std::to_string( i ), array::make_datatype(), array::make_shape( size ) ) ); + } + return set; +} + + CASE( "test_interpolation_grid_box_average" ) { + Log::info().precision( 16 ); + Grid gridA( "O32" ); Grid gridB( "O64" ); Grid gridC( "O32" ); - // the integral of a constant field = 1 is Earth's surface area struct reference_t { reference_t( double value, double tolerance ) : value_( value ), tolerance_( tolerance ) { Log::info() << "Reference: " << value_ << ", abs. tolerance: +-" << tolerance_ << std::endl; @@ -59,40 +92,87 @@ CASE( "test_interpolation_grid_box_average" ) { const double value_; const double tolerance_; - } reference( util::Earth::area(), 1.e2 ); + }; + + + // the integral of a constant field = 1 is Earth's surface area (and tolerance has to be large) + reference_t surface1( util::Earth::area(), 1.e3 ); + reference_t surface2( util::Earth::area() * 2., 1.e3 ); + reference_t surface3( util::Earth::area() * 3., 1.e3 ); + + // setup fields Field fieldA( "A", array::make_datatype(), array::make_shape( gridA.size() ) ); - array::make_view( fieldA ).assign( 1. ); - reference.check( "Integral A: ", integral( gridA, fieldA ) ); + Field fieldB( "B", array::make_datatype(), array::make_shape( gridB.size() ) ); + Field fieldC( "C", array::make_datatype(), array::make_shape( gridC.size() ) ); + FieldSet fieldsA( create_fieldset( "A", gridA.size(), 3 ) ); + FieldSet fieldsB( create_fieldset( "B", gridB.size(), 3 ) ); + FieldSet fieldsC( create_fieldset( "C", gridC.size(), 3 ) ); - SECTION( "matrix-based" ) { - Field fieldB( "B", array::make_datatype(), array::make_shape( gridB.size() ) ); - Field fieldC( "C", array::make_datatype(), array::make_shape( gridC.size() ) ); + + SECTION( "Earth's surface area, Field interpolation, matrix-based" ) { + reset_fields( fieldA, fieldB, fieldC ); + surface1.check( "Integral A: ", integral( gridA, fieldA ) ); // checked once auto config = option::type( "grid-box-average" ); config.set( "matrix_free", false ); Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); - reference.check( "Integral B: ", integral( gridB, fieldB ) ); + surface1.check( "Integral B: ", integral( gridB, fieldB ) ); Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); - reference.check( "Integral C: ", integral( gridC, fieldC ) ); + surface1.check( "Integral C: ", integral( gridC, fieldC ) ); + } + + + SECTION( "Earth's surface area, FieldSet interpolation, matrix-based" ) { + reset_fieldsets( fieldsA, fieldsB, fieldsC ); + + auto config = option::type( "grid-box-average" ); + config.set( "matrix_free", false ); + + Interpolation( config, gridA, gridB ).execute( fieldsA, fieldsB ); + surface1.check( "Integral B1: ", integral( gridB, fieldsB[0] ) ); + surface2.check( "Integral B2: ", integral( gridB, fieldsB[1] ) ); + surface3.check( "Integral B3: ", integral( gridB, fieldsB[2] ) ); + + Interpolation( config, gridB, gridC ).execute( fieldsB, fieldsC ); + surface1.check( "Integral C1: ", integral( gridC, fieldsC[0] ) ); + surface2.check( "Integral C2: ", integral( gridC, fieldsC[1] ) ); + surface3.check( "Integral C3: ", integral( gridC, fieldsC[2] ) ); } - SECTION( "matrix-free" ) { - Field fieldB( "B", array::make_datatype(), array::make_shape( gridB.size() ) ); - Field fieldC( "C", array::make_datatype(), array::make_shape( gridC.size() ) ); + SECTION( "Earth's surface area, Field interpolation, matrix-free" ) { + reset_fields( fieldA, fieldB, fieldC ); auto config = option::type( "grid-box-average" ); config.set( "matrix_free", true ); Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); - reference.check( "Integral B: ", integral( gridB, fieldB ) ); + surface1.check( "Integral B: ", integral( gridB, fieldB ) ); Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); - reference.check( "Integral C: ", integral( gridC, fieldC ) ); + surface1.check( "Integral C: ", integral( gridC, fieldC ) ); + } + + + SECTION( "Earth's surface area, FieldSet interpolation, matrix-free" ) { + reset_fieldsets( fieldsA, fieldsB, fieldsC ); + + auto config = option::type( "grid-box-average" ); + config.set( "matrix_free", true ); + + Interpolation( config, gridA, gridB ).execute( fieldsA, fieldsB ); + surface1.check( "Integral B1: ", integral( gridB, fieldsB[0] ) ); + surface2.check( "Integral B2: ", integral( gridB, fieldsB[1] ) ); + surface3.check( "Integral B3: ", integral( gridB, fieldsB[2] ) ); + + Interpolation( config, gridB, gridC ).execute( fieldsB, fieldsC ); + surface1.check( "Integral C1: ", integral( gridC, fieldsC[0] ) ); + surface2.check( "Integral C2: ", integral( gridC, fieldsC[1] ) ); + surface3.check( "Integral C3: ", integral( gridC, fieldsC[2] ) ); } } From a37e5e1d77a0ffd5acc19a54e11b87e09310823d Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Apr 2020 19:42:56 +0100 Subject: [PATCH 042/161] ATLAS-269, MIR-465: testing interpolation (real conservative this time) --- .../test_interpolation_grid_box_average.cc | 58 ++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_grid_box_average.cc b/src/tests/interpolation/test_interpolation_grid_box_average.cc index 7c01c3578..7b9c80aa4 100644 --- a/src/tests/interpolation/test_interpolation_grid_box_average.cc +++ b/src/tests/interpolation/test_interpolation_grid_box_average.cc @@ -8,7 +8,6 @@ * nor does it submit to any jurisdiction. */ -#include #include "eckit/types/FloatCompare.h" @@ -42,7 +41,7 @@ double integral( const Grid& grid, const Field& field ) { void reset_fields( Field& f1, Field& f2, Field& f3 ) { - array::make_view( f1 ).assign( 1. ); + array::make_view( f1 ).assign( 0. ); array::make_view( f2 ).assign( 0. ); array::make_view( f3 ).assign( 0. ); } @@ -100,6 +99,9 @@ CASE( "test_interpolation_grid_box_average" ) { reference_t surface2( util::Earth::area() * 2., 1.e3 ); reference_t surface3( util::Earth::area() * 3., 1.e3 ); + reference_t countA( double( gridA.size() ), 1.e-8 ); + reference_t countB( double( gridB.size() ), 1.e-8 ); + // setup fields Field fieldA( "A", array::make_datatype(), array::make_shape( gridA.size() ) ); @@ -113,6 +115,8 @@ CASE( "test_interpolation_grid_box_average" ) { SECTION( "Earth's surface area, Field interpolation, matrix-based" ) { reset_fields( fieldA, fieldB, fieldC ); + array::make_view( fieldA ).assign( 1. ); + surface1.check( "Integral A: ", integral( gridA, fieldA ) ); // checked once auto config = option::type( "grid-box-average" ); @@ -146,6 +150,7 @@ CASE( "test_interpolation_grid_box_average" ) { SECTION( "Earth's surface area, Field interpolation, matrix-free" ) { reset_fields( fieldA, fieldB, fieldC ); + array::make_view( fieldA ).assign( 1. ); auto config = option::type( "grid-box-average" ); config.set( "matrix_free", true ); @@ -174,6 +179,55 @@ CASE( "test_interpolation_grid_box_average" ) { surface2.check( "Integral C2: ", integral( gridC, fieldsC[1] ) ); surface3.check( "Integral C3: ", integral( gridC, fieldsC[2] ) ); } + + + SECTION( "Conserve count as integral value, low >> high >> low resolution, matrix-free" ) { + reset_fields( fieldA, fieldB, fieldC ); + + size_t i = 0; + auto values = array::make_view( fieldA ); + for ( auto& box : util::GridBoxes( gridA ) ) { + ATLAS_ASSERT( box.area() > 0. ); + values( i++ ) = 1. / box.area(); + } + + auto config = option::type( "grid-box-average" ); + config.set( "matrix_free", true ); + + countA.check( "Count A: ", integral( gridA, fieldA ) ); + + Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); + countA.check( "Count B: ", integral( gridB, fieldB ) ); + + Interpolation( config, gridB, gridA ).execute( fieldB, fieldC ); + countA.check( "Count A: ", integral( gridA, fieldC ) ); + } + + + SECTION( "Conserve count as integral value, high >> low >> high resolution, matrix-free" ) { + reset_fields( fieldA, fieldB, fieldC ); + + Field fieldD( "D", array::make_datatype(), array::make_shape( gridB.size() ) ); + array::make_view( fieldD ).assign( 0. ); + + size_t i = 0; + auto values = array::make_view( fieldB ); + for ( auto& box : util::GridBoxes( gridB ) ) { + ATLAS_ASSERT( box.area() > 0. ); + values( i++ ) = 1. / box.area(); + } + + auto config = option::type( "grid-box-average" ); + config.set( "matrix_free", true ); + + countB.check( "Count B: ", integral( gridB, fieldB ) ); + + Interpolation( config, gridB, gridA ).execute( fieldB, fieldA ); + countB.check( "Count A: ", integral( gridA, fieldA ) ); + + Interpolation( config, gridA, gridB ).execute( fieldA, fieldD ); + countB.check( "Count B: ", integral( gridB, fieldD ) ); + } } From cade217fbe96d645de11ebf1fe8080684e27276c Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Sat, 25 Apr 2020 00:47:01 +0100 Subject: [PATCH 043/161] ATLAS-269, MIR-465: testing interpolation refactoring (more robust, more tests) --- .../test_interpolation_grid_box_average.cc | 186 ++++++++---------- 1 file changed, 86 insertions(+), 100 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_grid_box_average.cc b/src/tests/interpolation/test_interpolation_grid_box_average.cc index 7b9c80aa4..09f8da462 100644 --- a/src/tests/interpolation/test_interpolation_grid_box_average.cc +++ b/src/tests/interpolation/test_interpolation_grid_box_average.cc @@ -9,6 +9,8 @@ */ +#include + #include "eckit/types/FloatCompare.h" #include "atlas/array.h" @@ -40,23 +42,10 @@ double integral( const Grid& grid, const Field& field ) { } -void reset_fields( Field& f1, Field& f2, Field& f3 ) { - array::make_view( f1 ).assign( 0. ); - array::make_view( f2 ).assign( 0. ); - array::make_view( f3 ).assign( 0. ); -} - - -void reset_fieldsets( FieldSet& f1, FieldSet& f2, FieldSet& f3 ) { - ATLAS_ASSERT( f1.size() == f2.size() ); - ATLAS_ASSERT( f1.size() == f3.size() ); - - double value = 1.; // or your money back :-) - for ( idx_t i = 0; i < f1.size(); ++i, value += 1. ) { - array::make_view( f1[i] ).assign( value ); - array::make_view( f2[i] ).assign( 0. ); - array::make_view( f3[i] ).assign( 0. ); - } +Field create_field( std::string name, idx_t size, double init = double() ) { + auto f = Field( name, array::make_datatype(), array::make_shape( size ) ); + array::make_view( f ).assign( init ); + return f; } @@ -65,7 +54,9 @@ FieldSet create_fieldset( std::string name, idx_t size, size_t number ) { FieldSet set; for ( size_t i = 1; i <= number; ++i ) { - set.add( Field( name + std::to_string( i ), array::make_datatype(), array::make_shape( size ) ) ); + auto f = + set.add( Field( name + std::to_string( i ), array::make_datatype(), array::make_shape( size ) ) ); + array::make_view( f ).assign( 0. ); } return set; } @@ -80,12 +71,13 @@ CASE( "test_interpolation_grid_box_average" ) { struct reference_t { - reference_t( double value, double tolerance ) : value_( value ), tolerance_( tolerance ) { - Log::info() << "Reference: " << value_ << ", abs. tolerance: +-" << tolerance_ << std::endl; - } + reference_t( double value, double tolerance ) : value_( value ), tolerance_( tolerance ) {} void check( std::string label, double value ) { - Log::info() << label << value << std::endl; + double relerr = + eckit::types::is_approximately_equal( value_, 0. ) ? 0. : std::abs( ( value - value_ ) / value_ ); + Log::info() << label << value << ", compared to " << value_ << " +- " << tolerance_ + << ", with relative error = " << relerr << std::endl; EXPECT( eckit::types::is_approximately_equal( value, value_, tolerance_ ) ); } @@ -95,6 +87,7 @@ CASE( "test_interpolation_grid_box_average" ) { // the integral of a constant field = 1 is Earth's surface area (and tolerance has to be large) + // (also field = 2 and 3 just to test a bit further, too) reference_t surface1( util::Earth::area(), 1.e3 ); reference_t surface2( util::Earth::area() * 2., 1.e3 ); reference_t surface3( util::Earth::area() * 3., 1.e3 ); @@ -102,87 +95,78 @@ CASE( "test_interpolation_grid_box_average" ) { reference_t countA( double( gridA.size() ), 1.e-8 ); reference_t countB( double( gridB.size() ), 1.e-8 ); + auto configMB = option::type( "grid-box-average" ).set( "matrix_free", false ); + auto configMF = option::type( "grid-box-average" ).set( "matrix_free", true ); - // setup fields - Field fieldA( "A", array::make_datatype(), array::make_shape( gridA.size() ) ); - Field fieldB( "B", array::make_datatype(), array::make_shape( gridB.size() ) ); - Field fieldC( "C", array::make_datatype(), array::make_shape( gridC.size() ) ); - FieldSet fieldsA( create_fieldset( "A", gridA.size(), 3 ) ); - FieldSet fieldsB( create_fieldset( "B", gridB.size(), 3 ) ); - FieldSet fieldsC( create_fieldset( "C", gridC.size(), 3 ) ); + SECTION( "Earth's surface area using Field interpolation" ) { + Field fieldA( create_field( "A", gridA.size(), 1. ) ); + Field fieldB( create_field( "B", gridB.size() ) ); + Field fieldC( create_field( "C", gridC.size() ) ); + surface1.check( "Integral A: ", integral( gridA, fieldA ) ); - SECTION( "Earth's surface area, Field interpolation, matrix-based" ) { - reset_fields( fieldA, fieldB, fieldC ); - array::make_view( fieldA ).assign( 1. ); + Interpolation( configMB, gridA, gridB ).execute( fieldA, fieldB ); + Interpolation( configMB, gridB, gridC ).execute( fieldB, fieldC ); - surface1.check( "Integral A: ", integral( gridA, fieldA ) ); // checked once + surface1.check( "Integral B (MB): ", integral( gridB, fieldB ) ); + surface1.check( "Integral C (MB): ", integral( gridC, fieldC ) ); - auto config = option::type( "grid-box-average" ); - config.set( "matrix_free", false ); + fieldB = create_field( "B", gridB.size() ); + fieldC = create_field( "C", gridC.size() ); - Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); - surface1.check( "Integral B: ", integral( gridB, fieldB ) ); + Interpolation( configMF, gridA, gridB ).execute( fieldA, fieldB ); + Interpolation( configMF, gridB, gridC ).execute( fieldB, fieldC ); - Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); - surface1.check( "Integral C: ", integral( gridC, fieldC ) ); + surface1.check( "Integral B (MF): ", integral( gridB, fieldB ) ); + surface1.check( "Integral C (MF): ", integral( gridC, fieldC ) ); } - SECTION( "Earth's surface area, FieldSet interpolation, matrix-based" ) { - reset_fieldsets( fieldsA, fieldsB, fieldsC ); - - auto config = option::type( "grid-box-average" ); - config.set( "matrix_free", false ); - - Interpolation( config, gridA, gridB ).execute( fieldsA, fieldsB ); - surface1.check( "Integral B1: ", integral( gridB, fieldsB[0] ) ); - surface2.check( "Integral B2: ", integral( gridB, fieldsB[1] ) ); - surface3.check( "Integral B3: ", integral( gridB, fieldsB[2] ) ); - - Interpolation( config, gridB, gridC ).execute( fieldsB, fieldsC ); - surface1.check( "Integral C1: ", integral( gridC, fieldsC[0] ) ); - surface2.check( "Integral C2: ", integral( gridC, fieldsC[1] ) ); - surface3.check( "Integral C3: ", integral( gridC, fieldsC[2] ) ); - } + SECTION( "Earth's surface area using FieldSet interpolation" ) { + FieldSet fieldsA( create_fieldset( "A", gridA.size(), 3 ) ); + array::make_view( fieldsA[0] ).assign( 1. ); + array::make_view( fieldsA[1] ).assign( 2. ); + array::make_view( fieldsA[2] ).assign( 3. ); + FieldSet fieldsB( create_fieldset( "B", gridB.size(), 3 ) ); + FieldSet fieldsC( create_fieldset( "C", gridC.size(), 3 ) ); - SECTION( "Earth's surface area, Field interpolation, matrix-free" ) { - reset_fields( fieldA, fieldB, fieldC ); - array::make_view( fieldA ).assign( 1. ); + surface1.check( "Integral A1: ", integral( gridA, fieldsA[0] ) ); + surface2.check( "Integral A2: ", integral( gridA, fieldsA[1] ) ); + surface3.check( "Integral A3: ", integral( gridA, fieldsA[2] ) ); - auto config = option::type( "grid-box-average" ); - config.set( "matrix_free", true ); + Interpolation( configMB, gridA, gridB ).execute( fieldsA, fieldsB ); + Interpolation( configMB, gridB, gridC ).execute( fieldsB, fieldsC ); - Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); - surface1.check( "Integral B: ", integral( gridB, fieldB ) ); + surface1.check( "Integral B1 (MB): ", integral( gridB, fieldsB[0] ) ); + surface2.check( "Integral B2 (MB): ", integral( gridB, fieldsB[1] ) ); + surface3.check( "Integral B3 (MB): ", integral( gridB, fieldsB[2] ) ); - Interpolation( config, gridB, gridC ).execute( fieldB, fieldC ); - surface1.check( "Integral C: ", integral( gridC, fieldC ) ); - } + surface1.check( "Integral C1 (MB): ", integral( gridC, fieldsC[0] ) ); + surface2.check( "Integral C2 (MB): ", integral( gridC, fieldsC[1] ) ); + surface3.check( "Integral C3 (MB): ", integral( gridC, fieldsC[2] ) ); + fieldsB = create_fieldset( "B", gridB.size(), 3 ); + fieldsC = create_fieldset( "C", gridC.size(), 3 ); - SECTION( "Earth's surface area, FieldSet interpolation, matrix-free" ) { - reset_fieldsets( fieldsA, fieldsB, fieldsC ); + Interpolation( configMF, gridA, gridB ).execute( fieldsA, fieldsB ); + Interpolation( configMF, gridB, gridC ).execute( fieldsB, fieldsC ); - auto config = option::type( "grid-box-average" ); - config.set( "matrix_free", true ); + surface1.check( "Integral B1 (MF): ", integral( gridB, fieldsB[0] ) ); + surface2.check( "Integral B2 (MF): ", integral( gridB, fieldsB[1] ) ); + surface3.check( "Integral B3 (MF): ", integral( gridB, fieldsB[2] ) ); - Interpolation( config, gridA, gridB ).execute( fieldsA, fieldsB ); - surface1.check( "Integral B1: ", integral( gridB, fieldsB[0] ) ); - surface2.check( "Integral B2: ", integral( gridB, fieldsB[1] ) ); - surface3.check( "Integral B3: ", integral( gridB, fieldsB[2] ) ); - - Interpolation( config, gridB, gridC ).execute( fieldsB, fieldsC ); - surface1.check( "Integral C1: ", integral( gridC, fieldsC[0] ) ); - surface2.check( "Integral C2: ", integral( gridC, fieldsC[1] ) ); - surface3.check( "Integral C3: ", integral( gridC, fieldsC[2] ) ); + surface1.check( "Integral C1 (MF): ", integral( gridC, fieldsC[0] ) ); + surface2.check( "Integral C2 (MF): ", integral( gridC, fieldsC[1] ) ); + surface3.check( "Integral C3 (MF): ", integral( gridC, fieldsC[2] ) ); } - SECTION( "Conserve count as integral value, low >> high >> low resolution, matrix-free" ) { - reset_fields( fieldA, fieldB, fieldC ); + SECTION( "Count as integral value, low >> high >> low resolution" ) { + Field fieldA( create_field( "A", gridA.size(), 1. ) ); + Field fieldB( create_field( "B", gridB.size() ) ); + Field fieldC( create_field( "C", gridA.size() ) ); size_t i = 0; auto values = array::make_view( fieldA ); @@ -191,42 +175,44 @@ CASE( "test_interpolation_grid_box_average" ) { values( i++ ) = 1. / box.area(); } - auto config = option::type( "grid-box-average" ); - config.set( "matrix_free", true ); - countA.check( "Count A: ", integral( gridA, fieldA ) ); - Interpolation( config, gridA, gridB ).execute( fieldA, fieldB ); - countA.check( "Count B: ", integral( gridB, fieldB ) ); + Interpolation( configMB, gridA, gridB ).execute( fieldA, fieldB ); + Interpolation( configMB, gridB, gridA ).execute( fieldB, fieldC ); - Interpolation( config, gridB, gridA ).execute( fieldB, fieldC ); - countA.check( "Count A: ", integral( gridA, fieldC ) ); + countA.check( "Count B (MB): ", integral( gridB, fieldB ) ); + countA.check( "Count A (MB): ", integral( gridA, fieldC ) ); } - SECTION( "Conserve count as integral value, high >> low >> high resolution, matrix-free" ) { - reset_fields( fieldA, fieldB, fieldC ); - - Field fieldD( "D", array::make_datatype(), array::make_shape( gridB.size() ) ); - array::make_view( fieldD ).assign( 0. ); + SECTION( "Count as integral value, high >> low >> high resolution" ) { + Field fieldA( create_field( "A", gridB.size(), 1. ) ); + Field fieldB( create_field( "B", gridA.size() ) ); + Field fieldC( create_field( "C", gridB.size() ) ); size_t i = 0; - auto values = array::make_view( fieldB ); + auto values = array::make_view( fieldA ); for ( auto& box : util::GridBoxes( gridB ) ) { ATLAS_ASSERT( box.area() > 0. ); values( i++ ) = 1. / box.area(); } - auto config = option::type( "grid-box-average" ); - config.set( "matrix_free", true ); + countB.check( "Count B: ", integral( gridB, fieldA ) ); + + Interpolation( configMB, gridB, gridA ).execute( fieldA, fieldB ); + Interpolation( configMB, gridA, gridB ).execute( fieldB, fieldC ); + + countB.check( "Count A (MB): ", integral( gridA, fieldB ) ); + countB.check( "Count B (MB): ", integral( gridB, fieldC ) ); - countB.check( "Count B: ", integral( gridB, fieldB ) ); + fieldB = create_field( "B", gridA.size() ); + fieldC = create_field( "C", gridB.size() ); - Interpolation( config, gridB, gridA ).execute( fieldB, fieldA ); - countB.check( "Count A: ", integral( gridA, fieldA ) ); + Interpolation( configMF, gridB, gridA ).execute( fieldA, fieldB ); + Interpolation( configMF, gridA, gridB ).execute( fieldB, fieldC ); - Interpolation( config, gridA, gridB ).execute( fieldA, fieldD ); - countB.check( "Count B: ", integral( gridB, fieldD ) ); + countB.check( "Count A (MF): ", integral( gridA, fieldB ) ); + countB.check( "Count B (MF): ", integral( gridB, fieldC ) ); } } From cc3cd512d545b903416d5b946ce218ee51da6b59 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Sat, 25 Apr 2020 11:35:18 +0100 Subject: [PATCH 044/161] ATLAS-269, MIR-465: re-enable assertion --- src/atlas/util/GridBox.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index ca9cc3666..150f66d7e 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -104,7 +104,7 @@ void GridBox::print( std::ostream& out ) const { GridBoxes::GridBoxes( const Grid& grid ) { StructuredGrid structured( grid ); - if ( !structured || /*!structured.domain().global() ||*/ grid.projection() ) { + if ( !structured || !structured.domain().global() || grid.projection() ) { throw_NotImplemented( "GridBoxes only support structured, unprojected/unrotated global grids", Here() ); } From 93e277b0c308b62edc024da11bdcf31970fa2cd4 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Mon, 27 Apr 2020 17:49:18 +0100 Subject: [PATCH 045/161] ATLAS-269, MIR-465: conservative interpolation support for non-global grids --- src/atlas/util/GridBox.cc | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index 150f66d7e..47430f03b 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -104,8 +104,8 @@ void GridBox::print( std::ostream& out ) const { GridBoxes::GridBoxes( const Grid& grid ) { StructuredGrid structured( grid ); - if ( !structured || !structured.domain().global() || grid.projection() ) { - throw_NotImplemented( "GridBoxes only support structured, unprojected/unrotated global grids", Here() ); + if ( !structured || grid.projection() ) { + throw_NotImplemented( "GridBoxes only support structured, unprojected/unrotated grids", Here() ); } @@ -115,11 +115,21 @@ GridBoxes::GridBoxes( const Grid& grid ) { std::vector lat; lat.reserve( y.size() + 1 ); - lat.push_back( NORTH_POLE ); + RectangularDomain domain = structured.domain(); + ATLAS_ASSERT( domain ); + + auto north = domain.ymax(); + auto south = domain.ymin(); + ATLAS_ASSERT( -90. <= south && south <= north && north <= 90. ); + + lat.push_back( north ); for ( auto b = y.begin(), a = b++; b != y.end(); a = b++ ) { lat.push_back( ( *b + *a ) / 2. ); } - lat.push_back( SOUTH_POLE ); + lat.push_back( south ); + + lat.front() = std::min( north, std::max( south, lat.front() ) ); // clip to domain + lat.back() = std::min( north, std::max( south, lat.back() ) ); // (...) // Calculate grid-box meridians (longitude midpoints) @@ -127,14 +137,17 @@ GridBoxes::GridBoxes( const Grid& grid ) { ATLAS_ASSERT( x.nx().size() == x.dx().size() ); ATLAS_ASSERT( x.nx().size() == x.xmin().size() ); + bool periodic( ZonalBandDomain( structured.domain() ) ); + clear(); reserve( grid.size() ); for ( size_t j = 0; j < x.nx().size(); ++j ) { eckit::Fraction dx( x.dx()[j] ); eckit::Fraction xmin( x.xmin()[j] ); + auto n = ( xmin / dx ).integralPart(); if ( n * dx < xmin ) { - n += 1; + n += 1; // (adjust double-fraction conversions) } eckit::Fraction lon1 = ( n * dx ) - ( dx / 2 ); @@ -143,6 +156,10 @@ GridBoxes::GridBoxes( const Grid& grid ) { lon1 += dx; emplace_back( GridBox( lat[j], lon0, lat[j + 1], lon1 ) ); } + + if ( periodic ) { + ATLAS_ASSERT( lon1 == xmin - ( dx / 2 ) + 360 ); + } } ATLAS_ASSERT( idx_t( size() ) == grid.size() ); From a96a8c09a8157a558f471ab183352bb79460cd5b Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 28 Apr 2020 09:40:06 +0100 Subject: [PATCH 046/161] ATLAS-269, MIR-465: conservative interpolation support for non-global grids --- src/atlas/util/GridBox.cc | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index 47430f03b..86f99ca72 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -14,6 +14,7 @@ #include #include +#include #include "eckit/types/FloatCompare.h" #include "eckit/types/Fraction.h" @@ -109,18 +110,26 @@ GridBoxes::GridBoxes( const Grid& grid ) { } - // Calculate grid-box parallels (latitude midpoints) - auto& y = structured.yspace(); - - std::vector lat; - lat.reserve( y.size() + 1 ); - + // Bounding box and periodicity RectangularDomain domain = structured.domain(); ATLAS_ASSERT( domain ); auto north = domain.ymax(); auto south = domain.ymin(); + auto west = domain.xmin(); + auto east = domain.xmax(); ATLAS_ASSERT( -90. <= south && south <= north && north <= 90. ); + ATLAS_ASSERT( west <= east && east <= west + 360. ); + + bool periodic( ZonalBandDomain( structured.domain() ) ); + const GridBox clip( north, west, south, east ); + + + // Calculate grid-box parallels (latitude midpoints, or accumulated Gaussian quadrature weights on those grids) + auto& y = structured.yspace(); + + std::vector lat; + lat.reserve( y.size() + 1 ); lat.push_back( north ); for ( auto b = y.begin(), a = b++; b != y.end(); a = b++ ) { @@ -128,17 +137,12 @@ GridBoxes::GridBoxes( const Grid& grid ) { } lat.push_back( south ); - lat.front() = std::min( north, std::max( south, lat.front() ) ); // clip to domain - lat.back() = std::min( north, std::max( south, lat.back() ) ); // (...) - // Calculate grid-box meridians (longitude midpoints) auto& x = structured.xspace(); ATLAS_ASSERT( x.nx().size() == x.dx().size() ); ATLAS_ASSERT( x.nx().size() == x.xmin().size() ); - bool periodic( ZonalBandDomain( structured.domain() ) ); - clear(); reserve( grid.size() ); for ( size_t j = 0; j < x.nx().size(); ++j ) { @@ -150,16 +154,24 @@ GridBoxes::GridBoxes( const Grid& grid ) { n += 1; // (adjust double-fraction conversions) } + // West- and East-most grid-boxes on non-global grids might need clipping eckit::Fraction lon1 = ( n * dx ) - ( dx / 2 ); for ( idx_t i = 0; i < x.nx()[j]; ++i ) { double lon0 = lon1; lon1 += dx; emplace_back( GridBox( lat[j], lon0, lat[j + 1], lon1 ) ); + + if ( !periodic && i == 0 ) { + clip.intersects( back() ); + } } if ( periodic ) { ATLAS_ASSERT( lon1 == xmin - ( dx / 2 ) + 360 ); } + else { + clip.intersects( back() ); + } } ATLAS_ASSERT( idx_t( size() ) == grid.size() ); From ae4779b49e37a9bb236b11e337c6561b5d329601 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 28 Apr 2020 01:03:29 +0100 Subject: [PATCH 047/161] ATLAS-269, MIR-465: conservative interpolation, grid-box-average and grid-box-maximum (wip) --- src/atlas/CMakeLists.txt | 4 + .../method/knn/GridBoxAverage.cc | 109 ++++++++++++++++++ .../interpolation/method/knn/GridBoxAverage.h | 34 ++++++ .../method/knn/GridBoxMaximum.cc | 51 ++++++++ .../interpolation/method/knn/GridBoxMaximum.h | 34 ++++++ .../interpolation/method/knn/GridBoxMethod.cc | 101 ++-------------- .../interpolation/method/knn/GridBoxMethod.h | 12 +- 7 files changed, 251 insertions(+), 94 deletions(-) create mode 100644 src/atlas/interpolation/method/knn/GridBoxAverage.cc create mode 100644 src/atlas/interpolation/method/knn/GridBoxAverage.h create mode 100644 src/atlas/interpolation/method/knn/GridBoxMaximum.cc create mode 100644 src/atlas/interpolation/method/knn/GridBoxMaximum.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 928d5bfa7..42bebdcef 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -499,6 +499,10 @@ interpolation/method/Ray.cc interpolation/method/Ray.h interpolation/method/fe/FiniteElement.cc interpolation/method/fe/FiniteElement.h +interpolation/method/knn/GridBoxAverage.cc +interpolation/method/knn/GridBoxAverage.h +interpolation/method/knn/GridBoxMaximum.cc +interpolation/method/knn/GridBoxMaximum.h interpolation/method/knn/GridBoxMethod.cc interpolation/method/knn/GridBoxMethod.h interpolation/method/knn/KNearestNeighbours.cc diff --git a/src/atlas/interpolation/method/knn/GridBoxAverage.cc b/src/atlas/interpolation/method/knn/GridBoxAverage.cc new file mode 100644 index 000000000..5125d8095 --- /dev/null +++ b/src/atlas/interpolation/method/knn/GridBoxAverage.cc @@ -0,0 +1,109 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. and Interpolation + */ + + +#include "atlas/interpolation/method/knn/GridBoxAverage.h" + +#include + +#include "eckit/log/ProgressTimer.h" + +#include "atlas/array.h" +#include "atlas/functionspace/Points.h" +#include "atlas/interpolation/method/MethodFactory.h" +#include "atlas/runtime/Exception.h" + + +namespace atlas { +namespace interpolation { +namespace method { + + +namespace { +MethodBuilder __builder( "grid-box-average" ); +} + + +void GridBoxAverage::do_execute( const FieldSet& source, FieldSet& target ) const { + ATLAS_ASSERT( source.size() == target.size() ); + + // Matrix-based interpolation is handled by base (Method) class + // TODO: exploit sparse/dense matrix multiplication + for ( idx_t i = 0; i < source.size(); ++i ) { + if ( matrixFree_ ) { + GridBoxAverage::do_execute( source[i], target[i] ); + } + else { + Method::do_execute( source[i], target[i] ); + } + } +} + + +void GridBoxAverage::do_execute( const Field& source, Field& target ) const { + ATLAS_TRACE( "atlas::interpolation::method::GridBoxAverage::do_execute()" ); + + // Matrix-based interpolation is handled by base (Method) class + if ( !matrixFree_ ) { + Method::do_execute( source, target ); + return; + } + + + // ensure GridBoxMethod::setup() + functionspace::Points tgt = target_; + ATLAS_ASSERT( tgt ); + + ATLAS_ASSERT( pTree_ != nullptr ); + ATLAS_ASSERT( searchRadius_ > 0. ); + ATLAS_ASSERT( !sourceBoxes_.empty() ); + ATLAS_ASSERT( !targetBoxes_.empty() ); + + + // set arrays + ATLAS_ASSERT( source.rank() == 1 ); + ATLAS_ASSERT( target.rank() == 1 ); + + auto xarray = atlas::array::make_view( source ); + auto yarray = atlas::array::make_view( target ); + ATLAS_ASSERT( xarray.size() == idx_t( sourceBoxes_.size() ) ); + ATLAS_ASSERT( yarray.size() == idx_t( targetBoxes_.size() ) ); + + yarray.assign( 0. ); + failures_.clear(); + + + // interpolate + eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ) ); + + std::vector triplets; + size_t i = 0; + for ( auto p : tgt.iterate().xyz() ) { + ++progress; + + if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { + auto& y = yarray[i]; + for ( auto& t : triplets ) { + y += xarray[t.col()] * t.value(); + } + } + + ++i; + } + + if ( !failures_.empty() ) { + giveUp( failures_ ); + } +} + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/knn/GridBoxAverage.h b/src/atlas/interpolation/method/knn/GridBoxAverage.h new file mode 100644 index 000000000..0cc14bd26 --- /dev/null +++ b/src/atlas/interpolation/method/knn/GridBoxAverage.h @@ -0,0 +1,34 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. and Interpolation + */ + + +#pragma once + +#include "atlas/interpolation/method/knn/GridBoxMethod.h" + + +namespace atlas { +namespace interpolation { +namespace method { + + +class GridBoxAverage final : public GridBoxMethod { +public: + using GridBoxMethod::GridBoxMethod; + +private: + void do_execute( const FieldSet& source, FieldSet& target ) const override; + void do_execute( const Field& source, Field& target ) const override; +}; + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc new file mode 100644 index 000000000..8e2a386cb --- /dev/null +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc @@ -0,0 +1,51 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. and Interpolation + */ + + +#include "atlas/interpolation/method/knn/GridBoxMaximum.h" + +#include "atlas/interpolation/method/MethodFactory.h" +#include "atlas/runtime/Exception.h" + + +namespace atlas { +namespace interpolation { +namespace method { + + +namespace { +MethodBuilder __builder( "grid-box-maximum" ); +} + + +void GridBoxMaximum::do_execute( const FieldSet& source, FieldSet& target ) const { + ATLAS_ASSERT( source.size() == target.size() ); + + // Matrix-based interpolation is handled by base (Method) class + // TODO: exploit sparse/dense matrix multiplication + for ( idx_t i = 0; i < source.size(); ++i ) { + if ( matrixFree_ ) { + GridBoxMaximum::do_execute( source[i], target[i] ); + } + else { + Method::do_execute( source[i], target[i] ); + } + } +} + + +void GridBoxMaximum::do_execute( const Field& /*source*/, Field& /*target*/ ) const { + ATLAS_NOTIMPLEMENTED; +} + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.h b/src/atlas/interpolation/method/knn/GridBoxMaximum.h new file mode 100644 index 000000000..b10009d72 --- /dev/null +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.h @@ -0,0 +1,34 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. and Interpolation + */ + + +#pragma once + +#include "atlas/interpolation/method/knn/GridBoxMethod.h" + + +namespace atlas { +namespace interpolation { +namespace method { + + +class GridBoxMaximum final : public GridBoxMethod { +public: + using GridBoxMethod::GridBoxMethod; + +private: + void do_execute( const FieldSet& source, FieldSet& target ) const override; + void do_execute( const Field& source, Field& target ) const override; +}; + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index b72a0a0cb..9474c210b 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2013 ECMWF. + * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -19,7 +19,6 @@ #include "eckit/types/FloatCompare.h" #include "atlas/grid.h" -#include "atlas/interpolation/method/MethodFactory.h" #include "atlas/parallel/mpi/mpi.h" #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" @@ -31,26 +30,6 @@ namespace interpolation { namespace method { -MethodBuilder __builder( "grid-box-average" ); - - -void give_up( const std::forward_list& failures ) { - Log::warning() << "Failed to intersect grid boxes: "; - - size_t count = 0; - auto sep = ""; - for ( const auto& f : failures ) { - if ( count++ < 10 ) { - Log::warning() << sep << f; - sep = ", "; - } - } - Log::warning() << "... (" << eckit::Plural( count, "total failure" ) << std::endl; - - throw_Exception( "Failed to intersect grid boxes" ); -} - - GridBoxMethod::GridBoxMethod( const Method::Config& config ) : KNearestNeighboursBase( config ) { config.get( "matrix_free", matrixFree_ = false ); config.get( "fail_early", failEarly_ = true ); @@ -163,7 +142,7 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { } if ( !failures_.empty() ) { - give_up( failures_ ); + giveUp( failures_ ); } } @@ -175,76 +154,20 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { } -void GridBoxMethod::do_execute( const FieldSet& source, FieldSet& target ) const { - ATLAS_ASSERT( source.size() == target.size() ); - - // Matrix-based interpolation is handled by base (Method) class - // TODO: exploit sparse/dense matrix multiplication - for ( idx_t i = 0; i < source.size(); ++i ) { - if ( matrixFree_ ) { - GridBoxMethod::do_execute( source[i], target[i] ); - } - else { - Method::do_execute( source[i], target[i] ); - } - } -} - - -void GridBoxMethod::do_execute( const Field& source, Field& target ) const { - ATLAS_TRACE( "atlas::interpolation::method::GridBoxMethod::do_execute()" ); - - // Matrix-based interpolation is handled by base (Method) class - if ( !matrixFree_ ) { - Method::do_execute( source, target ); - return; - } - - - // ensure setup() - functionspace::Points tgt = target_; - ATLAS_ASSERT( tgt ); - - ATLAS_ASSERT( pTree_ != nullptr ); - ATLAS_ASSERT( searchRadius_ > 0. ); - ATLAS_ASSERT( !sourceBoxes_.empty() ); - ATLAS_ASSERT( !targetBoxes_.empty() ); - - - // set arrays - ATLAS_ASSERT( source.rank() == 1 ); - ATLAS_ASSERT( target.rank() == 1 ); - - auto xarray = atlas::array::make_view( source ); - auto yarray = atlas::array::make_view( target ); - ATLAS_ASSERT( xarray.size() == idx_t( sourceBoxes_.size() ) ); - ATLAS_ASSERT( yarray.size() == idx_t( targetBoxes_.size() ) ); - - yarray.assign( 0. ); - failures_.clear(); - - - // interpolate - eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ) ); - - std::vector triplets; - size_t i = 0; - for ( auto p : tgt.iterate().xyz() ) { - ++progress; +void GridBoxMethod::giveUp( const std::forward_list& failures ) { + Log::warning() << "Failed to intersect grid boxes: "; - if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { - auto& y = yarray[i]; - for ( auto& t : triplets ) { - y += xarray[t.col()] * t.value(); - } + size_t count = 0; + auto sep = ""; + for ( const auto& f : failures ) { + if ( count++ < 10 ) { + Log::warning() << sep << f; + sep = ", "; } - - ++i; } + Log::warning() << "... (" << eckit::Plural( count, "total failure" ) << std::endl; - if ( !failures_.empty() ) { - give_up( failures_ ); - } + throw_Exception( "Failed to intersect grid boxes" ); } diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index 49b2d7a21..22fd9ee30 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2013 ECMWF. + * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -40,15 +40,17 @@ class GridBoxMethod : public KNearestNeighboursBase { virtual void do_setup( const FunctionSpace& source, const FunctionSpace& target ) override; virtual void do_setup( const Grid& source, const Grid& target ) override; - virtual void do_execute( const FieldSet& source, FieldSet& target ) const override; - virtual void do_execute( const Field& source, Field& target ) const override; - virtual const FunctionSpace& source() const override { return source_; } virtual const FunctionSpace& target() const override { return target_; } bool intersect( size_t i, const util::GridBox& iBox, const PointIndex3::NodeList&, std::vector& ) const; -private: + virtual void do_execute( const FieldSet& source, FieldSet& target ) const = 0; + virtual void do_execute( const Field& source, Field& target ) const = 0; + +protected: + static void giveUp( const std::forward_list& ); + FunctionSpace source_; FunctionSpace target_; From 43d3066411d47f541e7e296be9abf1d7646bd142 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 28 Apr 2020 09:36:56 +0100 Subject: [PATCH 048/161] ATLAS-269, MIR-465: conservative interpolation, grid-box-average and grid-box-maximum --- .../method/knn/GridBoxMaximum.cc | 93 +++++++++++++++++-- 1 file changed, 83 insertions(+), 10 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc index 8e2a386cb..7ad65049b 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc @@ -11,6 +11,15 @@ #include "atlas/interpolation/method/knn/GridBoxMaximum.h" +#include +#include +#include + +#include "eckit/log/ProgressTimer.h" +#include "eckit/types/FloatCompare.h" + +#include "atlas/array.h" +#include "atlas/functionspace/Points.h" #include "atlas/interpolation/method/MethodFactory.h" #include "atlas/runtime/Exception.h" @@ -28,21 +37,85 @@ MethodBuilder __builder( "grid-box-maximum" ); void GridBoxMaximum::do_execute( const FieldSet& source, FieldSet& target ) const { ATLAS_ASSERT( source.size() == target.size() ); - // Matrix-based interpolation is handled by base (Method) class - // TODO: exploit sparse/dense matrix multiplication for ( idx_t i = 0; i < source.size(); ++i ) { - if ( matrixFree_ ) { - GridBoxMaximum::do_execute( source[i], target[i] ); - } - else { - Method::do_execute( source[i], target[i] ); - } + do_execute( source[i], target[i] ); } } -void GridBoxMaximum::do_execute( const Field& /*source*/, Field& /*target*/ ) const { - ATLAS_NOTIMPLEMENTED; +void GridBoxMaximum::do_execute( const Field& source, Field& target ) const { + ATLAS_TRACE( "atlas::interpolation::method::GridBoxMaximum::do_execute()" ); + + // set arrays + ATLAS_ASSERT( source.rank() == 1 ); + ATLAS_ASSERT( target.rank() == 1 ); + + auto xarray = atlas::array::make_view( source ); + auto yarray = atlas::array::make_view( target ); + ATLAS_ASSERT( xarray.size() == idx_t( sourceBoxes_.size() ) ); + ATLAS_ASSERT( yarray.size() == idx_t( targetBoxes_.size() ) ); + + yarray.assign( 0. ); + failures_.clear(); + + + if ( !matrixFree_ ) { + Matrix::const_iterator k( matrix_ ); + + for ( decltype( matrix_.rows() ) i = 0, j = 0; i < matrix_.rows(); ++i ) { + double max = std::numeric_limits::lowest(); + bool found = false; + + for ( ; k != matrix_.end( i ); ++k ) { + ATLAS_ASSERT( k.col() < size_t( xarray.shape( 0 ) ) ); + auto value = xarray[k.col()]; + if ( max < value ) { + max = value; + j = k.col(); + } + found = true; + } + + ATLAS_ASSERT( found ); + yarray[i] = xarray[j]; + } + return; + } + + + // ensure GridBoxMethod::setup() + functionspace::Points tgt = target_; + ATLAS_ASSERT( tgt ); + + ATLAS_ASSERT( pTree_ != nullptr ); + ATLAS_ASSERT( searchRadius_ > 0. ); + ATLAS_ASSERT( !sourceBoxes_.empty() ); + ATLAS_ASSERT( !targetBoxes_.empty() ); + + + // interpolate + eckit::ProgressTimer progress( "Intersecting", targetBoxes_.size(), "grid box", double( 5. ) ); + + std::vector triplets; + size_t i = 0; + for ( auto p : tgt.iterate().xyz() ) { + ++progress; + + if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { + auto triplet = + std::max_element( triplets.begin(), triplets.end(), []( const Triplet& a, const Triplet& b ) { + return !eckit::types::is_approximately_greater_or_equal( a.value(), b.value() ); + } ); + + yarray[i] = xarray[triplet->col()]; + } + + ++i; + } + + if ( !failures_.empty() ) { + giveUp( failures_ ); + } } From e6bb98efd3ab6c1924dfdc96e93c2a0aa5f85e54 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 28 Apr 2020 19:19:29 +0100 Subject: [PATCH 049/161] ATLAS-269, MIR-465: Gaussian grids grid boxes --- src/atlas/util/GridBox.cc | 47 ++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index 86f99ca72..ce1db971a 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -21,7 +21,9 @@ #include "atlas/grid.h" #include "atlas/runtime/Exception.h" +#include "atlas/util/Constants.h" #include "atlas/util/Earth.h" +#include "atlas/util/GaussianLatitudes.h" #include "atlas/util/Point.h" @@ -125,17 +127,52 @@ GridBoxes::GridBoxes( const Grid& grid ) { const GridBox clip( north, west, south, east ); - // Calculate grid-box parallels (latitude midpoints, or accumulated Gaussian quadrature weights on those grids) + // Calculate grid-box parallels (latitude midpoints, or accumulated Gaussian quadrature weights) auto& y = structured.yspace(); std::vector lat; lat.reserve( y.size() + 1 ); - lat.push_back( north ); - for ( auto b = y.begin(), a = b++; b != y.end(); a = b++ ) { - lat.push_back( ( *b + *a ) / 2. ); + GaussianGrid gaussian( grid ); + if ( gaussian ) { + // FIXME: innefficient interface, latitudes are discarded + auto N = gaussian.N(); + std::vector latitudes( N * 2 ); + std::vector weights( N * 2 ); + util::gaussian_quadrature_npole_spole( N, latitudes.data(), weights.data() ); + + std::vector lat_global( y.size() + 1 ); + auto b = lat_global.rbegin(); + auto f = lat_global.begin(); + *( b++ ) = -90.; + *( f++ ) = 90.; + + double wacc = -1.; + for ( idx_t j = 0; j < N; ++j, ++b, ++f ) { + wacc += 2. * weights[j]; + double deg = util::Constants::radiansToDegrees() * std::asin( wacc ); + *b = deg; + *f = -( *b ); + } + lat_global[N] = 0.; // (equator) + + // grids not covering the poles need clipping + for ( auto l : lat_global ) { + if ( eckit::types::is_approximately_lesser_or_equal( l, north ) && + eckit::types::is_approximately_lesser_or_equal( south, l ) ) { + lat.push_back( l ); + } + } + } + else { + // non-Gaussian (but structured) grids + lat.push_back( north ); + for ( auto b = y.begin(), a = b++; b != y.end(); a = b++ ) { + lat.push_back( ( *b + *a ) / 2. ); + } + lat.push_back( south ); } - lat.push_back( south ); + ATLAS_ASSERT( !lat.empty() ); // Calculate grid-box meridians (longitude midpoints) From dc6957584b0adcc64001b6af7c297b476ed3570a Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 29 Apr 2020 00:40:59 +0100 Subject: [PATCH 050/161] ATLAS-269, MIR-465: conservative interpolation support for non-global grids --- src/atlas/util/GridBox.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index ce1db971a..f7c9fa3e3 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -124,7 +124,6 @@ GridBoxes::GridBoxes( const Grid& grid ) { ATLAS_ASSERT( west <= east && east <= west + 360. ); bool periodic( ZonalBandDomain( structured.domain() ) ); - const GridBox clip( north, west, south, east ); // Calculate grid-box parallels (latitude midpoints, or accumulated Gaussian quadrature weights) @@ -191,24 +190,25 @@ GridBoxes::GridBoxes( const Grid& grid ) { n += 1; // (adjust double-fraction conversions) } - // West- and East-most grid-boxes on non-global grids might need clipping + // On non-global grids, North- and South-most grid-boxes need clipping; + // West- and East-most grid-boxes need clipping on non-periodic grids + ATLAS_ASSERT( x.nx()[j] > 0 ); + eckit::Fraction lon1 = ( n * dx ) - ( dx / 2 ); for ( idx_t i = 0; i < x.nx()[j]; ++i ) { double lon0 = lon1; lon1 += dx; - emplace_back( GridBox( lat[j], lon0, lat[j + 1], lon1 ) ); - if ( !periodic && i == 0 ) { - clip.intersects( back() ); - } + double n = std::min( north, lat[j] ); + double s = std::max( south, lat[j + 1] ); + double w = periodic ? lon0 : std::max( west, lon0 ); + double e = periodic ? lon1 : std::min( east, lon1 ); + emplace_back( GridBox( n, w, s, e ) ); } if ( periodic ) { ATLAS_ASSERT( lon1 == xmin - ( dx / 2 ) + 360 ); } - else { - clip.intersects( back() ); - } } ATLAS_ASSERT( idx_t( size() ) == grid.size() ); From b89c6933736f019778eabaca9f1daf2d8636554e Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 29 Apr 2020 09:46:50 +0100 Subject: [PATCH 051/161] ATLAS-269, MIR-465: conservative interpolation support for non-global grids --- src/atlas/util/GridBox.cc | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/atlas/util/GridBox.cc b/src/atlas/util/GridBox.cc index f7c9fa3e3..f0a496a3a 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/util/GridBox.cc @@ -134,13 +134,12 @@ GridBoxes::GridBoxes( const Grid& grid ) { GaussianGrid gaussian( grid ); if ( gaussian ) { - // FIXME: innefficient interface, latitudes are discarded auto N = gaussian.N(); std::vector latitudes( N * 2 ); std::vector weights( N * 2 ); util::gaussian_quadrature_npole_spole( N, latitudes.data(), weights.data() ); - std::vector lat_global( y.size() + 1 ); + std::vector lat_global( N * 2 + 1 ); auto b = lat_global.rbegin(); auto f = lat_global.begin(); *( b++ ) = -90.; @@ -155,13 +154,19 @@ GridBoxes::GridBoxes( const Grid& grid ) { } lat_global[N] = 0.; // (equator) - // grids not covering the poles need clipping - for ( auto l : lat_global ) { - if ( eckit::types::is_approximately_lesser_or_equal( l, north ) && - eckit::types::is_approximately_lesser_or_equal( south, l ) ) { - lat.push_back( l ); + // Grids not covering the poles need clipping + size_t j = 0; + for ( ; j < latitudes.size(); ++j ) { + if ( eckit::types::is_approximately_lesser_or_equal( latitudes[j], north ) && + eckit::types::is_approximately_lesser_or_equal( south, latitudes[j] ) ) { + lat.push_back( std::min( north, lat_global[j] ) ); + } + else if ( !lat.empty() ) { + break; } } + ATLAS_ASSERT( !lat.empty() ); + lat.push_back( std::max( south, lat_global[j] ) ); } else { // non-Gaussian (but structured) grids @@ -171,7 +176,6 @@ GridBoxes::GridBoxes( const Grid& grid ) { } lat.push_back( south ); } - ATLAS_ASSERT( !lat.empty() ); // Calculate grid-box meridians (longitude midpoints) @@ -190,8 +194,7 @@ GridBoxes::GridBoxes( const Grid& grid ) { n += 1; // (adjust double-fraction conversions) } - // On non-global grids, North- and South-most grid-boxes need clipping; - // West- and East-most grid-boxes need clipping on non-periodic grids + // On non-periodic grids, West- and East-most grid-boxes need clipping ATLAS_ASSERT( x.nx()[j] > 0 ); eckit::Fraction lon1 = ( n * dx ) - ( dx / 2 ); @@ -199,8 +202,8 @@ GridBoxes::GridBoxes( const Grid& grid ) { double lon0 = lon1; lon1 += dx; - double n = std::min( north, lat[j] ); - double s = std::max( south, lat[j + 1] ); + double n = lat[j]; + double s = lat[j + 1]; double w = periodic ? lon0 : std::max( west, lon0 ); double e = periodic ? lon1 : std::min( east, lon1 ); emplace_back( GridBox( n, w, s, e ) ); From a3b0faaa240632ebae9e7fb8c1909e8dc3181698 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 29 Apr 2020 09:53:51 +0100 Subject: [PATCH 052/161] ATLAS-269, MIR-465: self-registrion static vs. anonymous namespaces --- src/atlas/interpolation/method/knn/GridBoxAverage.cc | 4 +--- src/atlas/interpolation/method/knn/GridBoxMaximum.cc | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxAverage.cc b/src/atlas/interpolation/method/knn/GridBoxAverage.cc index 5125d8095..ad7152fe9 100644 --- a/src/atlas/interpolation/method/knn/GridBoxAverage.cc +++ b/src/atlas/interpolation/method/knn/GridBoxAverage.cc @@ -26,9 +26,7 @@ namespace interpolation { namespace method { -namespace { -MethodBuilder __builder( "grid-box-average" ); -} +static MethodBuilder __builder( "grid-box-average" ); void GridBoxAverage::do_execute( const FieldSet& source, FieldSet& target ) const { diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc index 7ad65049b..3c1a71514 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc @@ -29,9 +29,7 @@ namespace interpolation { namespace method { -namespace { -MethodBuilder __builder( "grid-box-maximum" ); -} +static MethodBuilder __builder( "grid-box-maximum" ); void GridBoxMaximum::do_execute( const FieldSet& source, FieldSet& target ) const { From 86f740836a52963f86d1bae6ea6dcb304e86dcdf Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 29 Apr 2020 10:04:16 +0100 Subject: [PATCH 053/161] ATLAS-269, MIR-465: self-registrion static vs. anonymous namespaces (revert to match other code, but static version works well) --- src/atlas/interpolation/method/knn/GridBoxAverage.cc | 4 +++- src/atlas/interpolation/method/knn/GridBoxMaximum.cc | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxAverage.cc b/src/atlas/interpolation/method/knn/GridBoxAverage.cc index ad7152fe9..5125d8095 100644 --- a/src/atlas/interpolation/method/knn/GridBoxAverage.cc +++ b/src/atlas/interpolation/method/knn/GridBoxAverage.cc @@ -26,7 +26,9 @@ namespace interpolation { namespace method { -static MethodBuilder __builder( "grid-box-average" ); +namespace { +MethodBuilder __builder( "grid-box-average" ); +} void GridBoxAverage::do_execute( const FieldSet& source, FieldSet& target ) const { diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc index 3c1a71514..7ad65049b 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc @@ -29,7 +29,9 @@ namespace interpolation { namespace method { -static MethodBuilder __builder( "grid-box-maximum" ); +namespace { +MethodBuilder __builder( "grid-box-maximum" ); +} void GridBoxMaximum::do_execute( const FieldSet& source, FieldSet& target ) const { From 2c05818e39e1832405f06618b0f1f56bd9a2778e Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 29 Apr 2020 13:05:56 +0100 Subject: [PATCH 054/161] ATLAS-269, MIR-465: use a single namespace --- src/atlas/CMakeLists.txt | 4 ++-- .../{util => interpolation/method/knn}/GridBox.cc | 8 +++++--- .../{util => interpolation/method/knn}/GridBox.h | 6 ++++-- .../interpolation/method/knn/GridBoxAverage.cc | 2 +- src/atlas/interpolation/method/knn/GridBoxAverage.h | 2 +- .../interpolation/method/knn/GridBoxMaximum.cc | 2 +- src/atlas/interpolation/method/knn/GridBoxMaximum.h | 2 +- src/atlas/interpolation/method/knn/GridBoxMethod.cc | 8 ++++---- src/atlas/interpolation/method/knn/GridBoxMethod.h | 10 +++++----- .../test_interpolation_grid_box_average.cc | 13 ++++++++----- 10 files changed, 32 insertions(+), 25 deletions(-) rename src/atlas/{util => interpolation/method/knn}/GridBox.cc (97%) rename src/atlas/{util => interpolation/method/knn}/GridBox.h (94%) diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 42bebdcef..d8968ffbb 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -499,6 +499,8 @@ interpolation/method/Ray.cc interpolation/method/Ray.h interpolation/method/fe/FiniteElement.cc interpolation/method/fe/FiniteElement.h +interpolation/method/knn/GridBox.cc +interpolation/method/knn/GridBox.h interpolation/method/knn/GridBoxAverage.cc interpolation/method/knn/GridBoxAverage.h interpolation/method/knn/GridBoxMaximum.cc @@ -623,8 +625,6 @@ util/KDTree.cc util/KDTree.h util/PolygonXY.cc util/PolygonXY.h -util/GridBox.cc -util/GridBox.h util/Metadata.cc util/Metadata.h util/Point.cc diff --git a/src/atlas/util/GridBox.cc b/src/atlas/interpolation/method/knn/GridBox.cc similarity index 97% rename from src/atlas/util/GridBox.cc rename to src/atlas/interpolation/method/knn/GridBox.cc index f0a496a3a..f04faf25c 100644 --- a/src/atlas/util/GridBox.cc +++ b/src/atlas/interpolation/method/knn/GridBox.cc @@ -10,7 +10,7 @@ */ -#include "atlas/util/GridBox.h" +#include "atlas/interpolation/method/knn/GridBox.h" #include #include @@ -43,7 +43,8 @@ double normalise( double lon, double minimum ) { namespace atlas { -namespace util { +namespace interpolation { +namespace method { GridBox::GridBox( double north, double west, double south, double east ) : @@ -234,5 +235,6 @@ double GridBoxes::getLongestGridBoxDiagonal() const { } -} // namespace util +} // namespace method +} // namespace interpolation } // namespace atlas diff --git a/src/atlas/util/GridBox.h b/src/atlas/interpolation/method/knn/GridBox.h similarity index 94% rename from src/atlas/util/GridBox.h rename to src/atlas/interpolation/method/knn/GridBox.h index 8ba383c4f..291677c80 100644 --- a/src/atlas/util/GridBox.h +++ b/src/atlas/interpolation/method/knn/GridBox.h @@ -22,7 +22,8 @@ class Grid; namespace atlas { -namespace util { +namespace interpolation { +namespace method { class GridBox { @@ -115,5 +116,6 @@ struct GridBoxes : std::vector { }; -} // namespace util +} // namespace method +} // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/method/knn/GridBoxAverage.cc b/src/atlas/interpolation/method/knn/GridBoxAverage.cc index 5125d8095..20cbb510b 100644 --- a/src/atlas/interpolation/method/knn/GridBoxAverage.cc +++ b/src/atlas/interpolation/method/knn/GridBoxAverage.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 ECMWF. + * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. diff --git a/src/atlas/interpolation/method/knn/GridBoxAverage.h b/src/atlas/interpolation/method/knn/GridBoxAverage.h index 0cc14bd26..deb8ddb12 100644 --- a/src/atlas/interpolation/method/knn/GridBoxAverage.h +++ b/src/atlas/interpolation/method/knn/GridBoxAverage.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 ECMWF. + * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc index 7ad65049b..573fdb623 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 ECMWF. + * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.h b/src/atlas/interpolation/method/knn/GridBoxMaximum.h index b10009d72..9a1c68d8e 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMaximum.h +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 ECMWF. + * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 9474c210b..5cbc3ab28 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 ECMWF. + * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -49,7 +49,7 @@ void GridBoxMethod::do_setup( const FunctionSpace& /*source*/, const FunctionSpa } -bool GridBoxMethod::intersect( size_t i, const util::GridBox& box, const PointIndex3::NodeList& closest, +bool GridBoxMethod::intersect( size_t i, const GridBox& box, const PointIndex3::NodeList& closest, std::vector& triplets ) const { ASSERT( !closest.empty() ); @@ -108,8 +108,8 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { buildPointSearchTree( src ); ATLAS_ASSERT( pTree_ != nullptr ); - sourceBoxes_ = util::GridBoxes( source ); - targetBoxes_ = util::GridBoxes( target ); + sourceBoxes_ = GridBoxes( source ); + targetBoxes_ = GridBoxes( target ); searchRadius_ = sourceBoxes_.getLongestGridBoxDiagonal() + targetBoxes_.getLongestGridBoxDiagonal(); failures_.clear(); diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index 22fd9ee30..cb38e448f 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 ECMWF. + * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -16,7 +16,7 @@ #include #include "atlas/functionspace.h" -#include "atlas/util/GridBox.h" +#include "atlas/interpolation/method/knn/GridBox.h" namespace atlas { @@ -43,7 +43,7 @@ class GridBoxMethod : public KNearestNeighboursBase { virtual const FunctionSpace& source() const override { return source_; } virtual const FunctionSpace& target() const override { return target_; } - bool intersect( size_t i, const util::GridBox& iBox, const PointIndex3::NodeList&, std::vector& ) const; + bool intersect( size_t i, const GridBox& iBox, const PointIndex3::NodeList&, std::vector& ) const; virtual void do_execute( const FieldSet& source, FieldSet& target ) const = 0; virtual void do_execute( const Field& source, Field& target ) const = 0; @@ -54,8 +54,8 @@ class GridBoxMethod : public KNearestNeighboursBase { FunctionSpace source_; FunctionSpace target_; - util::GridBoxes sourceBoxes_; - util::GridBoxes targetBoxes_; + GridBoxes sourceBoxes_; + GridBoxes targetBoxes_; double searchRadius_; diff --git a/src/tests/interpolation/test_interpolation_grid_box_average.cc b/src/tests/interpolation/test_interpolation_grid_box_average.cc index 09f8da462..4d3041fe5 100644 --- a/src/tests/interpolation/test_interpolation_grid_box_average.cc +++ b/src/tests/interpolation/test_interpolation_grid_box_average.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2013 ECMWF. + * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -17,9 +17,9 @@ #include "atlas/field.h" #include "atlas/grid.h" #include "atlas/interpolation.h" +#include "atlas/interpolation/method/knn/GridBox.h" #include "atlas/option.h" #include "atlas/util/Config.h" -#include "atlas/util/GridBox.h" #include "tests/AtlasTestEnvironment.h" @@ -28,10 +28,13 @@ namespace atlas { namespace test { +using interpolation::method::GridBoxes; + + double integral( const Grid& grid, const Field& field ) { auto values = array::make_view( field ); - auto boxes = util::GridBoxes( grid ); + auto boxes = GridBoxes( grid ); ATLAS_ASSERT( boxes.size() == size_t( field.shape( 0 ) ) ); double i = 0.; @@ -170,7 +173,7 @@ CASE( "test_interpolation_grid_box_average" ) { size_t i = 0; auto values = array::make_view( fieldA ); - for ( auto& box : util::GridBoxes( gridA ) ) { + for ( auto& box : GridBoxes( gridA ) ) { ATLAS_ASSERT( box.area() > 0. ); values( i++ ) = 1. / box.area(); } @@ -192,7 +195,7 @@ CASE( "test_interpolation_grid_box_average" ) { size_t i = 0; auto values = array::make_view( fieldA ); - for ( auto& box : util::GridBoxes( gridB ) ) { + for ( auto& box : GridBoxes( gridB ) ) { ATLAS_ASSERT( box.area() > 0. ); values( i++ ) = 1. / box.area(); } From 946f0a78cf5a0e403e85f61dcf9e7cbf68879ddf Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 29 Apr 2020 17:49:38 +0100 Subject: [PATCH 055/161] ATLAS-269, MIR-465: Points::createField to support the grid-box method --- src/atlas/functionspace/Points.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/atlas/functionspace/Points.cc b/src/atlas/functionspace/Points.cc index 81fb4b546..0e68f3a90 100644 --- a/src/atlas/functionspace/Points.cc +++ b/src/atlas/functionspace/Points.cc @@ -52,8 +52,15 @@ Field Points::createField( const Field& other, const eckit::Configuration& confi } -Field Points::createField( const eckit::Configuration& ) const { - ATLAS_NOTIMPLEMENTED; +Field Points::createField( const eckit::Configuration& config ) const { + array::DataType::kind_t kind; + if ( !config.get( "datatype", kind ) ) { + throw_Exception( "datatype missing", Here() ); + } + + Field field( config.getString( "name", "" ), array::DataType( kind ), array::make_shape( size() ) ); + field.set_functionspace( this ); + return field; } From 47562f94dc98d9d98fef8799f0ce6cd65cbeb359 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 28 Aug 2020 23:11:07 +0100 Subject: [PATCH 056/161] ATLAS-269 Add test for k_nearest_neighbours --- src/atlas/interpolation/method/Method.h | 7 ++ src/tests/interpolation/CMakeLists.txt | 6 ++ ...test_interpolation_k_nearest_neighbours.cc | 84 +++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc diff --git a/src/atlas/interpolation/method/Method.h b/src/atlas/interpolation/method/Method.h index d78a5e25d..487d805eb 100644 --- a/src/atlas/interpolation/method/Method.h +++ b/src/atlas/interpolation/method/Method.h @@ -25,6 +25,12 @@ class FunctionSpace; class Grid; } // namespace atlas +namespace atlas { + namespace test { + class Access; + } +} + namespace atlas { namespace interpolation { @@ -72,6 +78,7 @@ class Method : public util::Object { // matrices, // so do not expose here, even though only linear operators are now // implemented. + friend class atlas::test::Access; Matrix matrix_; bool use_eckit_linalg_spmv_; diff --git a/src/tests/interpolation/CMakeLists.txt b/src/tests/interpolation/CMakeLists.txt index 0df8ae423..32ab36d2e 100644 --- a/src/tests/interpolation/CMakeLists.txt +++ b/src/tests/interpolation/CMakeLists.txt @@ -25,6 +25,12 @@ ecbuild_add_test( TARGET atlas_test_interpolation_grid_box_average ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_interpolation_k_nearest_neighbours + SOURCES test_interpolation_k_nearest_neighbours.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + ecbuild_add_test( TARGET atlas_test_interpolation_cubic_prototype SOURCES test_interpolation_cubic_prototype.cc CubicInterpolationPrototype.h LIBS atlas diff --git a/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc b/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc new file mode 100644 index 000000000..1df3e313c --- /dev/null +++ b/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc @@ -0,0 +1,84 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +#include + +#include "eckit/types/FloatCompare.h" +#include "eckit/utils/MD5.h" + +#include "atlas/array.h" +#include "atlas/field.h" +#include "atlas/grid.h" +#include "atlas/interpolation.h" +#include "atlas/option.h" +#include "atlas/util/Config.h" + +#include "tests/AtlasTestEnvironment.h" + + +namespace atlas { +namespace test { + +class Access { +public: + Access( const Interpolation& interpolation ) : interpolation_{interpolation} { + } + const Interpolation::Implementation::Matrix& matrix() const { + return interpolation_.get()->matrix_; + } + Interpolation interpolation_; + + std::string hash() + { + eckit::MD5 hash; + const auto& m = matrix(); + const auto outer = m.outer(); + const auto index = m.inner(); + const auto weight = m.data(); + + idx_t rows = static_cast( m.rows() ); + hash.add(rows); + for( idx_t r = 0; r < rows; ++r ) { + //v_tgt( r ) = 0.; + for ( idx_t c = outer[r]; c < outer[r + 1]; ++c ) { + hash.add(c); + idx_t n = index[c]; + hash.add(n); + //Value w = static_cast( weight[c] ); + //v_tgt( r ) += w * v_src( n ); + } + } + return hash.digest(); + } +}; + + +CASE( "test_interpolation_k_nearest_neighbours" ) { + Log::info().precision( 16 ); + + Grid gridA( "O32" ); + Grid gridB( "O64" ); + Grid gridC( "O32" ); + + Interpolation a( option::type("k-nearest-neighbours"), gridA, gridB ); + Interpolation b( option::type("k-nearest-neighbours"), gridB, gridC ); + EXPECT( Access{a}.hash() == "5ecc9d615dcf7112b5a97b38341099a5" ); + EXPECT( Access{b}.hash() == "bf4b4214ef50387ba00a1950b95e0d93" ); +} + + +} // namespace test +} // namespace atlas + + +int main( int argc, char** argv ) { + return atlas::test::run( argc, argv ); +} From 5513d202f1f46a3cf57c579e3ba1e0830c783914 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 3 Sep 2020 10:23:42 +0100 Subject: [PATCH 057/161] ATLAS-269 Improve PointCloud iterators --- src/atlas/functionspace/PointCloud.cc | 85 ++++++++-------- src/atlas/functionspace/PointCloud.h | 96 +++++++------------ src/tests/functionspace/test_pointcloud.cc | 87 ++++++++++++++++- .../test_interpolation_cubic_prototype.cc | 60 ++++++------ 4 files changed, 191 insertions(+), 137 deletions(-) diff --git a/src/atlas/functionspace/PointCloud.cc b/src/atlas/functionspace/PointCloud.cc index f33429524..66931fc57 100644 --- a/src/atlas/functionspace/PointCloud.cc +++ b/src/atlas/functionspace/PointCloud.cc @@ -16,6 +16,7 @@ #include "atlas/grid/Iterator.h" #include "atlas/option/Options.h" #include "atlas/runtime/Exception.h" +#include "atlas/util/CoordinateEnums.h" #if ATLAS_HAVE_FORTRAN #define REMOTE_IDX_BASE 1 @@ -28,8 +29,7 @@ namespace functionspace { namespace detail { -PointCloud::PointCloud( PointXY, const std::vector& points ) : PointCloud( points ) {} - +template <> PointCloud::PointCloud( const std::vector& points ) { lonlat_ = Field( "lonlat", array::make_datatype(), array::make_shape( points.size(), 2 ) ); auto lonlat = array::make_view( lonlat_ ); @@ -39,7 +39,8 @@ PointCloud::PointCloud( const std::vector& points ) { } } -PointCloud::PointCloud( PointXYZ, const std::vector& points ) { +template <> +PointCloud::PointCloud( const std::vector& points ) { lonlat_ = Field( "lonlat", array::make_datatype(), array::make_shape( points.size(), 2 ) ); vertical_ = Field( "vertical", array::make_datatype(), array::make_shape( points.size() ) ); auto lonlat = array::make_view( lonlat_ ); @@ -107,52 +108,47 @@ std::string PointCloud::distribution() const { return std::string( "serial" ); } -atlas::functionspace::detail::PointCloud::IteratorXYZ::IteratorXYZ( const atlas::functionspace::detail::PointCloud& fs, - bool begin ) : - fs_( fs ), - xy_( array::make_view( fs_.lonlat() ) ), - z_( array::make_view( fs_.vertical() ) ), - n_( begin ? 0 : fs_.size() ) {} - -bool atlas::functionspace::detail::PointCloud::IteratorXYZ::next( PointXYZ& xyz ) { - if ( n_ < fs_.size() ) { - xyz.x() = xy_( n_, 0 ); - xyz.y() = xy_( n_, 1 ); - xyz.z() = z_( n_ ); - ++n_; - return true; - } - return false; +static const array::Array& get_dummy() { + static array::ArrayT dummy_{1}; + return dummy_; } -atlas::functionspace::detail::PointCloud::IteratorXY::IteratorXY( const atlas::functionspace::detail::PointCloud& fs, - bool begin ) : - fs_( fs ), xy_( array::make_view( fs_.lonlat() ) ), n_( begin ? 0 : fs_.size() ) {} - -bool atlas::functionspace::detail::PointCloud::IteratorXY::next( PointXY& xyz ) { - if ( n_ < fs_.size() ) { - xyz.x() = xy_( n_, 0 ); - xyz.y() = xy_( n_, 1 ); +template +PointCloud::IteratorT::IteratorT( const atlas::functionspace::detail::PointCloud& fs, bool begin ) : + fs_( fs ), + xy_( array::make_view( fs_.lonlat() ) ), + z_( array::make_view( bool( fs_.vertical() ) ? fs_.vertical().array() : get_dummy() ) ), + n_( begin ? 0 : fs_.size() ), + size_( fs_.size() ) {} + +template +bool PointCloud::IteratorT::next( Point& p ) { + if ( n_ < size_ ) { + p[XX] = xy_( n_, XX ); + p[YY] = xy_( n_, YY ); + if ( Point::DIMS == 3 ) { + p[ZZ] = z_( n_ ); + } ++n_; return true; } return false; } -const PointXY atlas::functionspace::detail::PointCloud::IteratorXY::operator*() const { - PointXY xy; - xy.x() = xy_( n_, 0 ); - xy.y() = xy_( n_, 1 ); - return xy; +template +const Point PointCloud::IteratorT::operator*() const { + Point p; + p[XX] = xy_( n_, XX ); + p[YY] = xy_( n_, YY ); + if ( Point::DIMS == 3 ) { + p[ZZ] = z_( n_ ); + } + return p; } -const PointXYZ atlas::functionspace::detail::PointCloud::IteratorXYZ::operator*() const { - PointXYZ xyz; - xyz.x() = xy_( n_, 0 ); - xyz.y() = xy_( n_, 1 ); - xyz.z() = z_( n_ ); - return xyz; -} +template class PointCloud::IteratorT; +template class PointCloud::IteratorT; +template class PointCloud::IteratorT; } // namespace detail @@ -167,12 +163,15 @@ PointCloud::PointCloud( const std::vector& points ) : FunctionSpace( new detail::PointCloud( points ) ), functionspace_( dynamic_cast( get() ) ) {} -PointCloud::PointCloud( PointXY p, const std::vector& points ) : - FunctionSpace( new detail::PointCloud( p, points ) ), +PointCloud::PointCloud( const std::vector& points ) : + FunctionSpace( new detail::PointCloud( points ) ), functionspace_( dynamic_cast( get() ) ) {} -PointCloud::PointCloud( PointXYZ p, const std::vector& points ) : - FunctionSpace( new detail::PointCloud( p, points ) ), + +PointCloud::PointCloud( const std::initializer_list>& points ) : + FunctionSpace( ( points.begin()->size() == 2 + ? new detail::PointCloud{std::vector( points.begin(), points.end() )} + : new detail::PointCloud{std::vector( points.begin(), points.end() )} ) ), functionspace_( dynamic_cast( get() ) ) {} PointCloud::PointCloud( const Grid& grid ) : diff --git a/src/atlas/functionspace/PointCloud.h b/src/atlas/functionspace/PointCloud.h index 5c4ae0bef..d7f9b3442 100644 --- a/src/atlas/functionspace/PointCloud.h +++ b/src/atlas/functionspace/PointCloud.h @@ -14,6 +14,7 @@ #include "atlas/field/Field.h" #include "atlas/functionspace/FunctionSpace.h" #include "atlas/functionspace/detail/FunctionSpaceImpl.h" +#include "atlas/runtime/Exception.h" #include "atlas/util/Config.h" #include "atlas/util/Point.h" @@ -28,9 +29,8 @@ namespace detail { class PointCloud : public functionspace::FunctionSpaceImpl { public: - PointCloud( const std::vector& ); - PointCloud( PointXY, const std::vector& ); - PointCloud( PointXYZ, const std::vector& ); + template + PointCloud( const std::vector& ); PointCloud( const Field& lonlat ); PointCloud( const Field& lonlat, const Field& ghost ); PointCloud( const Grid& ); @@ -49,81 +49,49 @@ class PointCloud : public functionspace::FunctionSpaceImpl { virtual Field createField( const Field&, const eckit::Configuration& ) const override; - class IteratorXYZ { - public: - IteratorXYZ( const PointCloud& fs, bool begin = true ); - - bool next( PointXYZ& xyz ); - - const PointXYZ operator*() const; - - const IteratorXYZ& operator++() { - ++n_; - return *this; - } - - bool operator==( const IteratorXYZ& other ) const { return n_ == static_cast( other ).n_; } - - virtual bool operator!=( const IteratorXYZ& other ) const { - return n_ != static_cast( other ).n_; - } - - private: - const PointCloud& fs_; - const array::ArrayView xy_; - const array::ArrayView z_; - idx_t n_; - }; - - - class IterateXYZ { + template + class IteratorT { public: - using iterator = IteratorXYZ; - using const_iterator = iterator; - - public: - IterateXYZ( const PointCloud& fs ) : fs_( fs ) {} - iterator begin() const { return IteratorXYZ( fs_ ); } - iterator end() const { return IteratorXYZ( fs_, false ); } - - private: - const PointCloud& fs_; - }; + using difference_type = std::ptrdiff_t; + using value_type = Point; + using pointer = Point*; + using reference = Point&; + using iterator_category = std::output_iterator_tag; - class IteratorXY { - public: - IteratorXY( const PointCloud& fs, bool begin = true ); + IteratorT( const PointCloud& fs, bool begin = true ); - bool next( PointXY& xyz ); + bool next( Point& ); - const PointXY operator*() const; + const Point operator*() const; - const IteratorXY& operator++() { + const IteratorT& operator++() { ++n_; return *this; } - bool operator==( const IteratorXY& other ) const { return n_ == static_cast( other ).n_; } - - virtual bool operator!=( const IteratorXY& other ) const { - return n_ != static_cast( other ).n_; - } + bool operator==( const IteratorT& other ) const { return n_ == other.n_; } + bool operator!=( const IteratorT& other ) const { return n_ != other.n_; } private: const PointCloud& fs_; const array::ArrayView xy_; + const array::ArrayView z_; idx_t n_; + idx_t size_; }; - class IterateXY { + template + class IterateT { public: - using iterator = IteratorXY; + using value_type = Point; + using iterator = IteratorT; using const_iterator = iterator; public: - IterateXY( const PointCloud& fs ) : fs_( fs ) {} - iterator begin() const { return IteratorXY( fs_ ); } - iterator end() const { return IteratorXY( fs_, false ); } + IterateT( const PointCloud& fs ) : fs_( fs ) {} + iterator begin() const { return IteratorT( fs_ ); } + iterator end() const { return IteratorT( fs_, false ); } + idx_t size() const { return fs_.size(); } private: const PointCloud& fs_; @@ -133,8 +101,12 @@ class PointCloud : public functionspace::FunctionSpaceImpl { class Iterate { public: Iterate( const PointCloud& fs ) : fs_( fs ) {} - IterateXYZ xyz() const { return IterateXYZ( fs_ ); } - IterateXY xy() const { return IterateXY( fs_ ); } + IterateT xyz() const { + ATLAS_ASSERT( fs_.vertical() ); + return IterateT{fs_}; + } + IterateT xy() const { return IterateT{fs_}; } + IterateT lonlat() const { return IterateT{fs_}; } private: const PointCloud& fs_; @@ -160,8 +132,8 @@ class PointCloud : public FunctionSpace { PointCloud( const FunctionSpace& ); PointCloud( const Field& points ); PointCloud( const std::vector& ); - PointCloud( PointXY, const std::vector& ); - PointCloud( PointXYZ, const std::vector& ); + PointCloud( const std::vector& ); + PointCloud( const std::initializer_list>& ); PointCloud( const Grid& grid ); operator bool() const { return valid(); } diff --git a/src/tests/functionspace/test_pointcloud.cc b/src/tests/functionspace/test_pointcloud.cc index 874fcd98f..eb7a97242 100644 --- a/src/tests/functionspace/test_pointcloud.cc +++ b/src/tests/functionspace/test_pointcloud.cc @@ -23,7 +23,29 @@ namespace test { //----------------------------------------------------------------------------- -CASE( "test_functionspace_PointCloud" ) { +std::vector ref_lonlat() { + static std::vector v = {{00., 0.}, {10., 0.}, {20., 0.}, {30., 0.}, {40., 0.}, + {50., 0.}, {60., 0.}, {70., 0.}, {80., 0.}, {90., 0.}}; + return v; +} +std::vector ref_xy() { + static std::vector v = {{00., 0.}, {10., 0.}, {20., 0.}, {30., 0.}, {40., 0.}, + {50., 0.}, {60., 0.}, {70., 0.}, {80., 0.}, {90., 0.}}; + return v; +} +std::vector ref_xyz() { + static std::vector v = {{00., 0., 1.}, {10., 0., 2.}, {20., 0., 3.}, {30., 0., 4.}, {40., 0., 5.}, + {50., 0., 6.}, {60., 0., 7.}, {70., 0., 8.}, {80., 0., 9.}, {90., 0., 10.}}; + return v; +} + +template +bool equal( const C1& c1, const C2& c2 ) { + return c1.size() == c2.size() && std::equal( std::begin( c1 ), std::end( c1 ), std::begin( c2 ) ); +} + + +CASE( "test_functionspace_PointCloud create from field" ) { Field points( "points", array::make_datatype(), array::make_shape( 10, 2 ) ); auto xy = array::make_view( points ); xy.assign( {00., 0., 10., 0., 20., 0., 30., 0., 40., 0., 50., 0., 60., 0., 70., 0., 80., 0., 90., 0.} ); @@ -33,8 +55,71 @@ CASE( "test_functionspace_PointCloud" ) { points.dump( Log::info() ); Log::info() << std::endl; + + EXPECT_THROWS( pointcloud.iterate().xyz() ); + EXPECT( equal( pointcloud.iterate().xy(), ref_xy() ) ); + EXPECT( equal( pointcloud.iterate().lonlat(), ref_lonlat() ) ); +} + +CASE( "test_functionspace_PointCloud create from 2D vector" ) { + std::vector points{{00., 0.}, {10., 0.}, {20., 0.}, {30., 0.}, {40., 0.}, + {50., 0.}, {60., 0.}, {70., 0.}, {80., 0.}, {90., 0.}}; + + functionspace::PointCloud pointcloud( points ); + EXPECT( pointcloud.size() == 10 ); + + pointcloud.lonlat().dump( Log::info() ); + Log::info() << std::endl; + + EXPECT_THROWS( pointcloud.iterate().xyz() ); + EXPECT( equal( pointcloud.iterate().xy(), ref_xy() ) ); + EXPECT( equal( pointcloud.iterate().lonlat(), ref_lonlat() ) ); +} + +CASE( "test_functionspace_PointCloud create from 3D vector" ) { + std::vector points{{00., 0., 1.}, {10., 0., 2.}, {20., 0., 3.}, {30., 0., 4.}, {40., 0., 5.}, + {50., 0., 6.}, {60., 0., 7.}, {70., 0., 8.}, {80., 0., 9.}, {90., 0., 10.}}; + functionspace::PointCloud pointcloud( points ); + EXPECT( pointcloud.size() == 10 ); + + pointcloud.lonlat().dump( Log::info() ); + Log::info() << std::endl; + + EXPECT( equal( pointcloud.iterate().xyz(), ref_xyz() ) ); + EXPECT( equal( pointcloud.iterate().xy(), ref_xy() ) ); + EXPECT( equal( pointcloud.iterate().lonlat(), ref_lonlat() ) ); } + +CASE( "test_functionspace_PointCloud create from 2D initializer list" ) { + functionspace::PointCloud pointcloud = {{00., 0.}, {10., 0.}, {20., 0.}, {30., 0.}, {40., 0.}, + {50., 0.}, {60., 0.}, {70., 0.}, {80., 0.}, {90., 0.}}; + EXPECT( pointcloud.size() == 10 ); + + pointcloud.lonlat().dump( Log::info() ); + Log::info() << std::endl; + + EXPECT_THROWS( pointcloud.iterate().xyz() ); + EXPECT( equal( pointcloud.iterate().xy(), ref_xy() ) ); + EXPECT( equal( pointcloud.iterate().lonlat(), ref_lonlat() ) ); +} + +CASE( "test_functionspace_PointCloud create from 3D initializer list" ) { + functionspace::PointCloud pointcloud = {{00., 0., 1.}, {10., 0., 2.}, {20., 0., 3.}, {30., 0., 4.}, {40., 0., 5.}, + {50., 0., 6.}, {60., 0., 7.}, {70., 0., 8.}, {80., 0., 9.}, {90., 0., 10.}}; + EXPECT( pointcloud.size() == 10 ); + + pointcloud.lonlat().dump( Log::info() ); + Log::info() << std::endl; + pointcloud.vertical().dump( Log::info() ); + Log::info() << std::endl; + + EXPECT( equal( pointcloud.iterate().xyz(), ref_xyz() ) ); + EXPECT( equal( pointcloud.iterate().xy(), ref_xy() ) ); + EXPECT( equal( pointcloud.iterate().lonlat(), ref_lonlat() ) ); +} + + //----------------------------------------------------------------------------- CASE( "test_createField" ) { diff --git a/src/tests/interpolation/test_interpolation_cubic_prototype.cc b/src/tests/interpolation/test_interpolation_cubic_prototype.cc index 41ce7f4e0..f0d9343cf 100644 --- a/src/tests/interpolation/test_interpolation_cubic_prototype.cc +++ b/src/tests/interpolation/test_interpolation_cubic_prototype.cc @@ -197,11 +197,10 @@ CASE( "test horizontal cubic interpolation triplets" ) { CubicHorizontalInterpolation cubic_interpolation( fs ); - auto departure_points = PointCloud{PointXY(), - { - {0.13257, 45.6397}, - {360., -90.}, - }}; + auto departure_points = PointCloud{{ + {0.13257, 45.6397}, + {360., -90.}, + }}; auto departure_lonlat = make_view( departure_points.lonlat() ); CubicHorizontalInterpolation::WorkSpace ws; @@ -281,32 +280,31 @@ CASE( "test 3d cubic interpolation" ) { input.set_dirty( false ); // to avoid halo-exchange - auto departure_points = PointCloud( - PointXYZ(), { - {90., -45., 0.25}, {0., -45., 0.25}, {180., -45., 0.25}, {360., -45., 0.25}, {90., -90., 0.25}, - {90., 0., 0.25}, {90., 90., 0.25}, {10., -45., 0.25}, {20., -45., 0.25}, {30., -45., 0.25}, - {40., -45., 0.25}, {50., -45., 0.25}, {60., -45., 0.25}, {70., -45., 0.25}, {80., -45., 0.25}, - {90., -45., 0.25}, {10., -10., 0.25}, {20., -20., 0.25}, {30., -30., 0.25}, {40., -40., 0.25}, - {50., -50., 0.25}, {60., -60., 0.25}, {70., -70., 0.25}, {80., -80., 0.25}, {90., -90., 0.25}, - {60., -60., 0.6}, {90., -45., 0.16}, {90., -45., 0.6}, {90., -45., 1.}, {90., -45., 0.}, - {90., -45., 0.1}, {45., 33., 0.7}, {359., 20., 0.1}, {360., 88., 0.9}, {0.1, -89., 1.}, - - {90., -45., 0.25}, {0., -45., 0.25}, {180., -45., 0.25}, {360., -45., 0.25}, {90., -90., 0.25}, - {90., 0., 0.25}, {90., 90., 0.25}, {10., -45., 0.25}, {20., -45., 0.25}, {30., -45., 0.25}, - {40., -45., 0.25}, {50., -45., 0.25}, {60., -45., 0.25}, {70., -45., 0.25}, {80., -45., 0.25}, - {90., -45., 0.25}, {10., -10., 0.25}, {20., -20., 0.25}, {30., -30., 0.25}, {40., -40., 0.25}, - {50., -50., 0.25}, {60., -60., 0.25}, {70., -70., 0.25}, {80., -80., 0.25}, {90., -90., 0.25}, - {60., -60., 0.6}, {90., -45., 0.16}, {90., -45., 0.6}, {90., -45., 1.}, {90., -45., 0.}, - {90., -45., 0.1}, {45., 33., 0.7}, {359., 20., 0.1}, {360., 88., 0.9}, {0.1, -89., 1.}, - - {90., -45., 0.25}, {0., -45., 0.25}, {180., -45., 0.25}, {360., -45., 0.25}, {90., -90., 0.25}, - {90., 0., 0.25}, {90., 90., 0.25}, {10., -45., 0.25}, {20., -45., 0.25}, {30., -45., 0.25}, - {40., -45., 0.25}, {50., -45., 0.25}, {60., -45., 0.25}, {70., -45., 0.25}, {80., -45., 0.25}, - {90., -45., 0.25}, {10., -10., 0.25}, {20., -20., 0.25}, {30., -30., 0.25}, {40., -40., 0.25}, - {50., -50., 0.25}, {60., -60., 0.25}, {70., -70., 0.25}, {80., -80., 0.25}, {90., -90., 0.25}, - {60., -60., 0.6}, {90., -45., 0.16}, {90., -45., 0.6}, {90., -45., 1.}, {90., -45., 0.}, - {90., -45., 0.1}, {45., 33., 0.7}, {359., 20., 0.1}, {360., 88., 0.9}, {0.1, -89., 1.}, - } ); + auto departure_points = PointCloud( { + {90., -45., 0.25}, {0., -45., 0.25}, {180., -45., 0.25}, {360., -45., 0.25}, {90., -90., 0.25}, + {90., 0., 0.25}, {90., 90., 0.25}, {10., -45., 0.25}, {20., -45., 0.25}, {30., -45., 0.25}, + {40., -45., 0.25}, {50., -45., 0.25}, {60., -45., 0.25}, {70., -45., 0.25}, {80., -45., 0.25}, + {90., -45., 0.25}, {10., -10., 0.25}, {20., -20., 0.25}, {30., -30., 0.25}, {40., -40., 0.25}, + {50., -50., 0.25}, {60., -60., 0.25}, {70., -70., 0.25}, {80., -80., 0.25}, {90., -90., 0.25}, + {60., -60., 0.6}, {90., -45., 0.16}, {90., -45., 0.6}, {90., -45., 1.}, {90., -45., 0.}, + {90., -45., 0.1}, {45., 33., 0.7}, {359., 20., 0.1}, {360., 88., 0.9}, {0.1, -89., 1.}, + + {90., -45., 0.25}, {0., -45., 0.25}, {180., -45., 0.25}, {360., -45., 0.25}, {90., -90., 0.25}, + {90., 0., 0.25}, {90., 90., 0.25}, {10., -45., 0.25}, {20., -45., 0.25}, {30., -45., 0.25}, + {40., -45., 0.25}, {50., -45., 0.25}, {60., -45., 0.25}, {70., -45., 0.25}, {80., -45., 0.25}, + {90., -45., 0.25}, {10., -10., 0.25}, {20., -20., 0.25}, {30., -30., 0.25}, {40., -40., 0.25}, + {50., -50., 0.25}, {60., -60., 0.25}, {70., -70., 0.25}, {80., -80., 0.25}, {90., -90., 0.25}, + {60., -60., 0.6}, {90., -45., 0.16}, {90., -45., 0.6}, {90., -45., 1.}, {90., -45., 0.}, + {90., -45., 0.1}, {45., 33., 0.7}, {359., 20., 0.1}, {360., 88., 0.9}, {0.1, -89., 1.}, + + {90., -45., 0.25}, {0., -45., 0.25}, {180., -45., 0.25}, {360., -45., 0.25}, {90., -90., 0.25}, + {90., 0., 0.25}, {90., 90., 0.25}, {10., -45., 0.25}, {20., -45., 0.25}, {30., -45., 0.25}, + {40., -45., 0.25}, {50., -45., 0.25}, {60., -45., 0.25}, {70., -45., 0.25}, {80., -45., 0.25}, + {90., -45., 0.25}, {10., -10., 0.25}, {20., -20., 0.25}, {30., -30., 0.25}, {40., -40., 0.25}, + {50., -50., 0.25}, {60., -60., 0.25}, {70., -70., 0.25}, {80., -80., 0.25}, {90., -90., 0.25}, + {60., -60., 0.6}, {90., -45., 0.16}, {90., -45., 0.6}, {90., -45., 1.}, {90., -45., 0.}, + {90., -45., 0.1}, {45., 33., 0.7}, {359., 20., 0.1}, {360., 88., 0.9}, {0.1, -89., 1.}, + } ); SECTION( "prototype" ) { Cubic3DInterpolation cubic_interpolation( fs ); From 7d9d68d5941e9b867109cfc068582f36cb43a122 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 28 Aug 2020 23:49:53 +0100 Subject: [PATCH 058/161] ATLAS-296 Replace PointIndex3 with IndexKDTree in KNearestNeighboursBase and derivates --- .../method/knn/GridBoxAverage.cc | 3 +- .../method/knn/GridBoxMaximum.cc | 3 +- .../interpolation/method/knn/GridBoxMethod.cc | 11 ++-- .../interpolation/method/knn/GridBoxMethod.h | 2 +- .../method/knn/KNearestNeighbours.cc | 14 +++-- .../method/knn/KNearestNeighboursBase.cc | 51 ++++++------------- .../method/knn/KNearestNeighboursBase.h | 7 +-- .../method/knn/NearestNeighbour.cc | 11 ++-- 8 files changed, 37 insertions(+), 65 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxAverage.cc b/src/atlas/interpolation/method/knn/GridBoxAverage.cc index 20cbb510b..52852a1ee 100644 --- a/src/atlas/interpolation/method/knn/GridBoxAverage.cc +++ b/src/atlas/interpolation/method/knn/GridBoxAverage.cc @@ -61,7 +61,6 @@ void GridBoxAverage::do_execute( const Field& source, Field& target ) const { functionspace::Points tgt = target_; ATLAS_ASSERT( tgt ); - ATLAS_ASSERT( pTree_ != nullptr ); ATLAS_ASSERT( searchRadius_ > 0. ); ATLAS_ASSERT( !sourceBoxes_.empty() ); ATLAS_ASSERT( !targetBoxes_.empty() ); @@ -88,7 +87,7 @@ void GridBoxAverage::do_execute( const Field& source, Field& target ) const { for ( auto p : tgt.iterate().xyz() ) { ++progress; - if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { + if ( intersect( i, targetBoxes_.at( i ), pTree_.closestPointsWithinRadius( p, searchRadius_ ), triplets ) ) { auto& y = yarray[i]; for ( auto& t : triplets ) { y += xarray[t.col()] * t.value(); diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc index 573fdb623..ae3ba22aa 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc @@ -87,7 +87,6 @@ void GridBoxMaximum::do_execute( const Field& source, Field& target ) const { functionspace::Points tgt = target_; ATLAS_ASSERT( tgt ); - ATLAS_ASSERT( pTree_ != nullptr ); ATLAS_ASSERT( searchRadius_ > 0. ); ATLAS_ASSERT( !sourceBoxes_.empty() ); ATLAS_ASSERT( !targetBoxes_.empty() ); @@ -101,7 +100,7 @@ void GridBoxMaximum::do_execute( const Field& source, Field& target ) const { for ( auto p : tgt.iterate().xyz() ) { ++progress; - if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { + if ( intersect( i, targetBoxes_.at( i ), pTree_.closestPointsWithinRadius( p, searchRadius_ ), triplets ) ) { auto triplet = std::max_element( triplets.begin(), triplets.end(), []( const Triplet& a, const Triplet& b ) { return !eckit::types::is_approximately_greater_or_equal( a.value(), b.value() ); diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 5cbc3ab28..da9aa6604 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -48,8 +48,7 @@ void GridBoxMethod::do_setup( const FunctionSpace& /*source*/, const FunctionSpa ATLAS_NOTIMPLEMENTED; } - -bool GridBoxMethod::intersect( size_t i, const GridBox& box, const PointIndex3::NodeList& closest, +bool GridBoxMethod::intersect( size_t i, const GridBox& box, const util::IndexKDTree::ValueList& closest, std::vector& triplets ) const { ASSERT( !closest.empty() ); @@ -106,7 +105,6 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { target_ = tgt; buildPointSearchTree( src ); - ATLAS_ASSERT( pTree_ != nullptr ); sourceBoxes_ = GridBoxes( source ); targetBoxes_ = GridBoxes( target ); @@ -130,11 +128,10 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { std::vector triplets; size_t i = 0; for ( auto p : tgt.iterate().xyz() ) { - if ( ++progress ) { - Log::info() << "GridBoxMethod: " << *pTree_ << std::endl; - } + ++progress; - if ( intersect( i, targetBoxes_.at( i ), pTree_->findInSphere( p, searchRadius_ ), triplets ) ) { + if ( intersect( i, targetBoxes_.at( i ), pTree_.closestPointsWithinRadius( p, searchRadius_ ), + triplets ) ) { std::copy( triplets.begin(), triplets.end(), std::back_inserter( allTriplets ) ); } diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index cb38e448f..935e58572 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -43,7 +43,7 @@ class GridBoxMethod : public KNearestNeighboursBase { virtual const FunctionSpace& source() const override { return source_; } virtual const FunctionSpace& target() const override { return target_; } - bool intersect( size_t i, const GridBox& iBox, const PointIndex3::NodeList&, std::vector& ) const; + bool intersect( size_t i, const GridBox& iBox, const util::IndexKDTree::ValueList&, std::vector& ) const; virtual void do_execute( const FieldSet& source, FieldSet& target ) const = 0; virtual void do_execute( const Field& source, Field& target ) const = 0; diff --git a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc index 8924fdd7c..35e4d9afd 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc @@ -26,6 +26,7 @@ #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "atlas/runtime/Trace.h" +#include "atlas/util/CoordinateEnums.h" namespace atlas { namespace interpolation { @@ -74,11 +75,8 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp // build point-search tree buildPointSearchTree( meshSource ); - ATLAS_ASSERT( pTree_ != nullptr ); - // generate 3D point coordinates - mesh::actions::BuildXYZField( "xyz" )( meshTarget ); - array::ArrayView coords = array::make_view( meshTarget.nodes().field( "xyz" ) ); + array::ArrayView lonlat = array::make_view( meshTarget.nodes().lonlat() ); size_t inp_npts = meshSource.nodes().size(); size_t out_npts = meshTarget.nodes().size(); @@ -91,6 +89,7 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp std::vector weights; + Log::debug() << "Computing interpolation weights for " << out_npts << " points." << std::endl; for ( size_t ip = 0; ip < out_npts; ++ip ) { if ( ip && ( ip % 1000 == 0 ) ) { auto elapsed = timer.elapsed(); @@ -101,8 +100,7 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp } // find the closest input points to the output point - PointIndex3::Point p{coords( ip, (size_t)0 ), coords( ip, (size_t)1 ), coords( ip, (size_t)2 )}; - PointIndex3::NodeList nn = pTree_->kNearestNeighbours( p, k_ ); + auto nn = pTree_.closestPoints( PointLonLat{lonlat( ip, LON ), lonlat( ip, LAT )}, k_ ); // calculate weights (individual and total, to normalise) using distance // squared @@ -112,8 +110,8 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp double sum = 0; for ( size_t j = 0; j < npts; ++j ) { - PointIndex3::Point np = nn[j].point(); - const double d2 = eckit::geometry::Point3::distance2( p, np ); + const double d = nn[j].distance(); + const double d2 = d * d; weights[j] = 1. / ( 1. + d2 ); sum += weights[j]; diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc index e2dade3f7..106271866 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc @@ -17,6 +17,7 @@ #include "atlas/mesh/Nodes.h" #include "atlas/mesh/actions/BuildXYZField.h" #include "atlas/runtime/Trace.h" +#include "atlas/util/CoordinateEnums.h" namespace atlas { namespace interpolation { @@ -25,57 +26,37 @@ namespace method { void KNearestNeighboursBase::buildPointSearchTree( Mesh& meshSource ) { eckit::TraceTimer tim( "KNearestNeighboursBase::buildPointSearchTree()" ); - // generate 3D point coordinates - mesh::actions::BuildXYZField( "xyz" )( meshSource ); - array::ArrayView coords = array::make_view( meshSource.nodes().field( "xyz" ) ); - // build point-search tree - pTree_.reset( new PointIndex3 ); + array::ArrayView lonlat = array::make_view( meshSource.nodes().lonlat() ); static bool fastBuildKDTrees = eckit::Resource( "$ATLAS_FAST_BUILD_KDTREES", true ); if ( fastBuildKDTrees ) { - std::vector pidx; - pidx.reserve( meshSource.nodes().size() ); - for ( idx_t ip = 0; ip < meshSource.nodes().size(); ++ip ) { - PointIndex3::Point p{coords( ip, 0 ), coords( ip, 1 ), coords( ip, 2 )}; - pidx.emplace_back( p, ip ); - } - pTree_->build( pidx.begin(), pidx.end() ); + pTree_.reserve( lonlat.shape( 0 ) ); } - else { - for ( idx_t ip = 0; ip < meshSource.nodes().size(); ++ip ) { - PointIndex3::Point p{coords( ip, 0 ), coords( ip, 1 ), coords( ip, 2 )}; - pTree_->insert( PointIndex3::Value( p, ip ) ); - } + for ( idx_t ip = 0; ip < lonlat.shape( 0 ); ++ip ) { + pTree_.insert( PointLonLat( lonlat( ip, LON ), lonlat( ip, LAT ) ), ip ); } + pTree_.build(); + + + // generate 3D point coordinates + mesh::actions::BuildXYZField( "xyz" )( meshSource ); } -void KNearestNeighboursBase::buildPointSearchTree(const functionspace::Points& points ) { +void KNearestNeighboursBase::buildPointSearchTree( const functionspace::Points& points ) { eckit::TraceTimer tim( "KNearestNeighboursBase::buildPointSearchTree()" ); - // build point-search tree - pTree_.reset( new PointIndex3 ); - static bool fastBuildKDTrees = eckit::Resource( "$ATLAS_FAST_BUILD_KDTREES", true ); if ( fastBuildKDTrees ) { - std::vector pidx; - pidx.reserve( points.size() ); - - idx_t ip = 0; - for ( auto p : points.iterate().xyz() ) { - pidx.emplace_back( p, ip++ ); - } - - pTree_->build( pidx.begin(), pidx.end() ); + pTree_.reserve( points.size() ); } - else { - idx_t ip = 0; - for ( auto p : points.iterate().xyz() ) { - pTree_->insert( PointIndex3::Value( p, ip++ ) ); - } + size_t ip{0}; + for ( auto p : points.iterate().lonlat() ) { + pTree_.insert( p, ip++ ); } + pTree_.build(); } } // namespace method diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h index 0265fdd94..9faec90e4 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h @@ -12,9 +12,10 @@ #include -#include "atlas/interpolation/method/Method.h" -#include "atlas/interpolation/method/PointIndex3.h" #include "atlas/functionspace/Points.h" +#include "atlas/mesh/Mesh.h" +#include "atlas/interpolation/method/Method.h" +#include "atlas/util/KDTree.h" namespace atlas { namespace interpolation { @@ -29,7 +30,7 @@ class KNearestNeighboursBase : public Method { void buildPointSearchTree( Mesh& meshSource ); void buildPointSearchTree( const functionspace::Points& ); - std::unique_ptr pTree_; + util::IndexKDTree pTree_; }; } // namespace method diff --git a/src/atlas/interpolation/method/knn/NearestNeighbour.cc b/src/atlas/interpolation/method/knn/NearestNeighbour.cc index cb8f6159d..71cad310f 100644 --- a/src/atlas/interpolation/method/knn/NearestNeighbour.cc +++ b/src/atlas/interpolation/method/knn/NearestNeighbour.cc @@ -26,6 +26,7 @@ #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "atlas/runtime/Trace.h" +#include "atlas/util/CoordinateEnums.h" namespace atlas { namespace interpolation { @@ -68,11 +69,8 @@ void NearestNeighbour::do_setup( const FunctionSpace& source, const FunctionSpac // build point-search tree buildPointSearchTree( meshSource ); - ATLAS_ASSERT( pTree_ != nullptr ); - // generate 3D point coordinates - mesh::actions::BuildXYZField( "xyz" )( meshTarget ); - array::ArrayView coords = array::make_view( meshTarget.nodes().field( "xyz" ) ); + array::ArrayView lonlat = array::make_view( meshTarget.nodes().lonlat() ); size_t inp_npts = meshSource.nodes().size(); size_t out_npts = meshTarget.nodes().size(); @@ -92,9 +90,8 @@ void NearestNeighbour::do_setup( const FunctionSpace& source, const FunctionSpac } // find the closest input point to the output point - PointIndex3::Point p{coords( ip, (size_t)0 ), coords( ip, (size_t)1 ), coords( ip, (size_t)2 )}; - PointIndex3::NodeInfo nn = pTree_->nearestNeighbour( p ); - size_t jp = nn.payload(); + auto nn = pTree_.closestPoint( PointLonLat{lonlat( ip, LON ), lonlat( ip, LAT )} ); + size_t jp = nn.payload(); // insert the weights into the interpolant matrix ATLAS_ASSERT( jp < inp_npts ); From b3fa9a85b2701f93825c6c7e10f8b4162ff3a83d Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 2 Sep 2020 16:46:55 +0100 Subject: [PATCH 059/161] ATLAS-269 Replace use of functionspace::Points with functionspace::PointCloud --- src/atlas/functionspace.h | 1 - src/atlas/functionspace/PointCloud.h | 2 ++ .../method/knn/GridBoxAverage.cc | 6 ++-- .../method/knn/GridBoxMaximum.cc | 6 ++-- .../interpolation/method/knn/GridBoxMethod.cc | 6 ++-- .../method/knn/KNearestNeighboursBase.cc | 29 +++++++++++++++---- .../method/knn/KNearestNeighboursBase.h | 5 ++-- 7 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/atlas/functionspace.h b/src/atlas/functionspace.h index 907824424..01de76bc5 100644 --- a/src/atlas/functionspace.h +++ b/src/atlas/functionspace.h @@ -17,6 +17,5 @@ #include "atlas/functionspace/FunctionSpace.h" #include "atlas/functionspace/NodeColumns.h" #include "atlas/functionspace/PointCloud.h" -#include "atlas/functionspace/Points.h" #include "atlas/functionspace/Spectral.h" #include "atlas/functionspace/StructuredColumns.h" diff --git a/src/atlas/functionspace/PointCloud.h b/src/atlas/functionspace/PointCloud.h index d7f9b3442..edeb379ab 100644 --- a/src/atlas/functionspace/PointCloud.h +++ b/src/atlas/functionspace/PointCloud.h @@ -48,6 +48,8 @@ class PointCloud : public functionspace::FunctionSpaceImpl { virtual Field createField( const eckit::Configuration& ) const override; virtual Field createField( const Field&, const eckit::Configuration& ) const override; + void haloExchange( const FieldSet&, bool /*on_device*/ = false ) const override {} + void haloExchange( const Field&, bool /*on_device*/ = false ) const override {} template class IteratorT { diff --git a/src/atlas/interpolation/method/knn/GridBoxAverage.cc b/src/atlas/interpolation/method/knn/GridBoxAverage.cc index 52852a1ee..2cb91708e 100644 --- a/src/atlas/interpolation/method/knn/GridBoxAverage.cc +++ b/src/atlas/interpolation/method/knn/GridBoxAverage.cc @@ -16,7 +16,7 @@ #include "eckit/log/ProgressTimer.h" #include "atlas/array.h" -#include "atlas/functionspace/Points.h" +#include "atlas/functionspace/PointCloud.h" #include "atlas/interpolation/method/MethodFactory.h" #include "atlas/runtime/Exception.h" @@ -58,7 +58,7 @@ void GridBoxAverage::do_execute( const Field& source, Field& target ) const { // ensure GridBoxMethod::setup() - functionspace::Points tgt = target_; + functionspace::PointCloud tgt = target_; ATLAS_ASSERT( tgt ); ATLAS_ASSERT( searchRadius_ > 0. ); @@ -84,7 +84,7 @@ void GridBoxAverage::do_execute( const Field& source, Field& target ) const { std::vector triplets; size_t i = 0; - for ( auto p : tgt.iterate().xyz() ) { + for ( auto p : tgt.iterate().lonlat() ) { ++progress; if ( intersect( i, targetBoxes_.at( i ), pTree_.closestPointsWithinRadius( p, searchRadius_ ), triplets ) ) { diff --git a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc index ae3ba22aa..d06fd71ef 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMaximum.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMaximum.cc @@ -19,7 +19,7 @@ #include "eckit/types/FloatCompare.h" #include "atlas/array.h" -#include "atlas/functionspace/Points.h" +#include "atlas/functionspace/PointCloud.h" #include "atlas/interpolation/method/MethodFactory.h" #include "atlas/runtime/Exception.h" @@ -84,7 +84,7 @@ void GridBoxMaximum::do_execute( const Field& source, Field& target ) const { // ensure GridBoxMethod::setup() - functionspace::Points tgt = target_; + functionspace::PointCloud tgt = target_; ATLAS_ASSERT( tgt ); ATLAS_ASSERT( searchRadius_ > 0. ); @@ -97,7 +97,7 @@ void GridBoxMaximum::do_execute( const Field& source, Field& target ) const { std::vector triplets; size_t i = 0; - for ( auto p : tgt.iterate().xyz() ) { + for ( auto p : tgt.iterate().lonlat() ) { ++progress; if ( intersect( i, targetBoxes_.at( i ), pTree_.closestPointsWithinRadius( p, searchRadius_ ), triplets ) ) { diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index da9aa6604..49986f02f 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -97,8 +97,8 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { ATLAS_ASSERT( source ); ATLAS_ASSERT( target ); - functionspace::Points src( source ); - functionspace::Points tgt( target ); + functionspace::PointCloud src( source ); + functionspace::PointCloud tgt( target ); ATLAS_ASSERT( src ); ATLAS_ASSERT( tgt ); source_ = src; @@ -127,7 +127,7 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { std::vector triplets; size_t i = 0; - for ( auto p : tgt.iterate().xyz() ) { + for ( auto p : tgt.iterate().lonlat() ) { ++progress; if ( intersect( i, targetBoxes_.at( i ), pTree_.closestPointsWithinRadius( p, searchRadius_ ), diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc index 106271866..e661036ef 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc @@ -12,6 +12,8 @@ #include "eckit/log/TraceTimer.h" #include "atlas/array.h" +#include "atlas/functionspace/PointCloud.h" +#include "atlas/functionspace/Points.h" #include "atlas/interpolation/method/knn/KNearestNeighboursBase.h" #include "atlas/library/Library.h" #include "atlas/mesh/Nodes.h" @@ -44,17 +46,34 @@ void KNearestNeighboursBase::buildPointSearchTree( Mesh& meshSource ) { mesh::actions::BuildXYZField( "xyz" )( meshSource ); } -void KNearestNeighboursBase::buildPointSearchTree( const functionspace::Points& points ) { +namespace { + +template +void insert_tree( util::IndexKDTree& tree, const FunctionSpace_type& functionspace ) { + size_t ip{0}; + for ( auto p : functionspace.iterate().lonlat() ) { + tree.insert( p, ip++ ); + } +} + +} // namespace + +void KNearestNeighboursBase::buildPointSearchTree( const FunctionSpace& functionspace ) { eckit::TraceTimer tim( "KNearestNeighboursBase::buildPointSearchTree()" ); static bool fastBuildKDTrees = eckit::Resource( "$ATLAS_FAST_BUILD_KDTREES", true ); if ( fastBuildKDTrees ) { - pTree_.reserve( points.size() ); + pTree_.reserve( functionspace.size() ); } - size_t ip{0}; - for ( auto p : points.iterate().lonlat() ) { - pTree_.insert( p, ip++ ); + if ( functionspace::Points fs = functionspace ) { + insert_tree( pTree_, fs ); + } + else if ( functionspace::PointCloud fs = functionspace ) { + insert_tree( pTree_, fs ); + } + else { + ATLAS_NOTIMPLEMENTED; } pTree_.build(); } diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h index 9faec90e4..42d478c1f 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h @@ -12,9 +12,8 @@ #include -#include "atlas/functionspace/Points.h" -#include "atlas/mesh/Mesh.h" #include "atlas/interpolation/method/Method.h" +#include "atlas/mesh/Mesh.h" #include "atlas/util/KDTree.h" namespace atlas { @@ -28,7 +27,7 @@ class KNearestNeighboursBase : public Method { protected: void buildPointSearchTree( Mesh& meshSource ); - void buildPointSearchTree( const functionspace::Points& ); + void buildPointSearchTree( const FunctionSpace& ); util::IndexKDTree pTree_; }; From 9fd3214546686b1bfe96d2e8e68be6ea6e292f81 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 3 Sep 2020 10:59:38 +0100 Subject: [PATCH 060/161] ATLAS-269 Remove unused functionspace::Points --- src/atlas/CMakeLists.txt | 2 - src/atlas/functionspace/Points.cc | 141 ------------------ src/atlas/functionspace/Points.h | 140 ----------------- .../method/knn/KNearestNeighboursBase.cc | 6 +- 4 files changed, 1 insertion(+), 288 deletions(-) delete mode 100644 src/atlas/functionspace/Points.cc delete mode 100644 src/atlas/functionspace/Points.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index d8968ffbb..67e9fc004 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -404,8 +404,6 @@ functionspace/Spectral.h functionspace/Spectral.cc functionspace/PointCloud.h functionspace/PointCloud.cc -functionspace/Points.h -functionspace/Points.cc functionspace/detail/FunctionSpaceImpl.h functionspace/detail/FunctionSpaceImpl.cc functionspace/detail/FunctionSpaceInterface.h diff --git a/src/atlas/functionspace/Points.cc b/src/atlas/functionspace/Points.cc deleted file mode 100644 index 0e68f3a90..000000000 --- a/src/atlas/functionspace/Points.cc +++ /dev/null @@ -1,141 +0,0 @@ -/* - * (C) Copyright 2013 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - - -#include "atlas/functionspace/Points.h" - -// #include - -#include "atlas/array.h" -#include "atlas/grid/Grid.h" -#include "atlas/grid/Iterator.h" -#include "atlas/option/Options.h" -#include "atlas/runtime/Exception.h" -#include "atlas/runtime/Log.h" -#include "atlas/util/CoordinateEnums.h" -#include "atlas/util/Earth.h" - - -namespace atlas { -namespace functionspace { -namespace detail { - - -Points::Points( const Grid& grid ) : - lonlat_( "lonlat", array::make_datatype(), array::make_shape( grid.size(), 2 ) ) { - auto lonlat = array::make_view( lonlat_ ); - - idx_t j = 0; - for ( auto p : grid.lonlat() ) { - lonlat( j, LON ) = p.lon(); - lonlat( j, LAT ) = p.lat(); - - // shouldn't be necessary - // lonlat( j, LAT ) = std::min(90., std::max(-90., p.lat())); - ++j; - } -} - - -Points::~Points() = default; - - -Field Points::createField( const Field& other, const eckit::Configuration& config ) const { - return createField( option::datatype( other.datatype() ) | config ); -} - - -Field Points::createField( const eckit::Configuration& config ) const { - array::DataType::kind_t kind; - if ( !config.get( "datatype", kind ) ) { - throw_Exception( "datatype missing", Here() ); - } - - Field field( config.getString( "name", "" ), array::DataType( kind ), array::make_shape( size() ) ); - field.set_functionspace( this ); - return field; -} - - -void Points::haloExchange( const FieldSet&, bool ) const { - Log::debug() << "Points::haloExchange: ignored" << std::endl; -} - - -void Points::haloExchange( const Field&, bool ) const { - Log::debug() << "Points::haloExchange: ignored" << std::endl; -} - - -idx_t Points::size() const { - return lonlat_.shape( 0 ); -} - - -size_t Points::footprint() const { - return sizeof( *this ); -} - - -std::string Points::distribution() const { - return std::string( "serial" ); -} - - -std::string Points::type() const { - return "Points"; -} - - -atlas::Field Points::ghost() const { - if ( not ghost_ ) { - ghost_ = Field( "ghost", array::make_datatype(), array::make_shape( size() ) ); - array::make_view( ghost_ ).assign( 0 ); - } - return ghost_; -} - - -template -T Points::IteratorT::operator*() const { - ATLAS_NOTIMPLEMENTED; -} - - -template <> -PointXYZ Points::IteratorT::operator*() const { - PointXYZ q; - Point2 p = {view_( n_, LON ), view_( n_, LAT )}; - util::Earth::convertSphericalToCartesian( p, q ); - return q; -} - - -template <> -PointLonLat Points::IteratorT::operator*() const { - return {view_( n_, LON ), view_( n_, LAT )}; -} - - -} // namespace detail - - -Points::Points( const Grid& grid ) : - FunctionSpace( new detail::Points( grid ) ), - functionspace_( dynamic_cast( get() ) ) {} - - -Points::Points( const FunctionSpace& functionspace ) : - FunctionSpace( functionspace ), - functionspace_( dynamic_cast( get() ) ) {} - - -} // namespace functionspace -} // namespace atlas diff --git a/src/atlas/functionspace/Points.h b/src/atlas/functionspace/Points.h deleted file mode 100644 index 3b5714385..000000000 --- a/src/atlas/functionspace/Points.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * (C) Copyright 2013 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - - -#pragma once - -#include "atlas/array/ArrayView.h" -#include "atlas/field/Field.h" -#include "atlas/functionspace/FunctionSpace.h" -#include "atlas/functionspace/detail/FunctionSpaceImpl.h" -#include "atlas/util/Point.h" - - -namespace atlas { -class Grid; -} - - -namespace atlas { -namespace functionspace { - - -namespace detail { - -class Points final : public FunctionSpaceImpl { -public: - Points( const Grid& ); - ~Points() override; - - using FunctionSpaceImpl::createField; - Field createField( const Field&, const eckit::Configuration& ) const override; - Field createField( const eckit::Configuration& ) const override; - - void haloExchange( const FieldSet&, bool /*on_device*/ = false ) const override; - void haloExchange( const Field&, bool /*on_device*/ = false ) const override; - - idx_t size() const override; - size_t footprint() const override; - std::string distribution() const override; - std::string type() const override; - - atlas::Field lonlat() const override { return lonlat_; } - atlas::Field ghost() const override; - - - template - struct IteratorT { - IteratorT( const Field& field, bool begin ) : - lonlat_( field ), - view_( array::make_view( lonlat_ ) ), - m_( lonlat_.shape( 0 ) ), - n_( begin ? 0 : m_ ) {} - - bool next( T& p ) { - if ( n_ < m_ ) { - p = operator*(); - ++n_; - return true; - } - return false; - } - - const IteratorT& operator++() { - ++n_; - return *this; - } - - T operator*() const; - - bool operator==( const IteratorT& other ) const { return n_ == other.n_; } - bool operator!=( const IteratorT& other ) const { return n_ != other.n_; } - - private: - const Field lonlat_; - const array::ArrayView view_; - const idx_t m_; - idx_t n_; - }; - - - template - struct IterateT { - using iterator = IteratorT; - using const_iterator = iterator; - - IterateT( const Field& lonlat ) : lonlat_( lonlat ) {} - iterator begin() const { return {lonlat_, true}; } - iterator end() const { return {lonlat_, false}; } - - private: - const Field lonlat_; - }; - - - struct Iterate { - Iterate( const Points& points ) : lonlat_( points.lonlat() ) {} - IterateT xyz() const { return {lonlat_}; } - IterateT lonlat() const { return {lonlat_}; } - - private: - const Field lonlat_; - }; - - Iterate iterate() const { return Iterate( *this ); } - -private: - Field lonlat_; - mutable Field ghost_; -}; - - -} // namespace detail - - -class Points : public FunctionSpace { -public: - Points( const Grid& ); - Points( const FunctionSpace& ); - - operator bool() const { return functionspace_ != nullptr; } - - Field lonlat() const { return functionspace_->lonlat(); } - Field ghost() const { return functionspace_->ghost(); } - - detail::Points::Iterate iterate() const { return functionspace_->iterate(); } - -private: - const detail::Points* functionspace_; -}; - - -} // namespace functionspace -} // namespace atlas diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc index e661036ef..6caa945fa 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc @@ -13,7 +13,6 @@ #include "atlas/array.h" #include "atlas/functionspace/PointCloud.h" -#include "atlas/functionspace/Points.h" #include "atlas/interpolation/method/knn/KNearestNeighboursBase.h" #include "atlas/library/Library.h" #include "atlas/mesh/Nodes.h" @@ -66,10 +65,7 @@ void KNearestNeighboursBase::buildPointSearchTree( const FunctionSpace& function if ( fastBuildKDTrees ) { pTree_.reserve( functionspace.size() ); } - if ( functionspace::Points fs = functionspace ) { - insert_tree( pTree_, fs ); - } - else if ( functionspace::PointCloud fs = functionspace ) { + if ( functionspace::PointCloud fs = functionspace ) { insert_tree( pTree_, fs ); } else { From 44e39119e2ad6346c8d88633de9a63fbbd282bfa Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 3 Sep 2020 15:07:32 +0100 Subject: [PATCH 061/161] ATLAS-269 Disable checks in test_interpolation_k_nearest_neighbours as they are not reproducible across platforms --- ...test_interpolation_k_nearest_neighbours.cc | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc b/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc index 1df3e313c..034e796d6 100644 --- a/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc +++ b/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc @@ -29,29 +29,25 @@ namespace test { class Access { public: - Access( const Interpolation& interpolation ) : interpolation_{interpolation} { - } - const Interpolation::Implementation::Matrix& matrix() const { - return interpolation_.get()->matrix_; - } + Access( const Interpolation& interpolation ) : interpolation_{interpolation} {} + const Interpolation::Implementation::Matrix& matrix() const { return interpolation_.get()->matrix_; } Interpolation interpolation_; - std::string hash() - { + std::string hash() { eckit::MD5 hash; - const auto& m = matrix(); + const auto& m = matrix(); const auto outer = m.outer(); const auto index = m.inner(); const auto weight = m.data(); - idx_t rows = static_cast( m.rows() ); - hash.add(rows); - for( idx_t r = 0; r < rows; ++r ) { + idx_t rows = static_cast( m.rows() ); + hash.add( rows ); + for ( idx_t r = 0; r < rows; ++r ) { //v_tgt( r ) = 0.; for ( idx_t c = outer[r]; c < outer[r + 1]; ++c ) { - hash.add(c); + hash.add( c ); idx_t n = index[c]; - hash.add(n); + hash.add( n ); //Value w = static_cast( weight[c] ); //v_tgt( r ) += w * v_src( n ); } @@ -68,10 +64,15 @@ CASE( "test_interpolation_k_nearest_neighbours" ) { Grid gridB( "O64" ); Grid gridC( "O32" ); - Interpolation a( option::type("k-nearest-neighbours"), gridA, gridB ); - Interpolation b( option::type("k-nearest-neighbours"), gridB, gridC ); - EXPECT( Access{a}.hash() == "5ecc9d615dcf7112b5a97b38341099a5" ); - EXPECT( Access{b}.hash() == "bf4b4214ef50387ba00a1950b95e0d93" ); + Interpolation a( option::type( "k-nearest-neighbours" ), gridA, gridB ); + Interpolation b( option::type( "k-nearest-neighbours" ), gridB, gridC ); + ATLAS_DEBUG_VAR( Access{a}.hash() ); + ATLAS_DEBUG_VAR( Access{b}.hash() ); + + // Following EXPECT are commented because they are not bit-identical across platforms. + // This indicates that not every platform finds the same nearest neighbours!!! + //EXPECT( Access{a}.hash() == "5ecc9d615dcf7112b5a97b38341099a5" ); + //EXPECT( Access{b}.hash() == "bf4b4214ef50387ba00a1950b95e0d93" ); } From bc625c2c0b557e5a8d049b9ca36cb22d0d961446 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 9 Sep 2020 11:00:03 +0100 Subject: [PATCH 062/161] Update PGI community edition installation --- .travis.yml | 2 +- tools/install-pgi.sh | 131 +++++++++++++++++-------------------------- 2 files changed, 52 insertions(+), 81 deletions(-) diff --git a/.travis.yml b/.travis.yml index db94b6ba7..22062de59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -129,7 +129,7 @@ install: - | ### Install PGI community edition if [[ "${PGI_VERSION:-notset}" == "CommunityEdition" ]]; then - install-pgi.sh --mpi --prefix ${DEPS_DIR}/pgi + install-pgi.sh --prefix ${DEPS_DIR}/pgi source ${DEPS_DIR}/pgi/env.sh fi diff --git a/tools/install-pgi.sh b/tools/install-pgi.sh index 72448ecfa..a63c5bbeb 100755 --- a/tools/install-pgi.sh +++ b/tools/install-pgi.sh @@ -12,119 +12,90 @@ # See for # details. -TEMPORARY_FILES="/tmp" - -export PGI_SILENT=true -export PGI_ACCEPT_EULA=accept -export PGI_INSTALL_DIR="${HOME}/pgi" -export PGI_INSTALL_NVIDIA=false -export PGI_INSTALL_AMD=false -export PGI_INSTALL_JAVA=false -export PGI_INSTALL_MPI=false -export PGI_MPI_GPU_SUPPORT=false -export PGI_INSTALL_MANAGED=false - - -VERBOSE=false - +TEMPORARY_FILES="${TMPDIR:-/tmp}" +export NVHPC_INSTALL_DIR=$(pwd)/pgi-install +export NVHPC_SILENT=true while [ $# != 0 ]; do case "$1" in "--prefix") - export PGI_INSTALL_DIR="$2"; shift + export NVHPC_INSTALL_DIR="$2"; shift ;; "--tmpdir") TEMPORARY_FILES="$2"; shift ;; - "--nvidia") - export PGI_INSTALL_NVIDIA=true - ;; - "--amd") - export PGI_INSTALL_AMD=true - ;; - "--java") - export PGI_INSTALL_JAVA=true - ;; - "--mpi") - export PGI_INSTALL_MPI=true - ;; - "--mpi-gpu") - export PGI_INSTALL_MPI_GPU=true - ;; - "--managed") - export PGI_INSTALL_MANAGED=true - ;; "--verbose") - VERBOSE=true; + export NVHPC_SILENT=false; ;; - *) + *) echo "Unrecognized argument '$1'" exit 1 ;; - esac + esac shift done -if [ ! -e "${TEMPORARY_FILES}" ]; then - mkdir -p "${TEMPORARY_FILES}" -fi -cd "${TEMPORARY_FILES}" - -PGI_URL="https://www.pgroup.com/support/downloader.php?file=pgi-community-linux-x64" - -curl --location \ - --user-agent "pgi-travis (https://github.com/nemequ/pgi-travis; ${TRAVIS_REPO_SLUG})" \ - --referer "http://www.pgroup.com/products/community.htm" \ - --header "X-Travis-Build-Number: ${TRAVIS_BUILD_NUMBER}" \ - --header "X-Travis-Event-Type: ${TRAVIS_EVENT_TYPE}" \ - --header "X-Travis-Job-Number: ${TRAVIS_JOB_NUMBER}" \ - "${PGI_URL}" | tar zxf - +case "$(uname -m)" in + x86_64|ppc64le|aarch64) + ;; + *) + echo "Unknown architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +URL="$(curl -s 'https://developer.nvidia.com/nvidia-hpc-sdk-download' | grep -oP "https://developer.download.nvidia.com/hpc-sdk/nvhpc_([0-9]{4})_([0-9]+)_Linux_$(uname -m)_cuda_([0-9\.]+).tar.gz")" +FOLDER="$(basename "$(echo "${URL}" | grep -oP '[^/]+$')" .tar.gz)" + +if [ ! -z "${TRAVIS_REPO_SLUG}" ]; then + curl --location \ + --user-agent "pgi-travis (https://github.com/nemequ/pgi-travis; ${TRAVIS_REPO_SLUG})" \ + --referer "http://www.pgroup.com/products/community.htm" \ + --header "X-Travis-Build-Number: ${TRAVIS_BUILD_NUMBER}" \ + --header "X-Travis-Event-Type: ${TRAVIS_EVENT_TYPE}" \ + --header "X-Travis-Job-Number: ${TRAVIS_JOB_NUMBER}" \ + "${URL}" | tar zx -C "${TEMPORARY_FILES}" +else + + if [ ! -d "${TEMPORARY_FILES}/${FOLDER}" ]; then + echo "Downloading ${TEMPORARY_FILES}/${FOLDER} from URL [${URL}]" + curl --location \ + --user-agent "pgi-travis (https://github.com/nemequ/pgi-travis)" \ + "${URL}" | tar zx -C "${TEMPORARY_FILES}" + else + echo "Download already present in ${TEMPORARY_FILES}/${FOLDER}" + fi -if [ x"${VERBOSE}" = "xtrue" ]; then - VERBOSE_SHORT="-v" - VERBOSE_V="v" fi -cd "${TEMPORARY_FILES}"/install_components && ./install +echo "+ ${TEMPORARY_FILES}/${FOLDER}/install" +"${TEMPORARY_FILES}/${FOLDER}/install" -PGI_VERSION=$(basename "${PGI_INSTALL_DIR}"/linux86-64/*.*/) +#comment out to cleanup +#rm -rf "${TEMPORARY_FILES}/${FOLDER}" + +PGI_VERSION=$(basename "${NVHPC_INSTALL_DIR}"/Linux_x86_64/*.*/) # Use gcc which is available in PATH -${PGI_INSTALL_DIR}/linux86-64/${PGI_VERSION}/bin/makelocalrc \ - -x ${PGI_INSTALL_DIR}/linux86-64/${PGI_VERSION}/bin \ +${NVHPC_INSTALL_DIR}/Linux_x86_64/${PGI_VERSION}/compilers/bin/makelocalrc \ + -x ${NVHPC_INSTALL_DIR}/Linux_x86_64/${PGI_VERSION}/compilers/bin \ -gcc $(which gcc) \ -gpp $(which g++) \ -g77 $(which gfortran) -cat > ${PGI_INSTALL_DIR}/env.sh << EOF +cat > ${NVHPC_INSTALL_DIR}/env.sh << EOF ### Variables -PGI_INSTALL_DIR=${PGI_INSTALL_DIR} +PGI_INSTALL_DIR=${NVHPC_INSTALL_DIR} PGI_VERSION=${PGI_VERSION} ### Compilers -PGI_DIR=\${PGI_INSTALL_DIR}/linux86-64/\${PGI_VERSION} -export PATH=\${PGI_DIR}/bin:\${PATH} +PGI_DIR=\${PGI_INSTALL_DIR}/Linux_x86_64/\${PGI_VERSION} +export PATH=\${PGI_DIR}/compilers/bin:\${PATH} EOF -if ${PGI_INSTALL_MPI} ; then -cat >> ${PGI_INSTALL_DIR}/env.sh << EOF +cat >> ${NVHPC_INSTALL_DIR}/env.sh << EOF ### MPI -export MPI_HOME=\${PGI_DIR}/mpi/openmpi +export MPI_HOME=\${PGI_DIR}/comm_libs/mpi export PATH=\${MPI_HOME}/bin:\${PATH} EOF -fi - - -# INSTALL_BINDIR="${HOME}/bin" -# if [ ! -e "${INSTALL_BINDIR}" ]; then -# mkdir -p "${INSTALL_BINDIR}" -# fi -# for file in "${PGI_INSTALL_DIR}"/linux86-64/"${PGI_VERSION}"/bin/*; do -# dest="${INSTALL_BINDIR}/$(basename "${file}")" -# if [ -x "${file}" ]; then -# echo "#!/bin/sh" > "${dest}" -# echo "PGI=${PGI_INSTALL_DIR} PGI_INSTALL=\"\${PGI}\"/linux86-64/${PGI_VERSION} ${file} \$@" >> "${dest}" -# chmod 0755 "${dest}" -# fi -# done \ No newline at end of file From dce1ee48f74e77941645d162d1fd400e167e6acf Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 14 Sep 2020 13:59:18 +0100 Subject: [PATCH 063/161] install-pgi: create tmpdir if doesn't exist --- tools/install-pgi.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/install-pgi.sh b/tools/install-pgi.sh index a63c5bbeb..d416a12ec 100755 --- a/tools/install-pgi.sh +++ b/tools/install-pgi.sh @@ -57,10 +57,11 @@ if [ ! -z "${TRAVIS_REPO_SLUG}" ]; then else if [ ! -d "${TEMPORARY_FILES}/${FOLDER}" ]; then - echo "Downloading ${TEMPORARY_FILES}/${FOLDER} from URL [${URL}]" - curl --location \ - --user-agent "pgi-travis (https://github.com/nemequ/pgi-travis)" \ - "${URL}" | tar zx -C "${TEMPORARY_FILES}" + echo "Downloading ${TEMPORARY_FILES}/${FOLDER} from URL [${URL}]" + mkdir -p ${TEMPORARY_FILES} + curl --location \ + --user-agent "pgi-travis (https://github.com/nemequ/pgi-travis)" \ + "${URL}" | tar zx -C "${TEMPORARY_FILES}" else echo "Download already present in ${TEMPORARY_FILES}/${FOLDER}" fi From 53d062fc558dfb7736ebf4454440d39263bb1907 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 17 Sep 2020 17:03:53 +0100 Subject: [PATCH 064/161] Add Authors list --- AUTHORS | 31 +++++++++++++++++++++++ tools/generate-authors.py | 53 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 AUTHORS create mode 100755 tools/generate-authors.py diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 000000000..874ea2ce3 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,31 @@ +Authors +======= + +- Willem Deconinck +- Pedro Maciel +- Tiago Quintino + +Thanks for conributions from +============================ + +- Carlos Osuna +- Avi Bahra +- Andreas Mueller +- Baudouin Raoult +- Florian Rathgeber +- Daan Degrauwe +- Gianmarco Mengaldo +- Philippe Marguinaud +- James Hawkes +- Mats Hamrud +- Christian Kuehnlein +- Rahul Mahajan +- Olivier Iffrig +- Iain Russell +- Benjamin Menetrier +- Domokos Sármány +- Yannick Trémolet +- Mark J. Olah +- Michael Lange +- Marek Wlasak +- Peter Bispham diff --git a/tools/generate-authors.py b/tools/generate-authors.py new file mode 100755 index 000000000..4733b73ec --- /dev/null +++ b/tools/generate-authors.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Usage: +# generate-authors.py > AUTHORS + + +# Authors +authors = [ + "Willem Deconinck", + "Pedro Maciel", + "Tiago Quintino" +] + +def real_name(name): + # Because some contributors use different names or aliases + alias = { + "cosunae" : "Carlos Osuna", + "carlos osuna" : "Carlos Osuna", + "mengaldo" : "Gianmarco Mengaldo" + } + if name in alias: + return alias[name] + else: + return name + +def contributors(): + # Contributors sorted by number of commits + from subprocess import check_output + command=['git', 'shortlog', '-n', '-s'] + output=check_output(command, universal_newlines=True).strip() + contributors_dict = {} + for line in output.split('\n'): + [commits,name] = line.strip().split('\t',1) + name = real_name(name) + if name in authors: + continue + commits = int(commits) + if name in contributors_dict: + commits += contributors_dict[name] + contributors_dict[name] = commits + return [item[0] for item in sorted(contributors_dict.items(), key = lambda x: x[1], reverse = True)] + +print("Authors") +print("=======\n") +for name in authors: + print("- "+name) + +print("") +print("Thanks for conributions from") +print("============================\n") + +for name in contributors(): + print("- "+name) From 45a5463b6332d6da0e9fe38be72a873631e1bc14 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Thu, 10 Sep 2020 07:02:36 +0000 Subject: [PATCH 065/161] Try to change Atlas communicator using eckit setDefaultComm --- src/tests/parallel/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/tests/parallel/CMakeLists.txt b/src/tests/parallel/CMakeLists.txt index 09939af87..7922b7af5 100644 --- a/src/tests/parallel/CMakeLists.txt +++ b/src/tests/parallel/CMakeLists.txt @@ -14,6 +14,14 @@ ecbuild_add_test( TARGET atlas_test_haloexchange_adjoint ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_setcomm + MPI 3 + CONDITION eckit_HAVE_MPI + SOURCES test_setcomm.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + ecbuild_add_test( TARGET atlas_test_haloexchange MPI 3 CONDITION eckit_HAVE_MPI From 8412f5aa106513ff943ad1f08f9537206232c385 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Thu, 10 Sep 2020 10:02:22 +0000 Subject: [PATCH 066/161] Fix the issue by removing the static _comm variable in atlas/parallel/mpi.h --- src/atlas/parallel/mpi/mpi.h | 3 +- src/tests/parallel/test_setcomm.cc | 46 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 src/tests/parallel/test_setcomm.cc diff --git a/src/atlas/parallel/mpi/mpi.h b/src/atlas/parallel/mpi/mpi.h index 68ae606a6..ac76cb582 100644 --- a/src/atlas/parallel/mpi/mpi.h +++ b/src/atlas/parallel/mpi/mpi.h @@ -20,8 +20,7 @@ namespace mpi { using Comm = eckit::mpi::Comm; inline const Comm& comm() { - static const Comm& _comm = eckit::mpi::comm(); - return _comm; + return eckit::mpi::comm(); } inline idx_t rank() { diff --git a/src/tests/parallel/test_setcomm.cc b/src/tests/parallel/test_setcomm.cc new file mode 100644 index 000000000..c0e564443 --- /dev/null +++ b/src/tests/parallel/test_setcomm.cc @@ -0,0 +1,46 @@ +#include "eckit/mpi/Comm.h" +#include "atlas/library/Library.h" +#include "atlas/util/Config.h" +#include "atlas/parallel/mpi/mpi.h" +#include "tests/AtlasTestEnvironment.h" + +#include +#include + +namespace atlas { +namespace test { + +CASE ("test_setcomm") +{ + + printf (" eckit::mpi::comm () = 0x%llx, eckit::mpi::comm ().size () = %8d\n", &eckit::mpi::comm (), eckit::mpi::comm ().size ()); + std::cout << eckit::mpi::comm ().name () << std::endl; + printf (" atlas::mpi::comm () = 0x%llx, atlas::mpi::comm ().size () = %8d\n", &atlas::mpi::comm (), atlas::mpi::comm ().size ()); + std::cout << atlas::mpi::comm ().name () << std::endl; + + EXPECT_EQ (eckit::mpi::comm ().size (), atlas::mpi::comm ().size ()); + + int irank = eckit::mpi::comm ().rank (); + + auto & comm = eckit::mpi::comm ().split (irank, "SPLIT"); + + eckit::mpi::setCommDefault ("SPLIT"); + + printf (" eckit::mpi::comm () = 0x%llx, eckit::mpi::comm ().size () = %8d\n", &eckit::mpi::comm (), eckit::mpi::comm ().size ()); + std::cout << eckit::mpi::comm ().name () << std::endl; + printf (" atlas::mpi::comm () = 0x%llx, atlas::mpi::comm ().size () = %8d\n", &atlas::mpi::comm (), atlas::mpi::comm ().size ()); + std::cout << atlas::mpi::comm ().name () << std::endl; + + EXPECT_EQ (eckit::mpi::comm ().size (), atlas::mpi::comm ().size ()); + + std::cout << "----- STOP -----" << std::endl; + +} + +} +} + +int main (int argc, char* argv[]) +{ + return atlas::test::run (argc, argv); +} From 9785fbb6a4c260fabab9c3a40d851da016cf6708 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 18 Sep 2020 15:27:50 +0100 Subject: [PATCH 067/161] Fix accidental removal of ATLAS_RUN feature --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c2251258..79e1054f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,7 @@ include( features/CLANG_TIDY ) include( features/INCLUDE_WHAT_YOU_USE ) include( features/INIT_SNAN ) include( features/DOCS ) +include( features/ATLAS_RUN ) ################################################################################ # sources From 430d0880ffd2819f95eb1f9d65b0d0197aba5255 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 18 Sep 2020 17:22:43 +0100 Subject: [PATCH 068/161] ATLAS-310 Fortran API: Add Domain and grid%lonlat_bounding_box() --- src/atlas/domain/detail/Domain.cc | 33 +++ src/atlas/domain/detail/Domain.h | 13 ++ src/atlas/domain/detail/RectangularDomain.cc | 15 ++ src/atlas/grid/detail/grid/Grid.cc | 12 ++ src/atlas/grid/detail/grid/Grid.h | 1 + src/atlas_f/CMakeLists.txt | 4 + src/atlas_f/atlas_module.F90 | 3 + src/atlas_f/domain/atlas_Domain_module.F90 | 199 ++++++++++++++++++ src/atlas_f/grid/atlas_Grid_module.F90 | 12 ++ src/tests/grid/fctest_regional_grids.F90 | 20 ++ .../fctest_stretchedrotatedgaussiangrid.F90 | 8 + 11 files changed, 320 insertions(+) create mode 100644 src/atlas_f/domain/atlas_Domain_module.F90 diff --git a/src/atlas/domain/detail/Domain.cc b/src/atlas/domain/detail/Domain.cc index 74f712aae..d679d4776 100644 --- a/src/atlas/domain/detail/Domain.cc +++ b/src/atlas/domain/detail/Domain.cc @@ -8,6 +8,10 @@ * nor does it submit to any jurisdiction. */ +#include + +#include "eckit/utils/MD5.h" + #include "atlas/domain/detail/Domain.h" #include "atlas/domain/detail/DomainFactory.h" #include "atlas/projection/Projection.h" @@ -33,5 +37,34 @@ const Domain* Domain::create( const eckit::Parametrisation& p ) { throw_Exception( "type missing in Params", Here() ); } +//--------------------------------------------------------------------------------------------------------------------- + +extern "C" { +const Domain* atlas__Domain__ctor_config( const eckit::Parametrisation* config ) { + return Domain::create( *config ); +} +void atlas__Domain__type( const Domain* This, char*& type, int& size ) { + ATLAS_ASSERT( This != nullptr, "Cannot access uninitialised atlas_Domain" ); + std::string s = This->type(); + size = static_cast( s.size() + 1 ); + type = new char[size]; + strcpy( type, s.c_str() ); +} +void atlas__Domain__hash( const Domain* This, char*& hash, int& size ) { + ATLAS_ASSERT( This != nullptr, "Cannot access uninitialised atlas_Domain" ); + eckit::MD5 md5; + This->hash( md5 ); + std::string s = md5.digest(); + size = static_cast( s.size() + 1 ); + hash = new char[size]; + strcpy( hash, s.c_str() ); +} +Domain::Spec* atlas__Domain__spec( const Domain* This ) { + ATLAS_ASSERT( This != nullptr, "Cannot access uninitialised atlas_Domain" ); + return new Domain::Spec( This->spec() ); +} + +} // extern "C" + } // namespace domain } // namespace atlas diff --git a/src/atlas/domain/detail/Domain.h b/src/atlas/domain/detail/Domain.h index 5377789a4..15ba8983b 100644 --- a/src/atlas/domain/detail/Domain.h +++ b/src/atlas/domain/detail/Domain.h @@ -72,5 +72,18 @@ class Domain : public util::Object { virtual void hash( eckit::Hash& ) const = 0; }; +class RectangularDomain; + +extern "C" { +const Domain* atlas__Domain__ctor_config( const eckit::Parametrisation* config ); +void atlas__Domain__type( const Domain* This, char*& type, int& size ); +void atlas__Domain__hash( const Domain* This, char*& hash, int& size ); +Domain::Spec* atlas__Domain__spec( const Domain* This ); +double atlas__LonLatRectangularDomain__north( const RectangularDomain* This ); +double atlas__LonLatRectangularDomain__west( const RectangularDomain* This ); +double atlas__LonLatRectangularDomain__south( const RectangularDomain* This ); +double atlas__LonLatRectangularDomain__east( const RectangularDomain* This ); +} + } // namespace domain } // namespace atlas diff --git a/src/atlas/domain/detail/RectangularDomain.cc b/src/atlas/domain/detail/RectangularDomain.cc index d075ec96a..3c986069f 100644 --- a/src/atlas/domain/detail/RectangularDomain.cc +++ b/src/atlas/domain/detail/RectangularDomain.cc @@ -147,5 +147,20 @@ namespace { static DomainBuilder register_builder( RectangularDomain::static_type() ); } +extern "C" { +double atlas__LonLatRectangularDomain__north( const RectangularDomain* This ) { + return This->ymax(); +} +double atlas__LonLatRectangularDomain__west( const RectangularDomain* This ) { + return This->xmin(); +} +double atlas__LonLatRectangularDomain__south( const RectangularDomain* This ) { + return This->ymin(); +} +double atlas__LonLatRectangularDomain__east( const RectangularDomain* This ) { + return This->xmax(); +} +} + } // namespace domain } // namespace atlas diff --git a/src/atlas/grid/detail/grid/Grid.cc b/src/atlas/grid/detail/grid/Grid.cc index 0083f88cb..2c132a79e 100644 --- a/src/atlas/grid/detail/grid/Grid.cc +++ b/src/atlas/grid/detail/grid/Grid.cc @@ -14,6 +14,7 @@ #include "eckit/utils/MD5.h" +#include "atlas/domain/detail/Domain.h" #include "atlas/grid.h" #include "atlas/grid/detail/grid/GridBuilder.h" #include "atlas/grid/detail/grid/Structured.h" @@ -158,6 +159,17 @@ void atlas__grid__Grid__uid( const Grid* This, char*& uid, int& size ) { strcpy( uid, s.c_str() ); } +Grid::Domain::Implementation* atlas__grid__Grid__lonlat_bounding_box( const Grid* This ) { + Grid::Domain::Implementation* lonlatboundingbox; + { + auto handle = This->lonlatBoundingBox(); + lonlatboundingbox = handle.get(); + lonlatboundingbox->attach(); + } + lonlatboundingbox->detach(); + return lonlatboundingbox; +} + } // namespace grid } // namespace detail diff --git a/src/atlas/grid/detail/grid/Grid.h b/src/atlas/grid/detail/grid/Grid.h index a0e3354ad..d79a76364 100644 --- a/src/atlas/grid/detail/grid/Grid.h +++ b/src/atlas/grid/detail/grid/Grid.h @@ -183,6 +183,7 @@ extern "C" { idx_t atlas__grid__Grid__size( Grid* This ); Grid::Spec* atlas__grid__Grid__spec( Grid* This ); void atlas__grid__Grid__uid( const Grid* This, char*& uid, int& size ); +Grid::Domain::Implementation* atlas__grid__Grid__lonlat_bounding_box( const Grid* This ); } } // namespace grid diff --git a/src/atlas_f/CMakeLists.txt b/src/atlas_f/CMakeLists.txt index a329419c3..ccb6a92b0 100644 --- a/src/atlas_f/CMakeLists.txt +++ b/src/atlas_f/CMakeLists.txt @@ -74,6 +74,9 @@ generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/grid/Partitioner.h) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/projection/detail/ProjectionImpl.h MODULE atlas_projection_c_binding OUTPUT projection_c_binding.f90 ) +generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/domain/detail/Domain.h + MODULE atlas_domain_c_binding + OUTPUT domain_c_binding.f90 ) generate_fortran_bindings(FORTRAN_BINDINGS ../atlas/mesh/detail/MeshIntf.h MODULE atlas_mesh_c_binding OUTPUT atlas_mesh_c_binding.f90 ) @@ -158,6 +161,7 @@ ecbuild_add_library( TARGET atlas_f util/atlas_Metadata_module.F90 util/atlas_allocate_module.F90 output/atlas_output_module.F90 + domain/atlas_Domain_module.F90 functionspace/atlas_FunctionSpace_module.F90 functionspace/atlas_functionspace_EdgeColumns_module.F90 functionspace/atlas_functionspace_NodeColumns_module.fypp diff --git a/src/atlas_f/atlas_module.F90 b/src/atlas_f/atlas_module.F90 index ce7ec777c..9bd07bc58 100644 --- a/src/atlas_f/atlas_module.F90 +++ b/src/atlas_f/atlas_module.F90 @@ -56,6 +56,9 @@ module atlas_module & atlas_Connectivity, & & atlas_MultiBlockConnectivity, & & atlas_BlockConnectivity +use atlas_Domain_module, only : & + & atlas_Domain, & + & atlas_LonLatRectangularDomain use atlas_mesh_Nodes_module, only: & & atlas_mesh_Nodes use atlas_HaloExchange_module, only: & diff --git a/src/atlas_f/domain/atlas_Domain_module.F90 b/src/atlas_f/domain/atlas_Domain_module.F90 new file mode 100644 index 000000000..e2d12664f --- /dev/null +++ b/src/atlas_f/domain/atlas_Domain_module.F90 @@ -0,0 +1,199 @@ +! (C) Copyright 2013 ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to any jurisdiction. + +#include "atlas/atlas_f.h" + +module atlas_Domain_module + +use, intrinsic :: iso_c_binding, only : c_double +use fckit_owned_object_module, only : fckit_owned_object +use atlas_config_module, only : atlas_Config + +implicit none + +private :: fckit_owned_object +private :: atlas_Config + +public :: atlas_Domain +public :: atlas_LonLatRectangularDomain + +private + +!------------------------------------------------------------------------------ +TYPE, extends(fckit_owned_object) :: atlas_Domain + +! Purpose : +! ------- +! *atlas_Domain* : + +!------------------------------------------------------------------------------ +contains + procedure, public :: type => atlas_Domain__type + procedure, public :: hash + procedure, public :: spec + +#if FCKIT_FINAL_NOT_INHERITING + final :: atlas_Domain__final_auto +#endif + +END TYPE atlas_Domain + +interface atlas_Domain + module procedure atlas_Domain__ctor_cptr + module procedure atlas_Domain__ctor_config +end interface + +!------------------------------------------------------------------------------ +TYPE, extends(atlas_Domain) :: atlas_LonLatRectangularDomain + +! Purpose : +! ------- +! *atlas_LonLatRectangularDomain* : + +!------------------------------------------------------------------------------ +contains + procedure, public :: north + procedure, public :: west + procedure, public :: south + procedure, public :: east +#if FCKIT_FINAL_NOT_INHERITING + final :: atlas_LonLatRectangularDomain__final_auto +#endif + +END TYPE atlas_LonLatRectangularDomain + +interface atlas_LonLatRectangularDomain + module procedure atlas_LonLatRectangularDomain__ctor_cptr +end interface +!------------------------------------------------------------------------------ + + +!======================================================== +contains +!======================================================== + +function atlas_Domain__ctor_cptr(cptr) result(this) + use, intrinsic :: iso_c_binding, only : c_ptr + type(atlas_Domain) :: this + type(c_ptr), intent(in) :: cptr + call this%reset_c_ptr( cptr ) + call this%return() +end function + +function atlas_Domain__ctor_config(config) result(this) + use atlas_Domain_c_binding + use, intrinsic :: iso_c_binding, only : c_ptr + type(atlas_Domain) :: this + type(atlas_Config) :: config + call this%reset_c_ptr( atlas__Domain__ctor_config(config%CPTR_PGIBUG_B) ) + call this%return() +end function + +function atlas_Domain__type(this) result(type_) + use atlas_Domain_c_binding + use fckit_c_interop_module, only : c_ptr_to_string, c_ptr_free + use, intrinsic :: iso_c_binding, only : c_ptr + class(atlas_Domain), intent(in) :: this + character(len=:), allocatable :: type_ + type(c_ptr) :: type_c_str + integer :: size + call atlas__Domain__type(this%CPTR_PGIBUG_A, type_c_str, size ) + type_ = c_ptr_to_string(type_c_str) + call c_ptr_free(type_c_str) +end function + +function hash(this) + use atlas_Domain_c_binding + use fckit_c_interop_module, only : c_ptr_to_string, c_ptr_free + use, intrinsic :: iso_c_binding, only : c_ptr + class(atlas_Domain), intent(in) :: this + character(len=:), allocatable :: hash + type(c_ptr) :: hash_c_str + integer :: size + call atlas__Domain__hash(this%CPTR_PGIBUG_A, hash_c_str, size ) + hash = c_ptr_to_string(hash_c_str) + call c_ptr_free(hash_c_str) +end function + +function spec(this) + use atlas_Domain_c_binding + class(atlas_Domain), intent(in) :: this + type(atlas_Config) :: spec + spec = atlas_Config( atlas__Domain__spec(this%CPTR_PGIBUG_A) ) + call spec%return() +end function + + +!------------------------------------------------------------------------------ + +function atlas_LonLatRectangularDomain__ctor_cptr(cptr) result(this) + use, intrinsic :: iso_c_binding, only : c_ptr + type(atlas_LonLatRectangularDomain) :: this + type(c_ptr), intent(in) :: cptr + call this%reset_c_ptr( cptr ) + call this%return() +end function + +function north(this) result(value) + use atlas_Domain_c_binding + class(atlas_LonLatRectangularDomain), intent(in) :: this + real(c_double) :: value + value = atlas__LonLatRectangularDomain__north(this%CPTR_PGIBUG_A) +end function + +function west(this) result(value) + use atlas_Domain_c_binding + class(atlas_LonLatRectangularDomain), intent(in) :: this + real(c_double) :: value + value = atlas__LonLatRectangularDomain__west(this%CPTR_PGIBUG_A) +end function + +function south(this) result(value) + use atlas_Domain_c_binding + class(atlas_LonLatRectangularDomain), intent(in) :: this + real(c_double) :: value + value = atlas__LonLatRectangularDomain__south(this%CPTR_PGIBUG_A) +end function + +function east(this) result(value) + use atlas_Domain_c_binding + class(atlas_LonLatRectangularDomain), intent(in) :: this + real(c_double) :: value + value = atlas__LonLatRectangularDomain__east(this%CPTR_PGIBUG_A) +end function + +!------------------------------------------------------------------------------ + + +#if FCKIT_FINAL_NOT_INHERITING +ATLAS_FINAL subroutine atlas_Domain__final_auto(this) + type(atlas_Domain), intent(inout) :: this +#if FCKIT_FINAL_DEBUGGING + write(0,*) "atlas_Domain__final_auto" +#endif +#if FCKIT_FINAL_NOT_PROPAGATING + call this%final() +#endif + FCKIT_SUPPRESS_UNUSED( this ) +end subroutine + +ATLAS_FINAL subroutine atlas_LonLatRectangularDomain__final_auto(this) + type(atlas_LonLatRectangularDomain), intent(inout) :: this +#if FCKIT_FINAL_DEBUGGING + write(0,*) "atlas_LonLatRectangularDomain__final_auto" +#endif +#if FCKIT_FINAL_NOT_PROPAGATING + call this%final() +#endif + FCKIT_SUPPRESS_UNUSED( this ) +end subroutine +#endif + + +end module atlas_Domain_module + diff --git a/src/atlas_f/grid/atlas_Grid_module.F90 b/src/atlas_f/grid/atlas_Grid_module.F90 index aa9939bda..aae3e384d 100644 --- a/src/atlas_f/grid/atlas_Grid_module.F90 +++ b/src/atlas_f/grid/atlas_Grid_module.F90 @@ -13,6 +13,7 @@ module atlas_Grid_module use fckit_owned_object_module, only: fckit_owned_object use atlas_Config_module, only: atlas_Config use atlas_Projection_module, only : atlas_Projection +use atlas_Domain_module, only : atlas_LonLatRectangularDomain use atlas_kinds_module, only : ATLAS_KIND_IDX, ATLAS_KIND_GIDX use, intrinsic :: iso_c_binding, only : c_ptr @@ -53,6 +54,7 @@ module atlas_Grid_module procedure :: size => atlas_Grid__size procedure :: spec => atlas_Grid__spec procedure :: uid + procedure :: lonlat_bounding_box #if FCKIT_FINAL_NOT_INHERITING final :: atlas_Grid__final_auto @@ -610,6 +612,16 @@ function uid(this) call c_ptr_free(uid_c_str) end function +function lonlat_bounding_box (this) result (bb) + use, intrinsic :: iso_c_binding, only : c_double + use atlas_grid_Grid_c_binding + class(atlas_Grid), intent(in) :: this + type(atlas_LonLatRectangularDomain) :: bb + bb = atlas_LonLatRectangularDomain(atlas__grid__Grid__lonlat_bounding_box (this%CPTR_PGIBUG_A)) + call bb%return() +end function + + function Gaussian__N(this) result(N) use, intrinsic :: iso_c_binding, only: c_long use atlas_grid_Structured_c_binding diff --git a/src/tests/grid/fctest_regional_grids.F90 b/src/tests/grid/fctest_regional_grids.F90 index 7cb209c20..7c3dbe2c2 100644 --- a/src/tests/grid/fctest_regional_grids.F90 +++ b/src/tests/grid/fctest_regional_grids.F90 @@ -84,6 +84,7 @@ subroutine print_spec( grid ) #if 1 ! Grid provided by Philippe Marguinaud, Meteo France type(atlas_StructuredGrid) :: grid + type(atlas_LonLatRectangularDomain) :: bounding_box real(c_double), parameter :: zlonw = -20., zlone = +10., zlats = 10., zlatn = 50. integer(c_int), parameter :: ilons = 30, ilats = 40 real(c_double), parameter :: tol = 1.e-5_dp @@ -97,6 +98,16 @@ subroutine print_spec( grid ) FCTEST_CHECK_CLOSE( grid%lonlat(ilons,1), ([ 0.1000000000E+02_dp, 0.1000000000E+02_dp]), tol) FCTEST_CHECK_CLOSE( grid%lonlat(1,2), ([-0.2000000000E+02_dp, 0.1102564103E+02_dp]), tol) FCTEST_CHECK_CLOSE( grid%lonlat(ilons,ilats), ([ 0.1000000000E+02_dp, 0.5000000000E+02_dp]), tol) + + bounding_box = grid%lonlat_bounding_box() + FCTEST_CHECK_CLOSE( bounding_box%west(), -20.0_dp, tol ) + FCTEST_CHECK_CLOSE( bounding_box%east(), +10.0_dp, tol ) + FCTEST_CHECK_CLOSE( bounding_box%south(), +10.0_dp, tol ) + FCTEST_CHECK_CLOSE( bounding_box%north(), +50.0_dp, tol ) + FCTEST_CHECK_EQUAL( bounding_box%owners(), 2 ) + + call bounding_box%final() + call grid%final() #endif END_TEST @@ -106,6 +117,7 @@ subroutine print_spec( grid ) #if 1 ! Grid provided by Philippe Marguinaud, Meteo France type(atlas_StructuredGrid) :: grid + type(atlas_LonLatRectangularDomain) :: bounding_box integer(c_int), parameter :: ndlon=64 integer(c_int), parameter :: ndglg=64 @@ -134,6 +146,14 @@ subroutine print_spec( grid ) FCTEST_CHECK_CLOSE( grid%lonlat(1,2), ([-0.1187876836E+02_dp, 0.3402241700E+02_dp]), tol) FCTEST_CHECK_CLOSE( grid%lonlat(ndlon,ndglg), ([ 0.3452466369E+02_dp, 0.5925747619E+02_dp]), tol) + bounding_box = grid%lonlat_bounding_box() + FCTEST_CHECK_CLOSE( bounding_box%west(), -21.513504_dp, tol ) + FCTEST_CHECK_CLOSE( bounding_box%east(), 34.524664_dp, tol ) + FCTEST_CHECK_CLOSE( bounding_box%south(), 32.586516_dp, tol ) + FCTEST_CHECK_CLOSE( bounding_box%north(), 62.587002_dp, tol ) + FCTEST_CHECK_EQUAL( bounding_box%owners(), 1 ) + + call bounding_box%final() call grid%final() #endif END_TEST diff --git a/src/tests/grid/fctest_stretchedrotatedgaussiangrid.F90 b/src/tests/grid/fctest_stretchedrotatedgaussiangrid.F90 index 9ee020a2e..125322d8d 100644 --- a/src/tests/grid/fctest_stretchedrotatedgaussiangrid.F90 +++ b/src/tests/grid/fctest_stretchedrotatedgaussiangrid.F90 @@ -32,6 +32,7 @@ use atlas_module implicit none type(atlas_ReducedGaussianGrid) :: grid + type(atlas_LonLatRectangularDomain) :: bounding_box real(dp) :: lonlat(3376) @@ -927,6 +928,13 @@ enddo enddo + bounding_box = grid%lonlat_bounding_box() + FCTEST_CHECK_CLOSE( bounding_box%west(), 0._dp, 1.e-10_dp ) + FCTEST_CHECK_CLOSE( bounding_box%east(), 360._dp, 1.e-10_dp ) + FCTEST_CHECK_CLOSE( bounding_box%south(), -90._dp, 1.e-10_dp ) + FCTEST_CHECK_CLOSE( bounding_box%north(), 90._dp, 1.e-10_dp ) + + call bounding_box%final() call grid%final() END_TEST From afdf0eebada4fca135baa311a967564a99cf9e77 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 21 Sep 2020 15:58:10 +0100 Subject: [PATCH 069/161] Fix warnings --- src/atlas/grid/detail/grid/Structured.cc | 9 --------- .../grid/detail/partitioner/CheckerboardPartitioner.h | 1 + .../MatchingFunctionSpacePartitionerLonLatPolygon.h | 1 + .../partitioner/MatchingMeshPartitionerBruteForce.h | 1 + .../MatchingMeshPartitionerSphericalPolygon.h | 1 + src/atlas/grid/detail/partitioner/TransPartitioner.h | 1 + src/atlas/library/Library.cc | 4 ++-- 7 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/atlas/grid/detail/grid/Structured.cc b/src/atlas/grid/detail/grid/Structured.cc index 69bb9da0d..52fcc25b6 100644 --- a/src/atlas/grid/detail/grid/Structured.cc +++ b/src/atlas/grid/detail/grid/Structured.cc @@ -485,15 +485,6 @@ void Structured::crop( const Domain& dom ) { // Cropping in X bool endpoint = true; { - bool do_normalise = false; - for ( idx_t j = jmin, jcropped = 0; j <= jmax; ++j, ++jcropped ) { - double x_first = x( 0, j ); - double x_last = x( nx( j ) - 1, j ); - if ( normalise( x_first ) != x_first || normalise( x_last ) != x_last ) { - do_normalise = true; - } - } - if ( rect_domain.zonal_band() ) { for ( idx_t j = jmin, jcropped = 0; j <= jmax; ++j, ++jcropped ) { if ( is_regular && jcropped > 0 ) { diff --git a/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.h b/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.h index 9d1eda305..f7107ab89 100644 --- a/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.h +++ b/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.h @@ -52,6 +52,7 @@ class CheckerboardPartitioner : public Partitioner { // algorithm is used internally void partition( const Checkerboard& cb, int nb_nodes, NodeInt nodes[], int part[] ) const; + using Partitioner::partition; virtual void partition( const Grid&, int part[] ) const; void check() const; diff --git a/src/atlas/grid/detail/partitioner/MatchingFunctionSpacePartitionerLonLatPolygon.h b/src/atlas/grid/detail/partitioner/MatchingFunctionSpacePartitionerLonLatPolygon.h index bb7ac1681..1c68e9886 100644 --- a/src/atlas/grid/detail/partitioner/MatchingFunctionSpacePartitionerLonLatPolygon.h +++ b/src/atlas/grid/detail/partitioner/MatchingFunctionSpacePartitionerLonLatPolygon.h @@ -28,6 +28,7 @@ class MatchingFunctionSpacePartitionerLonLatPolygon : public MatchingFunctionSpa MatchingFunctionSpacePartitionerLonLatPolygon( const FunctionSpace& FunctionSpace ) : MatchingFunctionSpacePartitioner( FunctionSpace ) {} + using MatchingFunctionSpacePartitioner::partition; /** * @brief Partition a grid, using the same partitions from a pre-partitioned * FunctionSpace. diff --git a/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerBruteForce.h b/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerBruteForce.h index cd8cec44e..966e4ffa7 100644 --- a/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerBruteForce.h +++ b/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerBruteForce.h @@ -28,6 +28,7 @@ class MatchingMeshPartitionerBruteForce : public MatchingMeshPartitioner { MatchingMeshPartitioner( nb_partitions ) {} MatchingMeshPartitionerBruteForce( const Mesh& mesh ) : MatchingMeshPartitioner( mesh ) {} + using MatchingMeshPartitioner::partition; virtual void partition( const Grid& grid, int partitioning[] ) const; virtual std::string type() const { return static_type(); } diff --git a/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerSphericalPolygon.h b/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerSphericalPolygon.h index 275b454ee..ba39e3b0c 100644 --- a/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerSphericalPolygon.h +++ b/src/atlas/grid/detail/partitioner/MatchingMeshPartitionerSphericalPolygon.h @@ -28,6 +28,7 @@ class MatchingMeshPartitionerSphericalPolygon : public MatchingMeshPartitioner { MatchingMeshPartitioner( nb_partitions ) {} MatchingMeshPartitionerSphericalPolygon( const Mesh& mesh ) : MatchingMeshPartitioner( mesh ) {} + using MatchingMeshPartitioner::partition; /** * @brief Partition a grid, using the same partitions from a pre-partitioned * mesh. diff --git a/src/atlas/grid/detail/partitioner/TransPartitioner.h b/src/atlas/grid/detail/partitioner/TransPartitioner.h index fe6220693..27fbadaef 100644 --- a/src/atlas/grid/detail/partitioner/TransPartitioner.h +++ b/src/atlas/grid/detail/partitioner/TransPartitioner.h @@ -34,6 +34,7 @@ class TransPartitioner : public Partitioner { virtual ~TransPartitioner(); + using Partitioner::partition; /// Warning: this function temporariliy allocates a new Trans, but without the /// computations /// of the spectral coefficients (LDGRIDONLY=TRUE) diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index 181b7c22c..f43c14cf0 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -120,8 +120,8 @@ void load_library( const eckit::PathName& library_dir, const std::string& librar break; } } - void* plib; - if ( ( plib = ::dlopen( library_path.localPath(), RTLD_NOW | RTLD_GLOBAL ) ) == nullptr ) { + void* plib = ::dlopen( library_path.localPath(), RTLD_NOW | RTLD_GLOBAL ); + if ( plib == nullptr ) { // dlopen failed std::ostringstream ss; ss << "dlopen(" << library_path << ", ...)"; throw eckit::FailedSystemCall( ss.str().c_str(), Here(), errno ); From eda2c1669f0ccc6bf415956da54dd919cf41e4ea Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 21 Sep 2020 16:00:38 +0100 Subject: [PATCH 070/161] Fix PGI 20.7 compiler bug where it applied wrongly vectorization --- src/atlas/parallel/HaloExchange.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/atlas/parallel/HaloExchange.cc b/src/atlas/parallel/HaloExchange.cc index 55fa19d11..78627d684 100644 --- a/src/atlas/parallel/HaloExchange.cc +++ b/src/atlas/parallel/HaloExchange.cc @@ -124,14 +124,18 @@ void HaloExchange::setup( const int part[], const idx_t remote_idx[], const int std::vector send_requests( recvcnt_ ); std::vector recv_requests( sendcnt_ ); std::vector cnt( nproc, 0 ); - recvmap_.resize( recvcnt_ ); +#ifdef __PGI + // No idea why PGI compiler (20.7) in Release build ( -fast -O3 ) decides to vectorize following loop +#pragma loop novector +#endif for ( idx_t jghost = 0; jghost < nghost; ++jghost ) { - const auto jj = ghost_points[jghost]; - const int req_idx = recvdispls_[part[jj]] + cnt[part[jj]]; + const idx_t jj = ghost_points[jghost]; + const int p = part[jj]; + const int req_idx = recvdispls_[p] + cnt[p]; send_requests[req_idx] = remote_idx[jj] - base; recvmap_[req_idx] = jj; - ++cnt[part[jj]]; + cnt[p]++; } /* From 7201ff48d8d1b17bde774b6fbfa0da4da27bda6c Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 21 Sep 2020 16:34:04 +0100 Subject: [PATCH 071/161] Fix typo --- AUTHORS | 4 ++-- tools/generate-authors.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index 874ea2ce3..6e9da9fd7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -5,8 +5,8 @@ Authors - Pedro Maciel - Tiago Quintino -Thanks for conributions from -============================ +Thanks for contributions from +============================= - Carlos Osuna - Avi Bahra diff --git a/tools/generate-authors.py b/tools/generate-authors.py index 4733b73ec..1d4b405ab 100755 --- a/tools/generate-authors.py +++ b/tools/generate-authors.py @@ -46,8 +46,8 @@ def contributors(): print("- "+name) print("") -print("Thanks for conributions from") -print("============================\n") +print("Thanks for contributions from") +print("=============================\n") for name in contributors(): print("- "+name) From aba18c51d5b1bfa5555aeebe252f1787dc3104b6 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 22 Sep 2020 17:44:12 +0100 Subject: [PATCH 072/161] Add CUDA feature to control Gridtools backend used --- cmake/features/ACC.cmake | 2 +- cmake/features/GRIDTOOLS_STORAGE.cmake | 22 ++++++++++++---------- src/atlas/parallel/HaloExchange.cc | 2 +- src/tests/CMakeLists.txt | 4 ++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/cmake/features/ACC.cmake b/cmake/features/ACC.cmake index 6de04f613..9ee1fab4f 100644 --- a/cmake/features/ACC.cmake +++ b/cmake/features/ACC.cmake @@ -1,7 +1,7 @@ ### OpenACC set( ATLAS_ACC_CAPABLE FALSE ) -if( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA ) +if( HAVE_GPU ) if( CMAKE_Fortran_COMPILER_ID MATCHES "PGI" ) set( ATLAS_ACC_CAPABLE TRUE ) endif() diff --git a/cmake/features/GRIDTOOLS_STORAGE.cmake b/cmake/features/GRIDTOOLS_STORAGE.cmake index 9d2dbb73d..b5e10705f 100644 --- a/cmake/features/GRIDTOOLS_STORAGE.cmake +++ b/cmake/features/GRIDTOOLS_STORAGE.cmake @@ -20,12 +20,16 @@ ecbuild_add_option( DESCRIPTION "Arrays internally use GridTools storage layer" CONDITION GridTools_FOUND ) -if( atlas_HAVE_GRIDTOOLS_STORAGE ) +ecbuild_add_option( FEATURE CUDA + DESCRIPTION "Enable CUDA support via GridTools CUDA backend" + CONDITION GRIDTOOLS_HAS_BACKEND_CUDA ) + +set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 0 ) +set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA 0 ) - set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 1 ) - set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA 0 ) +if( atlas_HAVE_GRIDTOOLS_STORAGE ) - if( GRIDTOOLS_HAS_BACKEND_CUDA ) + if( atlas_HAVE_CUDA ) ecbuild_info( "GridTools found with CUDA support" ) @@ -48,14 +52,12 @@ if( atlas_HAVE_GRIDTOOLS_STORAGE ) find_package( CUDA ) endif() - set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 0 ) set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA 1 ) - endif() -else() + else() - set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 0 ) - set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA 0 ) + set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 1 ) -endif() + endif() +endif() diff --git a/src/atlas/parallel/HaloExchange.cc b/src/atlas/parallel/HaloExchange.cc index 78627d684..1c4b03993 100644 --- a/src/atlas/parallel/HaloExchange.cc +++ b/src/atlas/parallel/HaloExchange.cc @@ -90,7 +90,7 @@ void HaloExchange::setup( const int part[], const idx_t remote_idx[], const int atlas_omp_parallel_for( int jj = halo_begin; jj < parsize_; ++jj ) { if ( is_ghost( jj ) ) { - idx_t p = part[jj]; + int p = part[jj]; atlas_omp_critical { ++recvcounts_[p]; ghost_points[nghost] = jj; diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 25f4e3b73..ab2ea14cd 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -47,7 +47,7 @@ macro( atlas_add_cuda_test ) ecbuild_critical("Unknown keywords given to atlas_add_cuda_test(): \"${_PAR_UNPARSED_ARGUMENTS}\"") endif() - if( atlas_HAVE_GRIDTOOLS_STORAGE AND GRIDTOOLS_HAS_BACKEND_CUDA AND HAVE_TESTS ) + if( HAVE_CUDA AND HAVE_TESTS ) ecbuild_debug("atlas_add_cuda_test: Adding test ${_PAR_TARGET}") @@ -100,7 +100,7 @@ if( NOT DEFINED transi_VERSION ) endif() endif() -if( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA ) +if( HAVE_CUDA ) set( ATLAS_TEST_ENVIRONMENT "ATLAS_RUN_NGPUS=1" ) endif() From 13132baae89b82321f2d4dcb6fe9fb14f6bdb1f9 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 23 Sep 2020 08:30:08 +0000 Subject: [PATCH 073/161] Fixup for OpenACC compilation --- cmake/features/ACC.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/features/ACC.cmake b/cmake/features/ACC.cmake index 9ee1fab4f..162c67bd1 100644 --- a/cmake/features/ACC.cmake +++ b/cmake/features/ACC.cmake @@ -1,7 +1,7 @@ ### OpenACC set( ATLAS_ACC_CAPABLE FALSE ) -if( HAVE_GPU ) +if( HAVE_CUDA ) if( CMAKE_Fortran_COMPILER_ID MATCHES "PGI" ) set( ATLAS_ACC_CAPABLE TRUE ) endif() From 55750a9356d6fa86b882b292968bbfb3835dfc6c Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 23 Sep 2020 08:58:46 +0000 Subject: [PATCH 074/161] Fix installation of libatlas_acc_support.so to INSTALL_LIB_DIR (could be lib64) --- src/atlas_acc_support/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas_acc_support/CMakeLists.txt b/src/atlas_acc_support/CMakeLists.txt index d7c3d536f..ffe144a37 100644 --- a/src/atlas_acc_support/CMakeLists.txt +++ b/src/atlas_acc_support/CMakeLists.txt @@ -25,7 +25,7 @@ if( atlas_HAVE_ACC ) set_property( TARGET atlas_acc_support PROPERTY IMPORTED_NO_SONAME TRUE ) set_property( TARGET atlas_acc_support PROPERTY IMPORTED_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR} ) add_dependencies( atlas_acc_support build-atlas_acc_support ) - install( FILES ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so DESTINATION lib/ ) + install( FILES ${CMAKE_BINARY_DIR}/lib/libatlas_acc_support.so DESTINATION ${INSTALL_LIB_DIR}/ ) else() From 3e277560b525e0ce77794243b07678610784d663 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Tue, 22 Sep 2020 10:01:05 +0000 Subject: [PATCH 075/161] StructuredColumns with x/y bi-periodicity on regional grids --- .../detail/StructuredColumns_setup.cc | 72 ++++++----- src/tests/functionspace/CMakeLists.txt | 8 ++ .../test_structuredcolumns_biperiodic.cc | 118 ++++++++++++++++++ 3 files changed, 169 insertions(+), 29 deletions(-) create mode 100644 src/tests/functionspace/test_structuredcolumns_biperiodic.cc diff --git a/src/atlas/functionspace/detail/StructuredColumns_setup.cc b/src/atlas/functionspace/detail/StructuredColumns_setup.cc index a7f1e8c74..993702c87 100644 --- a/src/atlas/functionspace/detail/StructuredColumns_setup.cc +++ b/src/atlas/functionspace/detail/StructuredColumns_setup.cc @@ -90,6 +90,10 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck throw_Exception( "Grid is not a grid::Structured type", Here() ); } + bool periodic_x = false, periodic_y = false; + config.get ("periodic_x", periodic_x); + config.get ("periodic_y", periodic_y); + const double eps = 1.e-12; ny_ = grid_->ny(); @@ -244,17 +248,28 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck }; std::function compute_j; - compute_j = [this, &compute_j]( idx_t j ) -> idx_t { - if ( j < 0 ) { - j = ( grid_->y( 0 ) == 90. ) ? -j : -j - 1; - } - else if ( j >= grid_->ny() ) { - idx_t jlast = grid_->ny() - 1; - j = ( grid_->y( jlast ) == -90. ) ? jlast - 1 - ( j - grid_->ny() ) : jlast - ( j - grid_->ny() ); - } - if ( j < 0 or j >= grid_->ny() ) { - j = compute_j( j ); - } + compute_j = [this, &compute_j, &periodic_y]( idx_t j ) -> idx_t { + if (periodic_y) + { + const idx_t ny = grid_->ny(); + while (j < 0) + j += ny; + while (j >= ny) + j -= ny; + } + else + { + if ( j < 0 ) { + j = ( grid_->y( 0 ) == 90. ) ? -j : -j - 1; + } + else if ( j >= grid_->ny() ) { + idx_t jlast = grid_->ny() - 1; + j = ( grid_->y( jlast ) == -90. ) ? jlast - 1 - ( j - grid_->ny() ) : jlast - ( j - grid_->ny() ); + } + if ( j < 0 or j >= grid_->ny() ) { + j = compute_j( j ); + } + } return j; }; @@ -280,12 +295,19 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck return i; }; - auto compute_y = [this, &compute_j]( idx_t j ) -> double { - idx_t jj; + auto compute_y = [this, &compute_j, &periodic_y]( idx_t j ) -> double { double y; + idx_t jj; jj = compute_j( j ); - y = ( j < 0 ) ? 90. + ( 90. - grid_->y( jj ) ) - : ( j >= grid_->ny() ) ? -90. + ( -90. - grid_->y( jj ) ) : grid_->y( jj ); + if (periodic_y) + { + y = grid_->y( jj ); + } + else + { + y = ( j < 0 ) ? 90. + ( 90. - grid_->y( jj ) ) + : ( j >= grid_->ny() ) ? -90. + ( -90. - grid_->y( jj ) ) : grid_->y( jj ); + } return y; }; @@ -296,12 +318,14 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck grid_idx += grid_->nx( j ); } - auto compute_g = [this, &global_offsets, &compute_i, &compute_j]( idx_t i, idx_t j ) -> gidx_t { + auto compute_g = [this, &global_offsets, &compute_i, + &compute_j, &periodic_y]( idx_t i, idx_t j ) -> gidx_t { idx_t ii, jj; gidx_t g; jj = compute_j( j ); ii = compute_i( i, jj ); - if ( jj != j ) { + if (! periodic_y) + if (jj != j) { ATLAS_ASSERT( grid_->nx( jj ) % 2 == 0 ); // assert even number of points ii = ( ii < grid_->nx( jj ) / 2 ) ? ii + grid_->nx( jj ) / 2 : ( ii >= grid_->nx( jj ) / 2 ) ? ii - grid_->nx( jj ) / 2 : ii; @@ -310,18 +334,8 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck return g; }; - auto compute_p = [this, &global_offsets, &distribution, &compute_i, &compute_j]( idx_t i, idx_t j ) -> int { - idx_t ii, jj; - int p; - jj = compute_j( j ); - ii = compute_i( i, jj ); - if ( jj != j ) { - ATLAS_ASSERT( grid_->nx( jj ) % 2 == 0 ); // assert even number of points - ii = ( ii < grid_->nx( jj ) / 2 ) ? ii + grid_->nx( jj ) / 2 - : ( ii >= grid_->nx( jj ) / 2 ) ? ii - grid_->nx( jj ) / 2 : ii; - } - p = distribution.partition( global_offsets[jj] + ii ); - return p; + auto compute_p = [&compute_g, &distribution]( idx_t i, idx_t j ) -> int { + return distribution.partition (compute_g (i, j) - 1); }; GridPointSet gridpoints; diff --git a/src/tests/functionspace/CMakeLists.txt b/src/tests/functionspace/CMakeLists.txt index 9f3730375..b770eb55c 100644 --- a/src/tests/functionspace/CMakeLists.txt +++ b/src/tests/functionspace/CMakeLists.txt @@ -35,6 +35,14 @@ ecbuild_add_test( TARGET atlas_test_cellcolumns ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET test_structuredcolumns_biperiodic + SOURCES test_structuredcolumns_biperiodic.cc + LIBS atlas + MPI 5 + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} + CONDITION eckit_HAVE_MPI +) + ecbuild_add_test( TARGET atlas_test_structuredcolumns SOURCES test_structuredcolumns.cc LIBS atlas diff --git a/src/tests/functionspace/test_structuredcolumns_biperiodic.cc b/src/tests/functionspace/test_structuredcolumns_biperiodic.cc new file mode 100644 index 000000000..51e376c6f --- /dev/null +++ b/src/tests/functionspace/test_structuredcolumns_biperiodic.cc @@ -0,0 +1,118 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include +#include +#include + +#include "atlas/array.h" +#include "atlas/field.h" +#include "atlas/functionspace.h" +#include "atlas/grid.h" +#include "atlas/parallel/mpi/mpi.h" +#include "atlas/util/Config.h" + + +#include +#include +#include +#include + + +#include "tests/AtlasTestEnvironment.h" + +using Grid = atlas::Grid; +using Config = atlas::util::Config; + +namespace atlas { +namespace test { + +//----------------------------------------------------------------------------- + +CASE( "biperiodic_latlon") { + + auto & comm = atlas::mpi::comm (); + const int myproc = comm.rank (); + + + using namespace atlas::util; + using namespace atlas; + using namespace atlas::grid; + + const int Nx = 80, Ny = 80; + const double xmin = +20, xmax = +60, ymin = +20, ymax = +60; + + std::vector spacings (Ny); + + for (int i = 0; i < Ny; i++) + spacings[i] = Spacing (Config ("type", "linear") | Config ("N", Nx) + | Config ("start", xmin) | Config ("end", xmax)); + + StructuredGrid::XSpace xspace (spacings); + StructuredGrid::YSpace yspace (Config ("type", "linear") | Config ("N", Ny) | Config ("start", ymin) | Config ("end", ymax)); + Projection proj (Config ("type", "lonlat")); + + atlas::StructuredGrid grid (xspace, yspace, proj, Domain ()); + + atlas::grid::Distribution dist (grid, atlas::util::Config ("type", "checkerboard")); + atlas::functionspace::StructuredColumns fs (grid, dist, atlas::util::Config ("halo", 3) + | atlas::util::Config ("periodic_x", true) + | atlas::util::Config ("periodic_y", true)); + + + auto f = atlas::Field ("f", + atlas::array::DataType::kind (), + atlas::array::make_shape (fs.size ())); + + auto v = atlas::array::make_view (f); + + for (int i = 0; i < f.size (); i++) + v (i) = static_cast (myproc); + + fs.haloExchange (f); + + auto clamp = [] (int i, int n) + { + while (i < 0) + i += n; + while (i >= n) + i -= n; + return i; + }; + + auto gv = atlas::array::make_view( fs.global_index() ); + auto pv = atlas::array::make_view( fs.partition() ); + + for (int j = fs.j_begin_halo (); j < fs.j_end_halo (); j++) + for (int i = fs.i_begin_halo (j); i < fs.i_end_halo (j); i++) + { + int k = fs.index (i, j); + int jj = clamp (j, grid.ny ()); + int ii = clamp (i, grid.nx (jj)); + + int g = grid.index (ii, jj); + int p = dist.partition (g); + + EXPECT_EQ (v (k), p); + EXPECT_EQ (p, pv (k)); + EXPECT_EQ (gv (k)-1, g); + } + +} + +//----------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas + +int main( int argc, char** argv ) { + return atlas::test::run( argc, argv ); +} From 9f266ce2422b4c953efffe3af17d30b8ad27d9c8 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 21 Jul 2020 18:07:04 +0100 Subject: [PATCH 076/161] ATLAS-306 Non-linear interpolation matrix corrections: base class and factory --- src/atlas/CMakeLists.txt | 4 ++ .../interpolation/nonlinear/NonLinear.cc | 39 ++++++++++++ src/atlas/interpolation/nonlinear/NonLinear.h | 60 +++++++++++++++++++ .../nonlinear/NonLinearFactory.cc | 44 ++++++++++++++ .../nonlinear/NonLinearFactory.h | 52 ++++++++++++++++ 5 files changed, 199 insertions(+) create mode 100644 src/atlas/interpolation/nonlinear/NonLinear.cc create mode 100644 src/atlas/interpolation/nonlinear/NonLinear.h create mode 100644 src/atlas/interpolation/nonlinear/NonLinearFactory.cc create mode 100644 src/atlas/interpolation/nonlinear/NonLinearFactory.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index b34828152..46bb2f339 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -528,6 +528,10 @@ interpolation/method/structured/QuasiCubic2D.cc interpolation/method/structured/QuasiCubic2D.h interpolation/method/structured/QuasiCubic3D.cc interpolation/method/structured/QuasiCubic3D.h +interpolation/nonlinear/NonLinear.cc +interpolation/nonlinear/NonLinear.h +interpolation/nonlinear/NonLinearFactory.cc +interpolation/nonlinear/NonLinearFactory.h ) diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc new file mode 100644 index 000000000..a2814293a --- /dev/null +++ b/src/atlas/interpolation/nonlinear/NonLinear.cc @@ -0,0 +1,39 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/interpolation/nonlinear/NonLinear.h" + +#include "atlas/runtime/Exception.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +NonLinear::NonLinear( const Config& config ) : missingValue_( 0. ) { + config.get( "missingValue", missingValue_ ); + ATLAS_ASSERT( missingValue_ == missingValue_ ); +} + + +bool NonLinear::missingValue( const double& value ) const { + return value == missingValue_; +} + + +NonLinear::~NonLinear() = default; + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h new file mode 100644 index 000000000..a100d7416 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -0,0 +1,60 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include "eckit/config/Parametrisation.h" +#include "eckit/linalg/SparseMatrix.h" + + +namespace atlas { +class Field; +} + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +/** + * @brief NonLinear class applies non-linear corrections to an interpolation matrix, given a field with missing values. + * The interpolatation are re-weighted to factor those values out of the resulting field. + */ +class NonLinear { +public: + using Config = eckit::Parametrisation; + using Matrix = eckit::linalg::SparseMatrix; + using Scalar = eckit::linalg::Scalar; + using Size = eckit::linalg::Size; + + /// ctor + NonLinear( const Config& ); + + /// dtor + virtual ~NonLinear(); + + /// Update interpolation linear system to account for non-linearities + virtual bool treatment( Matrix&, const Field& ) const = 0; + + /// Check if value represents a missing value + virtual bool missingValue( const double& ) const; + +private: + /// Missing value to compare against + double missingValue_; +}; + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc b/src/atlas/interpolation/nonlinear/NonLinearFactory.cc new file mode 100644 index 000000000..ac7b8c73a --- /dev/null +++ b/src/atlas/interpolation/nonlinear/NonLinearFactory.cc @@ -0,0 +1,44 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include + +#include "atlas/interpolation/nonlinear/NonLinearFactory.h" +//#include "atlas/interpolation/nonlinear/.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +namespace { +void force_link() { + static struct Link { + Link() { + //NonLinearFactoryBuilder<...>(); + } + } link; +} +} // namespace + + +const NonLinear* NonLinearFactory::build( const std::string& builder, const util::Config& config ) { + force_link(); + auto factory = get( builder ); + return factory->make( config ); +} + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.h b/src/atlas/interpolation/nonlinear/NonLinearFactory.h new file mode 100644 index 000000000..82bb93601 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/NonLinearFactory.h @@ -0,0 +1,52 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include + +#include "atlas/util/Config.h" +#include "atlas/util/Factory.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +class NonLinear; + + +class NonLinearFactory : public util::Factory { +public: + static std::string className() { return "NonLinearFactory"; } + static const NonLinear* build( const std::string&, const util::Config& ); + using Factory::Factory; + +private: + virtual const NonLinear* make( const util::Config& ) = 0; +}; + + +template +class NonLinearFactoryBuilder : public NonLinearFactory { +private: + virtual const NonLinear* make( const util::Config& config ) override { return new T( config ); } + +public: + using NonLinearFactory::NonLinearFactory; +}; + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas From 71cfb420b30e0720d76f7a725618927a77402784 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 21 Jul 2020 18:26:48 +0100 Subject: [PATCH 077/161] ATLAS-306 Non-linear interpolation matrix corrections: if any/all/heaviest is missing implementations --- src/atlas/CMakeLists.txt | 6 ++ .../nonlinear/MissingIfAllMissing.cc | 95 ++++++++++++++++ .../nonlinear/MissingIfAllMissing.h | 31 ++++++ .../nonlinear/MissingIfAnyMissing.cc | 79 ++++++++++++++ .../nonlinear/MissingIfAnyMissing.h | 31 ++++++ .../nonlinear/MissingIfHeaviestMissing.cc | 101 ++++++++++++++++++ .../nonlinear/MissingIfHeaviestMissing.h | 31 ++++++ .../nonlinear/NonLinearFactory.cc | 11 +- 8 files changed, 382 insertions(+), 3 deletions(-) create mode 100644 src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc create mode 100644 src/atlas/interpolation/nonlinear/MissingIfAllMissing.h create mode 100644 src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc create mode 100644 src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h create mode 100644 src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc create mode 100644 src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 46bb2f339..d79a989b7 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -528,6 +528,12 @@ interpolation/method/structured/QuasiCubic2D.cc interpolation/method/structured/QuasiCubic2D.h interpolation/method/structured/QuasiCubic3D.cc interpolation/method/structured/QuasiCubic3D.h +interpolation/nonlinear/MissingIfAllMissing.cc +interpolation/nonlinear/MissingIfAllMissing.h +interpolation/nonlinear/MissingIfAnyMissing.cc +interpolation/nonlinear/MissingIfAnyMissing.h +interpolation/nonlinear/MissingIfHeaviestMissing.cc +interpolation/nonlinear/MissingIfHeaviestMissing.h interpolation/nonlinear/NonLinear.cc interpolation/nonlinear/NonLinear.h interpolation/nonlinear/NonLinearFactory.cc diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc new file mode 100644 index 000000000..56b3eaf87 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc @@ -0,0 +1,95 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/interpolation/nonlinear/MissingIfAllMissing.h" + +#include "eckit/types/FloatCompare.h" + +#include "atlas/array.h" +#include "atlas/field/Field.h" +#include "atlas/interpolation/nonlinear/NonLinearFactory.h" +#include "atlas/runtime/Exception.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +MissingIfAllMissing::MissingIfAllMissing( const Config& config ) : NonLinear( config ) {} + + +bool MissingIfAllMissing::treatment( NonLinear::Matrix& W, const Field& field ) const { + // NOTE only for scalars (for now) + auto values = array::make_view( field ); + + // correct matrix weigths for the missing values + // (force a missing value only if all row values are missing) + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + double sum = 0.; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + else { + sum += *it; + } + } + + // weights redistribution: zero-weight all missing values, linear re-weighting for the others; + // the result is missing value if all values in row are missing + if ( N_missing > 0 ) { + if ( N_missing == N_entries || eckit::types::is_approximately_equal( sum, 0. ) ) { + for ( Size j = k; j < k + N_entries; ++j ) { + data[j] = j == i_missing ? 1. : 0.; + } + } + else { + const double factor = 1. / sum; + for ( Size j = k; j < k + N_entries; ++j, ++kt ) { + const bool miss = missingValue( values[kt.col()] ); + data[j] = miss ? 0. : ( factor * data[j] ); + } + } + modif = true; + } + } + + return modif; +} + + +static NonLinearFactoryBuilder __nonlinear( "missing-if-all-missing" ); + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h new file mode 100644 index 000000000..e3e3d957d --- /dev/null +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h @@ -0,0 +1,31 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include "atlas/interpolation/nonlinear/NonLinear.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +struct MissingIfAllMissing : NonLinear { + MissingIfAllMissing( const Config& ); + bool treatment( NonLinear::Matrix&, const Field& ) const; +}; + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc new file mode 100644 index 000000000..00942ecf6 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc @@ -0,0 +1,79 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/interpolation/nonlinear/MissingIfAnyMissing.h" + +#include "atlas/array.h" +#include "atlas/field/Field.h" +#include "atlas/interpolation/nonlinear/NonLinearFactory.h" +#include "atlas/runtime/Exception.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +MissingIfAnyMissing::MissingIfAnyMissing( const Config& config ) : NonLinear( config ) {} + + +bool MissingIfAnyMissing::treatment( NonLinear::Matrix& W, const Field& field ) const { + // NOTE only for scalars (for now) + auto values = array::make_view( field ); + + // correct matrix weigths for the missing values + // (force a missing value only if any row values is missing) + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + } + + // if any values in row are missing, force missing value + if ( N_missing > 0 ) { + for ( Size j = k; j < k + N_entries; ++j ) { + data[j] = j == i_missing ? 1. : 0.; + } + modif = true; + } + } + + return modif; +} + + +static NonLinearFactoryBuilder __nonlinear( "missing-if-any-missing" ); + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h new file mode 100644 index 000000000..52279358e --- /dev/null +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h @@ -0,0 +1,31 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include "atlas/interpolation/nonlinear/NonLinear.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +struct MissingIfAnyMissing : NonLinear { + MissingIfAnyMissing( const Config& ); + bool treatment( NonLinear::Matrix&, const Field& ) const; +}; + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc new file mode 100644 index 000000000..1e2bbc480 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc @@ -0,0 +1,101 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h" + +#include "eckit/types/FloatCompare.h" + +#include "atlas/array.h" +#include "atlas/field/Field.h" +#include "atlas/interpolation/nonlinear/NonLinearFactory.h" +#include "atlas/runtime/Exception.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +MissingIfHeaviestMissing::MissingIfHeaviestMissing( const Config& config ) : NonLinear( config ) {} + + +bool MissingIfHeaviestMissing::treatment( NonLinear::Matrix& W, const Field& field ) const { + // NOTE only for scalars (for now) + auto values = array::make_view( field ); + + // correct matrix weigths for the missing values + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) and find maximum weight in row + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + double sum = 0.; + double heaviest = -1.; + bool heaviest_is_missing = false; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + else { + sum += *it; + } + + if ( heaviest < data[i] ) { + heaviest = data[i]; + heaviest_is_missing = miss; + } + } + + // weights redistribution: zero-weight all missing values, linear re-weighting for the others; + // if all values are missing, or the closest value is missing, force missing value + if ( N_missing > 0 ) { + if ( N_missing == N_entries || heaviest_is_missing || eckit::types::is_approximately_equal( sum, 0. ) ) { + for ( Size j = k; j < k + N_entries; ++j ) { + data[j] = j == i_missing ? 1. : 0.; + } + } + else { + const double factor = 1. / sum; + for ( Size j = k; j < k + N_entries; ++j, ++kt ) { + const bool miss = missingValue( values[kt.col()] ); + data[j] = miss ? 0. : ( factor * data[j] ); + } + } + modif = true; + } + } + + return modif; +} + + +static NonLinearFactoryBuilder __nonlinear( "missing-if-heaviest-missing" ); + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h new file mode 100644 index 000000000..28b2fae6a --- /dev/null +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h @@ -0,0 +1,31 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include "atlas/interpolation/nonlinear/NonLinear.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +struct MissingIfHeaviestMissing : NonLinear { + MissingIfHeaviestMissing( const Config& ); + bool treatment( NonLinear::Matrix&, const Field& ) const; +}; + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc b/src/atlas/interpolation/nonlinear/NonLinearFactory.cc index ac7b8c73a..4004b5448 100644 --- a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc +++ b/src/atlas/interpolation/nonlinear/NonLinearFactory.cc @@ -10,10 +10,13 @@ */ +#include "atlas/interpolation/nonlinear/NonLinearFactory.h" + #include -#include "atlas/interpolation/nonlinear/NonLinearFactory.h" -//#include "atlas/interpolation/nonlinear/.h" +#include "atlas/interpolation/nonlinear/MissingIfAllMissing.h" +#include "atlas/interpolation/nonlinear/MissingIfAnyMissing.h" +#include "atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h" namespace atlas { @@ -25,7 +28,9 @@ namespace { void force_link() { static struct Link { Link() { - //NonLinearFactoryBuilder<...>(); + NonLinearFactoryBuilder(); + NonLinearFactoryBuilder(); + NonLinearFactoryBuilder(); } } link; } From 53150a4ce231a4ab98bbee30ecbb6863a9605554 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 21 Jul 2020 19:16:43 +0100 Subject: [PATCH 078/161] ATLAS-306 Non-linear interpolation matrix corrections: pimpl idiom class at level atlas::interpolation::NonLinear --- src/atlas/CMakeLists.txt | 38 ++++++++++--------- src/atlas/interpolation/NonLinear.cc | 35 +++++++++++++++++ src/atlas/interpolation/NonLinear.h | 56 ++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 18 deletions(-) create mode 100644 src/atlas/interpolation/NonLinear.cc create mode 100644 src/atlas/interpolation/NonLinear.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index d79a989b7..c62074126 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -473,8 +473,10 @@ endif() list( APPEND atlas_interpolation_srcs interpolation.h -interpolation/Interpolation.h interpolation/Interpolation.cc +interpolation/Interpolation.h +interpolation/NonLinear.cc +interpolation/NonLinear.h interpolation/Vector2D.cc interpolation/Vector2D.h interpolation/Vector3D.cc @@ -503,31 +505,31 @@ interpolation/method/knn/KNearestNeighboursBase.cc interpolation/method/knn/KNearestNeighboursBase.h interpolation/method/knn/NearestNeighbour.cc interpolation/method/knn/NearestNeighbour.h -interpolation/method/structured/StructuredInterpolation2D.tcc -interpolation/method/structured/StructuredInterpolation2D.h -interpolation/method/structured/StructuredInterpolation3D.tcc -interpolation/method/structured/StructuredInterpolation3D.h -interpolation/method/structured/kernels/LinearHorizontalKernel.h -interpolation/method/structured/kernels/Linear3DKernel.h -interpolation/method/structured/kernels/LinearVerticalKernel.h -interpolation/method/structured/kernels/CubicHorizontalKernel.h -interpolation/method/structured/kernels/Cubic3DKernel.h -interpolation/method/structured/kernels/CubicVerticalKernel.h -interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h -interpolation/method/structured/kernels/QuasiCubic3DKernel.cc -interpolation/method/structured/kernels/QuasiCubic3DKernel.h -interpolation/method/structured/Linear2D.cc -interpolation/method/structured/Linear2D.h -interpolation/method/structured/Linear3D.cc -interpolation/method/structured/Linear3D.h interpolation/method/structured/Cubic2D.cc interpolation/method/structured/Cubic2D.h interpolation/method/structured/Cubic3D.cc interpolation/method/structured/Cubic3D.h +interpolation/method/structured/Linear2D.cc +interpolation/method/structured/Linear2D.h +interpolation/method/structured/Linear3D.cc +interpolation/method/structured/Linear3D.h interpolation/method/structured/QuasiCubic2D.cc interpolation/method/structured/QuasiCubic2D.h interpolation/method/structured/QuasiCubic3D.cc interpolation/method/structured/QuasiCubic3D.h +interpolation/method/structured/StructuredInterpolation2D.h +interpolation/method/structured/StructuredInterpolation2D.tcc +interpolation/method/structured/StructuredInterpolation3D.h +interpolation/method/structured/StructuredInterpolation3D.tcc +interpolation/method/structured/kernels/Cubic3DKernel.h +interpolation/method/structured/kernels/CubicHorizontalKernel.h +interpolation/method/structured/kernels/CubicVerticalKernel.h +interpolation/method/structured/kernels/Linear3DKernel.h +interpolation/method/structured/kernels/LinearHorizontalKernel.h +interpolation/method/structured/kernels/LinearVerticalKernel.h +interpolation/method/structured/kernels/QuasiCubic3DKernel.cc +interpolation/method/structured/kernels/QuasiCubic3DKernel.h +interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h interpolation/nonlinear/MissingIfAllMissing.cc interpolation/nonlinear/MissingIfAllMissing.h interpolation/nonlinear/MissingIfAnyMissing.cc diff --git a/src/atlas/interpolation/NonLinear.cc b/src/atlas/interpolation/NonLinear.cc new file mode 100644 index 000000000..6510094a0 --- /dev/null +++ b/src/atlas/interpolation/NonLinear.cc @@ -0,0 +1,35 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/interpolation/NonLinear.h" + +#include "atlas/interpolation/nonlinear/NonLinearFactory.h" +#include "atlas/util/Config.h" + + +namespace atlas { +namespace interpolation { + + +NonLinear::NonLinear() : Handle( nullptr ) {} + + +NonLinear::NonLinear( const util::Config& config ) : + Handle( nonlinear::NonLinearFactory::build( config.getString( "type" ), config ) ) {} + + +NonLinear::NonLinear( const std::string& type, const util::Config& config ) : + Handle( nonlinear::NonLinearFactory::build( type, config ) ) {} + + +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/NonLinear.h b/src/atlas/interpolation/NonLinear.h new file mode 100644 index 000000000..93e02b561 --- /dev/null +++ b/src/atlas/interpolation/NonLinear.h @@ -0,0 +1,56 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include + +#include "atlas/library/config.h" +#include "atlas/util/ObjectHandle.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { +class NonLinear; +} +} // namespace interpolation +namespace util { +class Config; +} +} // namespace atlas + + +namespace atlas { +namespace interpolation { + + +/** + * @brief NonLinear class applies non-linear corrections to an interpolation matrix, given a field with missing values. + * The interpolatation are re-weighted to factor those values out of the resulting field. + */ +struct NonLinear : DOXYGEN_HIDE( public util::ObjectHandle ) { + using Spec = util::Config; + + /// ctor + using Handle::Handle; + NonLinear(); + NonLinear( const util::Config& ); + NonLinear( const std::string& type, const util::Config& ); + + /// bool operator + using Handle::operator bool; // (ensure this exists) +}; + + +} // namespace interpolation +} // namespace atlas From f730ac74ff05f261478e78989d388b13cd570804 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Jul 2020 09:42:49 +0100 Subject: [PATCH 079/161] ATLAS-306 Non-linear interpolation matrix corrections: missing values: NaN (default) and fixed value ("missingValue", such as from GRIB) --- .../interpolation/nonlinear/NonLinear.cc | 28 ++++++++++++++++--- src/atlas/interpolation/nonlinear/NonLinear.h | 13 +++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc index a2814293a..8301b9a6a 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.cc +++ b/src/atlas/interpolation/nonlinear/NonLinear.cc @@ -12,6 +12,8 @@ #include "atlas/interpolation/nonlinear/NonLinear.h" +#include + #include "atlas/runtime/Exception.h" @@ -20,14 +22,32 @@ namespace interpolation { namespace nonlinear { -NonLinear::NonLinear( const Config& config ) : missingValue_( 0. ) { - config.get( "missingValue", missingValue_ ); - ATLAS_ASSERT( missingValue_ == missingValue_ ); +/// @brief CompareNaN Indicate missing value if NaN +struct CompareNaN : NonLinear::Compare { + bool operator()( const double& value ) const override { return std::isnan( value ); } +}; + + +/// @brief CompareValue Indicate missing value if compares equally to pre-defined value +struct CompareValue : NonLinear::Compare { + CompareValue( double missingValue ) : missingValue_( missingValue ) { ATLAS_ASSERT( !std::isnan( missingValue ) ); } + + bool operator()( const double& value ) const override { return value == missingValue_; } + + double missingValue_; +}; + + +NonLinear::NonLinear( const Config& config ) : missingValue_( new CompareNaN() ) { + double missingValue; + if ( config.get( "missingValue", missingValue ) ) { + missingValue_.reset( new CompareValue( missingValue ) ); + } } bool NonLinear::missingValue( const double& value ) const { - return value == missingValue_; + return ( *missingValue_ )( value ); } diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index a100d7416..b1b96987a 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -12,6 +12,8 @@ #pragma once +#include + #include "eckit/config/Parametrisation.h" #include "eckit/linalg/SparseMatrix.h" @@ -37,6 +39,11 @@ class NonLinear { using Scalar = eckit::linalg::Scalar; using Size = eckit::linalg::Size; + /// Missing values indicator + struct Compare { + virtual bool operator()( const double& ) const = 0; + }; + /// ctor NonLinear( const Config& ); @@ -49,9 +56,9 @@ class NonLinear { /// Check if value represents a missing value virtual bool missingValue( const double& ) const; -private: - /// Missing value to compare against - double missingValue_; +protected: + /// Missing value indicator + std::unique_ptr missingValue_; }; From 72d32bf3cfccd013f82532a2aa3f2b5f740e8130 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Jul 2020 10:10:23 +0100 Subject: [PATCH 080/161] ATLAS-306 Non-linear interpolation matrix corrections: documentation --- src/atlas/interpolation/nonlinear/NonLinear.h | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index b1b96987a..b6706e196 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -39,25 +39,39 @@ class NonLinear { using Scalar = eckit::linalg::Scalar; using Size = eckit::linalg::Size; - /// Missing values indicator + /** + * @brief Missing values indicator base class + */ struct Compare { virtual bool operator()( const double& ) const = 0; }; - /// ctor - NonLinear( const Config& ); + /** + * @brief NonLinear ctor + * @param [in] config with optional "missingValue", default NaN indicates missing value in field + */ + NonLinear( const Config& config); - /// dtor + /** + * @brief NonLinear dtor + */ virtual ~NonLinear(); - /// Update interpolation linear system to account for non-linearities - virtual bool treatment( Matrix&, const Field& ) const = 0; + /** + * @brief Apply non-linear corrections to interpolation matrix + * @param [inout] W interpolation matrix + * @param [in] field with missing values information + * @return if W was modified + */ + virtual bool treatment( Matrix& W, const Field& field ) const = 0; /// Check if value represents a missing value virtual bool missingValue( const double& ) const; protected: - /// Missing value indicator + /** + * @brief Missing values indicator + */ std::unique_ptr missingValue_; }; From fd9fd918a87358871b3d10b52f3db31b24944df9 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Jul 2020 10:12:42 +0100 Subject: [PATCH 081/161] ATLAS-306 Non-linear interpolation matrix corrections: choose NaN or fixed value --- src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc | 8 +++++--- src/atlas/interpolation/nonlinear/MissingIfAllMissing.h | 2 +- src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc | 6 ++++-- src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h | 2 +- .../interpolation/nonlinear/MissingIfHeaviestMissing.cc | 8 +++++--- .../interpolation/nonlinear/MissingIfHeaviestMissing.h | 2 +- src/atlas/interpolation/nonlinear/NonLinear.cc | 5 ----- src/atlas/interpolation/nonlinear/NonLinear.h | 5 +---- 8 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc index 56b3eaf87..7bfd6302c 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc @@ -28,10 +28,12 @@ namespace nonlinear { MissingIfAllMissing::MissingIfAllMissing( const Config& config ) : NonLinear( config ) {} -bool MissingIfAllMissing::treatment( NonLinear::Matrix& W, const Field& field ) const { +bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) const { // NOTE only for scalars (for now) auto values = array::make_view( field ); + ATLAS_ASSERT( missingValue_ ); + // correct matrix weigths for the missing values // (force a missing value only if all row values are missing) ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); @@ -53,7 +55,7 @@ bool MissingIfAllMissing::treatment( NonLinear::Matrix& W, const Field& field ) Matrix::iterator kt( it ); Size k = i; for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); + const bool miss = ( *missingValue_ )( values[it.col()] ); if ( miss ) { ++N_missing; @@ -75,7 +77,7 @@ bool MissingIfAllMissing::treatment( NonLinear::Matrix& W, const Field& field ) else { const double factor = 1. / sum; for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - const bool miss = missingValue( values[kt.col()] ); + const bool miss = ( *missingValue_ )( values[kt.col()] ); data[j] = miss ? 0. : ( factor * data[j] ); } } diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h index e3e3d957d..fabd84a9a 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h @@ -22,7 +22,7 @@ namespace nonlinear { struct MissingIfAllMissing : NonLinear { MissingIfAllMissing( const Config& ); - bool treatment( NonLinear::Matrix&, const Field& ) const; + bool execute( NonLinear::Matrix&, const Field& ) const; }; diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc index 00942ecf6..c0e3e7d86 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc @@ -26,10 +26,12 @@ namespace nonlinear { MissingIfAnyMissing::MissingIfAnyMissing( const Config& config ) : NonLinear( config ) {} -bool MissingIfAnyMissing::treatment( NonLinear::Matrix& W, const Field& field ) const { +bool MissingIfAnyMissing::execute( NonLinear::Matrix& W, const Field& field ) const { // NOTE only for scalars (for now) auto values = array::make_view( field ); + ATLAS_ASSERT( missingValue_ ); + // correct matrix weigths for the missing values // (force a missing value only if any row values is missing) ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); @@ -50,7 +52,7 @@ bool MissingIfAnyMissing::treatment( NonLinear::Matrix& W, const Field& field ) Matrix::iterator kt( it ); Size k = i; for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); + const bool miss = ( *missingValue_ )( values[it.col()] ); if ( miss ) { ++N_missing; diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h index 52279358e..a6af90b57 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h @@ -22,7 +22,7 @@ namespace nonlinear { struct MissingIfAnyMissing : NonLinear { MissingIfAnyMissing( const Config& ); - bool treatment( NonLinear::Matrix&, const Field& ) const; + bool execute( NonLinear::Matrix&, const Field& ) const; }; diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc index 1e2bbc480..17680c317 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc @@ -28,10 +28,12 @@ namespace nonlinear { MissingIfHeaviestMissing::MissingIfHeaviestMissing( const Config& config ) : NonLinear( config ) {} -bool MissingIfHeaviestMissing::treatment( NonLinear::Matrix& W, const Field& field ) const { +bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field ) const { // NOTE only for scalars (for now) auto values = array::make_view( field ); + ATLAS_ASSERT( missingValue_ ); + // correct matrix weigths for the missing values ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); @@ -54,7 +56,7 @@ bool MissingIfHeaviestMissing::treatment( NonLinear::Matrix& W, const Field& fie Matrix::iterator kt( it ); Size k = i; for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); + const bool miss = ( *missingValue_ )( values[it.col()] ); if ( miss ) { ++N_missing; @@ -81,7 +83,7 @@ bool MissingIfHeaviestMissing::treatment( NonLinear::Matrix& W, const Field& fie else { const double factor = 1. / sum; for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - const bool miss = missingValue( values[kt.col()] ); + const bool miss = ( *missingValue_ )( values[kt.col()] ); data[j] = miss ? 0. : ( factor * data[j] ); } } diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h index 28b2fae6a..6aa95b1fd 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h @@ -22,7 +22,7 @@ namespace nonlinear { struct MissingIfHeaviestMissing : NonLinear { MissingIfHeaviestMissing( const Config& ); - bool treatment( NonLinear::Matrix&, const Field& ) const; + bool execute( NonLinear::Matrix&, const Field& ) const; }; diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc index 8301b9a6a..feecc2da1 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.cc +++ b/src/atlas/interpolation/nonlinear/NonLinear.cc @@ -46,11 +46,6 @@ NonLinear::NonLinear( const Config& config ) : missingValue_( new CompareNaN() ) } -bool NonLinear::missingValue( const double& value ) const { - return ( *missingValue_ )( value ); -} - - NonLinear::~NonLinear() = default; diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index b6706e196..c35e3ff5f 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -63,10 +63,7 @@ class NonLinear { * @param [in] field with missing values information * @return if W was modified */ - virtual bool treatment( Matrix& W, const Field& field ) const = 0; - - /// Check if value represents a missing value - virtual bool missingValue( const double& ) const; + virtual bool execute( Matrix& W, const Field& field ) const = 0; protected: /** From 9ff5c83b6a04932ce220054d4b1e274afad0158b Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Jul 2020 10:34:37 +0100 Subject: [PATCH 082/161] ATLAS-306 Non-linear interpolation matrix corrections: refactoring --- src/atlas/interpolation/NonLinear.cc | 7 ++++++ src/atlas/interpolation/NonLinear.h | 24 ++++++++++++------- .../interpolation/nonlinear/NonLinear.cc | 22 +++++++++-------- src/atlas/interpolation/nonlinear/NonLinear.h | 10 ++++---- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/atlas/interpolation/NonLinear.cc b/src/atlas/interpolation/NonLinear.cc index 6510094a0..816319e7f 100644 --- a/src/atlas/interpolation/NonLinear.cc +++ b/src/atlas/interpolation/NonLinear.cc @@ -13,6 +13,7 @@ #include "atlas/interpolation/NonLinear.h" #include "atlas/interpolation/nonlinear/NonLinearFactory.h" +#include "atlas/runtime/Exception.h" #include "atlas/util/Config.h" @@ -31,5 +32,11 @@ NonLinear::NonLinear( const std::string& type, const util::Config& config ) : Handle( nonlinear::NonLinearFactory::build( type, config ) ) {} +bool NonLinear::execute( NonLinear::Matrix& W, const Field& f ) const { + ATLAS_ASSERT_MSG( this, "NonLinear: ObjectHandle not setup" ); + return get()->execute( W, f ); +} + + } // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/NonLinear.h b/src/atlas/interpolation/NonLinear.h index 93e02b561..a151554bd 100644 --- a/src/atlas/interpolation/NonLinear.h +++ b/src/atlas/interpolation/NonLinear.h @@ -14,16 +14,12 @@ #include +#include "atlas/interpolation/nonlinear/NonLinear.h" #include "atlas/library/config.h" #include "atlas/util/ObjectHandle.h" namespace atlas { -namespace interpolation { -namespace nonlinear { -class NonLinear; -} -} // namespace interpolation namespace util { class Config; } @@ -39,16 +35,28 @@ namespace interpolation { * The interpolatation are re-weighted to factor those values out of the resulting field. */ struct NonLinear : DOXYGEN_HIDE( public util::ObjectHandle ) { - using Spec = util::Config; + using Spec = util::Config; + using Matrix = nonlinear::NonLinear::Matrix; - /// ctor + // ctor using Handle::Handle; NonLinear(); NonLinear( const util::Config& ); NonLinear( const std::string& type, const util::Config& ); - /// bool operator + /** + * @brief bool operator + * @return if NonLinear object has been setup + */ using Handle::operator bool; // (ensure this exists) + + /** + * @brief Apply non-linear corrections to interpolation matrix + * @param [inout] W interpolation matrix + * @param [in] f field with missing values information + * @return if W was modified + */ + bool execute( Matrix& W, const Field& f ) const; }; diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc index feecc2da1..e9f56abb8 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.cc +++ b/src/atlas/interpolation/nonlinear/NonLinear.cc @@ -22,27 +22,29 @@ namespace interpolation { namespace nonlinear { -/// @brief CompareNaN Indicate missing value if NaN -struct CompareNaN : NonLinear::Compare { +/// @brief Indicate missing value if NaN +struct MissingValueNaN : NonLinear::MissingValue { bool operator()( const double& value ) const override { return std::isnan( value ); } }; -/// @brief CompareValue Indicate missing value if compares equally to pre-defined value -struct CompareValue : NonLinear::Compare { - CompareValue( double missingValue ) : missingValue_( missingValue ) { ATLAS_ASSERT( !std::isnan( missingValue ) ); } +/// @brief Indicate missing value if it compares equally to pre-defined value +struct MissingValueFixed : NonLinear::MissingValue { + MissingValueFixed( double missingValue ) : missingValue_( missingValue ) { + ATLAS_ASSERT( !std::isnan( missingValue ) ); + } bool operator()( const double& value ) const override { return value == missingValue_; } - double missingValue_; + const double missingValue_; }; -NonLinear::NonLinear( const Config& config ) : missingValue_( new CompareNaN() ) { +NonLinear::NonLinear( const Config& config ) { double missingValue; - if ( config.get( "missingValue", missingValue ) ) { - missingValue_.reset( new CompareValue( missingValue ) ); - } + missingValue_.reset( config.get( "missing_value", missingValue ) + ? static_cast( new MissingValueFixed( missingValue ) ) + : new MissingValueNaN() ); } diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index c35e3ff5f..b02ef0a0e 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -42,7 +42,7 @@ class NonLinear { /** * @brief Missing values indicator base class */ - struct Compare { + struct MissingValue { virtual bool operator()( const double& ) const = 0; }; @@ -50,7 +50,7 @@ class NonLinear { * @brief NonLinear ctor * @param [in] config with optional "missingValue", default NaN indicates missing value in field */ - NonLinear( const Config& config); + NonLinear( const Config& config ); /** * @brief NonLinear dtor @@ -60,16 +60,16 @@ class NonLinear { /** * @brief Apply non-linear corrections to interpolation matrix * @param [inout] W interpolation matrix - * @param [in] field with missing values information + * @param [in] f field with missing values information * @return if W was modified */ - virtual bool execute( Matrix& W, const Field& field ) const = 0; + virtual bool execute( Matrix& W, const Field& f ) const = 0; protected: /** * @brief Missing values indicator */ - std::unique_ptr missingValue_; + std::unique_ptr missingValue_; }; From ef839fa5673a8388d5a53fd74624b2a57975eba0 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Jul 2020 17:02:18 +0100 Subject: [PATCH 083/161] ATLAS-306 Non-linear interpolation matrix corrections: refactoring --- src/atlas/interpolation/NonLinear.cc | 15 ++++++++++++--- src/atlas/interpolation/NonLinear.h | 5 +++-- .../interpolation/nonlinear/NonLinearFactory.cc | 2 +- .../interpolation/nonlinear/NonLinearFactory.h | 10 ++++++---- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/atlas/interpolation/NonLinear.cc b/src/atlas/interpolation/NonLinear.cc index 816319e7f..e670ab435 100644 --- a/src/atlas/interpolation/NonLinear.cc +++ b/src/atlas/interpolation/NonLinear.cc @@ -21,14 +21,23 @@ namespace atlas { namespace interpolation { +namespace { +std::string get_string( const eckit::Parametrisation& p, const std::string& name ) { + std::string value; + ATLAS_ASSERT_MSG( p.get( name, value ), "" ); + return value; +} +} // namespace + + NonLinear::NonLinear() : Handle( nullptr ) {} -NonLinear::NonLinear( const util::Config& config ) : - Handle( nonlinear::NonLinearFactory::build( config.getString( "type" ), config ) ) {} +NonLinear::NonLinear( const NonLinear::Config& config ) : + Handle( nonlinear::NonLinearFactory::build( get_string( config, "type" ), config ) ) {} -NonLinear::NonLinear( const std::string& type, const util::Config& config ) : +NonLinear::NonLinear( const std::string& type, const NonLinear::Config& config ) : Handle( nonlinear::NonLinearFactory::build( type, config ) ) {} diff --git a/src/atlas/interpolation/NonLinear.h b/src/atlas/interpolation/NonLinear.h index a151554bd..c3d0f7ff2 100644 --- a/src/atlas/interpolation/NonLinear.h +++ b/src/atlas/interpolation/NonLinear.h @@ -36,13 +36,14 @@ namespace interpolation { */ struct NonLinear : DOXYGEN_HIDE( public util::ObjectHandle ) { using Spec = util::Config; + using Config = nonlinear::NonLinear::Config; using Matrix = nonlinear::NonLinear::Matrix; // ctor using Handle::Handle; NonLinear(); - NonLinear( const util::Config& ); - NonLinear( const std::string& type, const util::Config& ); + NonLinear( const Config& ); + NonLinear( const std::string& type, const Config& ); /** * @brief bool operator diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc b/src/atlas/interpolation/nonlinear/NonLinearFactory.cc index 4004b5448..e46dc4781 100644 --- a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc +++ b/src/atlas/interpolation/nonlinear/NonLinearFactory.cc @@ -37,7 +37,7 @@ void force_link() { } // namespace -const NonLinear* NonLinearFactory::build( const std::string& builder, const util::Config& config ) { +const NonLinear* NonLinearFactory::build( const std::string& builder, const Config& config ) { force_link(); auto factory = get( builder ); return factory->make( config ); diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.h b/src/atlas/interpolation/nonlinear/NonLinearFactory.h index 82bb93601..39e8670b9 100644 --- a/src/atlas/interpolation/nonlinear/NonLinearFactory.h +++ b/src/atlas/interpolation/nonlinear/NonLinearFactory.h @@ -14,7 +14,8 @@ #include -#include "atlas/util/Config.h" +#include "eckit/config/Parametrisation.h" + #include "atlas/util/Factory.h" @@ -24,23 +25,24 @@ namespace nonlinear { class NonLinear; +using Config = eckit::Parametrisation; class NonLinearFactory : public util::Factory { public: static std::string className() { return "NonLinearFactory"; } - static const NonLinear* build( const std::string&, const util::Config& ); + static const NonLinear* build( const std::string&, const Config& ); using Factory::Factory; private: - virtual const NonLinear* make( const util::Config& ) = 0; + virtual const NonLinear* make( const Config& ) = 0; }; template class NonLinearFactoryBuilder : public NonLinearFactory { private: - virtual const NonLinear* make( const util::Config& config ) override { return new T( config ); } + virtual const NonLinear* make( const Config& config ) override { return new T( config ); } public: using NonLinearFactory::NonLinearFactory; From a572de3b93356a81e46422261631845784cdfde1 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Jul 2020 19:56:27 +0100 Subject: [PATCH 084/161] ATLAS-306 Non-linear interpolation matrix corrections: refactoring --- .../nonlinear/MissingIfAllMissing.cc | 6 ++-- .../nonlinear/MissingIfAnyMissing.cc | 4 +-- .../nonlinear/MissingIfHeaviestMissing.cc | 6 ++-- .../interpolation/nonlinear/NonLinear.cc | 30 +++++-------------- src/atlas/interpolation/nonlinear/NonLinear.h | 20 +++++-------- 5 files changed, 21 insertions(+), 45 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc index 7bfd6302c..75f5993e9 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc @@ -32,8 +32,6 @@ bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) co // NOTE only for scalars (for now) auto values = array::make_view( field ); - ATLAS_ASSERT( missingValue_ ); - // correct matrix weigths for the missing values // (force a missing value only if all row values are missing) ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); @@ -55,7 +53,7 @@ bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) co Matrix::iterator kt( it ); Size k = i; for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = ( *missingValue_ )( values[it.col()] ); + const bool miss = missingValue( values[it.col()] ); if ( miss ) { ++N_missing; @@ -77,7 +75,7 @@ bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) co else { const double factor = 1. / sum; for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - const bool miss = ( *missingValue_ )( values[kt.col()] ); + const bool miss = missingValue( values[kt.col()] ); data[j] = miss ? 0. : ( factor * data[j] ); } } diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc index c0e3e7d86..92c150bbb 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc @@ -30,8 +30,6 @@ bool MissingIfAnyMissing::execute( NonLinear::Matrix& W, const Field& field ) co // NOTE only for scalars (for now) auto values = array::make_view( field ); - ATLAS_ASSERT( missingValue_ ); - // correct matrix weigths for the missing values // (force a missing value only if any row values is missing) ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); @@ -52,7 +50,7 @@ bool MissingIfAnyMissing::execute( NonLinear::Matrix& W, const Field& field ) co Matrix::iterator kt( it ); Size k = i; for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = ( *missingValue_ )( values[it.col()] ); + const bool miss = missingValue( values[it.col()] ); if ( miss ) { ++N_missing; diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc index 17680c317..e630e15e8 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc @@ -32,8 +32,6 @@ bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field // NOTE only for scalars (for now) auto values = array::make_view( field ); - ATLAS_ASSERT( missingValue_ ); - // correct matrix weigths for the missing values ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); @@ -56,7 +54,7 @@ bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field Matrix::iterator kt( it ); Size k = i; for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = ( *missingValue_ )( values[it.col()] ); + const bool miss = missingValue( values[it.col()] ); if ( miss ) { ++N_missing; @@ -83,7 +81,7 @@ bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field else { const double factor = 1. / sum; for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - const bool miss = ( *missingValue_ )( values[kt.col()] ); + const bool miss = missingValue( values[kt.col()] ); data[j] = miss ? 0. : ( factor * data[j] ); } } diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc index e9f56abb8..b349f8070 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.cc +++ b/src/atlas/interpolation/nonlinear/NonLinear.cc @@ -13,6 +13,7 @@ #include "atlas/interpolation/nonlinear/NonLinear.h" #include +#include #include "atlas/runtime/Exception.h" @@ -22,35 +23,20 @@ namespace interpolation { namespace nonlinear { -/// @brief Indicate missing value if NaN -struct MissingValueNaN : NonLinear::MissingValue { - bool operator()( const double& value ) const override { return std::isnan( value ); } -}; - - -/// @brief Indicate missing value if it compares equally to pre-defined value -struct MissingValueFixed : NonLinear::MissingValue { - MissingValueFixed( double missingValue ) : missingValue_( missingValue ) { - ATLAS_ASSERT( !std::isnan( missingValue ) ); - } +NonLinear::NonLinear( const Config& config ) : + missingValueIsNaN_( !config.get( "missing_value", missingValue_ = std::numeric_limits::quiet_NaN() ) ) { + ATLAS_ASSERT( missingValueIsNaN_ != !std::isnan( missingValue_ ) ); +} - bool operator()( const double& value ) const override { return value == missingValue_; } - const double missingValue_; -}; +NonLinear::~NonLinear() = default; -NonLinear::NonLinear( const Config& config ) { - double missingValue; - missingValue_.reset( config.get( "missing_value", missingValue ) - ? static_cast( new MissingValueFixed( missingValue ) ) - : new MissingValueNaN() ); +bool NonLinear::missingValue( const double& value ) const { + return missingValueIsNaN_ ? std::isnan( value ) : missingValue_ == value; } -NonLinear::~NonLinear() = default; - - } // namespace nonlinear } // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index b02ef0a0e..749f59736 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -12,8 +12,6 @@ #pragma once -#include - #include "eckit/config/Parametrisation.h" #include "eckit/linalg/SparseMatrix.h" @@ -39,13 +37,6 @@ class NonLinear { using Scalar = eckit::linalg::Scalar; using Size = eckit::linalg::Size; - /** - * @brief Missing values indicator base class - */ - struct MissingValue { - virtual bool operator()( const double& ) const = 0; - }; - /** * @brief NonLinear ctor * @param [in] config with optional "missingValue", default NaN indicates missing value in field @@ -65,11 +56,16 @@ class NonLinear { */ virtual bool execute( Matrix& W, const Field& f ) const = 0; -protected: /** - * @brief Missing values indicator + * @brief Return if value is considered a missing value + * @param [in] value to evaluate + * @return if value is considered a missing value */ - std::unique_ptr missingValue_; + bool missingValue( const double& ) const; + +private: + double missingValue_; + const bool missingValueIsNaN_; }; From 87faa8eeb9e1825c7e15bcad784ad4ec9c67306b Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 22 Jul 2020 19:57:18 +0100 Subject: [PATCH 085/161] ATLAS-306 Non-linear interpolation matrix corrections: integration of atlas::interpolation::NonLinear with atlas::interpolation::Method --- src/atlas/interpolation/method/Method.cc | 83 +++++++++++++++--------- src/atlas/interpolation/method/Method.h | 28 ++++---- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index ebdd1f7e7..e12efc063 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -29,37 +29,41 @@ namespace atlas { namespace interpolation { -void Method::check_compatibility( const Field& src, const Field& tgt ) const { +void Method::check_compatibility( const Field& src, const Field& tgt, const Matrix& W ) const { ATLAS_ASSERT( src.datatype() == tgt.datatype() ); ATLAS_ASSERT( src.rank() == tgt.rank() ); ATLAS_ASSERT( src.levels() == tgt.levels() ); ATLAS_ASSERT( src.variables() == tgt.variables() ); - ATLAS_ASSERT( !matrix_.empty() ); - ATLAS_ASSERT( tgt.shape( 0 ) >= static_cast( matrix_.rows() ) ); - ATLAS_ASSERT( src.shape( 0 ) >= static_cast( matrix_.cols() ) ); + ATLAS_ASSERT( !W.empty() ); + ATLAS_ASSERT( tgt.shape( 0 ) >= static_cast( W.rows() ) ); + ATLAS_ASSERT( src.shape( 0 ) >= static_cast( W.cols() ) ); } template -void Method::interpolate_field( const Field& src, Field& tgt ) const { - check_compatibility( src, tgt ); +void Method::interpolate_field( const Field& src, Field& tgt, const Matrix& W ) const { + check_compatibility( src, tgt, W ); + if ( src.rank() == 1 ) { - interpolate_field_rank1( src, tgt ); + interpolate_field_rank1( src, tgt, W ); + } + else if ( src.rank() == 2 ) { + interpolate_field_rank2( src, tgt, W ); } - if ( src.rank() == 2 ) { - interpolate_field_rank2( src, tgt ); + else if ( src.rank() == 3 ) { + interpolate_field_rank3( src, tgt, W ); } - if ( src.rank() == 3 ) { - interpolate_field_rank3( src, tgt ); + else { + ATLAS_NOTIMPLEMENTED; } } template -void Method::interpolate_field_rank1( const Field& src, Field& tgt ) const { - const auto outer = matrix_.outer(); - const auto index = matrix_.inner(); - const auto weight = matrix_.data(); - idx_t rows = static_cast( matrix_.rows() ); +void Method::interpolate_field_rank1( const Field& src, Field& tgt, const Matrix& W ) const { + const auto outer = W.outer(); + const auto index = W.inner(); + const auto weight = W.data(); + idx_t rows = static_cast( W.rows() ); if ( use_eckit_linalg_spmv_ ) { if ( src.datatype() != array::make_datatype() ) { @@ -71,7 +75,7 @@ void Method::interpolate_field_rank1( const Field& src, Field& tgt ) const { eckit::linalg::Vector v_src( const_cast( array::make_view( src ).data() ), src.shape( 0 ) ); eckit::linalg::Vector v_tgt( array::make_view( tgt ).data(), tgt.shape( 0 ) ); - eckit::linalg::LinearAlgebra::backend().spmv( matrix_, v_src, v_tgt ); + eckit::linalg::LinearAlgebra::backend().spmv( W, v_src, v_tgt ); } else { auto v_src = array::make_view( src ); @@ -89,11 +93,11 @@ void Method::interpolate_field_rank1( const Field& src, Field& tgt ) const { } template -void Method::interpolate_field_rank2( const Field& src, Field& tgt ) const { - const auto outer = matrix_.outer(); - const auto index = matrix_.inner(); - const auto weight = matrix_.data(); - idx_t rows = static_cast( matrix_.rows() ); +void Method::interpolate_field_rank2( const Field& src, Field& tgt, const Matrix& W ) const { + const auto outer = W.outer(); + const auto index = W.inner(); + const auto weight = W.data(); + idx_t rows = static_cast( W.rows() ); auto v_src = array::make_view( src ); auto v_tgt = array::make_view( tgt ); @@ -116,11 +120,11 @@ void Method::interpolate_field_rank2( const Field& src, Field& tgt ) const { template -void Method::interpolate_field_rank3( const Field& src, Field& tgt ) const { - const auto outer = matrix_.outer(); - const auto index = matrix_.inner(); - const auto weight = matrix_.data(); - idx_t rows = static_cast( matrix_.rows() ); +void Method::interpolate_field_rank3( const Field& src, Field& tgt, const Matrix& W ) const { + const auto outer = W.outer(); + const auto index = W.inner(); + const auto weight = W.data(); + idx_t rows = static_cast( W.rows() ); auto v_src = array::make_view( src ); auto v_tgt = array::make_view( tgt ); @@ -150,6 +154,11 @@ Method::Method( const Method::Config& config ) { std::string spmv = ""; config.get( "spmv", spmv ); use_eckit_linalg_spmv_ = ( spmv == "eckit" ); + + std::string non_linear; + if ( config.get( "non_linear", non_linear ) ) { + nonLinear_ = NonLinear( non_linear, config ); + } } void Method::setup( const FunctionSpace& source, const FunctionSpace& target ) { @@ -203,15 +212,27 @@ void Method::do_execute( const FieldSet& fieldsSource, FieldSet& fieldsTarget ) } void Method::do_execute( const Field& src, Field& tgt ) const { + ATLAS_TRACE( "atlas::interpolation::method::Method::do_execute()" ); + haloExchange( src ); - ATLAS_TRACE( "atlas::interpolation::method::Method::do_execute()" ); + // non-linearities: a non-empty M matrix contains the corrections applied to matrix_ + Matrix M; + if ( !matrix_.empty() && nonLinear_ ) { + Matrix W( matrix_ ); // copy (a big penalty -- copy-on-write would definitely be better) + if ( nonLinear_->execute( W, src ) ) { + M.swap( W ); + } + } if ( src.datatype().kind() == array::DataType::KIND_REAL64 ) { - interpolate_field( src, tgt ); + interpolate_field( src, tgt, M.empty() ? matrix_ : M ); + } + else if ( src.datatype().kind() == array::DataType::KIND_REAL32 ) { + interpolate_field( src, tgt, M.empty() ? matrix_ : M ); } - if ( src.datatype().kind() == array::DataType::KIND_REAL32 ) { - interpolate_field( src, tgt ); + else { + ATLAS_NOTIMPLEMENTED; } tgt.set_dirty(); diff --git a/src/atlas/interpolation/method/Method.h b/src/atlas/interpolation/method/Method.h index d78a5e25d..28f265c24 100644 --- a/src/atlas/interpolation/method/Method.h +++ b/src/atlas/interpolation/method/Method.h @@ -14,6 +14,7 @@ #include #include +#include "atlas/interpolation/NonLinear.h" #include "atlas/util/Object.h" #include "eckit/config/Configuration.h" #include "eckit/linalg/SparseMatrix.h" @@ -36,10 +37,10 @@ class Method : public util::Object { virtual ~Method() {} /** - * @brief Setup the interpolator relating two functionspaces - * @param source functionspace containing source elements - * @param target functionspace containing target points - */ + * @brief Setup the interpolator relating two functionspaces + * @param source functionspace containing source elements + * @param target functionspace containing target points + */ void setup( const FunctionSpace& source, const FunctionSpace& target ); void setup( const Grid& source, const Grid& target ); void setup( const FunctionSpace& source, const Field& target ); @@ -66,14 +67,9 @@ class Method : public util::Object { void haloExchange( const FieldSet& ) const; void haloExchange( const Field& ) const; - //const Config& config_; - - // NOTE : Matrix-free or non-linear interpolation operators do not have - // matrices, - // so do not expose here, even though only linear operators are now - // implemented. + // NOTE : Matrix-free operators do not have matrices (!), so do not expose here Matrix matrix_; - + NonLinear nonLinear_; bool use_eckit_linalg_spmv_; protected: @@ -84,18 +80,18 @@ class Method : public util::Object { private: template - void interpolate_field( const Field& src, Field& tgt ) const; + void interpolate_field( const Field& src, Field& tgt, const Matrix& ) const; template - void interpolate_field_rank1( const Field& src, Field& tgt ) const; + void interpolate_field_rank1( const Field& src, Field& tgt, const Matrix& ) const; template - void interpolate_field_rank2( const Field& src, Field& tgt ) const; + void interpolate_field_rank2( const Field& src, Field& tgt, const Matrix& ) const; template - void interpolate_field_rank3( const Field& src, Field& tgt ) const; + void interpolate_field_rank3( const Field& src, Field& tgt, const Matrix& ) const; - void check_compatibility( const Field& src, const Field& tgt ) const; + void check_compatibility( const Field& src, const Field& tgt, const Matrix& W ) const; }; } // namespace interpolation From 1a2f35ae7c41d09102ad1a52c9358af78e33cd49 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 23 Jul 2020 14:42:05 +0100 Subject: [PATCH 086/161] ATLAS-306 Non-linear interpolation matrix corrections: atlas::interpolation::MissingValue --- src/atlas/CMakeLists.txt | 4 + src/atlas/interpolation/MissingValue.cc | 63 +++++++++ src/atlas/interpolation/MissingValue.h | 52 ++++++++ .../interpolation/nonlinear/MissingValue.cc | 120 ++++++++++++++++++ .../interpolation/nonlinear/MissingValue.h | 60 +++++++++ 5 files changed, 299 insertions(+) create mode 100644 src/atlas/interpolation/MissingValue.cc create mode 100644 src/atlas/interpolation/MissingValue.h create mode 100644 src/atlas/interpolation/nonlinear/MissingValue.cc create mode 100644 src/atlas/interpolation/nonlinear/MissingValue.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index c62074126..29cd8f01b 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -475,6 +475,8 @@ list( APPEND atlas_interpolation_srcs interpolation.h interpolation/Interpolation.cc interpolation/Interpolation.h +interpolation/MissingValue.cc +interpolation/MissingValue.h interpolation/NonLinear.cc interpolation/NonLinear.h interpolation/Vector2D.cc @@ -536,6 +538,8 @@ interpolation/nonlinear/MissingIfAnyMissing.cc interpolation/nonlinear/MissingIfAnyMissing.h interpolation/nonlinear/MissingIfHeaviestMissing.cc interpolation/nonlinear/MissingIfHeaviestMissing.h +interpolation/nonlinear/MissingValue.cc +interpolation/nonlinear/MissingValue.h interpolation/nonlinear/NonLinear.cc interpolation/nonlinear/NonLinear.h interpolation/nonlinear/NonLinearFactory.cc diff --git a/src/atlas/interpolation/MissingValue.cc b/src/atlas/interpolation/MissingValue.cc new file mode 100644 index 000000000..a78d413e9 --- /dev/null +++ b/src/atlas/interpolation/MissingValue.cc @@ -0,0 +1,63 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/interpolation/MissingValue.h" + +#include + +#include "eckit/types/FloatCompare.h" + +#include "atlas/runtime/Exception.h" +#include "atlas/util/Config.h" + + +namespace atlas { +namespace interpolation { + + +namespace { +using Config = MissingValue::Config; + + +std::string config_type( const Config& c ) { + std::string value; + ATLAS_ASSERT( c.get( "type", value ) ); + return value; +} +} // namespace + + +MissingValue::MissingValue() : Handle( nullptr ) {} + + +MissingValue::MissingValue( const MissingValue::Config& config ) : + Handle( nonlinear::MissingValueFactory::build( config_type( config ), config ) ) {} + + +MissingValue::MissingValue( const std::string& type, const MissingValue::Config& config ) : + Handle( nonlinear::MissingValueFactory::build( type, config ) ) {} + + +bool MissingValue::operator()( const double& value ) const { + ATLAS_ASSERT_MSG( operator bool(), "MissingValue: ObjectHandle not setup" ); + return get()->operator()( value ); +} + + +const nonlinear::MissingValue& MissingValue::ref() const { + ATLAS_ASSERT_MSG( operator bool(), "MissingValue: ObjectHandle not setup" ); + return *get(); // (one dereferencing level less) +} + + +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/MissingValue.h b/src/atlas/interpolation/MissingValue.h new file mode 100644 index 000000000..3cc845120 --- /dev/null +++ b/src/atlas/interpolation/MissingValue.h @@ -0,0 +1,52 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include + +#include "atlas/interpolation/nonlinear/MissingValue.h" +#include "atlas/library/config.h" +#include "atlas/util/ObjectHandle.h" + + +namespace atlas { +namespace util { +class Config; +} +} // namespace atlas + + +namespace atlas { +namespace interpolation { + + +/// @brief Missing values indicator object +struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle ) { + using Spec = util::Config; + using Config = nonlinear::MissingValue::Config; + + using Handle::Handle; + MissingValue(); + MissingValue( const Config& ); + MissingValue( const std::string& type, const Config& ); + + using Handle::operator bool; // (ensure this exists) + + bool operator()( const double& ) const; + + const nonlinear::MissingValue& ref() const; +}; + + +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingValue.cc b/src/atlas/interpolation/nonlinear/MissingValue.cc new file mode 100644 index 000000000..adce996f4 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/MissingValue.cc @@ -0,0 +1,120 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/interpolation/nonlinear/MissingValue.h" + +#include + +#include "eckit/types/FloatCompare.h" + +#include "atlas/runtime/Exception.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +std::string config_type( const MissingValue::Config& c ) { + std::string value; + ATLAS_ASSERT( c.get( "type", value ) ); + return value; +} + + +double config_value( const MissingValue::Config& c ) { + double value; + ATLAS_ASSERT( c.get( "missing_value", value ) ); + return value; +} + + +double config_epsilon( const MissingValue::Config& c ) { + double value; + ATLAS_ASSERT( c.get( "missing_value_epsilon", value ) ); + return value; +} + + +/// @brief Missing value if NaN +struct MissingValueNaN : MissingValue { + MissingValueNaN( const Config& ) {} + bool operator()( const double& value ) const override { return std::isnan( value ); } +}; + + +/// @brief Missing value if comparing equally to pre-defined value +struct MissingValueEquals : MissingValue { + MissingValueEquals( const Config& config ) : missingValue_( config_value( config ) ) {} + MissingValueEquals( double missingValue ) : missingValue_( missingValue ) { + ATLAS_ASSERT( !std::isnan( missingValue_ ) ); + } + + bool operator()( const double& value ) const override { return value == missingValue_; } + + const double missingValue_; +}; + + +/// @brief Missing value if comparing approximately to pre-defined value +struct MissingValueApprox : MissingValue { + MissingValueApprox( const Config& config ) : + missingValue_( config_value( config ) ), epsilon_( config_epsilon( config ) ) {} + MissingValueApprox( double missingValue, double epsilon ) : missingValue_( missingValue ), epsilon_( epsilon ) { + ATLAS_ASSERT( !std::isnan( missingValue_ ) ); + ATLAS_ASSERT( epsilon_ >= 0. ); + } + + bool operator()( const double& value ) const override { + return eckit::types::is_approximately_equal( value, missingValue_, epsilon_ ); + } + + const double missingValue_; + const double epsilon_; +}; + + +MissingValue::~MissingValue() = default; + + +namespace { + + +static MissingValueFactoryBuilder __mv1( "nan" ); +static MissingValueFactoryBuilder __mv2( "equals" ); +static MissingValueFactoryBuilder __mv3( "approximately-equals" ); + + +void force_link() { + static struct Link { + Link() { + MissingValueFactoryBuilder(); + MissingValueFactoryBuilder(); + MissingValueFactoryBuilder(); + } + } link; +} + + +} // namespace + + +const MissingValue* MissingValueFactory::build( const std::string& builder, const Config& config ) { + force_link(); + auto factory = get( builder ); + return factory->make( config ); +} + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingValue.h b/src/atlas/interpolation/nonlinear/MissingValue.h new file mode 100644 index 000000000..b4ae7ed24 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/MissingValue.h @@ -0,0 +1,60 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include + +#include "eckit/config/Parametrisation.h" + +#include "atlas/util/Factory.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +/// @brief Missing values indicator base class +struct MissingValue { + using Config = eckit::Parametrisation; + virtual ~MissingValue(); + virtual bool operator()( const double& ) const = 0; +}; + + +/// @brief Missing values indicator factory +struct MissingValueFactory : util::Factory { + using Config = MissingValue::Config; + using Factory::Factory; + static std::string className() { return "MissingValueFactory"; } + static const MissingValue* build( const std::string&, const Config& ); + +private: + virtual const MissingValue* make( const Config& ) = 0; +}; + + +/// @brief Missing values indicator builder for factory registration +template +class MissingValueFactoryBuilder : public MissingValueFactory { +private: + virtual const MissingValue* make( const Config& config ) override { return new T( config ); } + +public: + using MissingValueFactory::MissingValueFactory; +}; + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas From 1ac70cdd96b9349b2fb647c66c7b75a966c31760 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 23 Jul 2020 15:06:46 +0100 Subject: [PATCH 087/161] ATLAS-306 Non-linear interpolation matrix corrections: atlas::interpolation::MissingValue integration --- src/atlas/interpolation/NonLinear.cc | 8 ++++---- .../nonlinear/MissingIfAllMissing.cc | 3 ++- .../nonlinear/MissingIfAnyMissing.cc | 3 ++- .../nonlinear/MissingIfHeaviestMissing.cc | 3 ++- .../interpolation/nonlinear/NonLinear.cc | 20 +++++++++---------- src/atlas/interpolation/nonlinear/NonLinear.h | 19 ++++++------------ 6 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/atlas/interpolation/NonLinear.cc b/src/atlas/interpolation/NonLinear.cc index e670ab435..e920f9b83 100644 --- a/src/atlas/interpolation/NonLinear.cc +++ b/src/atlas/interpolation/NonLinear.cc @@ -22,9 +22,9 @@ namespace interpolation { namespace { -std::string get_string( const eckit::Parametrisation& p, const std::string& name ) { +std::string config_type( const eckit::Parametrisation& config ) { std::string value; - ATLAS_ASSERT_MSG( p.get( name, value ), "" ); + ATLAS_ASSERT_MSG( config.get( "type", value ), "" ); return value; } } // namespace @@ -34,7 +34,7 @@ NonLinear::NonLinear() : Handle( nullptr ) {} NonLinear::NonLinear( const NonLinear::Config& config ) : - Handle( nonlinear::NonLinearFactory::build( get_string( config, "type" ), config ) ) {} + Handle( nonlinear::NonLinearFactory::build( config_type( config ), config ) ) {} NonLinear::NonLinear( const std::string& type, const NonLinear::Config& config ) : @@ -42,7 +42,7 @@ NonLinear::NonLinear( const std::string& type, const NonLinear::Config& config ) bool NonLinear::execute( NonLinear::Matrix& W, const Field& f ) const { - ATLAS_ASSERT_MSG( this, "NonLinear: ObjectHandle not setup" ); + ATLAS_ASSERT_MSG( operator bool(), "NonLinear: ObjectHandle not setup" ); return get()->execute( W, f ); } diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc index 75f5993e9..6885b2225 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc @@ -30,7 +30,8 @@ MissingIfAllMissing::MissingIfAllMissing( const Config& config ) : NonLinear( co bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) const { // NOTE only for scalars (for now) - auto values = array::make_view( field ); + auto values = array::make_view( field ); + auto& missingValue = missingValue_.ref(); // correct matrix weigths for the missing values // (force a missing value only if all row values are missing) diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc index 92c150bbb..0055215ad 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc @@ -28,7 +28,8 @@ MissingIfAnyMissing::MissingIfAnyMissing( const Config& config ) : NonLinear( co bool MissingIfAnyMissing::execute( NonLinear::Matrix& W, const Field& field ) const { // NOTE only for scalars (for now) - auto values = array::make_view( field ); + auto values = array::make_view( field ); + auto& missingValue = missingValue_.ref(); // correct matrix weigths for the missing values // (force a missing value only if any row values is missing) diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc index e630e15e8..c135fa8dc 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc @@ -30,7 +30,8 @@ MissingIfHeaviestMissing::MissingIfHeaviestMissing( const Config& config ) : Non bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field ) const { // NOTE only for scalars (for now) - auto values = array::make_view( field ); + auto values = array::make_view( field ); + auto& missingValue = missingValue_.ref(); // correct matrix weigths for the missing values ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc index b349f8070..63a70802d 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.cc +++ b/src/atlas/interpolation/nonlinear/NonLinear.cc @@ -12,9 +12,6 @@ #include "atlas/interpolation/nonlinear/NonLinear.h" -#include -#include - #include "atlas/runtime/Exception.h" @@ -23,18 +20,21 @@ namespace interpolation { namespace nonlinear { -NonLinear::NonLinear( const Config& config ) : - missingValueIsNaN_( !config.get( "missing_value", missingValue_ = std::numeric_limits::quiet_NaN() ) ) { - ATLAS_ASSERT( missingValueIsNaN_ != !std::isnan( missingValue_ ) ); +namespace { +std::string config_missing_value( const MissingValue::Config& c ) { + std::string value; + ATLAS_ASSERT_MSG( c.get( "missing_value_compare", value ), "NonLinear: expecting 'missing_value_compare'" ); + return value; } +} // namespace -NonLinear::~NonLinear() = default; +NonLinear::NonLinear( const Config& config ) : missingValue_( config_missing_value( config ), config ) { + ATLAS_ASSERT_MSG( missingValue_, "NonLinear: missingValue setup" ); +} -bool NonLinear::missingValue( const double& value ) const { - return missingValueIsNaN_ ? std::isnan( value ) : missingValue_ == value; -} +NonLinear::~NonLinear() = default; } // namespace nonlinear diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index 749f59736..eca6405b7 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -15,6 +15,8 @@ #include "eckit/config/Parametrisation.h" #include "eckit/linalg/SparseMatrix.h" +#include "atlas/interpolation/MissingValue.h" + namespace atlas { class Field; @@ -43,9 +45,7 @@ class NonLinear { */ NonLinear( const Config& config ); - /** - * @brief NonLinear dtor - */ + /// @brief NonLinear dtor virtual ~NonLinear(); /** @@ -56,16 +56,9 @@ class NonLinear { */ virtual bool execute( Matrix& W, const Field& f ) const = 0; - /** - * @brief Return if value is considered a missing value - * @param [in] value to evaluate - * @return if value is considered a missing value - */ - bool missingValue( const double& ) const; - -private: - double missingValue_; - const bool missingValueIsNaN_; +protected: + /// Evaluate if value is considered a missing value + interpolation::MissingValue missingValue_; }; From 8bdca779ac45a9b337479d0c4987c2c50a98609c Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 23 Jul 2020 18:48:01 +0100 Subject: [PATCH 088/161] ATLAS-306 Non-linear interpolation matrix corrections: atlas::interpolation::MissingValue missing_value_epsilon optional, default 0 --- src/atlas/interpolation/nonlinear/MissingValue.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/MissingValue.cc b/src/atlas/interpolation/nonlinear/MissingValue.cc index adce996f4..4a6b3c514 100644 --- a/src/atlas/interpolation/nonlinear/MissingValue.cc +++ b/src/atlas/interpolation/nonlinear/MissingValue.cc @@ -39,8 +39,8 @@ double config_value( const MissingValue::Config& c ) { double config_epsilon( const MissingValue::Config& c ) { - double value; - ATLAS_ASSERT( c.get( "missing_value_epsilon", value ) ); + double value = 0.; + c.get( "missing_value_epsilon", value ); return value; } From c379d0acaa5689e008d3924dba43bef93ba86023 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Jul 2020 08:59:15 +0100 Subject: [PATCH 089/161] ATLAS-306 Non-linear interpolation matrix corrections: unit test --- src/tests/interpolation/CMakeLists.txt | 6 + .../test_interpolation_non_linear.cc | 203 ++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 src/tests/interpolation/test_interpolation_non_linear.cc diff --git a/src/tests/interpolation/CMakeLists.txt b/src/tests/interpolation/CMakeLists.txt index 187af2b9b..f779926f6 100644 --- a/src/tests/interpolation/CMakeLists.txt +++ b/src/tests/interpolation/CMakeLists.txt @@ -31,6 +31,12 @@ ecbuild_add_executable( TARGET atlas_test_interpolation_structured2D NOINSTALL ) +ecbuild_add_test( TARGET atlas_test_interpolation_non_linear + SOURCES test_interpolation_non_linear.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + ecbuild_add_test( TARGET atlas_test_interpolation_bilinear COMMAND atlas_test_interpolation_structured2D ARGS --scheme linear ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc new file mode 100644 index 000000000..c5e31eee2 --- /dev/null +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -0,0 +1,203 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include + +#include "atlas/array.h" +#include "atlas/functionspace.h" +#include "atlas/grid.h" +#include "atlas/interpolation.h" +#include "atlas/interpolation/NonLinear.h" +#include "atlas/mesh.h" +#include "atlas/meshgenerator.h" +#include "atlas/runtime/Exception.h" + +#include "tests/AtlasTestEnvironment.h" + + +namespace atlas { +namespace test { + + +const double missingValue = 42.; +const double missingValueEps = 1e-9; +const double nan = std::numeric_limits::quiet_NaN(); + + +CASE( "test_interpolation_non_linear_missing_value" ) { + using interpolation::MissingValue; + + + SECTION( "nan" ) { + util::Config config; + auto mv_nan = MissingValue( "nan", config ); + EXPECT( bool( mv_nan ) ); + + EXPECT( mv_nan( nan ) ); + EXPECT( mv_nan( missingValue ) == false ); + + config.set( "type", "nan" ); + mv_nan = MissingValue( config ); + EXPECT( bool( mv_nan ) ); + + EXPECT( mv_nan( nan ) ); + EXPECT( mv_nan( missingValue ) == false ); + } + + + SECTION( "equals" ) { + util::Config config; + config.set( "missing_value", missingValue ); + + auto mv = MissingValue( "equals", config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( missingValue - 1 ) == false ); + EXPECT( mv( missingValue + 1 ) == false ); + // EXPECT( mv( missingValue ) ); // (fails due to internal conversions -- a bug?) + + config.set( "type", "equals" ); + mv = MissingValue( config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( missingValue - 1 ) == false ); + EXPECT( mv( missingValue + 1 ) == false ); + // EXPECT( mv( missingValue ) ); // (fails due to internal conversions -- a bug?) + } + + + SECTION( "approximately-equals" ) { + util::Config config; + config.set( "missing_value", missingValue ); + config.set( "missing_value_epsilon", missingValueEps ); + + auto mv = MissingValue( "approximately-equals", config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( missingValue - missingValueEps * 2 ) == false ); + EXPECT( mv( missingValue - missingValueEps / 2 ) ); + EXPECT( mv( missingValue ) ); + EXPECT( mv( missingValue + missingValueEps / 2 ) ); + EXPECT( mv( missingValue + missingValueEps * 2 ) == false ); + + config.set( "type", "approximately-equals" ); + mv = MissingValue( config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( missingValue - missingValueEps * 2 ) == false ); + EXPECT( mv( missingValue - missingValueEps / 2 ) ); + EXPECT( mv( missingValue ) ); + EXPECT( mv( missingValue + missingValueEps / 2 ) ); + EXPECT( mv( missingValue + missingValueEps * 2 ) == false ); + } +} + + +CASE( "test_interpolation_non_linear_matrix" ) { + /* + Set input field full of 1's, with 9 nodes + 1 ... 1 ... 1 + : : : + 1-----m ... 1 m: missing value + |i i| : i: interpolation on two points, this quadrilateral only + 1-----1 ... 1 + */ + RectangularDomain domain( {0, 2}, {0, 2}, "degrees" ); + Grid gridA( "L90", domain ); + + const idx_t nbNodes = 9; + ATLAS_ASSERT( gridA.size() == nbNodes ); + + Mesh meshA = MeshGenerator( "structured" ).generate( gridA ); + + functionspace::NodeColumns fsA( meshA ); + Field fieldA = fsA.createField( option::name( "A" ) ); + + auto viewA = array::make_view( fieldA ); + for ( idx_t j = 0; j < fsA.nodes().size(); ++j ) { + viewA( j ) = 1; + } + + + // Set output field (2 points) + functionspace::PointCloud fsB( {PointLonLat{0.1, 0.1}, PointLonLat{0.9, 0.9}} ); + Field fieldB( "B", array::make_datatype(), array::make_shape( fsB.size() ) ); + + auto viewB = array::make_view( fieldB ); + ATLAS_ASSERT( viewB.size() == 2 ); + + + util::Config cfg( "type", "finite-element" ); + cfg.set( "missing_value", missingValue ); + cfg.set( "missing_value_epsilon", missingValueEps ); + + + // NOTE: "equals" is not tested due to internal conversions + SECTION( "missing-if-all-missing" ) { + for ( std::string cmp : {"approximately-equals", "nan"} ) { + viewA( 4 ) = cmp == "nan" ? nan : missingValue; + + cfg.set( "non_linear", "missing-if-all-missing" ); + cfg.set( "missing_value_compare", cmp ); + + Interpolation interpolation( cfg, fsA, fsB ); + interpolation.execute( fieldA, fieldB ); + + interpolation::MissingValue miss( cmp, cfg ); + EXPECT( miss( viewB( 0 ) ) == false ); + EXPECT( miss( viewB( 1 ) ) == false ); + } + } + + + SECTION( "missing-if-any-missing" ) { + for ( std::string cmp : {"approximately-equals", "nan"} ) { + viewA( 4 ) = cmp == "nan" ? nan : missingValue; + + cfg.set( "non_linear", "missing-if-any-missing" ); + cfg.set( "missing_value_compare", cmp ); + + Interpolation interpolation( cfg, fsA, fsB ); + interpolation.execute( fieldA, fieldB ); + + interpolation::MissingValue miss( cmp, cfg ); + EXPECT( miss( viewB( 0 ) ) ); + EXPECT( miss( viewB( 1 ) ) ); + } + } + + + SECTION( "missing-if-heaviest-missing" ) { + for ( std::string cmp : {"approximately-equals", "nan"} ) { + viewA( 4 ) = cmp == "nan" ? nan : missingValue; + + cfg.set( "non_linear", "missing-if-heaviest-missing" ); + cfg.set( "missing_value_compare", cmp ); + + Interpolation interpolation( cfg, fsA, fsB ); + interpolation.execute( fieldA, fieldB ); + + interpolation::MissingValue miss( cmp, cfg ); + EXPECT( miss( viewB( 0 ) ) == false ); + EXPECT( miss( viewB( 1 ) ) ); + } + } +} + + +} // namespace test +} // namespace atlas + + +int main( int argc, char** argv ) { + return atlas::test::run( argc, argv ); +} From 4d13d11d4ec8b7d6406f28c6098182b0c4c447fd Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Jul 2020 09:44:40 +0100 Subject: [PATCH 090/161] ATLAS-306 Non-linear interpolation matrix corrections: atlas::interpolation::MissingValue::isnan --- src/atlas/interpolation/MissingValue.cc | 6 ++++++ src/atlas/interpolation/MissingValue.h | 2 ++ src/atlas/interpolation/nonlinear/MissingValue.cc | 5 +++++ src/atlas/interpolation/nonlinear/MissingValue.h | 1 + 4 files changed, 14 insertions(+) diff --git a/src/atlas/interpolation/MissingValue.cc b/src/atlas/interpolation/MissingValue.cc index a78d413e9..0e630fad2 100644 --- a/src/atlas/interpolation/MissingValue.cc +++ b/src/atlas/interpolation/MissingValue.cc @@ -53,6 +53,12 @@ bool MissingValue::operator()( const double& value ) const { } +bool MissingValue::isnan() const { + ATLAS_ASSERT_MSG( operator bool(), "MissingValue: ObjectHandle not setup" ); + return get()->isnan(); +} + + const nonlinear::MissingValue& MissingValue::ref() const { ATLAS_ASSERT_MSG( operator bool(), "MissingValue: ObjectHandle not setup" ); return *get(); // (one dereferencing level less) diff --git a/src/atlas/interpolation/MissingValue.h b/src/atlas/interpolation/MissingValue.h index 3cc845120..fb85a064c 100644 --- a/src/atlas/interpolation/MissingValue.h +++ b/src/atlas/interpolation/MissingValue.h @@ -44,6 +44,8 @@ struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle Date: Fri, 24 Jul 2020 09:54:20 +0100 Subject: [PATCH 091/161] ATLAS-306 Non-linear interpolation matrix corrections: unit test fix (NaN as missing values) --- .../interpolation/nonlinear/MissingIfAllMissing.cc | 14 ++++++++++++-- .../interpolation/nonlinear/MissingIfAnyMissing.cc | 13 ++++++++++++- .../nonlinear/MissingIfHeaviestMissing.cc | 14 ++++++++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc index 6885b2225..6c2d7aca7 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc @@ -39,6 +39,7 @@ bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) co auto data = const_cast( W.data() ); bool modif = false; + bool zeros = false; Size i = 0; Matrix::iterator it( W ); @@ -76,14 +77,23 @@ bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) co else { const double factor = 1. / sum; for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - const bool miss = missingValue( values[kt.col()] ); - data[j] = miss ? 0. : ( factor * data[j] ); + if ( missingValue( values[kt.col()] ) ) { + data[j] = 0.; + zeros = true; + } + else { + data[j] *= factor; + } } } modif = true; } } + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + return modif; } diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc index 0055215ad..39a7c221a 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc @@ -37,6 +37,7 @@ bool MissingIfAnyMissing::execute( NonLinear::Matrix& W, const Field& field ) co auto data = const_cast( W.data() ); bool modif = false; + bool zeros = false; Size i = 0; Matrix::iterator it( W ); @@ -62,12 +63,22 @@ bool MissingIfAnyMissing::execute( NonLinear::Matrix& W, const Field& field ) co // if any values in row are missing, force missing value if ( N_missing > 0 ) { for ( Size j = k; j < k + N_entries; ++j ) { - data[j] = j == i_missing ? 1. : 0.; + if ( j == i_missing ) { + data[j] = 1.; + } + else { + data[j] = 0.; + zeros = true; + } } modif = true; } } + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + return modif; } diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc index c135fa8dc..2f37b3cb4 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc @@ -38,6 +38,7 @@ bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field auto data = const_cast( W.data() ); bool modif = false; + bool zeros = false; Size i = 0; Matrix::iterator it( W ); @@ -82,14 +83,23 @@ bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field else { const double factor = 1. / sum; for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - const bool miss = missingValue( values[kt.col()] ); - data[j] = miss ? 0. : ( factor * data[j] ); + if ( missingValue( values[kt.col()] ) ) { + data[j] = 0.; + zeros = true; + } + else { + data[j] *= factor; + } } } modif = true; } } + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + return modif; } From 03102820436057038954ab21f1a367298d07c76a Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 24 Jul 2020 11:26:08 +0100 Subject: [PATCH 092/161] ATLAS-306 Non-linear interpolation matrix corrections: documentation --- src/atlas/interpolation/MissingValue.h | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/atlas/interpolation/MissingValue.h b/src/atlas/interpolation/MissingValue.h index fb85a064c..39a185c8f 100644 --- a/src/atlas/interpolation/MissingValue.h +++ b/src/atlas/interpolation/MissingValue.h @@ -30,22 +30,40 @@ namespace atlas { namespace interpolation { -/// @brief Missing values indicator object +/// @brief Missing values indicator struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle ) { using Spec = util::Config; using Config = nonlinear::MissingValue::Config; + // ctor using Handle::Handle; MissingValue(); MissingValue( const Config& ); MissingValue( const std::string& type, const Config& ); + /** + * @brief bool operator + * @return if MissingValue object has been setup + */ using Handle::operator bool; // (ensure this exists) - bool operator()( const double& ) const; - + /** + * @brief operator() on user-defined values + * @param [in] value user-defined value + * @return if user-defined value indicates a missing value + */ + bool operator()( const double& value ) const; + + /** + * @brief if missing value is represented by not-a-number + * @return if missing value is represented by not-a-number + */ bool isnan() const; + /** + * @brief bool operator + * @return reference to internal implementation + */ const nonlinear::MissingValue& ref() const; }; From c267c85bff354caad45a6be57c6ede1eec7d9318 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 16 Sep 2020 20:17:58 +0100 Subject: [PATCH 093/161] ATLAS-306 re-enable missing value exact 'equals' tests (failing) --- src/tests/interpolation/test_interpolation_non_linear.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index c5e31eee2..ba30fde39 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -63,7 +63,7 @@ CASE( "test_interpolation_non_linear_missing_value" ) { EXPECT( mv( missingValue - 1 ) == false ); EXPECT( mv( missingValue + 1 ) == false ); - // EXPECT( mv( missingValue ) ); // (fails due to internal conversions -- a bug?) + EXPECT( mv( missingValue ) ); config.set( "type", "equals" ); mv = MissingValue( config ); @@ -71,7 +71,7 @@ CASE( "test_interpolation_non_linear_missing_value" ) { EXPECT( mv( missingValue - 1 ) == false ); EXPECT( mv( missingValue + 1 ) == false ); - // EXPECT( mv( missingValue ) ); // (fails due to internal conversions -- a bug?) + EXPECT( mv( missingValue ) ); } From 6ad0cb84e9243dee642de84baaa8841ef23981c6 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 16 Sep 2020 20:20:46 +0100 Subject: [PATCH 094/161] ATLAS-306 re-enable missing value exact 'equals' tests (failing) fix by introducing a second (intermediate) variable to copy a value from a atlas::util::Config (eckit::LocalCOnfiguration), why it doesn't work without this isn't understood --- .../interpolation/nonlinear/MissingValue.cc | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/MissingValue.cc b/src/atlas/interpolation/nonlinear/MissingValue.cc index dd816e8aa..f7a871ce3 100644 --- a/src/atlas/interpolation/nonlinear/MissingValue.cc +++ b/src/atlas/interpolation/nonlinear/MissingValue.cc @@ -55,16 +55,25 @@ struct MissingValueNaN : MissingValue { /// @brief Missing value if comparing equally to pre-defined value struct MissingValueEquals : MissingValue { - MissingValueEquals( const Config& config ) : missingValue_( config_value( config ) ) {} - MissingValueEquals( double missingValue ) : missingValue_( missingValue ) { - ATLAS_ASSERT( !std::isnan( missingValue_ ) ); + MissingValueEquals( const Config& config ) : + missingValue_( config_value( config ) ), missingValue2_( missingValue_ ) { + ATLAS_ASSERT( missingValue_ == missingValue2_ ); // this succeeds + } + + MissingValueEquals( double missingValue ) : missingValue_( missingValue ), missingValue2_( missingValue_ ) { + ATLAS_ASSERT( !std::isnan( missingValue2_ ) ); } - bool operator()( const double& value ) const override { return value == missingValue_; } + bool operator()( const double& value ) const override { + // ATLAS_ASSERT(missingValue_ == missingValue2_); // this fails when not using missingValue2_ (copy ellision + // problem on POD!?) + return value == missingValue2_; + } bool isnan() const override { return false; } const double missingValue_; + const double missingValue2_; }; From 49fdcffceaed9660235d69e9e72a6970e8520643 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 18 Sep 2020 02:12:27 +0100 Subject: [PATCH 095/161] Move atlas_test_field from src/tests/grid/ to src/tests/field/ --- src/tests/field/CMakeLists.txt | 8 +++++++- src/tests/{grid => field}/test_field.cc | 0 src/tests/grid/CMakeLists.txt | 2 -- 3 files changed, 7 insertions(+), 3 deletions(-) rename src/tests/{grid => field}/test_field.cc (100%) diff --git a/src/tests/field/CMakeLists.txt b/src/tests/field/CMakeLists.txt index 8ede186cd..edbe47e83 100644 --- a/src/tests/field/CMakeLists.txt +++ b/src/tests/field/CMakeLists.txt @@ -14,7 +14,7 @@ if( HAVE_FCTEST ) LIBS atlas_f ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) - + add_fctest( TARGET atlas_fctest_field_wrap LINKER_LANGUAGE Fortran SOURCES fctest_field_wrap.F90 @@ -54,3 +54,9 @@ if( HAVE_FCTEST ) endif() +ecbuild_add_test( TARGET atlas_test_field + SOURCES test_field.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + diff --git a/src/tests/grid/test_field.cc b/src/tests/field/test_field.cc similarity index 100% rename from src/tests/grid/test_field.cc rename to src/tests/field/test_field.cc diff --git a/src/tests/grid/CMakeLists.txt b/src/tests/grid/CMakeLists.txt index 8fe0697c0..41442402f 100644 --- a/src/tests/grid/CMakeLists.txt +++ b/src/tests/grid/CMakeLists.txt @@ -22,7 +22,6 @@ endif() foreach(test test_domain - test_field test_grid_iterator test_grids test_stretchedrotatedgaussian @@ -31,7 +30,6 @@ foreach(test test_spacing test_state test_largegrid - test_grid_hash) ecbuild_add_test( TARGET atlas_${test} SOURCES ${test}.cc LIBS atlas ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) From 0be29275484070ddf515bab7c978c88a2965691c Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 18 Sep 2020 15:42:57 +0100 Subject: [PATCH 096/161] Revert "Move atlas_test_field from src/tests/grid/ to src/tests/field/" This reverts commit 7b7eae0a3e626c016241f3f4c0de49967510bc2b. --- src/tests/field/CMakeLists.txt | 8 +------- src/tests/grid/CMakeLists.txt | 2 ++ src/tests/{field => grid}/test_field.cc | 0 3 files changed, 3 insertions(+), 7 deletions(-) rename src/tests/{field => grid}/test_field.cc (100%) diff --git a/src/tests/field/CMakeLists.txt b/src/tests/field/CMakeLists.txt index edbe47e83..8ede186cd 100644 --- a/src/tests/field/CMakeLists.txt +++ b/src/tests/field/CMakeLists.txt @@ -14,7 +14,7 @@ if( HAVE_FCTEST ) LIBS atlas_f ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) - + add_fctest( TARGET atlas_fctest_field_wrap LINKER_LANGUAGE Fortran SOURCES fctest_field_wrap.F90 @@ -54,9 +54,3 @@ if( HAVE_FCTEST ) endif() -ecbuild_add_test( TARGET atlas_test_field - SOURCES test_field.cc - LIBS atlas - ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} -) - diff --git a/src/tests/grid/CMakeLists.txt b/src/tests/grid/CMakeLists.txt index 41442402f..8fe0697c0 100644 --- a/src/tests/grid/CMakeLists.txt +++ b/src/tests/grid/CMakeLists.txt @@ -22,6 +22,7 @@ endif() foreach(test test_domain + test_field test_grid_iterator test_grids test_stretchedrotatedgaussian @@ -30,6 +31,7 @@ foreach(test test_spacing test_state test_largegrid + test_grid_hash) ecbuild_add_test( TARGET atlas_${test} SOURCES ${test}.cc LIBS atlas ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) diff --git a/src/tests/field/test_field.cc b/src/tests/grid/test_field.cc similarity index 100% rename from src/tests/field/test_field.cc rename to src/tests/grid/test_field.cc From 3913505b935b46f227333d7254c8d86abea05d8e Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 18 Sep 2020 16:44:42 +0100 Subject: [PATCH 097/161] ATLAS-306 non-defined MissingValue --- src/atlas/interpolation/MissingValue.cc | 7 ++--- .../interpolation/nonlinear/MissingValue.cc | 12 ++------ .../test_interpolation_non_linear.cc | 28 +++++++++++++------ 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/atlas/interpolation/MissingValue.cc b/src/atlas/interpolation/MissingValue.cc index 0e630fad2..770c2c096 100644 --- a/src/atlas/interpolation/MissingValue.cc +++ b/src/atlas/interpolation/MissingValue.cc @@ -25,12 +25,9 @@ namespace interpolation { namespace { -using Config = MissingValue::Config; - - -std::string config_type( const Config& c ) { +std::string config_type( const MissingValue::Config& c ) { std::string value; - ATLAS_ASSERT( c.get( "type", value ) ); + c.get( "type", value ); return value; } } // namespace diff --git a/src/atlas/interpolation/nonlinear/MissingValue.cc b/src/atlas/interpolation/nonlinear/MissingValue.cc index f7a871ce3..2e823b1c3 100644 --- a/src/atlas/interpolation/nonlinear/MissingValue.cc +++ b/src/atlas/interpolation/nonlinear/MissingValue.cc @@ -24,13 +24,7 @@ namespace interpolation { namespace nonlinear { -std::string config_type( const MissingValue::Config& c ) { - std::string value; - ATLAS_ASSERT( c.get( "type", value ) ); - return value; -} - - +namespace { double config_value( const MissingValue::Config& c ) { double value; ATLAS_ASSERT( c.get( "missing_value", value ) ); @@ -43,6 +37,7 @@ double config_epsilon( const MissingValue::Config& c ) { c.get( "missing_value_epsilon", value ); return value; } +} // namespace /// @brief Missing value if NaN @@ -124,8 +119,7 @@ void force_link() { const MissingValue* MissingValueFactory::build( const std::string& builder, const Config& config ) { force_link(); - auto factory = get( builder ); - return factory->make( config ); + return has( builder ) ? get( builder )->make( config ) : nullptr; } diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index ba30fde39..d08008862 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -37,20 +37,32 @@ CASE( "test_interpolation_non_linear_missing_value" ) { using interpolation::MissingValue; + SECTION( "not defined" ) { + util::Config config; + + auto mv = MissingValue( config ); + EXPECT( !bool( mv ) ); + + mv = MissingValue( "not defined", config ); + EXPECT( !bool( mv ) ); + } + + SECTION( "nan" ) { util::Config config; - auto mv_nan = MissingValue( "nan", config ); - EXPECT( bool( mv_nan ) ); - EXPECT( mv_nan( nan ) ); - EXPECT( mv_nan( missingValue ) == false ); + auto mv = MissingValue( "nan", config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( nan ) ); + EXPECT( mv( missingValue ) == false ); config.set( "type", "nan" ); - mv_nan = MissingValue( config ); - EXPECT( bool( mv_nan ) ); + mv = MissingValue( config ); + EXPECT( bool( mv ) ); - EXPECT( mv_nan( nan ) ); - EXPECT( mv_nan( missingValue ) == false ); + EXPECT( mv( nan ) ); + EXPECT( mv( missingValue ) == false ); } From 963ed5ea1a247615100386edcd1055093f0b36ca Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 18 Sep 2020 16:46:48 +0100 Subject: [PATCH 098/161] ATLAS-306 MissingValue build from Field::metadata() (metadata key is 'missing_value_type', which can then trigger reading 'missing_value' and 'missing_value_epsilon') --- src/atlas/interpolation/MissingValue.cc | 17 +++++++++++++++++ src/atlas/interpolation/MissingValue.h | 3 +++ 2 files changed, 20 insertions(+) diff --git a/src/atlas/interpolation/MissingValue.cc b/src/atlas/interpolation/MissingValue.cc index 770c2c096..f84761af0 100644 --- a/src/atlas/interpolation/MissingValue.cc +++ b/src/atlas/interpolation/MissingValue.cc @@ -16,8 +16,10 @@ #include "eckit/types/FloatCompare.h" +#include "atlas/field/Field.h" #include "atlas/runtime/Exception.h" #include "atlas/util/Config.h" +#include "atlas/util/Metadata.h" namespace atlas { @@ -30,6 +32,13 @@ std::string config_type( const MissingValue::Config& c ) { c.get( "type", value ); return value; } + + +std::string field_type( const Field& f ) { + std::string value; + f.metadata().get( "missing_value_type", value ); + return value; +} } // namespace @@ -44,6 +53,14 @@ MissingValue::MissingValue( const std::string& type, const MissingValue::Config& Handle( nonlinear::MissingValueFactory::build( type, config ) ) {} +MissingValue::MissingValue( const Field& field ) : + Handle( nonlinear::MissingValueFactory::build( field_type( field ), field.metadata() ) ) {} + + +MissingValue::MissingValue( const std::string& type, const Field& field ) : + Handle( nonlinear::MissingValueFactory::build( type, field.metadata() ) ) {} + + bool MissingValue::operator()( const double& value ) const { ATLAS_ASSERT_MSG( operator bool(), "MissingValue: ObjectHandle not setup" ); return get()->operator()( value ); diff --git a/src/atlas/interpolation/MissingValue.h b/src/atlas/interpolation/MissingValue.h index 39a185c8f..3653df2ed 100644 --- a/src/atlas/interpolation/MissingValue.h +++ b/src/atlas/interpolation/MissingValue.h @@ -20,6 +20,7 @@ namespace atlas { +class Field; namespace util { class Config; } @@ -40,6 +41,8 @@ struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle Date: Fri, 18 Sep 2020 17:26:08 +0100 Subject: [PATCH 099/161] ATLAS-306 MissingValue build from Field::metadata() unit test --- .../test_interpolation_non_linear.cc | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index d08008862..de224330c 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -10,6 +10,7 @@ */ +#include #include #include "atlas/array.h" @@ -114,6 +115,55 @@ CASE( "test_interpolation_non_linear_missing_value" ) { } +CASE( "test_interpolation_non_linear_field_missing_value" ) { + using interpolation::MissingValue; + + std::vector values{1., nan, missingValue, missingValue, missingValue + missingValueEps / 2., 6., 7.}; + + + SECTION( "nan" ) { + Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); + EXPECT( !bool( MissingValue( field ) ) ); + + // missing value type from user + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "nan", field ) ) == 1 ); + + // missing value type from field + field.metadata().set( "missing_value_type", "nan" ); + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 1 ); + } + + + SECTION( "equals" ) { + Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); + field.metadata().set( "missing_value", missingValue ); + EXPECT( !bool( MissingValue( field ) ) ); + + // missing value type from user (value set from field) + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "equals", field ) ) == 2 ); + + // missing value type from field + field.metadata().set( "missing_value_type", "equals" ); + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 2 ); + } + + + SECTION( "approximately-equals" ) { + Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); + field.metadata().set( "missing_value", missingValue ); + field.metadata().set( "missing_value_epsilon", missingValueEps ); + EXPECT( !bool( MissingValue( field ) ) ); + + // missing value type from user (value set from field) + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "approximately-equals", field ) ) == 3 ); + + // missing value type from field + field.metadata().set( "missing_value_type", "approximately-equals" ); + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 3 ); + } +} + + CASE( "test_interpolation_non_linear_matrix" ) { /* Set input field full of 1's, with 9 nodes From 72237c67fbdc79433f9979ed661229885977141b Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 18 Sep 2020 17:48:23 +0100 Subject: [PATCH 100/161] ATLAS-306 Refactor --- .../test_interpolation_non_linear.cc | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index de224330c..b02726184 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -119,12 +119,16 @@ CASE( "test_interpolation_non_linear_field_missing_value" ) { using interpolation::MissingValue; std::vector values{1., nan, missingValue, missingValue, missingValue + missingValueEps / 2., 6., 7.}; + Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); + field.metadata().set( "missing_value_type", "not defined" ); + field.metadata().set( "missing_value", missingValue ); + field.metadata().set( "missing_value_epsilon", missingValueEps ); + + EXPECT( !bool( MissingValue( field ) ) ); - SECTION( "nan" ) { - Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); - EXPECT( !bool( MissingValue( field ) ) ); + SECTION( "nan" ) { // missing value type from user EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "nan", field ) ) == 1 ); @@ -135,10 +139,6 @@ CASE( "test_interpolation_non_linear_field_missing_value" ) { SECTION( "equals" ) { - Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); - field.metadata().set( "missing_value", missingValue ); - EXPECT( !bool( MissingValue( field ) ) ); - // missing value type from user (value set from field) EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "equals", field ) ) == 2 ); @@ -149,11 +149,6 @@ CASE( "test_interpolation_non_linear_field_missing_value" ) { SECTION( "approximately-equals" ) { - Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); - field.metadata().set( "missing_value", missingValue ); - field.metadata().set( "missing_value_epsilon", missingValueEps ); - EXPECT( !bool( MissingValue( field ) ) ); - // missing value type from user (value set from field) EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "approximately-equals", field ) ) == 3 ); From ec31d9dc851d034d10a984d460956f4a9beafe1f Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 18 Sep 2020 18:42:54 +0100 Subject: [PATCH 101/161] ATLAS-306 Consistency --- src/atlas/interpolation/nonlinear/NonLinear.cc | 2 +- src/tests/interpolation/test_interpolation_non_linear.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc index 63a70802d..c2f50358d 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.cc +++ b/src/atlas/interpolation/nonlinear/NonLinear.cc @@ -23,7 +23,7 @@ namespace nonlinear { namespace { std::string config_missing_value( const MissingValue::Config& c ) { std::string value; - ATLAS_ASSERT_MSG( c.get( "missing_value_compare", value ), "NonLinear: expecting 'missing_value_compare'" ); + ATLAS_ASSERT_MSG( c.get( "missing_value_type", value ), "NonLinear: expecting 'missing_value_type'" ); return value; } } // namespace diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index b02726184..a4d7ccf22 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -204,7 +204,7 @@ CASE( "test_interpolation_non_linear_matrix" ) { viewA( 4 ) = cmp == "nan" ? nan : missingValue; cfg.set( "non_linear", "missing-if-all-missing" ); - cfg.set( "missing_value_compare", cmp ); + cfg.set( "missing_value_type", cmp ); Interpolation interpolation( cfg, fsA, fsB ); interpolation.execute( fieldA, fieldB ); @@ -221,7 +221,7 @@ CASE( "test_interpolation_non_linear_matrix" ) { viewA( 4 ) = cmp == "nan" ? nan : missingValue; cfg.set( "non_linear", "missing-if-any-missing" ); - cfg.set( "missing_value_compare", cmp ); + cfg.set( "missing_value_type", cmp ); Interpolation interpolation( cfg, fsA, fsB ); interpolation.execute( fieldA, fieldB ); @@ -238,7 +238,7 @@ CASE( "test_interpolation_non_linear_matrix" ) { viewA( 4 ) = cmp == "nan" ? nan : missingValue; cfg.set( "non_linear", "missing-if-heaviest-missing" ); - cfg.set( "missing_value_compare", cmp ); + cfg.set( "missing_value_type", cmp ); Interpolation interpolation( cfg, fsA, fsB ); interpolation.execute( fieldA, fieldB ); From aa1347d7a8e7f8e4cc825aa3282082e47f88d2ac Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 18 Sep 2020 18:57:00 +0100 Subject: [PATCH 102/161] ATLAS-306 Non-linear matrix corrections only if field has set missing values (in field metadata) --- src/atlas/interpolation/method/Method.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index e12efc063..5696b606f 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -218,7 +218,7 @@ void Method::do_execute( const Field& src, Field& tgt ) const { // non-linearities: a non-empty M matrix contains the corrections applied to matrix_ Matrix M; - if ( !matrix_.empty() && nonLinear_ ) { + if ( !matrix_.empty() && nonLinear_ && MissingValue(src)) { Matrix W( matrix_ ); // copy (a big penalty -- copy-on-write would definitely be better) if ( nonLinear_->execute( W, src ) ) { M.swap( W ); From edc7ddb9b987e191bba9ca463616f2e5ebb72f24 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Mon, 21 Sep 2020 12:03:49 +0100 Subject: [PATCH 103/161] ATLAS-306 atlas::interpolation::nonlinear::MissingValue static_type method (to define the name once, remove force_link) --- .../interpolation/nonlinear/MissingValue.cc | 27 ++++++------------- .../interpolation/nonlinear/MissingValue.h | 2 +- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/MissingValue.cc b/src/atlas/interpolation/nonlinear/MissingValue.cc index 2e823b1c3..ec40779d0 100644 --- a/src/atlas/interpolation/nonlinear/MissingValue.cc +++ b/src/atlas/interpolation/nonlinear/MissingValue.cc @@ -45,6 +45,7 @@ struct MissingValueNaN : MissingValue { MissingValueNaN( const Config& ) {} bool operator()( const double& value ) const override { return std::isnan( value ); } bool isnan() const override { return true; } + static std::string static_type() { return "nan"; } }; @@ -67,6 +68,8 @@ struct MissingValueEquals : MissingValue { bool isnan() const override { return false; } + static std::string static_type() { return "equals"; } + const double missingValue_; const double missingValue2_; }; @@ -87,6 +90,8 @@ struct MissingValueApprox : MissingValue { bool isnan() const override { return false; } + static std::string static_type() { return "approximately-equals"; } + const double missingValue_; const double epsilon_; }; @@ -96,29 +101,13 @@ MissingValue::~MissingValue() = default; namespace { - - -static MissingValueFactoryBuilder __mv1( "nan" ); -static MissingValueFactoryBuilder __mv2( "equals" ); -static MissingValueFactoryBuilder __mv3( "approximately-equals" ); - - -void force_link() { - static struct Link { - Link() { - MissingValueFactoryBuilder(); - MissingValueFactoryBuilder(); - MissingValueFactoryBuilder(); - } - } link; -} - - +MissingValueFactoryBuilder __mv1; +MissingValueFactoryBuilder __mv2; +MissingValueFactoryBuilder __mv3; } // namespace const MissingValue* MissingValueFactory::build( const std::string& builder, const Config& config ) { - force_link(); return has( builder ) ? get( builder )->make( config ) : nullptr; } diff --git a/src/atlas/interpolation/nonlinear/MissingValue.h b/src/atlas/interpolation/nonlinear/MissingValue.h index 27dbef682..b9e7bf2d4 100644 --- a/src/atlas/interpolation/nonlinear/MissingValue.h +++ b/src/atlas/interpolation/nonlinear/MissingValue.h @@ -52,7 +52,7 @@ class MissingValueFactoryBuilder : public MissingValueFactory { virtual const MissingValue* make( const Config& config ) override { return new T( config ); } public: - using MissingValueFactory::MissingValueFactory; + MissingValueFactoryBuilder() : MissingValueFactory( T::static_type() ) {} }; From 4d809b882f7f451d0e5138364c43fbb5979bd3fb Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Mon, 21 Sep 2020 12:08:58 +0100 Subject: [PATCH 104/161] ATLAS-306 atlas::interpolation::nonlinear::MissingValue ability to set Field metadata --- src/atlas/interpolation/MissingValue.cc | 6 +++++ src/atlas/interpolation/MissingValue.h | 8 ++++++- src/atlas/interpolation/method/Method.cc | 6 +++++ .../interpolation/nonlinear/MissingValue.cc | 23 +++++++++++++++++-- .../interpolation/nonlinear/MissingValue.h | 6 +++++ 5 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/atlas/interpolation/MissingValue.cc b/src/atlas/interpolation/MissingValue.cc index f84761af0..dab70ff89 100644 --- a/src/atlas/interpolation/MissingValue.cc +++ b/src/atlas/interpolation/MissingValue.cc @@ -79,5 +79,11 @@ const nonlinear::MissingValue& MissingValue::ref() const { } +void MissingValue::metadata( Field& field ) const { + ATLAS_ASSERT_MSG( operator bool(), "MissingValue: ObjectHandle not setup" ); + get()->metadata( field ); +} + + } // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/MissingValue.h b/src/atlas/interpolation/MissingValue.h index 3653df2ed..e2d9407bd 100644 --- a/src/atlas/interpolation/MissingValue.h +++ b/src/atlas/interpolation/MissingValue.h @@ -64,10 +64,16 @@ struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle Date: Mon, 21 Sep 2020 12:14:21 +0100 Subject: [PATCH 105/161] ATLAS-306 atlas::interpolation::nonlinear::NonLinear make it non-configurable (nothing to configure) --- src/atlas/CMakeLists.txt | 1 - .../interpolation/nonlinear/NonLinear.cc | 42 ------------------- src/atlas/interpolation/nonlinear/NonLinear.h | 9 +--- .../nonlinear/NonLinearFactory.h | 2 +- 4 files changed, 3 insertions(+), 51 deletions(-) delete mode 100644 src/atlas/interpolation/nonlinear/NonLinear.cc diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 29cd8f01b..3b90a0003 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -540,7 +540,6 @@ interpolation/nonlinear/MissingIfHeaviestMissing.cc interpolation/nonlinear/MissingIfHeaviestMissing.h interpolation/nonlinear/MissingValue.cc interpolation/nonlinear/MissingValue.h -interpolation/nonlinear/NonLinear.cc interpolation/nonlinear/NonLinear.h interpolation/nonlinear/NonLinearFactory.cc interpolation/nonlinear/NonLinearFactory.h diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc deleted file mode 100644 index c2f50358d..000000000 --- a/src/atlas/interpolation/nonlinear/NonLinear.cc +++ /dev/null @@ -1,42 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#include "atlas/interpolation/nonlinear/NonLinear.h" - -#include "atlas/runtime/Exception.h" - - -namespace atlas { -namespace interpolation { -namespace nonlinear { - - -namespace { -std::string config_missing_value( const MissingValue::Config& c ) { - std::string value; - ATLAS_ASSERT_MSG( c.get( "missing_value_type", value ), "NonLinear: expecting 'missing_value_type'" ); - return value; -} -} // namespace - - -NonLinear::NonLinear( const Config& config ) : missingValue_( config_missing_value( config ), config ) { - ATLAS_ASSERT_MSG( missingValue_, "NonLinear: missingValue setup" ); -} - - -NonLinear::~NonLinear() = default; - - -} // namespace nonlinear -} // namespace interpolation -} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index eca6405b7..e6c210207 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -41,12 +41,11 @@ class NonLinear { /** * @brief NonLinear ctor - * @param [in] config with optional "missingValue", default NaN indicates missing value in field */ - NonLinear( const Config& config ); + NonLinear() = default; /// @brief NonLinear dtor - virtual ~NonLinear(); + virtual ~NonLinear() = default; /** * @brief Apply non-linear corrections to interpolation matrix @@ -55,10 +54,6 @@ class NonLinear { * @return if W was modified */ virtual bool execute( Matrix& W, const Field& f ) const = 0; - -protected: - /// Evaluate if value is considered a missing value - interpolation::MissingValue missingValue_; }; diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.h b/src/atlas/interpolation/nonlinear/NonLinearFactory.h index 39e8670b9..236d3e37c 100644 --- a/src/atlas/interpolation/nonlinear/NonLinearFactory.h +++ b/src/atlas/interpolation/nonlinear/NonLinearFactory.h @@ -42,7 +42,7 @@ class NonLinearFactory : public util::Factory { template class NonLinearFactoryBuilder : public NonLinearFactory { private: - virtual const NonLinear* make( const Config& config ) override { return new T( config ); } + virtual const NonLinear* make( const Config& /*config*/ ) override { return new T( /*config*/ ); } public: using NonLinearFactory::NonLinearFactory; From db807ab43584b185df2b0f627fc980346d030f73 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Mon, 21 Sep 2020 12:16:55 +0100 Subject: [PATCH 106/161] ATLAS-306 atlas::interpolation::Method delegate if non-linearities are applicable to the specific atlas::interpolation::nonlinear::NonLinear implementations --- src/atlas/interpolation/method/Method.cc | 2 +- .../interpolation/nonlinear/MissingIfAllMissing.cc | 10 ++++++---- .../interpolation/nonlinear/MissingIfAllMissing.h | 1 - .../interpolation/nonlinear/MissingIfAnyMissing.cc | 10 ++++++---- .../interpolation/nonlinear/MissingIfAnyMissing.h | 1 - .../nonlinear/MissingIfHeaviestMissing.cc | 10 ++++++---- .../interpolation/nonlinear/MissingIfHeaviestMissing.h | 1 - 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index ab9e1a7ca..d7364d3c0 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -218,7 +218,7 @@ void Method::do_execute( const Field& src, Field& tgt ) const { // non-linearities: a non-empty M matrix contains the corrections applied to matrix_ Matrix M; - if ( !matrix_.empty() && nonLinear_ && MissingValue(src)) { + if ( !matrix_.empty() && nonLinear_ ) { Matrix W( matrix_ ); // copy (a big penalty -- copy-on-write would definitely be better) if ( nonLinear_->execute( W, src ) ) { M.swap( W ); diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc index 6c2d7aca7..a057583df 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc @@ -25,13 +25,15 @@ namespace interpolation { namespace nonlinear { -MissingIfAllMissing::MissingIfAllMissing( const Config& config ) : NonLinear( config ) {} - - bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) const { + interpolation::MissingValue mv( field ); + if ( !mv ) { + return false; + } + // NOTE only for scalars (for now) auto values = array::make_view( field ); - auto& missingValue = missingValue_.ref(); + auto& missingValue = mv.ref(); // correct matrix weigths for the missing values // (force a missing value only if all row values are missing) diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h index fabd84a9a..104bcb0a5 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h @@ -21,7 +21,6 @@ namespace nonlinear { struct MissingIfAllMissing : NonLinear { - MissingIfAllMissing( const Config& ); bool execute( NonLinear::Matrix&, const Field& ) const; }; diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc index 39a7c221a..d9654a866 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc @@ -23,13 +23,15 @@ namespace interpolation { namespace nonlinear { -MissingIfAnyMissing::MissingIfAnyMissing( const Config& config ) : NonLinear( config ) {} - - bool MissingIfAnyMissing::execute( NonLinear::Matrix& W, const Field& field ) const { + interpolation::MissingValue mv( field ); + if ( !mv ) { + return false; + } + // NOTE only for scalars (for now) auto values = array::make_view( field ); - auto& missingValue = missingValue_.ref(); + auto& missingValue = mv.ref(); // correct matrix weigths for the missing values // (force a missing value only if any row values is missing) diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h index a6af90b57..552f01794 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h @@ -21,7 +21,6 @@ namespace nonlinear { struct MissingIfAnyMissing : NonLinear { - MissingIfAnyMissing( const Config& ); bool execute( NonLinear::Matrix&, const Field& ) const; }; diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc index 2f37b3cb4..77cb20f65 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc @@ -25,13 +25,15 @@ namespace interpolation { namespace nonlinear { -MissingIfHeaviestMissing::MissingIfHeaviestMissing( const Config& config ) : NonLinear( config ) {} - - bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field ) const { + interpolation::MissingValue mv( field ); + if ( !mv ) { + return false; + } + // NOTE only for scalars (for now) auto values = array::make_view( field ); - auto& missingValue = missingValue_.ref(); + auto& missingValue = mv.ref(); // correct matrix weigths for the missing values ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h index 6aa95b1fd..aa8211a74 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h @@ -21,7 +21,6 @@ namespace nonlinear { struct MissingIfHeaviestMissing : NonLinear { - MissingIfHeaviestMissing( const Config& ); bool execute( NonLinear::Matrix&, const Field& ) const; }; From 82ea0f1095d39ff348e35977d9a23e1ae45b5ec4 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Mon, 21 Sep 2020 13:02:32 +0100 Subject: [PATCH 107/161] ATLAS-306 fix unit test (example of attaching missing values metadata to fields) --- .../test_interpolation_non_linear.cc | 63 ++++++++++--------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index a4d7ccf22..6a3099c96 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -179,6 +179,9 @@ CASE( "test_interpolation_non_linear_matrix" ) { functionspace::NodeColumns fsA( meshA ); Field fieldA = fsA.createField( option::name( "A" ) ); + fieldA.metadata().set( "missing_value", missingValue ); + fieldA.metadata().set( "missing_value_epsilon", missingValueEps ); + auto viewA = array::make_view( fieldA ); for ( idx_t j = 0; j < fsA.nodes().size(); ++j ) { viewA( j ) = 1; @@ -193,59 +196,59 @@ CASE( "test_interpolation_non_linear_matrix" ) { ATLAS_ASSERT( viewB.size() == 2 ); - util::Config cfg( "type", "finite-element" ); - cfg.set( "missing_value", missingValue ); - cfg.set( "missing_value_epsilon", missingValueEps ); - - - // NOTE: "equals" is not tested due to internal conversions SECTION( "missing-if-all-missing" ) { - for ( std::string cmp : {"approximately-equals", "nan"} ) { - viewA( 4 ) = cmp == "nan" ? nan : missingValue; + Interpolation interpolation( + util::Config( "type", "finite-element" ).set( "non_linear", "missing-if-all-missing" ), fsA, fsB ); - cfg.set( "non_linear", "missing-if-all-missing" ); - cfg.set( "missing_value_type", cmp ); + for ( std::string type : {"equals", "approximately-equals", "nan"} ) { + fieldA.metadata().set( "missing_value_type", type ); + viewA( 4 ) = type == "nan" ? nan : missingValue; - Interpolation interpolation( cfg, fsA, fsB ); + EXPECT( interpolation::MissingValue( fieldA ) ); interpolation.execute( fieldA, fieldB ); - interpolation::MissingValue miss( cmp, cfg ); - EXPECT( miss( viewB( 0 ) ) == false ); - EXPECT( miss( viewB( 1 ) ) == false ); + interpolation::MissingValue mv( fieldB ); + EXPECT( mv ); + EXPECT( mv( viewB( 0 ) ) == false ); + EXPECT( mv( viewB( 1 ) ) == false ); } } SECTION( "missing-if-any-missing" ) { - for ( std::string cmp : {"approximately-equals", "nan"} ) { - viewA( 4 ) = cmp == "nan" ? nan : missingValue; + Interpolation interpolation( + util::Config( "type", "finite-element" ).set( "non_linear", "missing-if-any-missing" ), fsA, fsB ); - cfg.set( "non_linear", "missing-if-any-missing" ); - cfg.set( "missing_value_type", cmp ); + for ( std::string type : {"equals", "approximately-equals", "nan"} ) { + fieldA.metadata().set( "missing_value_type", type ); + viewA( 4 ) = type == "nan" ? nan : missingValue; - Interpolation interpolation( cfg, fsA, fsB ); + EXPECT( interpolation::MissingValue( fieldA ) ); interpolation.execute( fieldA, fieldB ); - interpolation::MissingValue miss( cmp, cfg ); - EXPECT( miss( viewB( 0 ) ) ); - EXPECT( miss( viewB( 1 ) ) ); + interpolation::MissingValue mv( fieldB ); + EXPECT( mv ); + EXPECT( mv( viewB( 0 ) ) ); + EXPECT( mv( viewB( 1 ) ) ); } } SECTION( "missing-if-heaviest-missing" ) { - for ( std::string cmp : {"approximately-equals", "nan"} ) { - viewA( 4 ) = cmp == "nan" ? nan : missingValue; + Interpolation interpolation( + util::Config( "type", "finite-element" ).set( "non_linear", "missing-if-heaviest-missing" ), fsA, fsB ); - cfg.set( "non_linear", "missing-if-heaviest-missing" ); - cfg.set( "missing_value_type", cmp ); + for ( std::string type : {"equals", "approximately-equals", "nan"} ) { + fieldA.metadata().set( "missing_value_type", type ); + viewA( 4 ) = type == "nan" ? nan : missingValue; - Interpolation interpolation( cfg, fsA, fsB ); + EXPECT( interpolation::MissingValue( fieldA ) ); interpolation.execute( fieldA, fieldB ); - interpolation::MissingValue miss( cmp, cfg ); - EXPECT( miss( viewB( 0 ) ) == false ); - EXPECT( miss( viewB( 1 ) ) ); + interpolation::MissingValue mv( fieldB ); + EXPECT( mv ); + EXPECT( mv( viewB( 0 ) ) == false ); + EXPECT( mv( viewB( 1 ) ) ); } } } From 7f66a8ac045a53f4b2a57314ec5a297e9278e56a Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 22 Sep 2020 07:08:08 +0100 Subject: [PATCH 108/161] ATLAS-306 generic atlas::interpolation::nonlinear::NonLinear implementations on DataType --- src/atlas/CMakeLists.txt | 3 - .../nonlinear/MissingIfAllMissing.cc | 108 ----------------- .../nonlinear/MissingIfAllMissing.h | 79 +++++++++++- .../nonlinear/MissingIfAnyMissing.cc | 93 -------------- .../nonlinear/MissingIfAnyMissing.h | 64 +++++++++- .../nonlinear/MissingIfHeaviestMissing.cc | 114 ------------------ .../nonlinear/MissingIfHeaviestMissing.h | 86 ++++++++++++- src/atlas/interpolation/nonlinear/NonLinear.h | 20 ++- .../nonlinear/NonLinearFactory.cc | 51 +++++--- 9 files changed, 276 insertions(+), 342 deletions(-) delete mode 100644 src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc delete mode 100644 src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc delete mode 100644 src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 3b90a0003..4a6ec5bfd 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -532,11 +532,8 @@ interpolation/method/structured/kernels/LinearVerticalKernel.h interpolation/method/structured/kernels/QuasiCubic3DKernel.cc interpolation/method/structured/kernels/QuasiCubic3DKernel.h interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h -interpolation/nonlinear/MissingIfAllMissing.cc interpolation/nonlinear/MissingIfAllMissing.h -interpolation/nonlinear/MissingIfAnyMissing.cc interpolation/nonlinear/MissingIfAnyMissing.h -interpolation/nonlinear/MissingIfHeaviestMissing.cc interpolation/nonlinear/MissingIfHeaviestMissing.h interpolation/nonlinear/MissingValue.cc interpolation/nonlinear/MissingValue.h diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc deleted file mode 100644 index a057583df..000000000 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.cc +++ /dev/null @@ -1,108 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#include "atlas/interpolation/nonlinear/MissingIfAllMissing.h" - -#include "eckit/types/FloatCompare.h" - -#include "atlas/array.h" -#include "atlas/field/Field.h" -#include "atlas/interpolation/nonlinear/NonLinearFactory.h" -#include "atlas/runtime/Exception.h" - - -namespace atlas { -namespace interpolation { -namespace nonlinear { - - -bool MissingIfAllMissing::execute( NonLinear::Matrix& W, const Field& field ) const { - interpolation::MissingValue mv( field ); - if ( !mv ) { - return false; - } - - // NOTE only for scalars (for now) - auto values = array::make_view( field ); - auto& missingValue = mv.ref(); - - // correct matrix weigths for the missing values - // (force a missing value only if all row values are missing) - ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); - - auto data = const_cast( W.data() ); - bool modif = false; - bool zeros = false; - - Size i = 0; - Matrix::iterator it( W ); - for ( Size r = 0; r < W.rows(); ++r ) { - const Matrix::iterator end = W.end( r ); - - // count missing values, accumulate weights (disregarding missing values) - size_t i_missing = i; - size_t N_missing = 0; - size_t N_entries = 0; - double sum = 0.; - - Matrix::iterator kt( it ); - Size k = i; - for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); - - if ( miss ) { - ++N_missing; - i_missing = i; - } - else { - sum += *it; - } - } - - // weights redistribution: zero-weight all missing values, linear re-weighting for the others; - // the result is missing value if all values in row are missing - if ( N_missing > 0 ) { - if ( N_missing == N_entries || eckit::types::is_approximately_equal( sum, 0. ) ) { - for ( Size j = k; j < k + N_entries; ++j ) { - data[j] = j == i_missing ? 1. : 0.; - } - } - else { - const double factor = 1. / sum; - for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - if ( missingValue( values[kt.col()] ) ) { - data[j] = 0.; - zeros = true; - } - else { - data[j] *= factor; - } - } - } - modif = true; - } - } - - if ( zeros && missingValue.isnan() ) { - W.prune( 0. ); - } - - return modif; -} - - -static NonLinearFactoryBuilder __nonlinear( "missing-if-all-missing" ); - - -} // namespace nonlinear -} // namespace interpolation -} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h index 104bcb0a5..5b592d949 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h @@ -14,14 +14,91 @@ #include "atlas/interpolation/nonlinear/NonLinear.h" +#include "eckit/types/FloatCompare.h" + namespace atlas { namespace interpolation { namespace nonlinear { +template struct MissingIfAllMissing : NonLinear { - bool execute( NonLinear::Matrix&, const Field& ) const; + bool execute( NonLinear::Matrix& W, const Field& field ) const { + interpolation::MissingValue mv( field ); + if ( !mv ) { + return false; + } + + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); + auto& missingValue = mv.ref(); + + // correct matrix weigths for the missing values + // (force a missing value only if all row values are missing) + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + bool zeros = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + Scalar sum = 0.; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + else { + sum += *it; + } + } + + // weights redistribution: zero-weight all missing values, linear re-weighting for the others; + // the result is missing value if all values in row are missing + if ( N_missing > 0 ) { + if ( N_missing == N_entries || eckit::types::is_approximately_equal( sum, 0. ) ) { + for ( Size j = k; j < k + N_entries; ++j ) { + data[j] = j == i_missing ? 1. : 0.; + } + } + else { + const Scalar factor = 1. / sum; + for ( Size j = k; j < k + N_entries; ++j, ++kt ) { + if ( missingValue( values[kt.col()] ) ) { + data[j] = 0.; + zeros = true; + } + else { + data[j] *= factor; + } + } + } + modif = true; + } + } + + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + + return modif; + } + + static std::string static_type() { return "missing-if-all-missing"; } }; diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc deleted file mode 100644 index d9654a866..000000000 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#include "atlas/interpolation/nonlinear/MissingIfAnyMissing.h" - -#include "atlas/array.h" -#include "atlas/field/Field.h" -#include "atlas/interpolation/nonlinear/NonLinearFactory.h" -#include "atlas/runtime/Exception.h" - - -namespace atlas { -namespace interpolation { -namespace nonlinear { - - -bool MissingIfAnyMissing::execute( NonLinear::Matrix& W, const Field& field ) const { - interpolation::MissingValue mv( field ); - if ( !mv ) { - return false; - } - - // NOTE only for scalars (for now) - auto values = array::make_view( field ); - auto& missingValue = mv.ref(); - - // correct matrix weigths for the missing values - // (force a missing value only if any row values is missing) - ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); - - auto data = const_cast( W.data() ); - bool modif = false; - bool zeros = false; - - Size i = 0; - Matrix::iterator it( W ); - for ( Size r = 0; r < W.rows(); ++r ) { - const Matrix::iterator end = W.end( r ); - - // count missing values, accumulate weights (disregarding missing values) - size_t i_missing = i; - size_t N_missing = 0; - size_t N_entries = 0; - - Matrix::iterator kt( it ); - Size k = i; - for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); - - if ( miss ) { - ++N_missing; - i_missing = i; - } - } - - // if any values in row are missing, force missing value - if ( N_missing > 0 ) { - for ( Size j = k; j < k + N_entries; ++j ) { - if ( j == i_missing ) { - data[j] = 1.; - } - else { - data[j] = 0.; - zeros = true; - } - } - modif = true; - } - } - - if ( zeros && missingValue.isnan() ) { - W.prune( 0. ); - } - - return modif; -} - - -static NonLinearFactoryBuilder __nonlinear( "missing-if-any-missing" ); - - -} // namespace nonlinear -} // namespace interpolation -} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h index 552f01794..178fa767c 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h @@ -20,8 +20,70 @@ namespace interpolation { namespace nonlinear { +template struct MissingIfAnyMissing : NonLinear { - bool execute( NonLinear::Matrix&, const Field& ) const; + bool execute( NonLinear::Matrix& W, const Field& field ) const { + interpolation::MissingValue mv( field ); + if ( !mv ) { + return false; + } + + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); + auto& missingValue = mv.ref(); + + // correct matrix weigths for the missing values + // (force a missing value only if any row values is missing) + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + bool zeros = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + } + + // if any values in row are missing, force missing value + if ( N_missing > 0 ) { + for ( Size j = k; j < k + N_entries; ++j ) { + if ( j == i_missing ) { + data[j] = 1.; + } + else { + data[j] = 0.; + zeros = true; + } + } + modif = true; + } + } + + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + + return modif; + } + + static std::string static_type() { return "missing-if-any-missing"; } }; diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc deleted file mode 100644 index 77cb20f65..000000000 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.cc +++ /dev/null @@ -1,114 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#include "atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h" - -#include "eckit/types/FloatCompare.h" - -#include "atlas/array.h" -#include "atlas/field/Field.h" -#include "atlas/interpolation/nonlinear/NonLinearFactory.h" -#include "atlas/runtime/Exception.h" - - -namespace atlas { -namespace interpolation { -namespace nonlinear { - - -bool MissingIfHeaviestMissing::execute( NonLinear::Matrix& W, const Field& field ) const { - interpolation::MissingValue mv( field ); - if ( !mv ) { - return false; - } - - // NOTE only for scalars (for now) - auto values = array::make_view( field ); - auto& missingValue = mv.ref(); - - // correct matrix weigths for the missing values - ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); - - auto data = const_cast( W.data() ); - bool modif = false; - bool zeros = false; - - Size i = 0; - Matrix::iterator it( W ); - for ( Size r = 0; r < W.rows(); ++r ) { - const Matrix::iterator end = W.end( r ); - - // count missing values, accumulate weights (disregarding missing values) and find maximum weight in row - size_t i_missing = i; - size_t N_missing = 0; - size_t N_entries = 0; - double sum = 0.; - double heaviest = -1.; - bool heaviest_is_missing = false; - - Matrix::iterator kt( it ); - Size k = i; - for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); - - if ( miss ) { - ++N_missing; - i_missing = i; - } - else { - sum += *it; - } - - if ( heaviest < data[i] ) { - heaviest = data[i]; - heaviest_is_missing = miss; - } - } - - // weights redistribution: zero-weight all missing values, linear re-weighting for the others; - // if all values are missing, or the closest value is missing, force missing value - if ( N_missing > 0 ) { - if ( N_missing == N_entries || heaviest_is_missing || eckit::types::is_approximately_equal( sum, 0. ) ) { - for ( Size j = k; j < k + N_entries; ++j ) { - data[j] = j == i_missing ? 1. : 0.; - } - } - else { - const double factor = 1. / sum; - for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - if ( missingValue( values[kt.col()] ) ) { - data[j] = 0.; - zeros = true; - } - else { - data[j] *= factor; - } - } - } - modif = true; - } - } - - if ( zeros && missingValue.isnan() ) { - W.prune( 0. ); - } - - return modif; -} - - -static NonLinearFactoryBuilder __nonlinear( "missing-if-heaviest-missing" ); - - -} // namespace nonlinear -} // namespace interpolation -} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h index aa8211a74..50e152eaa 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h @@ -14,14 +14,98 @@ #include "atlas/interpolation/nonlinear/NonLinear.h" +#include "eckit/types/FloatCompare.h" + namespace atlas { namespace interpolation { namespace nonlinear { +template struct MissingIfHeaviestMissing : NonLinear { - bool execute( NonLinear::Matrix&, const Field& ) const; + bool execute( NonLinear::Matrix& W, const Field& field ) const { + interpolation::MissingValue mv( field ); + if ( !mv ) { + return false; + } + + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); + auto& missingValue = mv.ref(); + + // correct matrix weigths for the missing values + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + bool zeros = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) and find maximum weight in row + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + Scalar sum = 0.; + Scalar heaviest = -1.; + bool heaviest_is_missing = false; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + else { + sum += *it; + } + + if ( heaviest < data[i] ) { + heaviest = data[i]; + heaviest_is_missing = miss; + } + } + + // weights redistribution: zero-weight all missing values, linear re-weighting for the others; + // if all values are missing, or the closest value is missing, force missing value + if ( N_missing > 0 ) { + if ( N_missing == N_entries || heaviest_is_missing || + eckit::types::is_approximately_equal( sum, 0. ) ) { + for ( Size j = k; j < k + N_entries; ++j ) { + data[j] = j == i_missing ? 1. : 0.; + } + } + else { + const Scalar factor = 1. / sum; + for ( Size j = k; j < k + N_entries; ++j, ++kt ) { + if ( missingValue( values[kt.col()] ) ) { + data[j] = 0.; + zeros = true; + } + else { + data[j] *= factor; + } + } + } + modif = true; + } + } + + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + + return modif; + } + + static std::string static_type() { return "missing-if-heaviest-missing"; } }; diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index e6c210207..49f11ba4e 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -12,15 +12,15 @@ #pragma once +#include + #include "eckit/config/Parametrisation.h" #include "eckit/linalg/SparseMatrix.h" +#include "atlas/array.h" +#include "atlas/field/Field.h" #include "atlas/interpolation/MissingValue.h" - - -namespace atlas { -class Field; -} +#include "atlas/runtime/Exception.h" namespace atlas { @@ -54,6 +54,16 @@ class NonLinear { * @return if W was modified */ virtual bool execute( Matrix& W, const Field& f ) const = 0; + +protected: + template + static array::ArrayView::type, Rank> make_view_field_values( const Field& field ) { + ATLAS_ASSERT( field ); + ATLAS_ASSERT_MSG( + field.datatype().kind() == array::DataType::kind(), + "Field(name:" + field.name() + ",DataType:" + field.datatype().str() + ") is not of required DataType" ); + return array::make_view::type, Rank>( field ); + } }; diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc b/src/atlas/interpolation/nonlinear/NonLinearFactory.cc index e46dc4781..69922d6e2 100644 --- a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc +++ b/src/atlas/interpolation/nonlinear/NonLinearFactory.cc @@ -12,8 +12,6 @@ #include "atlas/interpolation/nonlinear/NonLinearFactory.h" -#include - #include "atlas/interpolation/nonlinear/MissingIfAllMissing.h" #include "atlas/interpolation/nonlinear/MissingIfAnyMissing.h" #include "atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h" @@ -24,23 +22,44 @@ namespace interpolation { namespace nonlinear { -namespace { -void force_link() { - static struct Link { - Link() { - NonLinearFactoryBuilder(); - NonLinearFactoryBuilder(); - NonLinearFactoryBuilder(); - } - } link; -} -} // namespace +#define B NonLinearFactoryBuilder +#define M1 MissingIfAllMissing +#define M2 MissingIfAnyMissing +#define M3 MissingIfHeaviestMissing +#define T array::DataType::str + + +static B> __nl1( M1::static_type() ); +static B> __nl2( M1::static_type() + "-" + T() ); +static B> __nl3( M1::static_type() + "-" + T() ); +static B> __nl4( M1::static_type() + "-" + T() ); +static B> __nl5( M1::static_type() + "-" + T() ); +static B> __nl6( M1::static_type() + "-" + T() ); + +static B> __nl7( M2::static_type() ); +static B> __nl8( M2::static_type() + "-" + T() ); +static B> __nl9( M2::static_type() + "-" + T() ); +static B> __nl10( M2::static_type() + "-" + T() ); +static B> __nl11( M2::static_type() + "-" + T() ); +static B> __nl12( M2::static_type() + "-" + T() ); + +static B> __nl13( M3::static_type() ); +static B> __nl14( M3::static_type() + "-" + T() ); +static B> __nl15( M3::static_type() + "-" + T() ); +static B> __nl16( M3::static_type() + "-" + T() ); +static B> __nl17( M3::static_type() + "-" + T() ); +static B> __nl18( M3::static_type() + "-" + T() ); + + +#undef T +#undef M3 +#undef M2 +#undef M1 +#undef B const NonLinear* NonLinearFactory::build( const std::string& builder, const Config& config ) { - force_link(); - auto factory = get( builder ); - return factory->make( config ); + return get( builder )->make( config ); } From d6ee558edbdaadd5ce53ea9171ef9bfeab6d5d45 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 22 Sep 2020 08:05:16 +0100 Subject: [PATCH 109/161] ATLAS-306 generic atlas::interpolation::nonlinear::MissingValue implementations on DataType --- .../interpolation/nonlinear/MissingValue.cc | 87 ++++++++++++------- .../interpolation/nonlinear/MissingValue.h | 18 ++-- 2 files changed, 68 insertions(+), 37 deletions(-) diff --git a/src/atlas/interpolation/nonlinear/MissingValue.cc b/src/atlas/interpolation/nonlinear/MissingValue.cc index e679ad1fc..9a0fe9613 100644 --- a/src/atlas/interpolation/nonlinear/MissingValue.cc +++ b/src/atlas/interpolation/nonlinear/MissingValue.cc @@ -13,11 +13,12 @@ #include "atlas/interpolation/nonlinear/MissingValue.h" #include +#include #include "eckit/types/FloatCompare.h" +#include "atlas/array/DataType.h" #include "atlas/field/Field.h" -#include "atlas/runtime/Exception.h" #include "atlas/util/Metadata.h" @@ -32,15 +33,17 @@ static const std::string value_key = "missing_value"; static const std::string epsilon_key = "missing_value_epsilon"; -double config_value( const MissingValue::Config& c ) { - double value; +template +T config_value( const MissingValue::Config& c ) { + T value; ATLAS_ASSERT( c.get( value_key, value ) ); return value; } -double config_epsilon( const MissingValue::Config& c ) { - double value = 0.; +template +T config_epsilon( const MissingValue::Config& c ) { + T value = 0.; c.get( epsilon_key, value ); return value; } @@ -48,9 +51,10 @@ double config_epsilon( const MissingValue::Config& c ) { /// @brief Missing value if NaN +template struct MissingValueNaN : MissingValue { - MissingValueNaN( const Config& ) {} - bool operator()( const double& value ) const override { return std::isnan( value ); } + MissingValueNaN( const Config& ) { ATLAS_ASSERT( std::is_floating_point::value ); } + bool operator()( const T& value ) const override { return std::isnan( value ); } bool isnan() const override { return true; } void metadata( Field& field ) const override { field.metadata().set( type_key, static_type() ); } static std::string static_type() { return "nan"; } @@ -58,19 +62,17 @@ struct MissingValueNaN : MissingValue { /// @brief Missing value if comparing equally to pre-defined value +template struct MissingValueEquals : MissingValue { - MissingValueEquals( const Config& config ) : - missingValue_( config_value( config ) ), missingValue2_( missingValue_ ) { - ATLAS_ASSERT( missingValue_ == missingValue2_ ); // this succeeds - } + MissingValueEquals( const Config& config ) : MissingValueEquals( config_value( config ) ) {} - MissingValueEquals( double missingValue ) : missingValue_( missingValue ), missingValue2_( missingValue_ ) { + MissingValueEquals( T missingValue ) : missingValue_( missingValue ), missingValue2_( missingValue_ ) { + ATLAS_ASSERT( missingValue_ == missingValue2_ ); // FIXME this succeeds ATLAS_ASSERT( !std::isnan( missingValue2_ ) ); } - bool operator()( const double& value ) const override { - // ATLAS_ASSERT(missingValue_ == missingValue2_); // this fails when not using missingValue2_ (copy ellision - // problem on POD!?) + bool operator()( const T& value ) const override { + // ATLAS_ASSERT(missingValue_ == missingValue2_); // FIXME this fails (copy ellision problem on POD!?) return value == missingValue2_; } @@ -83,21 +85,24 @@ struct MissingValueEquals : MissingValue { static std::string static_type() { return "equals"; } - const double missingValue_; - const double missingValue2_; + const T missingValue_; + const T missingValue2_; }; /// @brief Missing value if comparing approximately to pre-defined value +template struct MissingValueApprox : MissingValue { MissingValueApprox( const Config& config ) : - missingValue_( config_value( config ) ), epsilon_( config_epsilon( config ) ) {} - MissingValueApprox( double missingValue, double epsilon ) : missingValue_( missingValue ), epsilon_( epsilon ) { + MissingValueApprox( config_value( config ), config_epsilon( config ) ) {} + + MissingValueApprox( T missingValue, T epsilon ) : missingValue_( missingValue ), epsilon_( epsilon ) { ATLAS_ASSERT( !std::isnan( missingValue_ ) ); + ATLAS_ASSERT( std::is_floating_point::value ); ATLAS_ASSERT( epsilon_ >= 0. ); } - bool operator()( const double& value ) const override { + bool operator()( const T& value ) const override { return eckit::types::is_approximately_equal( value, missingValue_, epsilon_ ); } @@ -111,24 +116,44 @@ struct MissingValueApprox : MissingValue { static std::string static_type() { return "approximately-equals"; } - const double missingValue_; - const double epsilon_; + const T missingValue_; + const T epsilon_; }; -MissingValue::~MissingValue() = default; +const MissingValue* MissingValueFactory::build( const std::string& builder, const Config& config ) { + return has( builder ) ? get( builder )->make( config ) : nullptr; +} -namespace { -MissingValueFactoryBuilder __mv1; -MissingValueFactoryBuilder __mv2; -MissingValueFactoryBuilder __mv3; -} // namespace +#define B MissingValueFactoryBuilder +#define M1 MissingValueNaN +#define M2 MissingValueEquals +#define M3 MissingValueApprox +#define T array::DataType::str -const MissingValue* MissingValueFactory::build( const std::string& builder, const Config& config ) { - return has( builder ) ? get( builder )->make( config ) : nullptr; -} +static B> __mv1( M1::static_type() ); +static B> __mv2( M1::static_type() + "-" + T() ); +static B> __mv3( M1::static_type() + "-" + T() ); + +static B> __mv4( M2::static_type() ); +static B> __mv5( M2::static_type() + "-" + T() ); +static B> __mv6( M2::static_type() + "-" + T() ); +static B> __mv7( M2::static_type() + "-" + T() ); +static B> __mv8( M2::static_type() + "-" + T() ); +static B> __mv9( M2::static_type() + "-" + T() ); + +static B> __mv10( M3::static_type() ); +static B> __mv11( M3::static_type() + "-" + T() ); +static B> __mv12( M3::static_type() + "-" + T() ); + + +#undef T +#undef M3 +#undef M2 +#undef M1 +#undef B } // namespace nonlinear diff --git a/src/atlas/interpolation/nonlinear/MissingValue.h b/src/atlas/interpolation/nonlinear/MissingValue.h index fd6774b61..80063aab7 100644 --- a/src/atlas/interpolation/nonlinear/MissingValue.h +++ b/src/atlas/interpolation/nonlinear/MissingValue.h @@ -16,6 +16,7 @@ #include "eckit/config/Parametrisation.h" +#include "atlas/runtime/Exception.h" #include "atlas/util/Factory.h" @@ -31,11 +32,16 @@ namespace nonlinear { /// @brief Missing values indicator base class struct MissingValue { - using Config = eckit::Parametrisation; - virtual ~MissingValue(); - virtual bool operator()( const double& ) const = 0; - virtual bool isnan() const = 0; - virtual void metadata( Field& ) const = 0; + using Config = eckit::Parametrisation; + virtual ~MissingValue() = default; + virtual bool isnan() const = 0; + virtual void metadata( Field& ) const = 0; + + virtual bool operator()( const double& ) const { ATLAS_NOTIMPLEMENTED; } + virtual bool operator()( const float& ) const { ATLAS_NOTIMPLEMENTED; } + virtual bool operator()( const int& ) const { ATLAS_NOTIMPLEMENTED; } + virtual bool operator()( const long& ) const { ATLAS_NOTIMPLEMENTED; } + virtual bool operator()( const unsigned long& ) const { ATLAS_NOTIMPLEMENTED; } }; @@ -58,7 +64,7 @@ class MissingValueFactoryBuilder : public MissingValueFactory { virtual const MissingValue* make( const Config& config ) override { return new T( config ); } public: - MissingValueFactoryBuilder() : MissingValueFactory( T::static_type() ) {} + using MissingValueFactory::MissingValueFactory; }; From d922655d9318e2519e44d36af8b359c4cc8d52ca Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 22 Sep 2020 08:33:38 +0100 Subject: [PATCH 110/161] ATLAS-306 move atlas::interpolation::nonlinear::MissingValue to atlas::field::MissingValue --- src/atlas/CMakeLists.txt | 24 +++++++++---------- .../{interpolation => field}/MissingValue.cc | 16 ++++++------- .../{interpolation => field}/MissingValue.h | 12 +++++----- .../detail}/MissingValue.cc | 10 ++++---- .../nonlinear => field/detail}/MissingValue.h | 8 +++---- src/atlas/interpolation/method/Method.cc | 3 ++- .../nonlinear/MissingIfAllMissing.h | 3 ++- .../nonlinear/MissingIfAnyMissing.h | 3 ++- .../nonlinear/MissingIfHeaviestMissing.h | 3 ++- src/atlas/interpolation/nonlinear/NonLinear.h | 1 - .../test_interpolation_non_linear.cc | 21 +++++++++------- 11 files changed, 56 insertions(+), 48 deletions(-) rename src/atlas/{interpolation => field}/MissingValue.cc (79%) rename src/atlas/{interpolation => field}/MissingValue.h (85%) rename src/atlas/{interpolation/nonlinear => field/detail}/MissingValue.cc (96%) rename src/atlas/{interpolation/nonlinear => field/detail}/MissingValue.h (95%) diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 4a6ec5bfd..252029303 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -370,22 +370,26 @@ output/detail/PointCloudIO.h list( APPEND atlas_field_srcs field.h -field/Field.h field/Field.cc -field/FieldCreator.h +field/Field.h field/FieldCreator.cc -field/FieldCreatorArraySpec.h +field/FieldCreator.h field/FieldCreatorArraySpec.cc -field/FieldCreatorIFS.h +field/FieldCreatorArraySpec.h field/FieldCreatorIFS.cc -field/FieldSet.h +field/FieldCreatorIFS.h field/FieldSet.cc -field/State.h +field/FieldSet.h +field/MissingValue.cc +field/MissingValue.h field/State.cc -field/detail/FieldImpl.h +field/State.h field/detail/FieldImpl.cc -field/detail/FieldInterface.h +field/detail/FieldImpl.h field/detail/FieldInterface.cc +field/detail/FieldInterface.h +field/detail/MissingValue.cc +field/detail/MissingValue.h ) list( APPEND atlas_functionspace_srcs @@ -475,8 +479,6 @@ list( APPEND atlas_interpolation_srcs interpolation.h interpolation/Interpolation.cc interpolation/Interpolation.h -interpolation/MissingValue.cc -interpolation/MissingValue.h interpolation/NonLinear.cc interpolation/NonLinear.h interpolation/Vector2D.cc @@ -535,8 +537,6 @@ interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h interpolation/nonlinear/MissingIfAllMissing.h interpolation/nonlinear/MissingIfAnyMissing.h interpolation/nonlinear/MissingIfHeaviestMissing.h -interpolation/nonlinear/MissingValue.cc -interpolation/nonlinear/MissingValue.h interpolation/nonlinear/NonLinear.h interpolation/nonlinear/NonLinearFactory.cc interpolation/nonlinear/NonLinearFactory.h diff --git a/src/atlas/interpolation/MissingValue.cc b/src/atlas/field/MissingValue.cc similarity index 79% rename from src/atlas/interpolation/MissingValue.cc rename to src/atlas/field/MissingValue.cc index dab70ff89..7d42f92f9 100644 --- a/src/atlas/interpolation/MissingValue.cc +++ b/src/atlas/field/MissingValue.cc @@ -10,7 +10,7 @@ */ -#include "atlas/interpolation/MissingValue.h" +#include "atlas/field/MissingValue.h" #include @@ -23,7 +23,7 @@ namespace atlas { -namespace interpolation { +namespace field { namespace { @@ -46,19 +46,19 @@ MissingValue::MissingValue() : Handle( nullptr ) {} MissingValue::MissingValue( const MissingValue::Config& config ) : - Handle( nonlinear::MissingValueFactory::build( config_type( config ), config ) ) {} + Handle( detail::MissingValueFactory::build( config_type( config ), config ) ) {} MissingValue::MissingValue( const std::string& type, const MissingValue::Config& config ) : - Handle( nonlinear::MissingValueFactory::build( type, config ) ) {} + Handle( detail::MissingValueFactory::build( type, config ) ) {} MissingValue::MissingValue( const Field& field ) : - Handle( nonlinear::MissingValueFactory::build( field_type( field ), field.metadata() ) ) {} + Handle( detail::MissingValueFactory::build( field_type( field ), field.metadata() ) ) {} MissingValue::MissingValue( const std::string& type, const Field& field ) : - Handle( nonlinear::MissingValueFactory::build( type, field.metadata() ) ) {} + Handle( detail::MissingValueFactory::build( type, field.metadata() ) ) {} bool MissingValue::operator()( const double& value ) const { @@ -73,7 +73,7 @@ bool MissingValue::isnan() const { } -const nonlinear::MissingValue& MissingValue::ref() const { +const detail::MissingValue& MissingValue::ref() const { ATLAS_ASSERT_MSG( operator bool(), "MissingValue: ObjectHandle not setup" ); return *get(); // (one dereferencing level less) } @@ -85,5 +85,5 @@ void MissingValue::metadata( Field& field ) const { } -} // namespace interpolation +} // namespace field } // namespace atlas diff --git a/src/atlas/interpolation/MissingValue.h b/src/atlas/field/MissingValue.h similarity index 85% rename from src/atlas/interpolation/MissingValue.h rename to src/atlas/field/MissingValue.h index e2d9407bd..614a0d130 100644 --- a/src/atlas/interpolation/MissingValue.h +++ b/src/atlas/field/MissingValue.h @@ -14,7 +14,7 @@ #include -#include "atlas/interpolation/nonlinear/MissingValue.h" +#include "atlas/field/detail/MissingValue.h" #include "atlas/library/config.h" #include "atlas/util/ObjectHandle.h" @@ -28,13 +28,13 @@ class Config; namespace atlas { -namespace interpolation { +namespace field { /// @brief Missing values indicator -struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle ) { +struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle ) { using Spec = util::Config; - using Config = nonlinear::MissingValue::Config; + using Config = detail::MissingValue::Config; // ctor using Handle::Handle; @@ -67,7 +67,7 @@ struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle #include @@ -23,8 +23,8 @@ namespace atlas { -namespace interpolation { -namespace nonlinear { +namespace field { +namespace detail { namespace { @@ -156,6 +156,6 @@ static B> __mv12( M3::static_type() + "-" + T() ); #undef B -} // namespace nonlinear -} // namespace interpolation +} // namespace detail +} // namespace field } // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingValue.h b/src/atlas/field/detail/MissingValue.h similarity index 95% rename from src/atlas/interpolation/nonlinear/MissingValue.h rename to src/atlas/field/detail/MissingValue.h index 80063aab7..9a9cf8f3e 100644 --- a/src/atlas/interpolation/nonlinear/MissingValue.h +++ b/src/atlas/field/detail/MissingValue.h @@ -26,8 +26,8 @@ class Field; namespace atlas { -namespace interpolation { -namespace nonlinear { +namespace field { +namespace detail { /// @brief Missing values indicator base class @@ -68,6 +68,6 @@ class MissingValueFactoryBuilder : public MissingValueFactory { }; -} // namespace nonlinear -} // namespace interpolation +} // namespace detail +} // namespace field } // namespace atlas diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index d7364d3c0..f961bc2f6 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -20,6 +20,7 @@ #include "atlas/array.h" #include "atlas/field/Field.h" #include "atlas/field/FieldSet.h" +#include "atlas/field/MissingValue.h" #include "atlas/functionspace/NodeColumns.h" #include "atlas/mesh/Nodes.h" #include "atlas/runtime/Exception.h" @@ -236,7 +237,7 @@ void Method::do_execute( const Field& src, Field& tgt ) const { } // carry over missing value metadata - MissingValue mv( src ); + field::MissingValue mv( src ); if ( mv ) { mv.metadata( tgt ); } diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h index 5b592d949..c101b565a 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h @@ -12,6 +12,7 @@ #pragma once +#include "atlas/field/MissingValue.h" #include "atlas/interpolation/nonlinear/NonLinear.h" #include "eckit/types/FloatCompare.h" @@ -25,7 +26,7 @@ namespace nonlinear { template struct MissingIfAllMissing : NonLinear { bool execute( NonLinear::Matrix& W, const Field& field ) const { - interpolation::MissingValue mv( field ); + field::MissingValue mv( field ); if ( !mv ) { return false; } diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h index 178fa767c..b731c231a 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h @@ -12,6 +12,7 @@ #pragma once +#include "atlas/field/MissingValue.h" #include "atlas/interpolation/nonlinear/NonLinear.h" @@ -23,7 +24,7 @@ namespace nonlinear { template struct MissingIfAnyMissing : NonLinear { bool execute( NonLinear::Matrix& W, const Field& field ) const { - interpolation::MissingValue mv( field ); + field::MissingValue mv( field ); if ( !mv ) { return false; } diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h index 50e152eaa..537b661b6 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h @@ -12,6 +12,7 @@ #pragma once +#include "atlas/field/MissingValue.h" #include "atlas/interpolation/nonlinear/NonLinear.h" #include "eckit/types/FloatCompare.h" @@ -25,7 +26,7 @@ namespace nonlinear { template struct MissingIfHeaviestMissing : NonLinear { bool execute( NonLinear::Matrix& W, const Field& field ) const { - interpolation::MissingValue mv( field ); + field::MissingValue mv( field ); if ( !mv ) { return false; } diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index 49f11ba4e..680027719 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -19,7 +19,6 @@ #include "atlas/array.h" #include "atlas/field/Field.h" -#include "atlas/interpolation/MissingValue.h" #include "atlas/runtime/Exception.h" diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index 6a3099c96..e8aef981f 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -14,6 +14,7 @@ #include #include "atlas/array.h" +#include "atlas/field/MissingValue.h" #include "atlas/functionspace.h" #include "atlas/grid.h" #include "atlas/interpolation.h" @@ -35,7 +36,7 @@ const double nan = std::numeric_limits::quiet_NaN(); CASE( "test_interpolation_non_linear_missing_value" ) { - using interpolation::MissingValue; + using field::MissingValue; SECTION( "not defined" ) { @@ -116,7 +117,8 @@ CASE( "test_interpolation_non_linear_missing_value" ) { CASE( "test_interpolation_non_linear_field_missing_value" ) { - using interpolation::MissingValue; + using field::MissingValue; + std::vector values{1., nan, missingValue, missingValue, missingValue + missingValueEps / 2., 6., 7.}; Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); @@ -160,6 +162,9 @@ CASE( "test_interpolation_non_linear_field_missing_value" ) { CASE( "test_interpolation_non_linear_matrix" ) { + using field::MissingValue; + + /* Set input field full of 1's, with 9 nodes 1 ... 1 ... 1 @@ -204,10 +209,10 @@ CASE( "test_interpolation_non_linear_matrix" ) { fieldA.metadata().set( "missing_value_type", type ); viewA( 4 ) = type == "nan" ? nan : missingValue; - EXPECT( interpolation::MissingValue( fieldA ) ); + EXPECT( MissingValue( fieldA ) ); interpolation.execute( fieldA, fieldB ); - interpolation::MissingValue mv( fieldB ); + MissingValue mv( fieldB ); EXPECT( mv ); EXPECT( mv( viewB( 0 ) ) == false ); EXPECT( mv( viewB( 1 ) ) == false ); @@ -223,10 +228,10 @@ CASE( "test_interpolation_non_linear_matrix" ) { fieldA.metadata().set( "missing_value_type", type ); viewA( 4 ) = type == "nan" ? nan : missingValue; - EXPECT( interpolation::MissingValue( fieldA ) ); + EXPECT( MissingValue( fieldA ) ); interpolation.execute( fieldA, fieldB ); - interpolation::MissingValue mv( fieldB ); + MissingValue mv( fieldB ); EXPECT( mv ); EXPECT( mv( viewB( 0 ) ) ); EXPECT( mv( viewB( 1 ) ) ); @@ -242,10 +247,10 @@ CASE( "test_interpolation_non_linear_matrix" ) { fieldA.metadata().set( "missing_value_type", type ); viewA( 4 ) = type == "nan" ? nan : missingValue; - EXPECT( interpolation::MissingValue( fieldA ) ); + EXPECT( MissingValue( fieldA ) ); interpolation.execute( fieldA, fieldB ); - interpolation::MissingValue mv( fieldB ); + MissingValue mv( fieldB ); EXPECT( mv ); EXPECT( mv( viewB( 0 ) ) == false ); EXPECT( mv( viewB( 1 ) ) ); From 8b2329bb0f07be855f40c8af3fa76eb5b0a311f1 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 22 Sep 2020 09:46:41 +0100 Subject: [PATCH 111/161] ATLAS-306 add missing interfacing methods --- src/atlas/field/MissingValue.cc | 26 +++++++++++++++++++++++++- src/atlas/field/MissingValue.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/atlas/field/MissingValue.cc b/src/atlas/field/MissingValue.cc index 7d42f92f9..c4a29fa4e 100644 --- a/src/atlas/field/MissingValue.cc +++ b/src/atlas/field/MissingValue.cc @@ -62,7 +62,31 @@ MissingValue::MissingValue( const std::string& type, const Field& field ) : bool MissingValue::operator()( const double& value ) const { - ATLAS_ASSERT_MSG( operator bool(), "MissingValue: ObjectHandle not setup" ); + ATLAS_ASSERT_MSG( operator bool(), "MissingValue::operator()( const double& ) ObjectHandle not setup" ); + return get()->operator()( value ); +} + + +bool MissingValue::operator()( const float& value ) const { + ATLAS_ASSERT_MSG( operator bool(), "MissingValue::operator()( const float& ): ObjectHandle not setup" ); + return get()->operator()( value ); +} + + +bool MissingValue::operator()( const int& value ) const { + ATLAS_ASSERT_MSG( operator bool(), "MissingValue::operator()( const int& ): ObjectHandle not setup" ); + return get()->operator()( value ); +} + + +bool MissingValue::operator()( const long& value ) const { + ATLAS_ASSERT_MSG( operator bool(), "MissingValue::operator()( const long& ): ObjectHandle not setup" ); + return get()->operator()( value ); +} + + +bool MissingValue::operator()( const unsigned long& value ) const { + ATLAS_ASSERT_MSG( operator bool(), "MissingValue::operator()( const unsigned long& ): ObjectHandle not setup" ); return get()->operator()( value ); } diff --git a/src/atlas/field/MissingValue.h b/src/atlas/field/MissingValue.h index 614a0d130..fff4b2e34 100644 --- a/src/atlas/field/MissingValue.h +++ b/src/atlas/field/MissingValue.h @@ -57,6 +57,34 @@ struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle Date: Tue, 22 Sep 2020 09:47:52 +0100 Subject: [PATCH 112/161] ATLAS-306 interpolation MissingValue transfer from source to target sanity assert --- src/atlas/interpolation/method/Method.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index f961bc2f6..79d20aaed 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -240,6 +240,7 @@ void Method::do_execute( const Field& src, Field& tgt ) const { field::MissingValue mv( src ); if ( mv ) { mv.metadata( tgt ); + ATLAS_ASSERT( field::MissingValue( tgt ) ); } tgt.set_dirty(); From 2bf4fbba63213e4d788a0dad72c47009b3137b19 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 22 Sep 2020 11:56:40 +0100 Subject: [PATCH 113/161] ATLAS-306 testing generic MissingValue --- .../test_interpolation_non_linear.cc | 95 +++++++++++++++---- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index e8aef981f..7b95bf560 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -34,13 +34,13 @@ const double missingValue = 42.; const double missingValueEps = 1e-9; const double nan = std::numeric_limits::quiet_NaN(); - -CASE( "test_interpolation_non_linear_missing_value" ) { - using field::MissingValue; +using field::MissingValue; +using util::Config; +CASE( "test_interpolation_non_linear_missing_value" ) { SECTION( "not defined" ) { - util::Config config; + Config config; auto mv = MissingValue( config ); EXPECT( !bool( mv ) ); @@ -51,7 +51,7 @@ CASE( "test_interpolation_non_linear_missing_value" ) { SECTION( "nan" ) { - util::Config config; + Config config; auto mv = MissingValue( "nan", config ); EXPECT( bool( mv ) ); @@ -69,7 +69,7 @@ CASE( "test_interpolation_non_linear_missing_value" ) { SECTION( "equals" ) { - util::Config config; + Config config; config.set( "missing_value", missingValue ); auto mv = MissingValue( "equals", config ); @@ -90,7 +90,7 @@ CASE( "test_interpolation_non_linear_missing_value" ) { SECTION( "approximately-equals" ) { - util::Config config; + Config config; config.set( "missing_value", missingValue ); config.set( "missing_value_epsilon", missingValueEps ); @@ -116,10 +116,74 @@ CASE( "test_interpolation_non_linear_missing_value" ) { } -CASE( "test_interpolation_non_linear_field_missing_value" ) { - using field::MissingValue; +CASE( "test_interpolation_non_linear_missing_value (DataType specialisations)" ) { + SECTION( "real64" ) { + auto n = static_cast( missingValue ); + auto eps = static_cast( missingValueEps ); + auto nan = std::numeric_limits::quiet_NaN(); + + Config config; + config.set( "missing_value", n ); + config.set( "missing_value_epsilon", eps ); + + for ( std::string type : {"nan", "equals", "approximately-equals"} ) { + auto mv = MissingValue( type + "-real64", config ); + EXPECT( bool( mv ) ); + EXPECT( mv( type == "nan" ? nan : n ) ); + EXPECT( mv( n ) != mv( nan ) ); + EXPECT( !mv( n + 1 ) ); + } + } + + + SECTION( "real32" ) { + auto n = static_cast( missingValue ); + auto eps = static_cast( missingValueEps ); + auto nan = std::numeric_limits::quiet_NaN(); + + Config config; + config.set( "missing_value", n ); + config.set( "missing_value_epsilon", eps ); + + for ( std::string type : {"nan", "equals", "approximately-equals"} ) { + auto mv = MissingValue( type + "-real32", config ); + EXPECT( bool( mv ) ); + EXPECT( mv( type == "nan" ? nan : n ) ); + EXPECT( mv( n ) != mv( nan ) ); + EXPECT( !mv( n + 1 ) ); + } + } + + + SECTION( "int32" ) { + auto n = static_cast( missingValue ); + auto mv = MissingValue( "equals-int32", Config( "missing_value", n ) ); + EXPECT( bool( mv ) ); + EXPECT( mv( n ) ); + EXPECT( !mv( n + 1 ) ); + } + + + SECTION( "int64" ) { + auto n = static_cast( missingValue ); + auto mv = MissingValue( "equals-int64", Config( "missing_value", n ) ); + EXPECT( bool( mv ) ); + EXPECT( mv( n ) ); + EXPECT( !mv( n + 1 ) ); + } + + + SECTION( "uint64" ) { + auto n = static_cast( missingValue ); + auto mv = MissingValue( "equals-uint64", Config( "missing_value", n ) ); + EXPECT( bool( mv ) ); + EXPECT( mv( n ) ); + EXPECT( !mv( n + 1 ) ); + } +} +CASE( "test_interpolation_non_linear_field_missing_value" ) { std::vector values{1., nan, missingValue, missingValue, missingValue + missingValueEps / 2., 6., 7.}; Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); @@ -162,9 +226,6 @@ CASE( "test_interpolation_non_linear_field_missing_value" ) { CASE( "test_interpolation_non_linear_matrix" ) { - using field::MissingValue; - - /* Set input field full of 1's, with 9 nodes 1 ... 1 ... 1 @@ -202,8 +263,8 @@ CASE( "test_interpolation_non_linear_matrix" ) { SECTION( "missing-if-all-missing" ) { - Interpolation interpolation( - util::Config( "type", "finite-element" ).set( "non_linear", "missing-if-all-missing" ), fsA, fsB ); + Interpolation interpolation( Config( "type", "finite-element" ).set( "non_linear", "missing-if-all-missing" ), + fsA, fsB ); for ( std::string type : {"equals", "approximately-equals", "nan"} ) { fieldA.metadata().set( "missing_value_type", type ); @@ -221,8 +282,8 @@ CASE( "test_interpolation_non_linear_matrix" ) { SECTION( "missing-if-any-missing" ) { - Interpolation interpolation( - util::Config( "type", "finite-element" ).set( "non_linear", "missing-if-any-missing" ), fsA, fsB ); + Interpolation interpolation( Config( "type", "finite-element" ).set( "non_linear", "missing-if-any-missing" ), + fsA, fsB ); for ( std::string type : {"equals", "approximately-equals", "nan"} ) { fieldA.metadata().set( "missing_value_type", type ); @@ -241,7 +302,7 @@ CASE( "test_interpolation_non_linear_matrix" ) { SECTION( "missing-if-heaviest-missing" ) { Interpolation interpolation( - util::Config( "type", "finite-element" ).set( "non_linear", "missing-if-heaviest-missing" ), fsA, fsB ); + Config( "type", "finite-element" ).set( "non_linear", "missing-if-heaviest-missing" ), fsA, fsB ); for ( std::string type : {"equals", "approximately-equals", "nan"} ) { fieldA.metadata().set( "missing_value_type", type ); From 70f8e42fc7169471993b2405bead0c64e477468f Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 22 Sep 2020 13:48:37 +0100 Subject: [PATCH 114/161] ATLAS-306 more specific unit tests for MissingValue("equals") --- .../test_interpolation_non_linear.cc | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index 7b95bf560..3c3f4729c 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -40,12 +40,10 @@ using util::Config; CASE( "test_interpolation_non_linear_missing_value" ) { SECTION( "not defined" ) { - Config config; - - auto mv = MissingValue( config ); + auto mv = MissingValue(); EXPECT( !bool( mv ) ); - mv = MissingValue( "not defined", config ); + mv = MissingValue( "not defined", Config() ); EXPECT( !bool( mv ) ); } @@ -76,16 +74,20 @@ CASE( "test_interpolation_non_linear_missing_value" ) { EXPECT( bool( mv ) ); EXPECT( mv( missingValue - 1 ) == false ); - EXPECT( mv( missingValue + 1 ) == false ); + EXPECT( mv( missingValue - missingValueEps / 2 ) == false ); EXPECT( mv( missingValue ) ); + EXPECT( mv( missingValue + missingValueEps / 2 ) == false ); + EXPECT( mv( missingValue + 1 ) == false ); config.set( "type", "equals" ); mv = MissingValue( config ); EXPECT( bool( mv ) ); EXPECT( mv( missingValue - 1 ) == false ); - EXPECT( mv( missingValue + 1 ) == false ); + EXPECT( mv( missingValue - missingValueEps / 2 ) == false ); EXPECT( mv( missingValue ) ); + EXPECT( mv( missingValue + missingValueEps / 2 ) == false ); + EXPECT( mv( missingValue + 1 ) == false ); } @@ -131,7 +133,7 @@ CASE( "test_interpolation_non_linear_missing_value (DataType specialisations)" ) EXPECT( bool( mv ) ); EXPECT( mv( type == "nan" ? nan : n ) ); EXPECT( mv( n ) != mv( nan ) ); - EXPECT( !mv( n + 1 ) ); + EXPECT( mv( n + 1 ) == false ); } } @@ -150,7 +152,7 @@ CASE( "test_interpolation_non_linear_missing_value (DataType specialisations)" ) EXPECT( bool( mv ) ); EXPECT( mv( type == "nan" ? nan : n ) ); EXPECT( mv( n ) != mv( nan ) ); - EXPECT( !mv( n + 1 ) ); + EXPECT( mv( n + 1 ) == false ); } } @@ -160,7 +162,7 @@ CASE( "test_interpolation_non_linear_missing_value (DataType specialisations)" ) auto mv = MissingValue( "equals-int32", Config( "missing_value", n ) ); EXPECT( bool( mv ) ); EXPECT( mv( n ) ); - EXPECT( !mv( n + 1 ) ); + EXPECT( mv( n + 1 ) == false ); } @@ -169,7 +171,7 @@ CASE( "test_interpolation_non_linear_missing_value (DataType specialisations)" ) auto mv = MissingValue( "equals-int64", Config( "missing_value", n ) ); EXPECT( bool( mv ) ); EXPECT( mv( n ) ); - EXPECT( !mv( n + 1 ) ); + EXPECT( mv( n + 1 ) == false ); } @@ -178,7 +180,7 @@ CASE( "test_interpolation_non_linear_missing_value (DataType specialisations)" ) auto mv = MissingValue( "equals-uint64", Config( "missing_value", n ) ); EXPECT( bool( mv ) ); EXPECT( mv( n ) ); - EXPECT( !mv( n + 1 ) ); + EXPECT( mv( n + 1 ) == false ); } } From 293dbbe627527f6cca333149525e1a1fdac94278 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 22 Sep 2020 13:51:09 +0000 Subject: [PATCH 115/161] ATLAS-306 Fix memory corruption --- src/atlas/field/detail/MissingValue.h | 3 ++- src/atlas/functionspace/detail/FunctionSpaceImpl.h | 2 +- src/atlas/interpolation/nonlinear/NonLinear.h | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/atlas/field/detail/MissingValue.h b/src/atlas/field/detail/MissingValue.h index 9a9cf8f3e..7a0fbc8d7 100644 --- a/src/atlas/field/detail/MissingValue.h +++ b/src/atlas/field/detail/MissingValue.h @@ -17,6 +17,7 @@ #include "eckit/config/Parametrisation.h" #include "atlas/runtime/Exception.h" +#include "atlas/util/Object.h" #include "atlas/util/Factory.h" @@ -31,7 +32,7 @@ namespace detail { /// @brief Missing values indicator base class -struct MissingValue { +struct MissingValue : util::Object { using Config = eckit::Parametrisation; virtual ~MissingValue() = default; virtual bool isnan() const = 0; diff --git a/src/atlas/functionspace/detail/FunctionSpaceImpl.h b/src/atlas/functionspace/detail/FunctionSpaceImpl.h index 3826c37c9..46aa59125 100644 --- a/src/atlas/functionspace/detail/FunctionSpaceImpl.h +++ b/src/atlas/functionspace/detail/FunctionSpaceImpl.h @@ -114,7 +114,7 @@ inline FunctionspaceT_const* FunctionSpaceImpl::cast() const { /// @brief Dummy Functionspace class that evaluates to false class NoFunctionSpace : public FunctionSpaceImpl { public: - NoFunctionSpace() {} + NoFunctionSpace(): FunctionSpaceImpl() {} virtual ~NoFunctionSpace() {} virtual std::string type() const { return "NoFunctionSpace"; } virtual operator bool() const { return false; } diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index 680027719..f04196bb5 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -20,6 +20,7 @@ #include "atlas/array.h" #include "atlas/field/Field.h" #include "atlas/runtime/Exception.h" +#include "atlas/util/ObjectHandle.h" namespace atlas { @@ -31,7 +32,7 @@ namespace nonlinear { * @brief NonLinear class applies non-linear corrections to an interpolation matrix, given a field with missing values. * The interpolatation are re-weighted to factor those values out of the resulting field. */ -class NonLinear { +class NonLinear : public util::Object { public: using Config = eckit::Parametrisation; using Matrix = eckit::linalg::SparseMatrix; From 9b96069ac05b152b9ee4183692fbc781d8b58c90 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Tue, 22 Sep 2020 15:05:27 +0100 Subject: [PATCH 116/161] clang-format --- src/atlas/field/detail/MissingValue.h | 2 +- src/atlas/functionspace/detail/FunctionSpaceImpl.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/atlas/field/detail/MissingValue.h b/src/atlas/field/detail/MissingValue.h index 7a0fbc8d7..b158e3ec1 100644 --- a/src/atlas/field/detail/MissingValue.h +++ b/src/atlas/field/detail/MissingValue.h @@ -17,8 +17,8 @@ #include "eckit/config/Parametrisation.h" #include "atlas/runtime/Exception.h" -#include "atlas/util/Object.h" #include "atlas/util/Factory.h" +#include "atlas/util/Object.h" namespace atlas { diff --git a/src/atlas/functionspace/detail/FunctionSpaceImpl.h b/src/atlas/functionspace/detail/FunctionSpaceImpl.h index 46aa59125..f069fd9af 100644 --- a/src/atlas/functionspace/detail/FunctionSpaceImpl.h +++ b/src/atlas/functionspace/detail/FunctionSpaceImpl.h @@ -114,7 +114,7 @@ inline FunctionspaceT_const* FunctionSpaceImpl::cast() const { /// @brief Dummy Functionspace class that evaluates to false class NoFunctionSpace : public FunctionSpaceImpl { public: - NoFunctionSpace(): FunctionSpaceImpl() {} + NoFunctionSpace() : FunctionSpaceImpl() {} virtual ~NoFunctionSpace() {} virtual std::string type() const { return "NoFunctionSpace"; } virtual operator bool() const { return false; } From 71c77c3a60fc93ae0ed16a8da7678e5bcff19fc6 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Wed, 23 Sep 2020 13:43:04 +0100 Subject: [PATCH 117/161] ATLAS-306 MissingValue(const Field&) now detects Field DataType to associate correct builder --- src/atlas/field/MissingValue.cc | 4 +- .../test_interpolation_non_linear.cc | 68 +++++++++++++++++-- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/src/atlas/field/MissingValue.cc b/src/atlas/field/MissingValue.cc index c4a29fa4e..521bc23f0 100644 --- a/src/atlas/field/MissingValue.cc +++ b/src/atlas/field/MissingValue.cc @@ -36,7 +36,9 @@ std::string config_type( const MissingValue::Config& c ) { std::string field_type( const Field& f ) { std::string value; - f.metadata().get( "missing_value_type", value ); + if ( f.metadata().get( "missing_value_type", value ) ) { + value += "-" + f.datatype().str(); + } return value; } } // namespace diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index 3c3f4729c..0847aedf6 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -38,7 +38,7 @@ using field::MissingValue; using util::Config; -CASE( "test_interpolation_non_linear_missing_value" ) { +CASE( "MissingValue (basic)" ) { SECTION( "not defined" ) { auto mv = MissingValue(); EXPECT( !bool( mv ) ); @@ -118,7 +118,7 @@ CASE( "test_interpolation_non_linear_missing_value" ) { } -CASE( "test_interpolation_non_linear_missing_value (DataType specialisations)" ) { +CASE( "MissingValue (DataType specialisations)" ) { SECTION( "real64" ) { auto n = static_cast( missingValue ); auto eps = static_cast( missingValueEps ); @@ -185,7 +185,7 @@ CASE( "test_interpolation_non_linear_missing_value (DataType specialisations)" ) } -CASE( "test_interpolation_non_linear_field_missing_value" ) { +CASE( "MissingValue from Field (basic)" ) { std::vector values{1., nan, missingValue, missingValue, missingValue + missingValueEps / 2., 6., 7.}; Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); @@ -227,7 +227,67 @@ CASE( "test_interpolation_non_linear_field_missing_value" ) { } -CASE( "test_interpolation_non_linear_matrix" ) { +CASE( "MissingValue from Field (DataType specialisations)" ) { + SECTION( "real64" ) { + std::vector values( 3, 1. ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::real64().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "nan" ); + EXPECT( MissingValue( field ) ); + } + + + SECTION( "real32" ) { + std::vector values( 3, 1. ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::real32().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "nan" ); + EXPECT( MissingValue( field ) ); + } + + + SECTION( "int32" ) { + std::vector values( 3, 1 ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::int32().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "equals" ); + field.metadata().set( "missing_value", static_cast( missingValue ) ); + EXPECT( MissingValue( field ) ); + } + + + SECTION( "int64" ) { + std::vector values( 3, 1 ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::int64().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "equals" ); + field.metadata().set( "missing_value", static_cast( missingValue ) ); + EXPECT( MissingValue( field ) ); + } + + + SECTION( "uint64" ) { + std::vector values( 3, 1 ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::uint64().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "equals" ); + field.metadata().set( "missing_value", static_cast( missingValue ) ); + EXPECT( MissingValue( field ) ); + } +} + + +CASE( "Interpolation with MissingValue" ) { /* Set input field full of 1's, with 9 nodes 1 ... 1 ... 1 From 3134ac2ffb47a04715431c40e01678616b17c101 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 24 Sep 2020 02:36:08 +0100 Subject: [PATCH 118/161] ATLAS-306 atlas::interpolation::NonLinear(const Field&) checks if interpolation non-linearities are applicable to field (in current case, in missing values are present, interpolant matrix is copied and adjusted accordingly) --- src/atlas/CMakeLists.txt | 1 + src/atlas/field/MissingValue.h | 8 +++-- src/atlas/field/detail/MissingValue.cc | 12 +++++-- src/atlas/field/detail/MissingValue.h | 12 +++++-- src/atlas/interpolation/NonLinear.cc | 5 +++ src/atlas/interpolation/NonLinear.h | 17 +++++++--- src/atlas/interpolation/method/Method.cc | 2 +- src/atlas/interpolation/nonlinear/Missing.h | 32 +++++++++++++++++++ .../nonlinear/MissingIfAllMissing.h | 14 +++----- .../nonlinear/MissingIfAnyMissing.h | 14 +++----- .../nonlinear/MissingIfHeaviestMissing.h | 13 +++----- src/atlas/interpolation/nonlinear/NonLinear.h | 13 ++++++-- 12 files changed, 99 insertions(+), 44 deletions(-) create mode 100644 src/atlas/interpolation/nonlinear/Missing.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 252029303..b812dfe64 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -534,6 +534,7 @@ interpolation/method/structured/kernels/LinearVerticalKernel.h interpolation/method/structured/kernels/QuasiCubic3DKernel.cc interpolation/method/structured/kernels/QuasiCubic3DKernel.h interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h +interpolation/nonlinear/Missing.h interpolation/nonlinear/MissingIfAllMissing.h interpolation/nonlinear/MissingIfAnyMissing.h interpolation/nonlinear/MissingIfHeaviestMissing.h diff --git a/src/atlas/field/MissingValue.h b/src/atlas/field/MissingValue.h index fff4b2e34..dbd84b66c 100644 --- a/src/atlas/field/MissingValue.h +++ b/src/atlas/field/MissingValue.h @@ -31,12 +31,16 @@ namespace atlas { namespace field { -/// @brief Missing values indicator +/** + * @brief Missing values indicator + */ struct MissingValue : DOXYGEN_HIDE( public util::ObjectHandle ) { using Spec = util::Config; using Config = detail::MissingValue::Config; - // ctor + /** + * @brief ctor + */ using Handle::Handle; MissingValue(); MissingValue( const Config& ); diff --git a/src/atlas/field/detail/MissingValue.cc b/src/atlas/field/detail/MissingValue.cc index 600eb7333..d03795c68 100644 --- a/src/atlas/field/detail/MissingValue.cc +++ b/src/atlas/field/detail/MissingValue.cc @@ -50,7 +50,9 @@ T config_epsilon( const MissingValue::Config& c ) { } // namespace -/// @brief Missing value if NaN +/** + * @brief Missing value if NaN + */ template struct MissingValueNaN : MissingValue { MissingValueNaN( const Config& ) { ATLAS_ASSERT( std::is_floating_point::value ); } @@ -61,7 +63,9 @@ struct MissingValueNaN : MissingValue { }; -/// @brief Missing value if comparing equally to pre-defined value +/** + * @brief Missing value if comparing equally to pre-defined value + */ template struct MissingValueEquals : MissingValue { MissingValueEquals( const Config& config ) : MissingValueEquals( config_value( config ) ) {} @@ -90,7 +94,9 @@ struct MissingValueEquals : MissingValue { }; -/// @brief Missing value if comparing approximately to pre-defined value +/** + * @brief Missing value if comparing approximately to pre-defined value + */ template struct MissingValueApprox : MissingValue { MissingValueApprox( const Config& config ) : diff --git a/src/atlas/field/detail/MissingValue.h b/src/atlas/field/detail/MissingValue.h index b158e3ec1..3e68be718 100644 --- a/src/atlas/field/detail/MissingValue.h +++ b/src/atlas/field/detail/MissingValue.h @@ -31,7 +31,9 @@ namespace field { namespace detail { -/// @brief Missing values indicator base class +/** + * @brief Missing values indicator base class + */ struct MissingValue : util::Object { using Config = eckit::Parametrisation; virtual ~MissingValue() = default; @@ -46,7 +48,9 @@ struct MissingValue : util::Object { }; -/// @brief Missing values indicator factory +/** + * @brief Missing values indicator factory + */ struct MissingValueFactory : util::Factory { using Config = MissingValue::Config; using Factory::Factory; @@ -58,7 +62,9 @@ struct MissingValueFactory : util::Factory { }; -/// @brief Missing values indicator builder for factory registration +/** + * @brief Missing values indicator builder for factory registration + */ template class MissingValueFactoryBuilder : public MissingValueFactory { private: diff --git a/src/atlas/interpolation/NonLinear.cc b/src/atlas/interpolation/NonLinear.cc index e920f9b83..f9d0d645a 100644 --- a/src/atlas/interpolation/NonLinear.cc +++ b/src/atlas/interpolation/NonLinear.cc @@ -41,6 +41,11 @@ NonLinear::NonLinear( const std::string& type, const NonLinear::Config& config ) Handle( nonlinear::NonLinearFactory::build( type, config ) ) {} +bool NonLinear::operator()( const Field& f ) const { + return operator bool() && get()->applicable( f ); +} + + bool NonLinear::execute( NonLinear::Matrix& W, const Field& f ) const { ATLAS_ASSERT_MSG( operator bool(), "NonLinear: ObjectHandle not setup" ); return get()->execute( W, f ); diff --git a/src/atlas/interpolation/NonLinear.h b/src/atlas/interpolation/NonLinear.h index c3d0f7ff2..126403f1c 100644 --- a/src/atlas/interpolation/NonLinear.h +++ b/src/atlas/interpolation/NonLinear.h @@ -20,6 +20,7 @@ namespace atlas { +class Field; namespace util { class Config; } @@ -31,15 +32,16 @@ namespace interpolation { /** - * @brief NonLinear class applies non-linear corrections to an interpolation matrix, given a field with missing values. - * The interpolatation are re-weighted to factor those values out of the resulting field. + * @brief NonLinear class applies non-linear corrections to an interpolation matrix */ struct NonLinear : DOXYGEN_HIDE( public util::ObjectHandle ) { using Spec = util::Config; using Config = nonlinear::NonLinear::Config; using Matrix = nonlinear::NonLinear::Matrix; - // ctor + /** + * @brief ctor + */ using Handle::Handle; NonLinear(); NonLinear( const Config& ); @@ -51,10 +53,17 @@ struct NonLinear : DOXYGEN_HIDE( public util::ObjectHandle */ using Handle::operator bool; // (ensure this exists) + /** + * @bried if NonLinear applies to field + * @param [in] f field + * @return if NonLinear applies to field + */ + bool operator()( const Field& f ) const; + /** * @brief Apply non-linear corrections to interpolation matrix * @param [inout] W interpolation matrix - * @param [in] f field with missing values information + * @param [in] f field * @return if W was modified */ bool execute( Matrix& W, const Field& f ) const; diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index 79d20aaed..289aea6b2 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -219,7 +219,7 @@ void Method::do_execute( const Field& src, Field& tgt ) const { // non-linearities: a non-empty M matrix contains the corrections applied to matrix_ Matrix M; - if ( !matrix_.empty() && nonLinear_ ) { + if ( !matrix_.empty() && nonLinear_( src ) ) { Matrix W( matrix_ ); // copy (a big penalty -- copy-on-write would definitely be better) if ( nonLinear_->execute( W, src ) ) { M.swap( W ); diff --git a/src/atlas/interpolation/nonlinear/Missing.h b/src/atlas/interpolation/nonlinear/Missing.h new file mode 100644 index 000000000..76d7011a2 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/Missing.h @@ -0,0 +1,32 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include "atlas/field/MissingValue.h" +#include "atlas/interpolation/nonlinear/NonLinear.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +struct Missing : NonLinear { +private: + bool applicable( const Field& f ) const override { return field::MissingValue( f ); } +}; + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h index c101b565a..f4e72ffd0 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h @@ -13,7 +13,7 @@ #pragma once #include "atlas/field/MissingValue.h" -#include "atlas/interpolation/nonlinear/NonLinear.h" +#include "atlas/interpolation/nonlinear/Missing.h" #include "eckit/types/FloatCompare.h" @@ -24,19 +24,13 @@ namespace nonlinear { template -struct MissingIfAllMissing : NonLinear { +struct MissingIfAllMissing : Missing { bool execute( NonLinear::Matrix& W, const Field& field ) const { field::MissingValue mv( field ); - if ( !mv ) { - return false; - } - - // NOTE only for scalars (for now) - auto values = make_view_field_values( field ); auto& missingValue = mv.ref(); - // correct matrix weigths for the missing values - // (force a missing value only if all row values are missing) + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); auto data = const_cast( W.data() ); diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h index b731c231a..abffc9d27 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h @@ -13,7 +13,7 @@ #pragma once #include "atlas/field/MissingValue.h" -#include "atlas/interpolation/nonlinear/NonLinear.h" +#include "atlas/interpolation/nonlinear/Missing.h" namespace atlas { @@ -22,19 +22,13 @@ namespace nonlinear { template -struct MissingIfAnyMissing : NonLinear { +struct MissingIfAnyMissing : Missing { bool execute( NonLinear::Matrix& W, const Field& field ) const { field::MissingValue mv( field ); - if ( !mv ) { - return false; - } - - // NOTE only for scalars (for now) - auto values = make_view_field_values( field ); auto& missingValue = mv.ref(); - // correct matrix weigths for the missing values - // (force a missing value only if any row values is missing) + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); auto data = const_cast( W.data() ); diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h index 537b661b6..90869c1a7 100644 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h +++ b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h @@ -13,7 +13,7 @@ #pragma once #include "atlas/field/MissingValue.h" -#include "atlas/interpolation/nonlinear/NonLinear.h" +#include "atlas/interpolation/nonlinear/Missing.h" #include "eckit/types/FloatCompare.h" @@ -24,18 +24,13 @@ namespace nonlinear { template -struct MissingIfHeaviestMissing : NonLinear { +struct MissingIfHeaviestMissing : Missing { bool execute( NonLinear::Matrix& W, const Field& field ) const { field::MissingValue mv( field ); - if ( !mv ) { - return false; - } - - // NOTE only for scalars (for now) - auto values = make_view_field_values( field ); auto& missingValue = mv.ref(); - // correct matrix weigths for the missing values + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); auto data = const_cast( W.data() ); diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index f04196bb5..11a72fddb 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -40,13 +40,22 @@ class NonLinear : public util::Object { using Size = eckit::linalg::Size; /** - * @brief NonLinear ctor + * @brief ctor */ NonLinear() = default; - /// @brief NonLinear dtor + /** + * @brief dtor + */ virtual ~NonLinear() = default; + /** + * @bried if NonLinear applies to field + * @param [in] f field + * @return if NonLinear applies to field + */ + virtual bool applicable( const Field& f ) const = 0; + /** * @brief Apply non-linear corrections to interpolation matrix * @param [inout] W interpolation matrix From 2ac6fb03d4288e85f05f0107eca7bf5d0d40ea21 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 24 Sep 2020 11:29:03 +0100 Subject: [PATCH 119/161] ATLAS-306 Move MissingValue tests from test_interpolation_non_linear.cc to test_field_missingvalue.cc --- src/tests/field/CMakeLists.txt | 6 + src/tests/field/test_field_missingvalue.cc | 290 ++++++++++++++++++ .../test_interpolation_non_linear.cc | 250 --------------- 3 files changed, 296 insertions(+), 250 deletions(-) create mode 100644 src/tests/field/test_field_missingvalue.cc diff --git a/src/tests/field/CMakeLists.txt b/src/tests/field/CMakeLists.txt index 8ede186cd..93e025b24 100644 --- a/src/tests/field/CMakeLists.txt +++ b/src/tests/field/CMakeLists.txt @@ -6,6 +6,12 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +ecbuild_add_test( TARGET atlas_test_field_missingvalue + SOURCES test_field_missingvalue.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + if( HAVE_FCTEST ) add_fctest( TARGET atlas_fctest_field diff --git a/src/tests/field/test_field_missingvalue.cc b/src/tests/field/test_field_missingvalue.cc new file mode 100644 index 000000000..33fb1d647 --- /dev/null +++ b/src/tests/field/test_field_missingvalue.cc @@ -0,0 +1,290 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include +#include + +#include "atlas/array.h" +#include "atlas/field/Field.h" +#include "atlas/field/MissingValue.h" +#include "atlas/util/Metadata.h" + +#include "tests/AtlasTestEnvironment.h" + + +namespace atlas { +namespace test { + + +const double missingValue = 42.; +const double missingValueEps = 1e-9; +const double nan = std::numeric_limits::quiet_NaN(); + +using field::MissingValue; +using util::Config; + + +CASE( "MissingValue (basic)" ) { + SECTION( "not defined" ) { + auto mv = MissingValue(); + EXPECT( !bool( mv ) ); + + mv = MissingValue( "not defined", Config() ); + EXPECT( !bool( mv ) ); + } + + + SECTION( "nan" ) { + Config config; + + auto mv = MissingValue( "nan", config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( nan ) ); + EXPECT( mv( missingValue ) == false ); + + config.set( "type", "nan" ); + mv = MissingValue( config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( nan ) ); + EXPECT( mv( missingValue ) == false ); + } + + + SECTION( "equals" ) { + Config config; + config.set( "missing_value", missingValue ); + + auto mv = MissingValue( "equals", config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( missingValue - 1 ) == false ); + EXPECT( mv( missingValue - missingValueEps / 2 ) == false ); + EXPECT( mv( missingValue ) ); + EXPECT( mv( missingValue + missingValueEps / 2 ) == false ); + EXPECT( mv( missingValue + 1 ) == false ); + + config.set( "type", "equals" ); + mv = MissingValue( config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( missingValue - 1 ) == false ); + EXPECT( mv( missingValue - missingValueEps / 2 ) == false ); + EXPECT( mv( missingValue ) ); + EXPECT( mv( missingValue + missingValueEps / 2 ) == false ); + EXPECT( mv( missingValue + 1 ) == false ); + } + + + SECTION( "approximately-equals" ) { + Config config; + config.set( "missing_value", missingValue ); + config.set( "missing_value_epsilon", missingValueEps ); + + auto mv = MissingValue( "approximately-equals", config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( missingValue - missingValueEps * 2 ) == false ); + EXPECT( mv( missingValue - missingValueEps / 2 ) ); + EXPECT( mv( missingValue ) ); + EXPECT( mv( missingValue + missingValueEps / 2 ) ); + EXPECT( mv( missingValue + missingValueEps * 2 ) == false ); + + config.set( "type", "approximately-equals" ); + mv = MissingValue( config ); + EXPECT( bool( mv ) ); + + EXPECT( mv( missingValue - missingValueEps * 2 ) == false ); + EXPECT( mv( missingValue - missingValueEps / 2 ) ); + EXPECT( mv( missingValue ) ); + EXPECT( mv( missingValue + missingValueEps / 2 ) ); + EXPECT( mv( missingValue + missingValueEps * 2 ) == false ); + } +} + + +CASE( "MissingValue (DataType specialisations)" ) { + SECTION( "real64" ) { + auto n = static_cast( missingValue ); + auto eps = static_cast( missingValueEps ); + auto nan = std::numeric_limits::quiet_NaN(); + + Config config; + config.set( "missing_value", n ); + config.set( "missing_value_epsilon", eps ); + + for ( std::string type : {"nan", "equals", "approximately-equals"} ) { + auto mv = MissingValue( type + "-real64", config ); + EXPECT( bool( mv ) ); + EXPECT( mv( type == "nan" ? nan : n ) ); + EXPECT( mv( n ) != mv( nan ) ); + EXPECT( mv( n + 1 ) == false ); + } + } + + + SECTION( "real32" ) { + auto n = static_cast( missingValue ); + auto eps = static_cast( missingValueEps ); + auto nan = std::numeric_limits::quiet_NaN(); + + Config config; + config.set( "missing_value", n ); + config.set( "missing_value_epsilon", eps ); + + for ( std::string type : {"nan", "equals", "approximately-equals"} ) { + auto mv = MissingValue( type + "-real32", config ); + EXPECT( bool( mv ) ); + EXPECT( mv( type == "nan" ? nan : n ) ); + EXPECT( mv( n ) != mv( nan ) ); + EXPECT( mv( n + 1 ) == false ); + } + } + + + SECTION( "int32" ) { + auto n = static_cast( missingValue ); + auto mv = MissingValue( "equals-int32", Config( "missing_value", n ) ); + EXPECT( bool( mv ) ); + EXPECT( mv( n ) ); + EXPECT( mv( n + 1 ) == false ); + } + + + SECTION( "int64" ) { + auto n = static_cast( missingValue ); + auto mv = MissingValue( "equals-int64", Config( "missing_value", n ) ); + EXPECT( bool( mv ) ); + EXPECT( mv( n ) ); + EXPECT( mv( n + 1 ) == false ); + } + + + SECTION( "uint64" ) { + auto n = static_cast( missingValue ); + auto mv = MissingValue( "equals-uint64", Config( "missing_value", n ) ); + EXPECT( bool( mv ) ); + EXPECT( mv( n ) ); + EXPECT( mv( n + 1 ) == false ); + } +} + + +CASE( "MissingValue from Field (basic)" ) { + std::vector values{1., nan, missingValue, missingValue, missingValue + missingValueEps / 2., 6., 7.}; + Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); + + field.metadata().set( "missing_value_type", "not defined" ); + field.metadata().set( "missing_value", missingValue ); + field.metadata().set( "missing_value_epsilon", missingValueEps ); + + EXPECT( !bool( MissingValue( field ) ) ); + + + SECTION( "nan" ) { + // missing value type from user + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "nan", field ) ) == 1 ); + + // missing value type from field + field.metadata().set( "missing_value_type", "nan" ); + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 1 ); + } + + + SECTION( "equals" ) { + // missing value type from user (value set from field) + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "equals", field ) ) == 2 ); + + // missing value type from field + field.metadata().set( "missing_value_type", "equals" ); + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 2 ); + } + + + SECTION( "approximately-equals" ) { + // missing value type from user (value set from field) + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "approximately-equals", field ) ) == 3 ); + + // missing value type from field + field.metadata().set( "missing_value_type", "approximately-equals" ); + EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 3 ); + } +} + + +CASE( "MissingValue from Field (DataType specialisations)" ) { + SECTION( "real64" ) { + std::vector values( 3, 1. ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::real64().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "nan" ); + EXPECT( MissingValue( field ) ); + } + + + SECTION( "real32" ) { + std::vector values( 3, 1. ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::real32().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "nan" ); + EXPECT( MissingValue( field ) ); + } + + + SECTION( "int32" ) { + std::vector values( 3, 1 ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::int32().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "equals" ); + field.metadata().set( "missing_value", static_cast( missingValue ) ); + EXPECT( MissingValue( field ) ); + } + + + SECTION( "int64" ) { + std::vector values( 3, 1 ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::int64().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "equals" ); + field.metadata().set( "missing_value", static_cast( missingValue ) ); + EXPECT( MissingValue( field ) ); + } + + + SECTION( "uint64" ) { + std::vector values( 3, 1 ); + Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); + EXPECT( field.datatype().str() == array::DataType::uint64().str() ); + EXPECT( !MissingValue( field ) ); + + field.metadata().set( "missing_value_type", "equals" ); + field.metadata().set( "missing_value", static_cast( missingValue ) ); + EXPECT( MissingValue( field ) ); + } +} + +} // namespace test +} // namespace atlas + + +int main( int argc, char** argv ) { + return atlas::test::run( argc, argv ); +} diff --git a/src/tests/interpolation/test_interpolation_non_linear.cc b/src/tests/interpolation/test_interpolation_non_linear.cc index 0847aedf6..7a55a747f 100644 --- a/src/tests/interpolation/test_interpolation_non_linear.cc +++ b/src/tests/interpolation/test_interpolation_non_linear.cc @@ -37,256 +37,6 @@ const double nan = std::numeric_limits::quiet_NaN(); using field::MissingValue; using util::Config; - -CASE( "MissingValue (basic)" ) { - SECTION( "not defined" ) { - auto mv = MissingValue(); - EXPECT( !bool( mv ) ); - - mv = MissingValue( "not defined", Config() ); - EXPECT( !bool( mv ) ); - } - - - SECTION( "nan" ) { - Config config; - - auto mv = MissingValue( "nan", config ); - EXPECT( bool( mv ) ); - - EXPECT( mv( nan ) ); - EXPECT( mv( missingValue ) == false ); - - config.set( "type", "nan" ); - mv = MissingValue( config ); - EXPECT( bool( mv ) ); - - EXPECT( mv( nan ) ); - EXPECT( mv( missingValue ) == false ); - } - - - SECTION( "equals" ) { - Config config; - config.set( "missing_value", missingValue ); - - auto mv = MissingValue( "equals", config ); - EXPECT( bool( mv ) ); - - EXPECT( mv( missingValue - 1 ) == false ); - EXPECT( mv( missingValue - missingValueEps / 2 ) == false ); - EXPECT( mv( missingValue ) ); - EXPECT( mv( missingValue + missingValueEps / 2 ) == false ); - EXPECT( mv( missingValue + 1 ) == false ); - - config.set( "type", "equals" ); - mv = MissingValue( config ); - EXPECT( bool( mv ) ); - - EXPECT( mv( missingValue - 1 ) == false ); - EXPECT( mv( missingValue - missingValueEps / 2 ) == false ); - EXPECT( mv( missingValue ) ); - EXPECT( mv( missingValue + missingValueEps / 2 ) == false ); - EXPECT( mv( missingValue + 1 ) == false ); - } - - - SECTION( "approximately-equals" ) { - Config config; - config.set( "missing_value", missingValue ); - config.set( "missing_value_epsilon", missingValueEps ); - - auto mv = MissingValue( "approximately-equals", config ); - EXPECT( bool( mv ) ); - - EXPECT( mv( missingValue - missingValueEps * 2 ) == false ); - EXPECT( mv( missingValue - missingValueEps / 2 ) ); - EXPECT( mv( missingValue ) ); - EXPECT( mv( missingValue + missingValueEps / 2 ) ); - EXPECT( mv( missingValue + missingValueEps * 2 ) == false ); - - config.set( "type", "approximately-equals" ); - mv = MissingValue( config ); - EXPECT( bool( mv ) ); - - EXPECT( mv( missingValue - missingValueEps * 2 ) == false ); - EXPECT( mv( missingValue - missingValueEps / 2 ) ); - EXPECT( mv( missingValue ) ); - EXPECT( mv( missingValue + missingValueEps / 2 ) ); - EXPECT( mv( missingValue + missingValueEps * 2 ) == false ); - } -} - - -CASE( "MissingValue (DataType specialisations)" ) { - SECTION( "real64" ) { - auto n = static_cast( missingValue ); - auto eps = static_cast( missingValueEps ); - auto nan = std::numeric_limits::quiet_NaN(); - - Config config; - config.set( "missing_value", n ); - config.set( "missing_value_epsilon", eps ); - - for ( std::string type : {"nan", "equals", "approximately-equals"} ) { - auto mv = MissingValue( type + "-real64", config ); - EXPECT( bool( mv ) ); - EXPECT( mv( type == "nan" ? nan : n ) ); - EXPECT( mv( n ) != mv( nan ) ); - EXPECT( mv( n + 1 ) == false ); - } - } - - - SECTION( "real32" ) { - auto n = static_cast( missingValue ); - auto eps = static_cast( missingValueEps ); - auto nan = std::numeric_limits::quiet_NaN(); - - Config config; - config.set( "missing_value", n ); - config.set( "missing_value_epsilon", eps ); - - for ( std::string type : {"nan", "equals", "approximately-equals"} ) { - auto mv = MissingValue( type + "-real32", config ); - EXPECT( bool( mv ) ); - EXPECT( mv( type == "nan" ? nan : n ) ); - EXPECT( mv( n ) != mv( nan ) ); - EXPECT( mv( n + 1 ) == false ); - } - } - - - SECTION( "int32" ) { - auto n = static_cast( missingValue ); - auto mv = MissingValue( "equals-int32", Config( "missing_value", n ) ); - EXPECT( bool( mv ) ); - EXPECT( mv( n ) ); - EXPECT( mv( n + 1 ) == false ); - } - - - SECTION( "int64" ) { - auto n = static_cast( missingValue ); - auto mv = MissingValue( "equals-int64", Config( "missing_value", n ) ); - EXPECT( bool( mv ) ); - EXPECT( mv( n ) ); - EXPECT( mv( n + 1 ) == false ); - } - - - SECTION( "uint64" ) { - auto n = static_cast( missingValue ); - auto mv = MissingValue( "equals-uint64", Config( "missing_value", n ) ); - EXPECT( bool( mv ) ); - EXPECT( mv( n ) ); - EXPECT( mv( n + 1 ) == false ); - } -} - - -CASE( "MissingValue from Field (basic)" ) { - std::vector values{1., nan, missingValue, missingValue, missingValue + missingValueEps / 2., 6., 7.}; - Field field( "field", values.data(), array::make_shape( values.size(), 1 ) ); - - field.metadata().set( "missing_value_type", "not defined" ); - field.metadata().set( "missing_value", missingValue ); - field.metadata().set( "missing_value_epsilon", missingValueEps ); - - EXPECT( !bool( MissingValue( field ) ) ); - - - SECTION( "nan" ) { - // missing value type from user - EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "nan", field ) ) == 1 ); - - // missing value type from field - field.metadata().set( "missing_value_type", "nan" ); - EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 1 ); - } - - - SECTION( "equals" ) { - // missing value type from user (value set from field) - EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "equals", field ) ) == 2 ); - - // missing value type from field - field.metadata().set( "missing_value_type", "equals" ); - EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 2 ); - } - - - SECTION( "approximately-equals" ) { - // missing value type from user (value set from field) - EXPECT( std::count_if( values.begin(), values.end(), MissingValue( "approximately-equals", field ) ) == 3 ); - - // missing value type from field - field.metadata().set( "missing_value_type", "approximately-equals" ); - EXPECT( std::count_if( values.begin(), values.end(), MissingValue( field ) ) == 3 ); - } -} - - -CASE( "MissingValue from Field (DataType specialisations)" ) { - SECTION( "real64" ) { - std::vector values( 3, 1. ); - Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); - EXPECT( field.datatype().str() == array::DataType::real64().str() ); - EXPECT( !MissingValue( field ) ); - - field.metadata().set( "missing_value_type", "nan" ); - EXPECT( MissingValue( field ) ); - } - - - SECTION( "real32" ) { - std::vector values( 3, 1. ); - Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); - EXPECT( field.datatype().str() == array::DataType::real32().str() ); - EXPECT( !MissingValue( field ) ); - - field.metadata().set( "missing_value_type", "nan" ); - EXPECT( MissingValue( field ) ); - } - - - SECTION( "int32" ) { - std::vector values( 3, 1 ); - Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); - EXPECT( field.datatype().str() == array::DataType::int32().str() ); - EXPECT( !MissingValue( field ) ); - - field.metadata().set( "missing_value_type", "equals" ); - field.metadata().set( "missing_value", static_cast( missingValue ) ); - EXPECT( MissingValue( field ) ); - } - - - SECTION( "int64" ) { - std::vector values( 3, 1 ); - Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); - EXPECT( field.datatype().str() == array::DataType::int64().str() ); - EXPECT( !MissingValue( field ) ); - - field.metadata().set( "missing_value_type", "equals" ); - field.metadata().set( "missing_value", static_cast( missingValue ) ); - EXPECT( MissingValue( field ) ); - } - - - SECTION( "uint64" ) { - std::vector values( 3, 1 ); - Field field( "field", array::make_datatype(), array::make_shape( values.size(), 1 ) ); - EXPECT( field.datatype().str() == array::DataType::uint64().str() ); - EXPECT( !MissingValue( field ) ); - - field.metadata().set( "missing_value_type", "equals" ); - field.metadata().set( "missing_value", static_cast( missingValue ) ); - EXPECT( MissingValue( field ) ); - } -} - - CASE( "Interpolation with MissingValue" ) { /* Set input field full of 1's, with 9 nodes From e6a74fc2490a068da1c2354e826b09a0773f0baa Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Thu, 20 Aug 2020 07:53:59 +0000 Subject: [PATCH 120/161] Add test to demonstrate a problem in interpolation2D --- .../test_interpolation_structured2D.cc | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/tests/interpolation/test_interpolation_structured2D.cc b/src/tests/interpolation/test_interpolation_structured2D.cc index cb01468c5..29899be0f 100644 --- a/src/tests/interpolation/test_interpolation_structured2D.cc +++ b/src/tests/interpolation/test_interpolation_structured2D.cc @@ -389,6 +389,29 @@ CASE( "test_interpolation_structured for vectors" ) { } } +CASE ("test_multiple_fs") { + + Grid grid1 ("L90x45"); + Grid grid2 ("O8"); + + Mesh mesh1 = StructuredMeshGenerator ().generate (grid1); + Mesh mesh2 = StructuredMeshGenerator ().generate (grid2); + + functionspace::NodeColumns fs11 (mesh1, option::halo (1)); + functionspace::NodeColumns fs12 (mesh1, option::halo (2)); + + auto fs1 = fs11; + functionspace::NodeColumns fs2 (mesh2, option::halo (1)); + + Interpolation interpolation12 (util::Config ("type", "k-nearest-neighbours") | util::Config ("k-nearest-neighbours", 5), fs1, fs2); + + auto f1 = fs1.createField (util::Config ("name", "source")); + auto f2 = fs2.createField (util::Config ("name", "target")); + + interpolation12.execute (f1, f2); + +} + } // namespace test } // namespace atlas From d9aa52bc6debf90197f3ed650ffe4ab5d604208c Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 23 Sep 2020 13:43:22 +0100 Subject: [PATCH 121/161] ATLAS-311 Fix bug where (k-)NearestNeighbours interpolation was sometimes building too large KDTree --- .../method/knn/KNearestNeighbours.cc | 6 +- .../method/knn/KNearestNeighboursBase.cc | 18 ++++-- .../method/knn/KNearestNeighboursBase.h | 4 +- .../method/knn/NearestNeighbour.cc | 6 +- src/tests/interpolation/CMakeLists.txt | 5 ++ ...test_interpolation_k_nearest_neighbours.cc | 59 +++++++++++++++++++ .../test_interpolation_structured2D.cc | 24 -------- 7 files changed, 87 insertions(+), 35 deletions(-) create mode 100644 src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc diff --git a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc index 8924fdd7c..2f46c3e50 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc @@ -73,7 +73,7 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp Mesh meshTarget = tgt.mesh(); // build point-search tree - buildPointSearchTree( meshSource ); + buildPointSearchTree( meshSource, src.halo() ); ATLAS_ASSERT( pTree_ != nullptr ); // generate 3D point coordinates @@ -81,6 +81,7 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp array::ArrayView coords = array::make_view( meshTarget.nodes().field( "xyz" ) ); size_t inp_npts = meshSource.nodes().size(); + meshSource.metadata().get( "nb_nodes_including_halo[" + std::to_string( src.halo().size() ) + "]", inp_npts ); size_t out_npts = meshTarget.nodes().size(); // fill the sparse matrix @@ -123,7 +124,8 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp // insert weights into the matrix for ( size_t j = 0; j < npts; ++j ) { size_t jp = nn[j].payload(); - ATLAS_ASSERT( jp < inp_npts ); + ATLAS_ASSERT( jp < inp_npts, + "point found which is not covered within the halo of the source function space" ); weights_triplets.emplace_back( ip, jp, weights[j] / sum ); } } diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc index f85a65a2c..999992506 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.cc @@ -22,32 +22,38 @@ namespace atlas { namespace interpolation { namespace method { -void KNearestNeighboursBase::buildPointSearchTree( Mesh& meshSource ) { +void KNearestNeighboursBase::buildPointSearchTree( Mesh& meshSource, const mesh::Halo& _halo ) { using namespace atlas; eckit::TraceTimer tim( "atlas::interpolation::method::KNearestNeighboursBase::setup()" ); // generate 3D point coordinates mesh::actions::BuildXYZField( "xyz" )( meshSource ); - array::ArrayView coords = array::make_view( meshSource.nodes().field( "xyz" ) ); + auto coords = array::make_view( meshSource.nodes().field( "xyz" ) ); + auto halo = array::make_view( meshSource.nodes().halo() ); // build point-search tree pTree_.reset( new PointIndex3 ); static bool fastBuildKDTrees = eckit::Resource( "$ATLAS_FAST_BUILD_KDTREES", true ); + int h = _halo.size(); if ( fastBuildKDTrees ) { std::vector pidx; pidx.reserve( meshSource.nodes().size() ); for ( idx_t ip = 0; ip < meshSource.nodes().size(); ++ip ) { - PointIndex3::Point p{coords( ip, 0 ), coords( ip, 1 ), coords( ip, 2 )}; - pidx.emplace_back( p, ip ); + if ( halo( ip ) <= h ) { + PointIndex3::Point p{coords( ip, 0 ), coords( ip, 1 ), coords( ip, 2 )}; + pidx.emplace_back( p, ip ); + } } pTree_->build( pidx.begin(), pidx.end() ); } else { for ( idx_t ip = 0; ip < meshSource.nodes().size(); ++ip ) { - PointIndex3::Point p{coords( ip, 0 ), coords( ip, 1 ), coords( ip, 2 )}; - pTree_->insert( PointIndex3::Value( p, ip ) ); + if ( halo( ip ) <= h ) { + PointIndex3::Point p{coords( ip, 0 ), coords( ip, 1 ), coords( ip, 2 )}; + pTree_->insert( PointIndex3::Value( p, ip ) ); + } } } } diff --git a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h index 0e92070ff..3a6645056 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h +++ b/src/atlas/interpolation/method/knn/KNearestNeighboursBase.h @@ -14,6 +14,7 @@ #include "atlas/interpolation/method/Method.h" #include "atlas/interpolation/method/PointIndex3.h" +#include "atlas/mesh/Halo.h" namespace atlas { namespace interpolation { @@ -25,7 +26,8 @@ class KNearestNeighboursBase : public Method { virtual ~KNearestNeighboursBase() override {} protected: - void buildPointSearchTree( Mesh& meshSource ); + void buildPointSearchTree( Mesh& meshSource ) { buildPointSearchTree( meshSource, mesh::Halo( meshSource ) ); } + void buildPointSearchTree( Mesh& meshSource, const mesh::Halo& ); std::unique_ptr pTree_; }; diff --git a/src/atlas/interpolation/method/knn/NearestNeighbour.cc b/src/atlas/interpolation/method/knn/NearestNeighbour.cc index cb8f6159d..733047b5a 100644 --- a/src/atlas/interpolation/method/knn/NearestNeighbour.cc +++ b/src/atlas/interpolation/method/knn/NearestNeighbour.cc @@ -67,7 +67,7 @@ void NearestNeighbour::do_setup( const FunctionSpace& source, const FunctionSpac Mesh meshTarget = tgt.mesh(); // build point-search tree - buildPointSearchTree( meshSource ); + buildPointSearchTree( meshSource, src.halo() ); ATLAS_ASSERT( pTree_ != nullptr ); // generate 3D point coordinates @@ -75,6 +75,7 @@ void NearestNeighbour::do_setup( const FunctionSpace& source, const FunctionSpac array::ArrayView coords = array::make_view( meshTarget.nodes().field( "xyz" ) ); size_t inp_npts = meshSource.nodes().size(); + meshSource.metadata().get( "nb_nodes_including_halo[" + std::to_string( src.halo().size() ) + "]", inp_npts ); size_t out_npts = meshTarget.nodes().size(); // fill the sparse matrix @@ -97,7 +98,8 @@ void NearestNeighbour::do_setup( const FunctionSpace& source, const FunctionSpac size_t jp = nn.payload(); // insert the weights into the interpolant matrix - ATLAS_ASSERT( jp < inp_npts ); + ATLAS_ASSERT( jp < inp_npts, + "point found which is not covered within the halo of the source function space" ); weights_triplets.emplace_back( ip, jp, 1 ); } } diff --git a/src/tests/interpolation/CMakeLists.txt b/src/tests/interpolation/CMakeLists.txt index f779926f6..a7c19dd86 100644 --- a/src/tests/interpolation/CMakeLists.txt +++ b/src/tests/interpolation/CMakeLists.txt @@ -60,3 +60,8 @@ ecbuild_add_test( TARGET atlas_test_interpolation_structured2D_to_unstructured ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_interpolation_k_nearest_neighbours + SOURCES test_interpolation_k_nearest_neighbours.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) diff --git a/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc b/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc new file mode 100644 index 000000000..cd5340633 --- /dev/null +++ b/src/tests/interpolation/test_interpolation_k_nearest_neighbours.cc @@ -0,0 +1,59 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "atlas/array.h" +#include "atlas/field/Field.h" +#include "atlas/functionspace/NodeColumns.h" +#include "atlas/grid/Grid.h" +#include "atlas/interpolation.h" +#include "atlas/mesh/Mesh.h" +#include "atlas/meshgenerator.h" + +#include "tests/AtlasTestEnvironment.h" + +using atlas::functionspace::NodeColumns; +using atlas::util::Config; + +namespace atlas { +namespace test { + +//----------------------------------------------------------------------------- + +CASE( "test_multiple_fs" ) { + Grid grid1( "L90x45" ); + Grid grid2( "O8" ); + + Mesh mesh1 = StructuredMeshGenerator().generate( grid1 ); + Mesh mesh2 = StructuredMeshGenerator().generate( grid2 ); + + functionspace::NodeColumns fs11( mesh1, option::halo( 1 ) ); + functionspace::NodeColumns fs12( mesh1, option::halo( 2 ) ); + + auto fs1 = fs11; + functionspace::NodeColumns fs2( mesh2, option::halo( 1 ) ); + + Interpolation interpolation12( Config( "type", "k-nearest-neighbours" ) | Config( "k-nearest-neighbours", 5 ), fs1, + fs2 ); + + auto f1 = fs1.createField( Config( "name", "source" ) ); + auto f2 = fs2.createField( Config( "name", "target" ) ); + + auto v1 = array::make_view( f1 ); + v1.assign( 1. ); + + interpolation12.execute( f1, f2 ); +} + +} // namespace test +} // namespace atlas + +int main( int argc, char** argv ) { + return atlas::test::run( argc, argv ); +} diff --git a/src/tests/interpolation/test_interpolation_structured2D.cc b/src/tests/interpolation/test_interpolation_structured2D.cc index 29899be0f..c8b3b2ffa 100644 --- a/src/tests/interpolation/test_interpolation_structured2D.cc +++ b/src/tests/interpolation/test_interpolation_structured2D.cc @@ -389,30 +389,6 @@ CASE( "test_interpolation_structured for vectors" ) { } } -CASE ("test_multiple_fs") { - - Grid grid1 ("L90x45"); - Grid grid2 ("O8"); - - Mesh mesh1 = StructuredMeshGenerator ().generate (grid1); - Mesh mesh2 = StructuredMeshGenerator ().generate (grid2); - - functionspace::NodeColumns fs11 (mesh1, option::halo (1)); - functionspace::NodeColumns fs12 (mesh1, option::halo (2)); - - auto fs1 = fs11; - functionspace::NodeColumns fs2 (mesh2, option::halo (1)); - - Interpolation interpolation12 (util::Config ("type", "k-nearest-neighbours") | util::Config ("k-nearest-neighbours", 5), fs1, fs2); - - auto f1 = fs1.createField (util::Config ("name", "source")); - auto f2 = fs2.createField (util::Config ("name", "target")); - - interpolation12.execute (f1, f2); - -} - - } // namespace test } // namespace atlas From d5fad2ade74976c001d617b09286220822e6b31a Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 24 Sep 2020 12:00:25 +0100 Subject: [PATCH 122/161] ATLAS-209 grid-box methods option 'gaussian_weighted_latitudes' (default true, false is latitude mid-point) applicable to source/target grids --- src/atlas/interpolation/method/knn/GridBoxMethod.cc | 1 + src/atlas/interpolation/method/knn/GridBoxMethod.h | 1 + 2 files changed, 2 insertions(+) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 49986f02f..56eeefcce 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -33,6 +33,7 @@ namespace method { GridBoxMethod::GridBoxMethod( const Method::Config& config ) : KNearestNeighboursBase( config ) { config.get( "matrix_free", matrixFree_ = false ); config.get( "fail_early", failEarly_ = true ); + config.get( "gaussian_weighted_latitudes", gaussianWeightedLatitudes_ = true ); } diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index 935e58572..889041336 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -63,6 +63,7 @@ class GridBoxMethod : public KNearestNeighboursBase { bool matrixFree_; bool failEarly_; + bool gaussianWeightedLatitudes_; }; From d31465ea8d0b7df28eb9e22a44c37fad80dacdd9 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 24 Sep 2020 13:04:36 +0100 Subject: [PATCH 123/161] ATLAS-209 grid-box methods option 'gaussian_weighted_latitudes' implementation --- src/atlas/interpolation/method/knn/GridBox.cc | 11 ++++------- src/atlas/interpolation/method/knn/GridBox.h | 2 +- src/atlas/interpolation/method/knn/GridBoxMethod.cc | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBox.cc b/src/atlas/interpolation/method/knn/GridBox.cc index f04faf25c..bf10f0442 100644 --- a/src/atlas/interpolation/method/knn/GridBox.cc +++ b/src/atlas/interpolation/method/knn/GridBox.cc @@ -48,10 +48,7 @@ namespace method { GridBox::GridBox( double north, double west, double south, double east ) : - north_( north ), - west_( west ), - south_( south ), - east_( east ) { + north_( north ), west_( west ), south_( south ), east_( east ) { ATLAS_ASSERT( SOUTH_POLE <= south_ && south_ <= north_ && north_ <= NORTH_POLE ); ATLAS_ASSERT( west_ <= east_ && east_ <= west_ + GLOBE ); } @@ -106,7 +103,7 @@ void GridBox::print( std::ostream& out ) const { } -GridBoxes::GridBoxes( const Grid& grid ) { +GridBoxes::GridBoxes( const Grid& grid, bool gaussianWeightedLatitudes ) { StructuredGrid structured( grid ); if ( !structured || grid.projection() ) { throw_NotImplemented( "GridBoxes only support structured, unprojected/unrotated grids", Here() ); @@ -133,8 +130,8 @@ GridBoxes::GridBoxes( const Grid& grid ) { std::vector lat; lat.reserve( y.size() + 1 ); - GaussianGrid gaussian( grid ); - if ( gaussian ) { + GaussianGrid gaussian; + if ( gaussianWeightedLatitudes && ( gaussian = grid ) ) { auto N = gaussian.N(); std::vector latitudes( N * 2 ); std::vector weights( N * 2 ); diff --git a/src/atlas/interpolation/method/knn/GridBox.h b/src/atlas/interpolation/method/knn/GridBox.h index 291677c80..133d27009 100644 --- a/src/atlas/interpolation/method/knn/GridBox.h +++ b/src/atlas/interpolation/method/knn/GridBox.h @@ -109,7 +109,7 @@ class GridBox { struct GridBoxes : std::vector { - GridBoxes( const Grid& ); + GridBoxes( const Grid&, bool gaussianWeightedLatitudes = true ); GridBoxes(); using vector::vector; double getLongestGridBoxDiagonal() const; diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 56eeefcce..b4989f6cb 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -107,8 +107,8 @@ void GridBoxMethod::do_setup( const Grid& source, const Grid& target ) { buildPointSearchTree( src ); - sourceBoxes_ = GridBoxes( source ); - targetBoxes_ = GridBoxes( target ); + sourceBoxes_ = GridBoxes( source, gaussianWeightedLatitudes_ ); + targetBoxes_ = GridBoxes( target, gaussianWeightedLatitudes_ ); searchRadius_ = sourceBoxes_.getLongestGridBoxDiagonal() + targetBoxes_.getLongestGridBoxDiagonal(); failures_.clear(); From b35cdbe2a03c8dfc7979c4d9c47383fd137315f4 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Thu, 24 Sep 2020 13:05:16 +0100 Subject: [PATCH 124/161] clang-format --- src/atlas/interpolation/method/Method.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/atlas/interpolation/method/Method.h b/src/atlas/interpolation/method/Method.h index 487d805eb..26f6d12e2 100644 --- a/src/atlas/interpolation/method/Method.h +++ b/src/atlas/interpolation/method/Method.h @@ -26,10 +26,10 @@ class Grid; } // namespace atlas namespace atlas { - namespace test { - class Access; - } +namespace test { +class Access; } +} // namespace atlas namespace atlas { namespace interpolation { From be7fd4781bd084f057b558f8ca08f13ce3682d77 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 25 Sep 2020 11:43:11 +0100 Subject: [PATCH 125/161] ATLAS-308 Fortran API: Add Partitioner or Config as argument to Distribution Co-authored-by: Philippe Marguinaud --- .../detail/distribution/DistributionArray.h | 2 + .../detail/distribution/DistributionImpl.cc | 33 ++++++++++++- .../detail/distribution/DistributionImpl.h | 12 +++++ .../grid/atlas_GridDistribution_module.F90 | 26 ++++++++++ src/tests/grid/fctest_griddistribution.F90 | 48 ++++++++++--------- 5 files changed, 98 insertions(+), 23 deletions(-) diff --git a/src/atlas/grid/detail/distribution/DistributionArray.h b/src/atlas/grid/detail/distribution/DistributionArray.h index 57f663757..29bd24b42 100644 --- a/src/atlas/grid/detail/distribution/DistributionArray.h +++ b/src/atlas/grid/detail/distribution/DistributionArray.h @@ -20,6 +20,8 @@ namespace atlas { namespace grid { +class Partitioner; + namespace detail { namespace distribution { diff --git a/src/atlas/grid/detail/distribution/DistributionImpl.cc b/src/atlas/grid/detail/distribution/DistributionImpl.cc index 4f93ab8a5..1aab3d2b3 100644 --- a/src/atlas/grid/detail/distribution/DistributionImpl.cc +++ b/src/atlas/grid/detail/distribution/DistributionImpl.cc @@ -12,8 +12,11 @@ #include +#include "atlas/grid/Distribution.h" +#include "atlas/grid/Grid.h" +#include "atlas/grid/Partitioner.h" #include "atlas/grid/detail/distribution/DistributionArray.h" - +#include "atlas/runtime/Exception.h" namespace atlas { namespace grid { @@ -35,6 +38,34 @@ idx_t atlas__atlas__GridDistribution__nb_partitions( DistributionImpl* This ) { return This->nb_partitions(); } +DistributionImpl* atlas__GridDistribution__new__Grid_Config( const detail::grid::Grid* grid, + const eckit::Parametrisation* config ) { + ATLAS_ASSERT( grid != nullptr, "grid is an invalid pointer" ); + ATLAS_ASSERT( config != nullptr, "config is an invalid pointer" ); + DistributionImpl* distribution; + { + Distribution d{Grid{grid}, *config}; + distribution = d.get(); + distribution->attach(); + } + distribution->detach(); + return distribution; +} + +DistributionImpl* atlas__GridDistribution__new__Grid_Partitioner( + const detail::grid::Grid* grid, const detail::partitioner::Partitioner* partitioner ) { + ATLAS_ASSERT( grid != nullptr, "grid is an invalid pointer" ); + ATLAS_ASSERT( partitioner != nullptr, "partitioner is an invalid pointer" ); + DistributionImpl* distribution; + { + Distribution d{Grid{grid}, Partitioner{partitioner}}; + distribution = d.get(); + distribution->attach(); + } + distribution->detach(); + return distribution; +} + } // namespace grid } // namespace atlas diff --git a/src/atlas/grid/detail/distribution/DistributionImpl.h b/src/atlas/grid/detail/distribution/DistributionImpl.h index 9502d7d64..40d08b71f 100644 --- a/src/atlas/grid/detail/distribution/DistributionImpl.h +++ b/src/atlas/grid/detail/distribution/DistributionImpl.h @@ -26,8 +26,16 @@ namespace atlas { class Grid; namespace grid { +namespace detail { + +namespace partitioner { class Partitioner; } +namespace grid { +class Grid; +} +} // namespace detail +} // namespace grid } // namespace atlas namespace atlas { @@ -61,6 +69,10 @@ class DistributionImpl : public util::Object { extern "C" { DistributionImpl* atlas__GridDistribution__new( idx_t size, int part[], int part0 ); +DistributionImpl* atlas__GridDistribution__new__Grid_Config( const detail::grid::Grid* grid, + const eckit::Parametrisation* config ); +DistributionImpl* atlas__GridDistribution__new__Grid_Partitioner( const detail::grid::Grid* grid, + const detail::partitioner::Partitioner* partitioner ); void atlas__GridDistribution__delete( DistributionImpl* This ); void atlas__GridDistribution__nb_pts( DistributionImpl* This, idx_t nb_pts[] ); idx_t atlas__atlas__GridDistribution__nb_partitions( DistributionImpl* This ); diff --git a/src/atlas_f/grid/atlas_GridDistribution_module.F90 b/src/atlas_f/grid/atlas_GridDistribution_module.F90 index e1eb39ce3..a22dd147f 100644 --- a/src/atlas_f/grid/atlas_GridDistribution_module.F90 +++ b/src/atlas_f/grid/atlas_GridDistribution_module.F90 @@ -52,6 +52,8 @@ module atlas_GridDistribution_module interface atlas_GridDistribution module procedure atlas_GridDistribution__cptr module procedure atlas_GridDistribution__ctor + module procedure atlas_GridDistribution__ctor_Grid_Config + module procedure atlas_GridDistribution__ctor_Grid_Partitioner end interface private :: c_ptr @@ -86,6 +88,30 @@ function atlas_GridDistribution__ctor( part, part0 ) result(this) call this%return() end function +!fckit_owned_object +function atlas_GridDistribution__ctor_Grid_Config( grid, config ) result(this) + use atlas_distribution_c_binding + use atlas_Grid_module, only : atlas_Grid + use atlas_Config_module, only : atlas_Config + type(atlas_GridDistribution) :: this + class(atlas_Grid), intent (in) :: grid + type(atlas_Config), intent(in) :: config + call this%reset_c_ptr( atlas__GridDistribution__new__Grid_Config(grid%CPTR_PGIBUG_A, config%CPTR_PGIBUG_B) ) + call this%return() +end function + +function atlas_GridDistribution__ctor_Grid_Partitioner( grid, partitioner ) result(this) + use atlas_distribution_c_binding + use atlas_Grid_module, only : atlas_Grid + use atlas_Config_module, only : atlas_Config + type(atlas_GridDistribution) :: this + class(atlas_Grid), intent (in) :: grid + class(fckit_owned_object), intent(in) :: partitioner ! cannot use atlas_Partitioner as it would cause cyclic module dependencies + call this%reset_c_ptr( atlas__GridDistribution__new__Grid_Partitioner(grid%CPTR_PGIBUG_A, partitioner%CPTR_PGIBUG_A) ) + call this%return() +end function + + function atlas_GridDistribution__nb_pts(this) result(nb_pts) use atlas_distribution_c_binding use atlas_kinds_module, only : ATLAS_KIND_IDX diff --git a/src/tests/grid/fctest_griddistribution.F90 b/src/tests/grid/fctest_griddistribution.F90 index 9dc88bec6..3cc6f7b3f 100644 --- a/src/tests/grid/fctest_griddistribution.F90 +++ b/src/tests/grid/fctest_griddistribution.F90 @@ -29,31 +29,10 @@ call atlas_library%finalise() END_TESTSUITE_FINALIZE -! ---------------------------------------------------------------------------- - -! T E S T( test_reducedgaussian ) -! use atlas_module -! use, intrinsic :: iso_c_binding -! implicit none -! type(atlas_StructuredGrid) :: N640 -! type(atlas_ReducedGaussianGrid) :: custom -! integer(c_long), pointer :: pl(:) -! -! N640 = atlas_StructuredGrid("N640") -! FCTEST_CHECK_EQUAL(N640%size(),2140702_c_long) -! pl => N640%pl() -! -! custom = atlas_ReducedGaussianGrid( N640%N(), pl ) -! FCTEST_CHECK_EQUAL(N640%size(),custom%size() ) -! -! call N640%final() -! call custom%final() -! -! E N D _ T E S T - ! ----------------------------------------------------------------------------- TEST( test_griddist ) +#if 1 use atlas_module implicit none type(atlas_StructuredGrid) :: grid @@ -101,6 +80,31 @@ call gmsh%final() call grid%final() call meshgenerator%final() +#endif +END_TEST + +! ----------------------------------------------------------------------------- + +TEST( test_griddist_from_partitioner ) + use atlas_module + implicit none + type(atlas_StructuredGrid) :: grid + type(atlas_Partitioner) :: partitioner + type(atlas_GridDistribution) :: griddistribution + + integer(ATLAS_KIND_IDX) :: jnode + + grid = atlas_StructuredGrid("O16") + + griddistribution = atlas_GridDistribution(grid,atlas_Partitioner("serial")) + + FCTEST_CHECK_EQUAL( griddistribution%nb_partitions(), 1 ) + FCTEST_CHECK_EQUAL( griddistribution%nb_pts(), [ int(grid%size(),ATLAS_KIND_IDX) ] ) + + FCTEST_CHECK_EQUAL( grid%owners(), 1 ) + + call griddistribution%final() + call grid%final() END_TEST ! ----------------------------------------------------------------------------- From deb61f3031a1d1274aaacb2c3448e312cc3b96a1 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 25 Sep 2020 12:07:00 +0100 Subject: [PATCH 126/161] ATLAS-308 Fortran API: distribution%partition(index) Co-authored-by: Philippe Marguinaud --- .../detail/distribution/DistributionImpl.cc | 8 +++++++ .../detail/distribution/DistributionImpl.h | 2 ++ .../grid/atlas_GridDistribution_module.F90 | 23 +++++++++++++++++++ src/tests/grid/fctest_griddistribution.F90 | 12 +++++++--- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/atlas/grid/detail/distribution/DistributionImpl.cc b/src/atlas/grid/detail/distribution/DistributionImpl.cc index 1aab3d2b3..583c73647 100644 --- a/src/atlas/grid/detail/distribution/DistributionImpl.cc +++ b/src/atlas/grid/detail/distribution/DistributionImpl.cc @@ -38,6 +38,14 @@ idx_t atlas__atlas__GridDistribution__nb_partitions( DistributionImpl* This ) { return This->nb_partitions(); } +int atlas__GridDistribution__partition_int32( DistributionImpl* dist, int i ) { + return dist->partition( i ); +} + +int atlas__GridDistribution__partition_int64( DistributionImpl* dist, long i ) { + return dist->partition( i ); +} + DistributionImpl* atlas__GridDistribution__new__Grid_Config( const detail::grid::Grid* grid, const eckit::Parametrisation* config ) { ATLAS_ASSERT( grid != nullptr, "grid is an invalid pointer" ); diff --git a/src/atlas/grid/detail/distribution/DistributionImpl.h b/src/atlas/grid/detail/distribution/DistributionImpl.h index 40d08b71f..2cb68edf5 100644 --- a/src/atlas/grid/detail/distribution/DistributionImpl.h +++ b/src/atlas/grid/detail/distribution/DistributionImpl.h @@ -73,6 +73,8 @@ DistributionImpl* atlas__GridDistribution__new__Grid_Config( const detail::grid: const eckit::Parametrisation* config ); DistributionImpl* atlas__GridDistribution__new__Grid_Partitioner( const detail::grid::Grid* grid, const detail::partitioner::Partitioner* partitioner ); +int atlas__GridDistribution__partition_int32( DistributionImpl* dist, int i ); +int atlas__GridDistribution__partition_int64( DistributionImpl* dist, long i ); void atlas__GridDistribution__delete( DistributionImpl* This ); void atlas__GridDistribution__nb_pts( DistributionImpl* This, idx_t nb_pts[] ); idx_t atlas__atlas__GridDistribution__nb_partitions( DistributionImpl* This ); diff --git a/src/atlas_f/grid/atlas_GridDistribution_module.F90 b/src/atlas_f/grid/atlas_GridDistribution_module.F90 index a22dd147f..f7c7e2af1 100644 --- a/src/atlas_f/grid/atlas_GridDistribution_module.F90 +++ b/src/atlas_f/grid/atlas_GridDistribution_module.F90 @@ -42,6 +42,10 @@ module atlas_GridDistribution_module contains procedure :: nb_partitions => atlas_GridDistribution__nb_partitions procedure :: nb_pts => atlas_GridDistribution__nb_pts + procedure, private :: partition_int32 + procedure, private :: partition_int64 + generic :: partition => partition_int32, partition_int64 + #if FCKIT_FINAL_NOT_INHERITING final :: atlas_GridDistribution__final_auto #endif @@ -112,6 +116,25 @@ function atlas_GridDistribution__ctor_Grid_Partitioner( grid, partitioner ) resu end function +function partition_int32(this, i) result(partition) + use, intrinsic :: iso_c_binding, only: c_long, c_int + use atlas_distribution_c_binding + integer(c_int) :: partition + class(atlas_GridDistribution), intent(in) :: this + integer(c_int), intent(in) :: i + partition = atlas__GridDistribution__partition_int32(this%CPTR_PGIBUG_A, i-1) +end function + +function partition_int64(this, i) result(partition) + use, intrinsic :: iso_c_binding, only: c_long, c_int + use atlas_distribution_c_binding + integer(c_int) :: partition + class(atlas_GridDistribution), intent(in) :: this + integer(c_long), intent(in) :: i + partition = atlas__GridDistribution__partition_int64(this%CPTR_PGIBUG_A, i-1) +end function + + function atlas_GridDistribution__nb_pts(this) result(nb_pts) use atlas_distribution_c_binding use atlas_kinds_module, only : ATLAS_KIND_IDX diff --git a/src/tests/grid/fctest_griddistribution.F90 b/src/tests/grid/fctest_griddistribution.F90 index 3cc6f7b3f..0582d4954 100644 --- a/src/tests/grid/fctest_griddistribution.F90 +++ b/src/tests/grid/fctest_griddistribution.F90 @@ -66,14 +66,18 @@ mesh = meshgenerator%generate(grid,griddistribution) FCTEST_CHECK_EQUAL( mesh%owners(), 1 ) - call griddistribution%final() + deallocate(part) FCTEST_CHECK_EQUAL( grid%owners(), 2 ) gmsh = atlas_output_Gmsh("testf3.msh") call gmsh%write(mesh) - deallocate(part) + do jnode=1,grid%size() + FCTEST_CHECK_EQUAL( griddistribution%partition(jnode), 0 ) + enddo + + call griddistribution%final() call mesh%final() FCTEST_CHECK_EQUAL( grid%owners(), 1 ) @@ -101,7 +105,9 @@ FCTEST_CHECK_EQUAL( griddistribution%nb_partitions(), 1 ) FCTEST_CHECK_EQUAL( griddistribution%nb_pts(), [ int(grid%size(),ATLAS_KIND_IDX) ] ) - FCTEST_CHECK_EQUAL( grid%owners(), 1 ) + do jnode=1,grid%size() + FCTEST_CHECK_EQUAL( griddistribution%partition(jnode), 0 ) + enddo call griddistribution%final() call grid%final() From 8c21548616d6d0c2601d16fb40e308e744a8aeea Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Sat, 26 Sep 2020 23:33:26 +0100 Subject: [PATCH 127/161] ATLAS-269, MIR-465 Correct assertion grid-boxes for periodic shifted regular_ll --- src/atlas/interpolation/method/knn/GridBox.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBox.cc b/src/atlas/interpolation/method/knn/GridBox.cc index bf10f0442..cbfc6dcea 100644 --- a/src/atlas/interpolation/method/knn/GridBox.cc +++ b/src/atlas/interpolation/method/knn/GridBox.cc @@ -195,7 +195,8 @@ GridBoxes::GridBoxes( const Grid& grid, bool gaussianWeightedLatitudes ) { // On non-periodic grids, West- and East-most grid-boxes need clipping ATLAS_ASSERT( x.nx()[j] > 0 ); - eckit::Fraction lon1 = ( n * dx ) - ( dx / 2 ); + eckit::Fraction lon0 = ( n * dx ) - ( dx / 2 ); + eckit::Fraction lon1 = lon0; for ( idx_t i = 0; i < x.nx()[j]; ++i ) { double lon0 = lon1; lon1 += dx; @@ -208,7 +209,7 @@ GridBoxes::GridBoxes( const Grid& grid, bool gaussianWeightedLatitudes ) { } if ( periodic ) { - ATLAS_ASSERT( lon1 == xmin - ( dx / 2 ) + 360 ); + ATLAS_ASSERT( lon1 == lon0 + 360 ); } } From adfcabe2fcb542b4d77979e28195a8c6ec4a88e0 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Sun, 27 Sep 2020 00:08:10 +0100 Subject: [PATCH 128/161] Improved code style --- src/atlas/interpolation/method/knn/GridBoxMethod.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.h b/src/atlas/interpolation/method/knn/GridBoxMethod.h index 889041336..e814413c8 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.h +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.h @@ -45,8 +45,8 @@ class GridBoxMethod : public KNearestNeighboursBase { bool intersect( size_t i, const GridBox& iBox, const util::IndexKDTree::ValueList&, std::vector& ) const; - virtual void do_execute( const FieldSet& source, FieldSet& target ) const = 0; - virtual void do_execute( const Field& source, Field& target ) const = 0; + virtual void do_execute( const FieldSet& source, FieldSet& target ) const override = 0; + virtual void do_execute( const Field& source, Field& target ) const override = 0; protected: static void giveUp( const std::forward_list& ); From db9503fffc8aa9c78a658f7fc9bac4e78cca7a22 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 28 Sep 2020 01:22:13 +0100 Subject: [PATCH 129/161] Update doc --- doc/pages/building.dot | 96 -- doc/pages/building.md | 11 - doc/pages/custom-buildsystems.md | 4 - .../{cmake.md => developer_information.md} | 4 +- doc/pages/getting-started.md | 125 --- doc/pages/getting-started_installation.tex | 883 ------------------ doc/pages/mainpage.md | 4 - 7 files changed, 2 insertions(+), 1125 deletions(-) delete mode 100644 doc/pages/building.dot delete mode 100644 doc/pages/building.md delete mode 100644 doc/pages/custom-buildsystems.md rename doc/pages/{cmake.md => developer_information.md} (94%) delete mode 100644 doc/pages/getting-started.md delete mode 100644 doc/pages/getting-started_installation.tex diff --git a/doc/pages/building.dot b/doc/pages/building.dot deleted file mode 100644 index 90612d138..000000000 --- a/doc/pages/building.dot +++ /dev/null @@ -1,96 +0,0 @@ -digraph "Atlas library dependency order" { - compound=true - rankdir=BT - ranksep=0.5 - nodesep=0.5 - style=solid - node [style=solid shape=rect margin="0.03,0.03"] - edge [arrowsize=0.8] - - - - - subgraph cluster_ecmwf { - nodesep=1.0 - - style=invisible - subgraph cluster_fckit { - style=dashed - fckit [class="m-primary"]; - } - subgraph cluster_atlas { - style=dashed - nodesep=0.5 - atlas [class="m-node m-primary"]; - atlas_f [m_class="m-node m-primary"]; - } - subgraph cluster_eckit { - style=dashed - nodesep=0.5 - eckit [m_class="m-primary"] - eckit_mpi [m_class="m-primary"] - eckit_geometry [m_class="m-primary"] - } - subgraph cluster_transi { - nodesep=0.5 - style=dashed - trans [class="m-info" style=dotted] - transi [class="m-info" style=dotted] - } - - - } - - subgraph cluster_optional { - compound=true - group=a - style=invisible - MPI [class="m-node m-info" style=dotted] - OpenMP [class="m-info" style=dotted] - BLAS [class="m-info" style=dotted] - Eigen [class="m-info" style=dotted] - CGAL [class="m-info" style=dotted] - Proj [class="m-info" style=dotted] - fftw [class="m-info" style=dotted] - CUDA [class="m-info" style=dotted] - GridTools [class="m-info" style=dotted] - Boost [class="m-info" style=dotted] - - }; - - GridTools -> Boost [style=dashed] - GridTools -> CUDA [style=dotted] - CGAL -> Boost [style=dashed] - trans -> fftw [style=dashed] - trans -> OpenMP [style=dotted] - trans -> MPI [style=dashed] - trans -> BLAS [style=dashed] - #BLAS -> MPI [style="invisible" dir="none"] - OpenMP -> fftw [style="invisible" dir="none"] - transi -> trans [style=dashed] - - eckit_mpi -> MPI [style=dotted group=a] - eckit_mpi -> eckit - eckit_geometry -> eckit - - fckit -> eckit_mpi [lhead="cluster_eckit"] #[style=dotted] - atlas -> eckit_mpi [lhead="cluster_eckit"] - atlas_f -> atlas #[style=dotted] - atlas_f -> fckit [lhead="cluster_fckit"] - - atlas -> transi [style=dotted group=a lhead="cluster_transi"] - atlas -> Eigen [style=dotted group=a] - atlas -> OpenMP [style=dotted] - atlas -> CGAL [style=dotted group=a] - atlas -> Proj [style=dotted group=a] - atlas -> fftw [style=dotted] - atlas -> CUDA [style=dotted group=a] - atlas -> GridTools [style=dotted group=a] - - #{ rank=same atlas fftw } - #{ rank=same atlas Eigen3 } - - #{rank=same; MPI; OpenMP;}; - #{rank=same; atlas; atlas_f;}; -} - diff --git a/doc/pages/building.md b/doc/pages/building.md deleted file mode 100644 index c26c772ef..000000000 --- a/doc/pages/building.md +++ /dev/null @@ -1,11 +0,0 @@ -Downloading and Building {#building} -======================== - -Requirements ------------- - -Atlas has following requirements: - -@m_div{m-container-inflate} @m_div{m-row} @m_div{m-col-m-12 m-nopadt} -@dotfile building.dot -@m_enddiv @m_enddiv @m_enddiv diff --git a/doc/pages/custom-buildsystems.md b/doc/pages/custom-buildsystems.md deleted file mode 100644 index 66c98432b..000000000 --- a/doc/pages/custom-buildsystems.md +++ /dev/null @@ -1,4 +0,0 @@ -Using Atlas with custom buildsystems {#custom-buildsystems} -==================================== - -todo \ No newline at end of file diff --git a/doc/pages/cmake.md b/doc/pages/developer_information.md similarity index 94% rename from doc/pages/cmake.md rename to doc/pages/developer_information.md index 4acc6a208..e90a36445 100644 --- a/doc/pages/cmake.md +++ b/doc/pages/developer_information.md @@ -1,5 +1,5 @@ -Usage with CMake {#cmake} -================ +Developer information {#developer_information} +===================== Enabling AddressSanitizer (for debugging) ----------------------------------------- diff --git a/doc/pages/getting-started.md b/doc/pages/getting-started.md deleted file mode 100644 index 0d707eca3..000000000 --- a/doc/pages/getting-started.md +++ /dev/null @@ -1,125 +0,0 @@ -Getting started {#getting-started} -=============== - -@tableofcontents - -Requirements {#getting-started-requirements} ------------- - -Tested compilers include: - -- GCC 4.9.1, 5.3.0, 6.3.0, 7.2.0 -- Intel 15.0.2, 16.0.3, 17, 18 -- CCE 8.4.5, 8.5.8, 8.6.2 -- PGI-Fortran 17.7 combined with GNU-C/C++ 6.3 -- PGI 17.7 - -Known compilers to fail include: - -- PGI-Fortran 17.10, 18.1 - -Required dependencies: - -- CMake --- For use and installation see http://www.cmake.org/ -- ecbuild --- ECMWF library of CMake macros -- eckit (with MPI support) --- C++ support library - -Recommended dependencies: - -- fckit --- For enabling Fortran interfaces -- python (only when Fortran bindings are required) - -Optional dependencies: - -- gridtools-storage --- For GPU interoperability -- transi --- For enabling IFS spherical harmonics transforms ( not open-source ) -- CGAL --- For enabling Delaunay triangulation of unstructured grids -- Eigen3 -- For certain linear algebra operations -- FFTW -- For enabling inverse spherical harmonics transforms (TransLocal) - -Installation {#getting-started-installation} ------------- - -Atlas employs an out-of-source build/install based on CMake. - -Make sure ecbuild, eckit and fckit are installed and the ecbuild -executable script is found ( `which ecbuild` ). Following environment variables -help the build system to detect the right dependencies: - -```bash -# For finding eckit -ECKIT_PATH # Path to eckit prefix - -# For finding fckit -FCKIT_PATH # Path to fckit prefix -``` - -Other environment variables that could be required for optional features: - -```bash -# For finding gridtools-storage -GRIDTOOLS_STORAGE_PATH # Path to gridtools-storage prefix - -# For finding transi -TRANSI_PATH # Path to transi prefix - -# For finding CGAL -BOOST_ROOT # Path to Boost prefix -CGAL_DIR # Path to directory containing CGALConfig.cmake -Eigen3_DIR # Path to directory containing Eigen3Config.cmake -FFTW_PATH # Path to FFTW prefix -``` - -Now proceed with installation as follows - -```bash -# Environment --- Edit as needed -ATLAS_SRC=$(pwd) -ATLAS_BUILD=build -ATLAS_INSTALL=$HOME/local - -# 1. Create the build directory: -mkdir $ATLAS_BUILD -cd $ATLAS_BUILD - -# 2. Run CMake -ecbuild --prefix=$ATLAS_INSTALL -- $ATLAS_SRC - -# 3. Compile / Install -make -j10 -make install - -# 4. Check installation -$ATLAS_INSTALL/bin/atlas --info -``` - -Extra flags maybe added to step 2 to fine-tune configuration. -- `--build=DEBUG|RELEASE|BIT` --- Optimisation level
-
DEBUG
No optimisation (`-O0 -g`)
-
BIT
Maximum optimisation while remaning bit-reproducible (`-O2 -g`)
-
RELEASE
Maximum optimisation (`-O3`)
-- `-DENABLE_OMP=OFF` --- Disable OpenMP -- `-DENABLE_FORTRAN=OFF` --- Disable Compilation of Fortran bindings - -@note - By default compilation is done using shared libraries. Some systems have linking - problems with static libraries that have not been compiled with `-fPIC`. - In this case, also compile atlas using static linking, by adding to step 2: - `--static` - -Runtime Configuration {#getting-started-runtime-configuration} ---------------------- - -Atlas behaviour can be configured through some environment variables with defaults marked in square brackets - -- `ATLAS_INFO=<0|[1]>` --- Control printing of Atlas standard information -- `ATLAS_DEBUG=<[0]|1>` --- Control printing of Atlas debug information -- `ATLAS_TRACE=<[0]|1>` --- Control printing of Atlas traces (includes timings) - - -Additional information {#getting-started-more} ----------------------- - -- @subpage building --- @copybrief building -- @subpage cmake --- @copybrief cmake -- @subpage custom-buildsystems --- @copybrief custom-buildsystems diff --git a/doc/pages/getting-started_installation.tex b/doc/pages/getting-started_installation.tex deleted file mode 100644 index 7c981a5d0..000000000 --- a/doc/pages/getting-started_installation.tex +++ /dev/null @@ -1,883 +0,0 @@ -/** -\chapter{Download and installation} -\label{chap:installation} -This chapter is intended to be a general introduction -on how to download, install and use \Atlas. In particular, -in \sect{s:requirements} we will present the general -requirements of the library. In \sect{s:installation} we -will first describe how to install the third-party packages -required by \Atlas (if supported by ECMWF) and successively -we will outline how to install \Atlas. Finally, in \sect{s:using} -we show how to use \Atlas by creating a simple example -that initializes and finalizes the library. - - -\section{General requirements} -\label{s:requirements} - -\Atlas is distributed as Git repository and is available at the ECMWF Stash -git hosting service: \url{https://software.ecmwf.int/stash/projects}. -This can only be accessed by ECMWF staff within the internal intranet. -Occasionally, access can be granted to external partners working -on specific projects. - -\Atlas is currently available under Stash as \Atlas project -itself or under the European project ESCAPE. These two repositories -are separated - specifically the \Atlas project under ESCAPE is -a so-called Git fork of the main \Atlas project. -Given this structure of the \Atlas project, one can find the -library either in the main repository:\\ -\url{https://software.ecmwf.int/wiki/display/ATLAS/Atlas}\\ -or in the ESCAPE repository:\\ -\url{https://software.ecmwf.int/stash/projects/ESCAPE/repos/atlas/browse}.\\ -If you encounter any problem accessing these pages, please contact -Willem Deconinck (\url{willem.deconinck@ecmwf.int}). - -Note that the main \Atlas project is intended for ECMWF internal -developments, while the \Atlas project under ESCAPE is intended -for experimental developments within the ESCAPE project. -Note also that \Atlas requires third-party libraries as described -in the \sect{s:installation} below. - -Finally, \Atlas has been tested and works correctly with the -following compilers: GCC 4.8.1, Intel 13.0.1, 14.0.1 and CCE -8.2.7, 8.3.1. - - -\section{Installation} -\label{s:installation} -\Atlas requires a number of third-party libraries. -Some of them are external third-party libraries not maintained -by ECMWF - these external libraries are briefly described -in \sect{s:external-libs}, where we also provide some -useful links on how to download and install them. - -Some other third-party libraries are developed and maintained -by ECMWF. For this set of libraries we provide a download and -installation instructions in \sect{s:ecmwf-libs}. - -\subsection{External third-party dependencies} -\label{s:external-libs} -\Atlas requires the following external third-party libraries -(some of the links provided may have changed, so we suggest -the reader to lookup on the web for these packages): -% -\begin{itemize} -\item \textbf{Git}: Required for project management and to download -the repository. For use and installation see \url{https://git-scm.com/} -\item \textbf{CMake}: Required for configuration and cross-compilation -purposes. For use and installation see \url{http://www.cmake.org/} -\item \textbf{MPI}: Required for distributed memory parallelisation. -For use and installation see for instance \url{https://www.open-mpi.org/} -\item \textbf{Python}: Required for certain components of the build system. -For use and installation see \url{https://www.python.org/}. -\item \textbf{OpenMP} (optional): Required for shared memory -parallelisation. For use and installation see \url{http://openmp.org/wp/} -\item \textbf{boost\_unit\_test} (optional): Required for unit testing -for C++. For use and installation see \url{http://www.boost.org/} -\end{itemize} -% -Note that if you are an ECMWF staff member, you have some of the above -libraries already available through the module system. In particular -you can load the following packages as follows: -% -\begin{lstlisting}[style=BashStyle] -module load git cmake python -\end{lstlisting} -% -If you are not an ECMWF staff member you need to either install them -manually following the links above or ask your system administrator -to verify whether these packages are already available within your -working environment. - - -\subsection{ECMWF third-party dependencies} -\label{s:ecmwf-libs} -\Atlas additionally requires the following projects developed -at ECMWF: -% -\begin{itemize} -\item \textbf{ecbuild}: It implements some CMake macros that -are useful for configuring and cross-compiling \Atlas and the -other ECMWF third-party libraries required by \Atlas. -For further information, please visit: -\url{https://software.ecmwf.int/wiki/display/ECBUILD/ecBuild}. -\item \textbf{eckit}: It implements some useful C++ -functionalities widely used in ECMWF C++ projects. -For further information, please -visit: \url{https://software.ecmwf.int/wiki/display/ECKIT/ecKit} -\item \textbf{fckit} (optional): It implements some useful -Fortran functionalities. For further information, please -visit: \url{https://software.ecmwf.int/stash/projects/ECSDK/repos/fckit/browse} -\item \textbf{transi} (optional): It implements the spectral -transform. For further information, please visit: -\url{https://software.ecmwf.int/stash/projects/ATLAS/repos/transi/browse} -\end{itemize} -% -In the following we will outline how to install each of the -projects above. - -The first step is to create a folder where to download, build -and install all the third-party projects required as well as -where to build and install \Atlas. Let us call this folder -\inlsh{myproject}, create it and enter into the folder: -% -\begin{lstlisting}[style=BashStyle] -mkdir -p $(pwd)/myproject -cd myproject -\end{lstlisting} -% -We then need to create the following folder tree: -% -\begin{lstlisting}[style=BashStyle] -SRC=$(pwd)/sources -BUILDS=$(pwd)/builds -INSTALL=$(pwd)/install -mkdir -p $SRC -mkdir -p $BUILDS -mkdir -p $INSTALL -\end{lstlisting} -% -where the sources directory will contain the source files -of each project, the builds directory will contain the -built projects and the install directory will contain -the installation of each project.\\ -% -It is guaranteed at any point in time that all ECMWF projects have -a git branches called ``master'' and ``develop''. These branches -respectively in each project are guaranteed to be compatible. The -``master'' branch contains the latest fixed release version of each -project, whereas the ``develop'' branch contains the latest daily -contributions to each project in preparation for future release -versions. It is not guaranteed that the ``develop'' branch of e.g. -\Atlas would be compatible with the ``master'' branch of one if its -dependencies (e.g. eckit).\\ -When updating the ``develop'' branch of \Atlas, it might therefore -be advisable to also update the ``develop'' branches of all of its -dependencies. -With the following, we can specify which branch in every project -will be built. -\begin{lstlisting}[style=BashStyle] -BRANCH=master -\end{lstlisting} -% -All ECMWF projects can be built with different optimisation options. -There are the following -three recommended options:\\ -\begin{itemize} -\setlength\itemsep{0.1em} -\item \inlsh{DEBUG}: No optimisation - used for debugging or development - purposes only. This option may enable additional - boundschecking. -\item \inlsh{BIT}: Maximum optimisation while remaining bit-reproducible. -\item \inlsh{RELEASE}: Maximum optimisation. -\end{itemize} -With the following, we can specify which optimisation to use for the -installation of all projects. -\begin{lstlisting}[style=BashStyle] -BUILD_TYPE=RELEASE -\end{lstlisting} -% -We can now proceed to the download and install -each of the ECMWF projects required by \Atlas. -% -\subsubsection{ecbuild} -To download the project and switch to the correct branch, -we can type on the terminal the commands reported below: -% -\begin{lstlisting}[style=BashStyle] -git clone ssh://git@software.ecmwf.int:7999/ecsdk/ecbuild.git $SRC/ecbuild -cd $SRC/ecbuild -git checkout $BRANCH -\end{lstlisting} -% -This project is constituted by just a few CMake macros -and it does not need to be compiled nor installed. -We do not need to to any additional step for ecbuild!\\ -% -In the ecbuild project resides an executable script called \inlsh{ecbuild} -to aid installation of all following projects. To make this script -easily accessible, prepend it to the \inlsh{PATH}. -\begin{lstlisting}[style=BashStyle] -export PATH=$SRC/ecbuild/bin:$PATH -\end{lstlisting} -% -This executable script \inlsh{ecbuild} acts as a wrapper around the -\inlsh{cmake} executable. More information on this script can be -obtained: -\begin{lstlisting}[style=BashStyle] -ecbuild --help -\end{lstlisting} -Particular options of the \inlsh{ecbuild} script noteworthy are -\inlsh{\ddash{build}} and \inlsh{\ddash{prefix}}. -\begin{itemize} -\setlength\itemsep{0.1em} -\item \inlsh{\ddash{build}=\$BUILD\_TYPE} sets the build type to specified optimisation -\item \inlsh{\ddash{install}=\$INSTALL} sets the install prefix to the specified path -\end{itemize} - -\subsubsection{eckit} -To download the project and switch to the correct branch, -we can type on the terminal the commands reported below: -% -\begin{lstlisting}[style=BashStyle] -git clone ssh://git@software.ecmwf.int:7999/ecsdk/eckit.git $SRC/eckit -cd $SRC/eckit -git checkout $BRANCH -\end{lstlisting} -% -Now that we have downloaded the project and switched -to the correct branch, we can proceed to build the -project and install it. We first need to create the -following folder where the files will be built: -% -\begin{lstlisting}[style=BashStyle] -mkdir $BUILDS/eckit -cd $BUILDS/eckit -\end{lstlisting} -% -Then, we need to run ecbuild in order to configure the -library - i.e. to find the various dependencies required, -etc. - and finally we need to run \inlsh{make install} -to compile and install the library. These two steps are -reported below: -% -\begin{lstlisting}[style=BashStyle] -ecbuild --build=$BUILD_TYPE --prefix=$INSTALL/eckit -- $SRC/eckit -make -j4 install -\end{lstlisting} -% -Note that if the folder \inlsh{\$INSTALL/eckit} -is not already present it will be automatically created. - -\subsubsection{fckit (optional)} -To download the library and switch to the correct branch called -\inlsh{develop}, we can type on the terminal the commands reported -below: -% -\begin{lstlisting}[style=BashStyle] -git clone ssh://git@software.ecmwf.int:7999/ecsdk/fckit.git $SRC/fckit -cd $SRC/fckit -git checkout $BRANCH -\end{lstlisting} -% -Now that we have downloaded the library and switched -to the develop branch, we can proceed to build the -library and install it. We first need to create the -following folder where the files will be built: -% -\begin{lstlisting}[style=BashStyle] -mkdir $BUILDS/fckit -cd $BUILDS/fckit -\end{lstlisting} -% -Then, we need to run ecbuild in order to configure the -library - i.e. to find the various dependencies required, -etc. - and finally we need to run \inlsh{make install} -to compile and install the library. These two steps are -reported below: -% -\begin{lstlisting}[style=BashStyle] -ecbuild --build=$BUILD_TYPE --prefix=$INSTALL/fckit -- $SRC/fckit -make -j4 install -\end{lstlisting} -% -Note that if the folder \inlsh{\$INSTALL/fckit} -is not already present it will be automatically created. - - - -\subsubsection{transi (optional)} -To download the library and switch to the correct branch called -\inlsh{develop}, we can type on the terminal the commands reported -below: -% -\begin{lstlisting}[style=BashStyle] -git clone ssh://git@software.ecmwf.int:7999/atlas/transi.git $SRC/transi -cd $SRC/transi -git checkout $BRANCH -\end{lstlisting} -% -Now that we have downloaded the library and switched -to the develop branch, we can proceed to build the -library and install it. We first need to create the -following folder where the files will be built: -% -\begin{lstlisting}[style=BashStyle] -mkdir $BUILDS/transi -cd $BUILDS/transi -\end{lstlisting} -% -Then, we need to run ecbuild in order to configure the -library - i.e. to find the various dependencies required, -etc. - and finally we need to run \inlsh{make install} -to compile and install the library. These two steps are -reported below: -% -\begin{lstlisting}[style=BashStyle] -ecbuild --build=$BUILD_TYPE --prefix=$INSTALL/transi -- $SRC/transi -make -j4 install -\end{lstlisting} -% -Note that if the folder \inlsh{\$INSTALL/transi} is not already -present it will be automatically created. - - - -\subsection{\Atlas installation} -Once we have downloaded, compiled and installed the third-party -dependencies described above, we can now download and install -\Atlas. In particular, to download the library and switch -to the correct branch called \inlsh{develop}, we can type -on the terminal the commands reported below: -% -\begin{lstlisting}[style=BashStyle] -git clone ssh://git@software.ecmwf.int:7999/atlas/atlas.git $SRC/atlas -cd $SRC/atlas -git checkout $BRANCH -\end{lstlisting} -% -Now that we have downloaded the library and switched -to the develop branch, we can proceed to build the -library and install it. We first need to create the -following folder where the files will be built: -% -\begin{lstlisting}[style=BashStyle] -mkdir $BUILDS/atlas -cd $BUILDS/atlas -\end{lstlisting} -% -Then, we need to run \inlsh{ecbuild} in order to configure -the library - i.e. to find the various dependencies required, -etc. - and finally we need to run \inlsh{make install} -to compile and install the library. These two steps are -reported below: -% -\begin{lstlisting}[style=BashStyle] -$SRC/ecbuild/bin/ecbuild --build=$BUILD_TYPE --prefix=$INSTALL/atlas -- \ - -DECKIT_PATH=$INSTALL/eckit \ - -DFCKIT_PATH=$INSTALL/fckit \ - -DTRANSI_PATH=$INSTALL/transi \ - $SRC/atlas -make -j4 install -\end{lstlisting} -% -Note that if the folder \inlsh{\$INSTALL/atlas} is not already -present it will be automatically created. - -The following extra flags may be added to the \inlsh{ecbuild} -step to fine-tune configuration: -\begin{itemize} -\setlength\itemsep{0.1em} -\item \inlsh{-DENABLE\_OMP=OFF} --- Disable OpenMP -\item \inlsh{-DENABLE\_MPI=OFF} --- Disable MPI -\item \inlsh{-DENABLE\_FORTRAN=OFF} --- Disable Compilation of Fortran bindings -\item \inlsh{-DENABLE\_TRANS=OFF} --- Disable compilation of the spectral transforms - functionality. This is automatically disabled if the optional \emph{transi} - dependency is not compiled or found. In this case it is also unnecessary to - provide \inlsh{-DTRANSI\_PATH=\$INSTALL/transi}. -\end{itemize} -% -% -\begin{notebox} -By default compilation is done using shared libraries. Some systems have -linking problems with static libraries that have not been compiled with -the flag \inlsh{-fPIC}. In this case, also compile atlas using static -linking, by adding to the ecbuild step the flag: \inlsh{\ddash{static}} -\end{notebox} -% -The building and installation of \Atlas should now be complete -and you can start using it. With this purpose, in the next -section we show a simple example on how to create a simple -program to initialize and finalize the library. - - -\section{Inspecting your \Atlas installation} \label{s:inspect} -Once installation of atlas is complete, an executable called "atlas" -can be found in \inlsh{\$INSTALL/bin/atlas}. Executing -\begin{lstlisting}[style=BashStyle] ->>> $INSTALL/bin/atlas --version -0.7.0 - ->>> $INSTALL/bin/atlas --git -2d683ab4aa0c - ->>> $INSTALL/bin/atlas --info -atlas version (0.7.0), git-sha1 2d683ab - - Build: - build type : Release - timestamp : 20160215122606 - op. system : Darwin-14.5.0 (macosx.64) - processor : x86_64 - c compiler : Clang 7.0.2.7000181 - flags : -O3 -DNDEBUG - c++ compiler : Clang 7.0.2.7000181 - flags : -O3 -DNDEBUG - fortran compiler: GNU 5.2.0 - flags : -fno-openmp -O3 -funroll-all-loops -finline-functions - - Features: - Fortran : ON - MPI : ON - OpenMP : OFF - BoundsChecking : OFF - Trans : ON - Tesselation : ON - gidx_t : 64 bit integer - - Dependencies: - eckit version (0.12.3), git-sha1 7b76818 - transi version (0.3.2), git-sha1 bf33f60 -\end{lstlisting} -% -gives you information respectively on the macro version, -a more detailed git-version-controlled identifier, and finally -a more complete view on all the features that Atlas has been compiled with, -as well as compiler and compile flag information. -Also printed is the versions of used dependencies such as eckit and transi. - -\section{Using \Atlas in your project} -\label{s:using} -In this section, we provide a simple example on how to use \Atlas. -The objective here is not to get familiar with the main -functionalities of \Atlas, but rather to show how to get started! -Specifically, we will show a simple ``Hello world'' program that -initialises and finalises the library, and uses the internal \Atlas -logging facilities to print ``Hello world!''. -The steps necessary to compile and run the program will be detailed -in this section. - -Note that the \Atlas supports both C++ and Fortran, therefore, -in the following, we will show both an example using C++ and -an example using Fortran. Before starting, we create a folder -called \inlsh{project1} in the \inlsh{sources} directory: -% -\begin{lstlisting}[style=BashStyle] -mkdir -p $SRC/project1 -\end{lstlisting} -% -Here, we will add both the C++ and Fortran files of this -simple example. Note that there are (at least) two ways -to compile the code we are going to write. The first involves -just using a C compiler for the C++ version and a Fortran -compiler for the Fortran version, without using any cmake -files. The second involves using cmake files. In the following, -we will detail both possibilities. - -\subsection{C++ version} -\label{s:atlas-hello-world-C} - -\subsubsection*{Program description} -The C++ version of the \Atlas initialization and finalization -calls is depicted in \lista{code1-C}. -% -\lstinputlisting[caption=Initialization and finalization -of \Atlas using C++, style=CStyle, label=code1-C]{hello-world.cc} -% -We can create a new file in the folder \inlsh{project1} just generated: -% -\begin{lstlisting}[style=BashStyle] -touch $SRC/project1/hello-world.cc -\end{lstlisting} -% -and copy the content of the code in \lista{code1-C} into it. -We can now have a closer look at the code. -On line 1, we include the \Atlas header file, we successively -specify a simple main function, in which we call the initialization -of the \Atlas library on line 6. -Note that we passed the two arguments of the main function -\inltc{argc} and \inltc{argv} to the \inltc{atlas\_init} -function. -We finally call the \Atlas \inltc{atlas\_finalize()} function -at line 8 without passing any argument to it. - -The function \inltc{atlas\_init()} is responsible for -setting up the logging facility and for the initialization -of MPI (Message Passage Interface), -while the function \inltc{atlas\_finalize()} -is responsible for finalizing MPI and closing the program. -On line 7, we log ``Hello world!'' to the \inltc{info} log -channel.\\ -% -\Atlas provides 4 different log channels which can be configured -separately: \inltc{debug}, \inltc{info}, \inltc{warning}, and -\inltc{error}. By default, the \inltc{debug} channel does not -get printed; the \inltc{info} and \inltc{warning} channel get -printed to the std::cout stream, and the \inltc{error} channel -gets printed to std::cerr. For more information on the logging -facility, the reader is referred to section~\ref{s:utilities-logging}. - -\subsubsection*{Code compilation} -We now need to compile the code. We first create a new directory -into the \inlsh{\$BUILDS} folder, where we will compile the code -% -\begin{lstlisting}[style=BashStyle] -mkdir -p $BUILDS/project1 -\end{lstlisting} -% -As mentioned above, there are (at least) two ways for compiling -the source code above. These are detailed below. -% -\begin{description} -% -\item \underline{Directly with C++ compiler}\\[0.5em] -% -The first possibility is to -avoid using cmake and ecbuild and directly run a C++ compiler, -such as g++. For doing so, especially when \Atlas is linked statically, -we need to know all \Atlas dependent libraries. This step can be easily -achieved by inspecting the file -% -\begin{lstlisting}[style=BashStyle] -vi $INSTALL/atlas/lib/pkgconfig/atlas.pc -\end{lstlisting} -% -Here, all the flags necessary for the correct compilation -of the C++ code in \lista{code1-C} are reported. For -compiling the code, we first go into the builds directory -just created and we generate a new folder where the executables -will be stored: -% -\begin{lstlisting}[style=BashStyle] -cd $BUILDS/project1 -mkdir -p bin -\end{lstlisting} -% -Note that, when using the cmake compilation route, it is not -necessary to generate the bin directory since it will automatically -created during compilation. -After having generated the bin folder, we can run the following -command: -% -\begin{lstlisting}[style=BashStyle] -g++ $SRC/project1/hello-world.cc -o bin/atlas_c-hello-world \ - $(pkg-config $INSTALL/atlas/lib/pkgconfig/atlas.pc --libs --cflags) -\end{lstlisting} -% -This will compile our hello-world.cc file and it will automatically -link all the static and dynamic libraries required by the program. -The executable, as mentioned, is generated into the folder bin. -% -\item \underline{CMake/ecbuild}\\[0.5em] -% -The second possibility is to create an ecbuild project, which is -especially beneficial for projects with multiple files, libraries, -and executables. - In particular, we need to create the following -cmake file -% -\begin{lstlisting}[style=CMakeStyle] -# File: CMakeLists.txt -cmake_minimum_required(VERSION 2.8.4 FATAL_ERROR) -project(usage_example) - -include(ecbuild_system NO_POLICY_SCOPE) -ecbuild_requires_macro_version(1.9) -ecbuild_declare_project() -ecbuild_use_package(PROJECT atlas REQUIRED) -ecbuild_add_executable(TARGET atlas_c-usage_example - SOURCES hello-world.cc - INCLUDES ${ATLAS_INCLUDE_DIRS} - LIBS atlas) -ecbuild_print_summary() -\end{lstlisting} -in the sources folder of our project \inlsh{\$SRC/project1}. -We can create the \inlsh{CMakeLists.txt} file in the correct -directory following the two steps below: -% -\begin{lstlisting}[style=BashStyle] -cd $SRC/project1 -touch CMakeLists.txt -\end{lstlisting} -% -and copy the CMake code above into it. -In the second line of the CMake file above, we declare the minimum -cmake version required to compile the code, while in the second -line we declare the name of our ecbuild project. -From line 5 to line 7 we include some required ecbuild macros -necessary for using ecbuild. On line 8 we specify that the -\Atlas library is required for this project. Finally, on line -9 we add the instruction to compile the executable. -Line 13 prints just a compilation summary. -We can build this simple ecbuild project by going into our builds -directory -% -\begin{lstlisting}[style=BashStyle] -cd $BUILDS/project1 -\end{lstlisting} -% -and by typing the following command: -% -\begin{lstlisting}[style=BashStyle] -ecbuild -DATLAS_PATH=$INSTALL/atlas $SRC/project1/ -make -\end{lstlisting} -% -Note that in the above command we needed to provide the path -to the \Atlas library installation. Alternatively, -\inlsh{ATLAS\_PATH} may be defined as an environment variable. -This completes the compilation of our first example that -uses \Atlas and generates an executable into the bin folder -(automatically generated by cmake) inside our builds directory -for project1. -\end{description} -% - -\subsubsection*{Run the code} -After the compilation of the source code is completed, -we have an executable file into the folder \inlsh{\$BUILDS/project1/bin/}. -If we simply run the executable file as follows: -% -\begin{lstlisting}[style=BashStyle] -./atlas_c-hello-world -\end{lstlisting} -% -we obtain the output -\begin{lstlisting}[style=BashStyle] -[0] (2016-03-09 T 15:07:15) (I) -- Hello world! -\end{lstlisting} -% -However, by adding \inlsh{\ddash{debug}} to the command line, -also debug information is printed. -In particular, if we type: -% -\begin{lstlisting}[style=BashStyle] -./atlas_c-hello-world --debug -\end{lstlisting} -% -we should obtain something similar to the following output: -% -\begin{lstlisting}[style=BashStyle] -[0] (2016-03-09 T 15:09:42) (D) -- Atlas program [atlas_c-hello-world] -[0] (2016-03-09 T 15:09:42) (D) -- atlas version [0.6.0] -[0] (2016-03-09 T 15:09:42) (D) -- atlas git - [dabb76e9b696c57fbe7e595b16f292f45547d628] -[0] (2016-03-09 T 15:09:42) (D) -- eckit version [0.11.0] -[0] (2016-03-09 T 15:09:42) (D) -- eckit git - [ac7f6a0b3cb4f60d9dc01c8d33ed8a44a4c6de27] -[0] (2016-03-09 T 15:09:42) (D) -- Configuration read from scripts: -[0] (2016-03-09 T 15:09:42) (D) -- rundir : - /home/na/nagm/myproject/builds/project1 -[0] (2016-03-09 T 15:09:42) (I) -- Hello world! -[0] (2016-03-09 T 15:09:42) (D) -- Atlas finalized -\end{lstlisting} -% -which gives us some information such as the version of \Atlas we are -running, the identifier of the commit and the path of the executable. - - - -\subsection{Fortran version} -\label{s:atlas-hello-world-F} - -\subsubsection*{Program description} -The Fortran version of the \Atlas initialization and finalization -calls is depicted in \lista{code1-F}. -% -\lstinputlisting[caption=Initialization and finalization of -\Atlas using Fortran, style=FStyle, label=code1-F]{hello-world.F90} -% -We can create a new file in the folder \inlsh{project1} just generated: -% -\begin{lstlisting}[style=BashStyle] -touch $SRC/project1/hello-world.F90 -\end{lstlisting} -% -and copy the content of the code in \lista{code1-F} into it. -We can now have a closer look at the code. -On line 1, we define the program, called \inltf{usage\_example}. -On line 3, we include the required \Atlas libraries -(note that we include only the three functions required -for this example - i.e. \inltf{atlas\_init}, \inltf{atlas\_finalize}), -and \inltf{atlas\_log}. -The function \inltf{atlas\_init()} on line 8 is responsible for setting up the -logging and for the initialization of MPI (Message Passage Interface), -while the function \inltf{atlas\_finalize()} on line 10 is responsible for -finalizing MPI and closing the program. -On line 9, we log ``Hello world!'' to the \inltf{info} log channel.\\ -% -\Atlas provides 4 different log channels which can be configured -separately: \inltc{debug}, \inltc{info}, \inltc{warning}, and -\inltc{error}. By default, the \inltc{debug} channel does not -get printed; the \inltc{info} and \inltc{warning} channel get -printed to the std::cout stream, and the \inltc{error} channel -gets printed to std::cerr. For more information on the logging -facility, the reader is referred to section~\ref{s:utilities-logging}. - - -\subsubsection*{Code compilation} -We now need to compile the code. We first create a new directory -into the \inlsh{\$BUILDS} folder, where we will compile the code -% -\begin{lstlisting}[style=BashStyle] -mkdir -p $BUILDS/project1 -\end{lstlisting} -% -As mentioned above, there are (at least) two ways for compiling -the source code above. These are detailed below. -% -\begin{description} -% -\item \underline{Directly with Fortran compiler}\\[0.5em] -% -The first possibility is to avoid using cmake and ecbuild and -directly run a Fortran compiler, such as gfortran. -For doing so, especially when \Atlas is linked statically, -we need to know all \Atlas dependent libraries. This step can be easily -achieved by inspecting the file. This step can be easily achieved by inspecting -the file -% -\begin{lstlisting}[style=BashStyle] -vi $INSTALL/atlas/lib/pkgconfig/atlas.pc -\end{lstlisting} -% -Here, all the flags necessary for the correct compilation -of the Fortran code in \lista{code1-F} are reported. For -compiling the code, we first go into the builds directory -just created and we generate a new folder where the executables -will be stored: -% -\begin{lstlisting}[style=BashStyle] -cd $BUILDS/project1 -mkdir -p bin -\end{lstlisting} -% -Note that, when using the cmake compilation route, it is not -necessary to generate the bin directory since it will automatically -created during compilation. -After having generated the bin folder, we can run the following -command: -% -\begin{lstlisting}[style=BashStyle] -gfortran $SRC/project1/hello-world.F90 -o bin/atlas_f-hello-world \ -$(pkg-config $INSTALL/atlas/lib/pkgconfig/atlas.pc --libs --cflags) -\end{lstlisting} -% -This will compile our hello-world.F90 file and it will automatically -link all the static and dynamic libraries required by the program. -The executable, as mentioned, is generated into the folder bin. -% -\item \underline{CMake/ecbuild}\\[0.5em] -% -The second possibility is to use a cmake file that uses some -ecbuild macros. In particular, we need to create the following -cmake file: -% -\begin{lstlisting}[style=CMakeStyle] -# File: CMakeLists.txt -cmake_minimum_required(VERSION 2.8.4 FATAL_ERROR) -project(usage_example) - -include(ecbuild_system NO_POLICY_SCOPE) -ecbuild_requires_macro_version(1.9) -ecbuild_declare_project() -ecbuild_enable_fortran(MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module - REQUIRED) -ecbuild_use_package(PROJECT atlas REQUIRED) -ecbuild_add_executable(TARGET atlas_f-usage_example - SOURCES hello-world.F90 - INCLUDES ${ATLAS_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR} - LIBS atlas_f) -ecbuild_print_summary() -\end{lstlisting} -% -in the sources folder of our project \inlsh{\$SRC/project1}. -We can create the \inlsh{CMakeLists.txt} file in the correct -directory following the two steps below: -% -\begin{lstlisting}[style=BashStyle] -cd $SRC/project1 -touch CMakeLists.txt -\end{lstlisting} -% -and copy the cmake code above into it. -In the second line of the cmake file, we declare the minimum cmake -version required to compile the code, while in the second line -we declare the name of our cmake project. -From line 5 to line 7 we include some required ecbuild macros -necessary for using ecbuild. On line 8 we enable Fortran compilation, -while on line 10 we specify that the \Atlas library is required for -this project. Finally, on line 11 we add the instruction to -compile the executable. -Line 15 prints just a compilation summary. We can now run this simple -cmake file by going into our builds directory -% -\begin{lstlisting}[style=BashStyle] -cd $BUILDS/project1 -\end{lstlisting} -% -and by typing the following command: -% -\begin{lstlisting}[style=BashStyle] -$SRC/ecbuild/bin/ecbuild -DATLAS_PATH=$INSTALL/atlas $SRC/project1/ -make -\end{lstlisting} -% -Note that in the above command we needed to provide the path -to the \Atlas library installation. Alternatively, -\inlsh{ATLAS\_PATH} may be defined as an environment variable. -This completes the compilation of our first example that uses -\Atlas and generates an executable file into the bin folder -(automatically generated by CMake) inside our builds directory -for project1. -\end{description} -% - -\subsubsection*{Run the code} -After the compilation of the source code is completed, -we have an executable file into the folder \inlsh{\$BUILDS/project1/bin/}. -If we simply run the executable file as follows: -% -\begin{lstlisting}[style=BashStyle] -./atlas_c-hello-world -\end{lstlisting} -% -we obtain the output -\begin{lstlisting}[style=BashStyle] -[0] (2016-03-09 T 15:27:00) (I) -- Hello world! -\end{lstlisting} -% -However, by setting the environment variable \inlsh{DEBUG=1}, -also debug information is printed. -In particular, if we type: -% -\begin{lstlisting}[style=BashStyle] -export DEBUG=1 -./atlas_c-hello-world -\end{lstlisting} -% -we should obtain something similar to the following output: -% -\begin{lstlisting}[style=BashStyle] -[0] (2016-03-09 T 15:27:04) (D) -- Atlas program [atlas_f-hello-world] -[0] (2016-03-09 T 15:27:04) (D) -- atlas version [0.6.0] -[0] (2016-03-09 T 15:27:04) (D) -- atlas git - [dabb76e9b696c57fbe7e595b16f292f45547d628] -[0] (2016-03-09 T 15:27:04) (D) -- eckit version [0.11.0] -[0] (2016-03-09 T 15:27:04) (D) -- eckit git - [ac7f6a0b3cb4f60d9dc01c8d33ed8a44a4c6de27] -[0] (2016-03-09 T 15:27:04) (D) -- Configuration read from scripts: -[0] (2016-03-09 T 15:27:04) (D) -- rundir : - /home/na/nagm/myproject/builds/project1 -[0] (2016-03-09 T 15:27:04) (I) -- Hello world! -[0] (2016-03-09 T 15:27:04) (D) -- Atlas finalized - -\end{lstlisting} -% -which gives us some information such as the version of \Atlas we are -running, the identifier of the commit and the path of the executable. -\begin{tipbox} -The outputs obtained for the Fortran and C++ versions should be identical -since they call exactly the same routines. -\end{tipbox} - -This completes your first project that uses the \Atlas library. - - - - - - -*/ \ No newline at end of file diff --git a/doc/pages/mainpage.md b/doc/pages/mainpage.md index fae14fa4b..9a6e63992 100644 --- a/doc/pages/mainpage.md +++ b/doc/pages/mainpage.md @@ -28,10 +28,6 @@ What's new? {#mainpage-whats-new} Curious about what was added or improved recently? Check out the [Changelog](https://github.com/ecmwf/atlas/blob/master/CHANGELOG.md) -Getting started {#mainpage-getting-started} ---------------- - -The best way to get started is to read the @ref getting-started guide. Contributing {#mainpage-contributing} ------------ From e27c4169b8f3109840975c161b25740d72d56051 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 28 Sep 2020 01:22:42 +0100 Subject: [PATCH 130/161] Make Plugin architecture robust to lib64 install dirs --- src/atlas/library/Library.cc | 36 ++++++++++++++---------------------- src/atlas/library/Plugin.cc | 4 ++++ src/atlas/library/Plugin.h | 9 +++++++++ 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index f43c14cf0..13fb622ae 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -91,35 +91,28 @@ int getEnv( const std::string& env, int default_value ) { } -bool library_exists( const eckit::PathName& library_dir, const std::string& library_name, - eckit::PathName& library_path ) { +bool plugin_library_exists( const eckit::PathName& plugin_dir, const std::string& library_name, + eckit::PathName& library_path ) { // WARNING: Mostly copy-paste from eckit develop before 1.13 release std::vector library_file_names; library_file_names.push_back( "lib" + library_name + ".so" ); library_file_names.push_back( "lib" + library_name + ".dylib" ); - for ( const auto& library_file_name : library_file_names ) { - library_path = library_dir / library_file_name; - if ( library_path.exists() ) { - return true; + std::vector library_dirs; + library_dirs.push_back( plugin_dir / "lib64" ); + library_dirs.push_back( plugin_dir / "lib" ); + for ( const auto& library_dir : library_dirs ) { + for ( const auto& library_file_name : library_file_names ) { + library_path = library_dir / library_file_name; + if ( library_path.exists() ) { + return true; + } } } return false; } -void load_library( const eckit::PathName& library_dir, const std::string& library_name ) { - // WARNING: Mostly copy-paste from eckit develop before 1.13 release - std::vector library_file_names; - library_file_names.push_back( "lib" + library_name + ".so" ); - library_file_names.push_back( "lib" + library_name + ".dylib" ); - - eckit::PathName library_path; - for ( const auto& library_file_name : library_file_names ) { - library_path = library_dir / library_file_name; - if ( library_path.exists() ) { - break; - } - } +void load_library( const eckit::PathName& library_path, const std::string& library_name ) { void* plib = ::dlopen( library_path.localPath(), RTLD_NOW | RTLD_GLOBAL ); if ( plib == nullptr ) { // dlopen failed std::ostringstream ss; @@ -270,10 +263,9 @@ void Library::initialise( const eckit::Parametrisation& config ) { ATLAS_ASSERT( plugin_config.get( "library", library_name ) ); bool library_loaded = eckit::system::Library::exists( library_name ); if ( not library_loaded ) { - eckit::PathName library_dir = plugin_dir / "lib"; - ATLAS_ASSERT( library_exists( library_dir, library_name, library_path ) ); + ATLAS_ASSERT( plugin_library_exists( plugin_dir, library_name, library_path ) ); Log::debug() << "Loading plugin [" << plugin_name << "] library " << library_path << std::endl; - load_library( library_dir, library_name ); + load_library( library_path, library_name ); library_loaded = true; } ATLAS_ASSERT( is_plugin_loaded( plugin_name ) ); diff --git a/src/atlas/library/Plugin.cc b/src/atlas/library/Plugin.cc index 306daab76..7dae617e8 100644 --- a/src/atlas/library/Plugin.cc +++ b/src/atlas/library/Plugin.cc @@ -23,6 +23,10 @@ Plugin::Plugin( const std::string& name, const std::string& libname ) : atlas::Library::instance().registerPlugin( *this ); } +const void* Plugin::addr() const { + return this; +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace atlas diff --git a/src/atlas/library/Plugin.h b/src/atlas/library/Plugin.h index eaed192e8..28a53638b 100644 --- a/src/atlas/library/Plugin.h +++ b/src/atlas/library/Plugin.h @@ -20,10 +20,19 @@ namespace atlas { class Plugin : public eckit::system::Library { public: + /// @param [in] name Plugin name + /// @param [in] libname Library name as will be used in file system Plugin( const std::string& name, const std::string& libname = "" ); + + /// @brief Plugin name const std::string& name() const { return name_; } + + /// @brief Library name as will be used in file system const std::string& libraryName() const { return libname_; } +private: + const void* addr() const override; + private: std::string name_; std::string libname_; From ec94aa296b40d4533b0533298e994b69f12900e3 Mon Sep 17 00:00:00 2001 From: Pedro Maciel Date: Fri, 25 Sep 2020 10:36:30 +0100 Subject: [PATCH 131/161] ATLAS-306 refactor: no sources separation of NonLinear/NonLinearFactory classes (like in the rest of Atlas), non-linear missing values implementation in template .h (one place) with the builders in .cc (as expected) --- src/atlas/CMakeLists.txt | 7 +- src/atlas/interpolation/NonLinear.cc | 2 +- .../{NonLinearFactory.cc => Missing.cc} | 11 +- src/atlas/interpolation/nonlinear/Missing.h | 219 ++++++++++++++++++ .../nonlinear/MissingIfAllMissing.h | 102 -------- .../nonlinear/MissingIfAnyMissing.h | 87 ------- .../nonlinear/MissingIfHeaviestMissing.h | 110 --------- .../interpolation/nonlinear/NonLinear.cc | 28 +++ src/atlas/interpolation/nonlinear/NonLinear.h | 25 ++ .../nonlinear/NonLinearFactory.h | 54 ----- 10 files changed, 276 insertions(+), 369 deletions(-) rename src/atlas/interpolation/nonlinear/{NonLinearFactory.cc => Missing.cc} (84%) delete mode 100644 src/atlas/interpolation/nonlinear/MissingIfAllMissing.h delete mode 100644 src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h delete mode 100644 src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h create mode 100644 src/atlas/interpolation/nonlinear/NonLinear.cc delete mode 100644 src/atlas/interpolation/nonlinear/NonLinearFactory.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index b812dfe64..734e10535 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -534,13 +534,10 @@ interpolation/method/structured/kernels/LinearVerticalKernel.h interpolation/method/structured/kernels/QuasiCubic3DKernel.cc interpolation/method/structured/kernels/QuasiCubic3DKernel.h interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h +interpolation/nonlinear/Missing.cc interpolation/nonlinear/Missing.h -interpolation/nonlinear/MissingIfAllMissing.h -interpolation/nonlinear/MissingIfAnyMissing.h -interpolation/nonlinear/MissingIfHeaviestMissing.h +interpolation/nonlinear/NonLinear.cc interpolation/nonlinear/NonLinear.h -interpolation/nonlinear/NonLinearFactory.cc -interpolation/nonlinear/NonLinearFactory.h ) diff --git a/src/atlas/interpolation/NonLinear.cc b/src/atlas/interpolation/NonLinear.cc index f9d0d645a..33cc9b92a 100644 --- a/src/atlas/interpolation/NonLinear.cc +++ b/src/atlas/interpolation/NonLinear.cc @@ -12,7 +12,7 @@ #include "atlas/interpolation/NonLinear.h" -#include "atlas/interpolation/nonlinear/NonLinearFactory.h" +#include "atlas/interpolation/nonlinear/NonLinear.h" #include "atlas/runtime/Exception.h" #include "atlas/util/Config.h" diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc b/src/atlas/interpolation/nonlinear/Missing.cc similarity index 84% rename from src/atlas/interpolation/nonlinear/NonLinearFactory.cc rename to src/atlas/interpolation/nonlinear/Missing.cc index 69922d6e2..200db4c3a 100644 --- a/src/atlas/interpolation/nonlinear/NonLinearFactory.cc +++ b/src/atlas/interpolation/nonlinear/Missing.cc @@ -10,11 +10,7 @@ */ -#include "atlas/interpolation/nonlinear/NonLinearFactory.h" - -#include "atlas/interpolation/nonlinear/MissingIfAllMissing.h" -#include "atlas/interpolation/nonlinear/MissingIfAnyMissing.h" -#include "atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h" +#include "atlas/interpolation/nonlinear/Missing.h" namespace atlas { @@ -58,11 +54,6 @@ static B> __nl18( M3::static_type() + "-" + Tmake( config ); -} - - } // namespace nonlinear } // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/Missing.h b/src/atlas/interpolation/nonlinear/Missing.h index 76d7011a2..b09c8f5dc 100644 --- a/src/atlas/interpolation/nonlinear/Missing.h +++ b/src/atlas/interpolation/nonlinear/Missing.h @@ -12,6 +12,8 @@ #pragma once +#include "eckit/types/FloatCompare.h" + #include "atlas/field/MissingValue.h" #include "atlas/interpolation/nonlinear/NonLinear.h" @@ -27,6 +29,223 @@ struct Missing : NonLinear { }; +template +struct MissingIfAllMissing : Missing { + bool execute( NonLinear::Matrix& W, const Field& field ) const { + field::MissingValue mv( field ); + auto& missingValue = mv.ref(); + + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + bool zeros = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + Scalar sum = 0.; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + else { + sum += *it; + } + } + + // weights redistribution: zero-weight all missing values, linear re-weighting for the others; + // the result is missing value if all values in row are missing + if ( N_missing > 0 ) { + if ( N_missing == N_entries || eckit::types::is_approximately_equal( sum, 0. ) ) { + for ( Size j = k; j < k + N_entries; ++j ) { + data[j] = j == i_missing ? 1. : 0.; + } + } + else { + const Scalar factor = 1. / sum; + for ( Size j = k; j < k + N_entries; ++j, ++kt ) { + if ( missingValue( values[kt.col()] ) ) { + data[j] = 0.; + zeros = true; + } + else { + data[j] *= factor; + } + } + } + modif = true; + } + } + + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + + return modif; + } + + static std::string static_type() { return "missing-if-all-missing"; } +}; + + +template +struct MissingIfAnyMissing : Missing { + bool execute( NonLinear::Matrix& W, const Field& field ) const { + field::MissingValue mv( field ); + auto& missingValue = mv.ref(); + + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + bool zeros = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + } + + // if any values in row are missing, force missing value + if ( N_missing > 0 ) { + for ( Size j = k; j < k + N_entries; ++j ) { + if ( j == i_missing ) { + data[j] = 1.; + } + else { + data[j] = 0.; + zeros = true; + } + } + modif = true; + } + } + + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + + return modif; + } + + static std::string static_type() { return "missing-if-any-missing"; } +}; + + +template +struct MissingIfHeaviestMissing : Missing { + bool execute( NonLinear::Matrix& W, const Field& field ) const { + field::MissingValue mv( field ); + auto& missingValue = mv.ref(); + + // NOTE only for scalars (for now) + auto values = make_view_field_values( field ); + ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); + + auto data = const_cast( W.data() ); + bool modif = false; + bool zeros = false; + + Size i = 0; + Matrix::iterator it( W ); + for ( Size r = 0; r < W.rows(); ++r ) { + const Matrix::iterator end = W.end( r ); + + // count missing values, accumulate weights (disregarding missing values) and find maximum weight in row + size_t i_missing = i; + size_t N_missing = 0; + size_t N_entries = 0; + Scalar sum = 0.; + Scalar heaviest = -1.; + bool heaviest_is_missing = false; + + Matrix::iterator kt( it ); + Size k = i; + for ( ; it != end; ++it, ++i, ++N_entries ) { + const bool miss = missingValue( values[it.col()] ); + + if ( miss ) { + ++N_missing; + i_missing = i; + } + else { + sum += *it; + } + + if ( heaviest < data[i] ) { + heaviest = data[i]; + heaviest_is_missing = miss; + } + } + + // weights redistribution: zero-weight all missing values, linear re-weighting for the others; + // if all values are missing, or the closest value is missing, force missing value + if ( N_missing > 0 ) { + if ( N_missing == N_entries || heaviest_is_missing || + eckit::types::is_approximately_equal( sum, 0. ) ) { + for ( Size j = k; j < k + N_entries; ++j ) { + data[j] = j == i_missing ? 1. : 0.; + } + } + else { + const Scalar factor = 1. / sum; + for ( Size j = k; j < k + N_entries; ++j, ++kt ) { + if ( missingValue( values[kt.col()] ) ) { + data[j] = 0.; + zeros = true; + } + else { + data[j] *= factor; + } + } + } + modif = true; + } + } + + if ( zeros && missingValue.isnan() ) { + W.prune( 0. ); + } + + return modif; + } + + static std::string static_type() { return "missing-if-heaviest-missing"; } +}; + + } // namespace nonlinear } // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h deleted file mode 100644 index f4e72ffd0..000000000 --- a/src/atlas/interpolation/nonlinear/MissingIfAllMissing.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#pragma once - -#include "atlas/field/MissingValue.h" -#include "atlas/interpolation/nonlinear/Missing.h" - -#include "eckit/types/FloatCompare.h" - - -namespace atlas { -namespace interpolation { -namespace nonlinear { - - -template -struct MissingIfAllMissing : Missing { - bool execute( NonLinear::Matrix& W, const Field& field ) const { - field::MissingValue mv( field ); - auto& missingValue = mv.ref(); - - // NOTE only for scalars (for now) - auto values = make_view_field_values( field ); - ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); - - auto data = const_cast( W.data() ); - bool modif = false; - bool zeros = false; - - Size i = 0; - Matrix::iterator it( W ); - for ( Size r = 0; r < W.rows(); ++r ) { - const Matrix::iterator end = W.end( r ); - - // count missing values, accumulate weights (disregarding missing values) - size_t i_missing = i; - size_t N_missing = 0; - size_t N_entries = 0; - Scalar sum = 0.; - - Matrix::iterator kt( it ); - Size k = i; - for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); - - if ( miss ) { - ++N_missing; - i_missing = i; - } - else { - sum += *it; - } - } - - // weights redistribution: zero-weight all missing values, linear re-weighting for the others; - // the result is missing value if all values in row are missing - if ( N_missing > 0 ) { - if ( N_missing == N_entries || eckit::types::is_approximately_equal( sum, 0. ) ) { - for ( Size j = k; j < k + N_entries; ++j ) { - data[j] = j == i_missing ? 1. : 0.; - } - } - else { - const Scalar factor = 1. / sum; - for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - if ( missingValue( values[kt.col()] ) ) { - data[j] = 0.; - zeros = true; - } - else { - data[j] *= factor; - } - } - } - modif = true; - } - } - - if ( zeros && missingValue.isnan() ) { - W.prune( 0. ); - } - - return modif; - } - - static std::string static_type() { return "missing-if-all-missing"; } -}; - - -} // namespace nonlinear -} // namespace interpolation -} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h b/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h deleted file mode 100644 index abffc9d27..000000000 --- a/src/atlas/interpolation/nonlinear/MissingIfAnyMissing.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#pragma once - -#include "atlas/field/MissingValue.h" -#include "atlas/interpolation/nonlinear/Missing.h" - - -namespace atlas { -namespace interpolation { -namespace nonlinear { - - -template -struct MissingIfAnyMissing : Missing { - bool execute( NonLinear::Matrix& W, const Field& field ) const { - field::MissingValue mv( field ); - auto& missingValue = mv.ref(); - - // NOTE only for scalars (for now) - auto values = make_view_field_values( field ); - ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); - - auto data = const_cast( W.data() ); - bool modif = false; - bool zeros = false; - - Size i = 0; - Matrix::iterator it( W ); - for ( Size r = 0; r < W.rows(); ++r ) { - const Matrix::iterator end = W.end( r ); - - // count missing values, accumulate weights (disregarding missing values) - size_t i_missing = i; - size_t N_missing = 0; - size_t N_entries = 0; - - Matrix::iterator kt( it ); - Size k = i; - for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); - - if ( miss ) { - ++N_missing; - i_missing = i; - } - } - - // if any values in row are missing, force missing value - if ( N_missing > 0 ) { - for ( Size j = k; j < k + N_entries; ++j ) { - if ( j == i_missing ) { - data[j] = 1.; - } - else { - data[j] = 0.; - zeros = true; - } - } - modif = true; - } - } - - if ( zeros && missingValue.isnan() ) { - W.prune( 0. ); - } - - return modif; - } - - static std::string static_type() { return "missing-if-any-missing"; } -}; - - -} // namespace nonlinear -} // namespace interpolation -} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h b/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h deleted file mode 100644 index 90869c1a7..000000000 --- a/src/atlas/interpolation/nonlinear/MissingIfHeaviestMissing.h +++ /dev/null @@ -1,110 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#pragma once - -#include "atlas/field/MissingValue.h" -#include "atlas/interpolation/nonlinear/Missing.h" - -#include "eckit/types/FloatCompare.h" - - -namespace atlas { -namespace interpolation { -namespace nonlinear { - - -template -struct MissingIfHeaviestMissing : Missing { - bool execute( NonLinear::Matrix& W, const Field& field ) const { - field::MissingValue mv( field ); - auto& missingValue = mv.ref(); - - // NOTE only for scalars (for now) - auto values = make_view_field_values( field ); - ATLAS_ASSERT( idx_t( W.cols() ) == values.size() ); - - auto data = const_cast( W.data() ); - bool modif = false; - bool zeros = false; - - Size i = 0; - Matrix::iterator it( W ); - for ( Size r = 0; r < W.rows(); ++r ) { - const Matrix::iterator end = W.end( r ); - - // count missing values, accumulate weights (disregarding missing values) and find maximum weight in row - size_t i_missing = i; - size_t N_missing = 0; - size_t N_entries = 0; - Scalar sum = 0.; - Scalar heaviest = -1.; - bool heaviest_is_missing = false; - - Matrix::iterator kt( it ); - Size k = i; - for ( ; it != end; ++it, ++i, ++N_entries ) { - const bool miss = missingValue( values[it.col()] ); - - if ( miss ) { - ++N_missing; - i_missing = i; - } - else { - sum += *it; - } - - if ( heaviest < data[i] ) { - heaviest = data[i]; - heaviest_is_missing = miss; - } - } - - // weights redistribution: zero-weight all missing values, linear re-weighting for the others; - // if all values are missing, or the closest value is missing, force missing value - if ( N_missing > 0 ) { - if ( N_missing == N_entries || heaviest_is_missing || - eckit::types::is_approximately_equal( sum, 0. ) ) { - for ( Size j = k; j < k + N_entries; ++j ) { - data[j] = j == i_missing ? 1. : 0.; - } - } - else { - const Scalar factor = 1. / sum; - for ( Size j = k; j < k + N_entries; ++j, ++kt ) { - if ( missingValue( values[kt.col()] ) ) { - data[j] = 0.; - zeros = true; - } - else { - data[j] *= factor; - } - } - } - modif = true; - } - } - - if ( zeros && missingValue.isnan() ) { - W.prune( 0. ); - } - - return modif; - } - - static std::string static_type() { return "missing-if-heaviest-missing"; } -}; - - -} // namespace nonlinear -} // namespace interpolation -} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinear.cc b/src/atlas/interpolation/nonlinear/NonLinear.cc new file mode 100644 index 000000000..e23650300 --- /dev/null +++ b/src/atlas/interpolation/nonlinear/NonLinear.cc @@ -0,0 +1,28 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "atlas/interpolation/nonlinear/NonLinear.h" + + +namespace atlas { +namespace interpolation { +namespace nonlinear { + + +const NonLinear* NonLinearFactory::build( const std::string& builder, const NonLinearFactory::Config& config ) { + return get( builder )->make( config ); +} + + +} // namespace nonlinear +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinear.h b/src/atlas/interpolation/nonlinear/NonLinear.h index 11a72fddb..0d92712ba 100644 --- a/src/atlas/interpolation/nonlinear/NonLinear.h +++ b/src/atlas/interpolation/nonlinear/NonLinear.h @@ -12,6 +12,7 @@ #pragma once +#include #include #include "eckit/config/Parametrisation.h" @@ -20,6 +21,7 @@ #include "atlas/array.h" #include "atlas/field/Field.h" #include "atlas/runtime/Exception.h" +#include "atlas/util/Factory.h" #include "atlas/util/ObjectHandle.h" @@ -76,6 +78,29 @@ class NonLinear : public util::Object { }; +class NonLinearFactory : public util::Factory { +public: + using Config = NonLinear::Config; + + static std::string className() { return "NonLinearFactory"; } + static const NonLinear* build( const std::string&, const Config& ); + using Factory::Factory; + +private: + virtual const NonLinear* make( const Config& ) = 0; +}; + + +template +class NonLinearFactoryBuilder : public NonLinearFactory { +private: + virtual const NonLinear* make( const Config& /*config*/ ) override { return new T( /*config*/ ); } + +public: + using NonLinearFactory::NonLinearFactory; +}; + + } // namespace nonlinear } // namespace interpolation } // namespace atlas diff --git a/src/atlas/interpolation/nonlinear/NonLinearFactory.h b/src/atlas/interpolation/nonlinear/NonLinearFactory.h deleted file mode 100644 index 236d3e37c..000000000 --- a/src/atlas/interpolation/nonlinear/NonLinearFactory.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#pragma once - -#include - -#include "eckit/config/Parametrisation.h" - -#include "atlas/util/Factory.h" - - -namespace atlas { -namespace interpolation { -namespace nonlinear { - - -class NonLinear; -using Config = eckit::Parametrisation; - - -class NonLinearFactory : public util::Factory { -public: - static std::string className() { return "NonLinearFactory"; } - static const NonLinear* build( const std::string&, const Config& ); - using Factory::Factory; - -private: - virtual const NonLinear* make( const Config& ) = 0; -}; - - -template -class NonLinearFactoryBuilder : public NonLinearFactory { -private: - virtual const NonLinear* make( const Config& /*config*/ ) override { return new T( /*config*/ ); } - -public: - using NonLinearFactory::NonLinearFactory; -}; - - -} // namespace nonlinear -} // namespace interpolation -} // namespace atlas From cff600ea329a47e4d1d42dd75d570d20c9bfb8ea Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 17 Sep 2020 17:11:23 +0100 Subject: [PATCH 132/161] ATLAS-309 Migrate to ecbuild-3.4 --- CMakeLists.txt | 12 ++--- atlas.sublime-project | 53 ------------------- bamboo/CLANG-env.sh | 2 +- bamboo/GCC-env.sh | 2 +- bamboo/INTEL-env.sh | 2 +- .../atlas-import.cmake.in | 0 cmake/features/EIGEN.cmake | 1 + .../project_summary.cmake | 0 .../installation/installation.tex | 4 +- 9 files changed, 9 insertions(+), 67 deletions(-) delete mode 100644 atlas.sublime-project rename atlas-import.cmake.in => cmake/atlas-import.cmake.in (100%) rename project_summary.cmake => cmake/project_summary.cmake (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79e1054f8..ce5aeb401 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,15 +9,9 @@ ############################################################################################ # Atlas -cmake_minimum_required( VERSION 3.6 FATAL_ERROR ) -if( POLICY CMP0074 ) - cmake_policy( SET CMP0074 NEW ) - # This policy allows to search for packages with _ROOT variables - # (only supported with CMake 3.12 and above) - # This policy can be removed once cmake_minimum_required( VERSION 3.12 ) is used -endif() - -find_package( ecbuild 3.1.0 REQUIRED ) +cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) + +find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR}) ################################################################################ # Initialise project Atlas diff --git a/atlas.sublime-project b/atlas.sublime-project deleted file mode 100644 index 486aa3e56..000000000 --- a/atlas.sublime-project +++ /dev/null @@ -1,53 +0,0 @@ -{ - "SublimeLinter": - { - "linters": - { - "cpplint": - { - "filter": "-whitespace/line_length,-whitespace/blank_line,-runtime/references" - } - } - }, - "build_systems": - [ - { - "file_regex": "(.+[^:]):(\\d+):(\\d+): (?:fatal )?((?:error|warning): .+)$", - "name": "atlas", - "shell_cmd": "make -j4", - "syntax": "Packages/CMakeBuilder/Syntax/Make.sublime-syntax", - "working_dir": "${project_path}/../../build/atlas" - }, - { - "file_regex": "^[ ]*File \"(...*?)\", line ([0-9]*)", - "name": "Anaconda Python Builder", - "selector": "source.python", - "shell_cmd": "\"python\" -u \"$file\"" - } - ], - "folders": - [ - { - "file_exclude_patterns": - [ - ".tags", - ".tags_sorted_by_file", - ".gemtags", - "CMakeLists.txt.user*" - ], - "follow_symlinks": true, - "path": "." - } - ], - "settings": - { - "cmake": - { - "build_folder": "${project_path}/../../build/atlas", - "command_line_overrides": - { - "DEVELOPER_MODE": 1 - } - } - } -} diff --git a/bamboo/CLANG-env.sh b/bamboo/CLANG-env.sh index f16fb80cb..f7765804b 100644 --- a/bamboo/CLANG-env.sh +++ b/bamboo/CLANG-env.sh @@ -20,6 +20,6 @@ module unload libemos module unload metview module unload netcdf4 -module load cmake/3.10.2 +module load cmake/3.16.5 module switch gnu clang diff --git a/bamboo/GCC-env.sh b/bamboo/GCC-env.sh index 8ebb1ad62..3397bd06c 100644 --- a/bamboo/GCC-env.sh +++ b/bamboo/GCC-env.sh @@ -15,5 +15,5 @@ module unload libemos module unload metview module unload netcdf4 -module load cmake/3.10.2 +module load cmake/3.16.5 module load proj/6.1.1 diff --git a/bamboo/INTEL-env.sh b/bamboo/INTEL-env.sh index 3f77166ef..2a6eb1c40 100644 --- a/bamboo/INTEL-env.sh +++ b/bamboo/INTEL-env.sh @@ -15,6 +15,6 @@ module unload libemos module unload metview module unload netcdf4 -module load cmake/3.10.2 +module load cmake/3.16.5 module switch gnu intel/17.0.3 diff --git a/atlas-import.cmake.in b/cmake/atlas-import.cmake.in similarity index 100% rename from atlas-import.cmake.in rename to cmake/atlas-import.cmake.in diff --git a/cmake/features/EIGEN.cmake b/cmake/features/EIGEN.cmake index 35adc22cc..81abf47fe 100644 --- a/cmake/features/EIGEN.cmake +++ b/cmake/features/EIGEN.cmake @@ -1,5 +1,6 @@ ### Eigen +set( Eigen3_NO_MODULE ON ) ecbuild_find_package( NAME Eigen3 VERSION 3.3 QUIET ) ecbuild_add_option( FEATURE EIGEN DESCRIPTION "Use Eigen linear algebra library" diff --git a/project_summary.cmake b/cmake/project_summary.cmake similarity index 100% rename from project_summary.cmake rename to cmake/project_summary.cmake diff --git a/doc/user-guide/getting-started/installation/installation.tex b/doc/user-guide/getting-started/installation/installation.tex index 15aea1bdb..312a4540b 100644 --- a/doc/user-guide/getting-started/installation/installation.tex +++ b/doc/user-guide/getting-started/installation/installation.tex @@ -574,7 +574,7 @@ \subsubsection*{Code compilation} include(ecbuild_system NO_POLICY_SCOPE) ecbuild_requires_macro_version(1.9) ecbuild_declare_project() -ecbuild_use_package(PROJECT atlas REQUIRED) +ecbuild_find_package(NAME atlas REQUIRED) ecbuild_add_executable(TARGET atlas_c-usage_example SOURCES hello-world.cc INCLUDES ${ATLAS_INCLUDE_DIRS} @@ -773,7 +773,7 @@ \subsubsection*{Code compilation} ecbuild_declare_project() ecbuild_enable_fortran(MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module REQUIRED) -ecbuild_use_package(PROJECT atlas REQUIRED) +ecbuild_find_package(NAME atlas REQUIRED) ecbuild_add_executable(TARGET atlas_f-usage_example SOURCES hello-world.F90 INCLUDES ${ATLAS_INCLUDE_DIRS} From 85b3f47bd32e51dcea8a9c4bf86d962c8aea3ed7 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 17 Sep 2020 17:15:49 +0100 Subject: [PATCH 133/161] ATLAS-309 Compatibility with eckit at feature/ecbuild_3.4 -- will require new minimum required eckit version! --- src/apps/atlas-grids.cc | 3 +-- src/atlas/library/Library.cc | 10 +++++----- src/atlas/trans/ifs/TransIFS.cc | 1 - src/atlas/util/Config.cc | 5 +---- src/atlas/util/Metadata.cc | 2 -- src/tests/grid/test_state.cc | 2 -- 6 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/apps/atlas-grids.cc b/src/apps/atlas-grids.cc index 930ef648c..17441040d 100644 --- a/src/apps/atlas-grids.cc +++ b/src/apps/atlas-grids.cc @@ -15,12 +15,11 @@ #include #include -#include "eckit/eckit_version.h" -#include "eckit/log/JSON.h" #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Bytes.h" +#include "eckit/log/JSON.h" #include "eckit/log/Log.h" #include "eckit/types/FloatCompare.h" diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index 13fb622ae..8368aee27 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -409,11 +409,11 @@ void Library::Information::print( std::ostream& out ) const { << " flags : " << ATLAS_C_FLAGS << '\n' << " c++ compiler : " << ATLAS_CXX_COMPILER_ID << " " << ATLAS_CXX_COMPILER_VERSION << '\n' << " flags : " << ATLAS_CXX_FLAGS << '\n' -#ifndef EC_HAVE_FORTRAN - << " fortran : NO " << '\n' -#else +#ifdef ATLAS_Fortran_COMPILER << " fortran compiler: " << ATLAS_Fortran_COMPILER_ID << " " << ATLAS_Fortran_COMPILER_VERSION << '\n' << " flags : " << ATLAS_Fortran_FLAGS << '\n' +#else + << " fortran : NO " << '\n' #endif << " \n"; @@ -426,11 +426,11 @@ void Library::Information::print( std::ostream& out ) const { bool feature_BoundsChecking( ATLAS_ARRAYVIEW_BOUNDS_CHECKING ); bool feature_Init_sNaN( ATLAS_INIT_SNAN ); bool feature_MPI( false ); -#ifdef ECKIT_HAVE_MPI +#if ECKIT_HAVE_MPI feature_MPI = true; #endif bool feature_MKL( false ); -#ifdef ECKIT_HAVE_MKL +#if ECKIT_HAVE_MKL feature_MKL = true; #endif std::string array_data_store = "Native"; diff --git a/src/atlas/trans/ifs/TransIFS.cc b/src/atlas/trans/ifs/TransIFS.cc index 4fb482acc..6e39b2c35 100644 --- a/src/atlas/trans/ifs/TransIFS.cc +++ b/src/atlas/trans/ifs/TransIFS.cc @@ -8,7 +8,6 @@ * nor does it submit to any jurisdiction. */ -#include "eckit/eckit_version.h" #include "eckit/log/JSON.h" #include "transi/trans.h" diff --git a/src/atlas/util/Config.cc b/src/atlas/util/Config.cc index 215ccaf99..ad81ad93e 100644 --- a/src/atlas/util/Config.cc +++ b/src/atlas/util/Config.cc @@ -15,11 +15,8 @@ #include #include - -#include "eckit/eckit_version.h" -#include "eckit/log/JSON.h" - #include "eckit/filesystem/PathName.h" +#include "eckit/log/JSON.h" #include "eckit/parser/YAMLParser.h" #include "atlas/grid/Grid.h" diff --git a/src/atlas/util/Metadata.cc b/src/atlas/util/Metadata.cc index f8e8d6ad4..9a6086b6d 100644 --- a/src/atlas/util/Metadata.cc +++ b/src/atlas/util/Metadata.cc @@ -14,9 +14,7 @@ #include #include -#include "eckit/eckit_version.h" #include "eckit/log/JSON.h" - #include "eckit/parser/JSONParser.h" #include "eckit/utils/Hash.h" diff --git a/src/tests/grid/test_state.cc b/src/tests/grid/test_state.cc index 54e790ee6..7a371a9e2 100644 --- a/src/tests/grid/test_state.cc +++ b/src/tests/grid/test_state.cc @@ -11,9 +11,7 @@ #include #include -#include "eckit/eckit_version.h" #include "eckit/log/JSON.h" - #include "eckit/parser/JSONParser.h" #include "atlas/array/ArrayView.h" From ce9240e87c5f4ed34e6336c5a234e9d18f2abff1 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 29 Sep 2020 09:51:18 +0100 Subject: [PATCH 134/161] Fix compilation with Cray 8.5 --- src/atlas/interpolation/method/knn/GridBox.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/interpolation/method/knn/GridBox.h b/src/atlas/interpolation/method/knn/GridBox.h index 133d27009..c56845ae2 100644 --- a/src/atlas/interpolation/method/knn/GridBox.h +++ b/src/atlas/interpolation/method/knn/GridBox.h @@ -111,7 +111,7 @@ class GridBox { struct GridBoxes : std::vector { GridBoxes( const Grid&, bool gaussianWeightedLatitudes = true ); GridBoxes(); - using vector::vector; + using std::vector::vector; double getLongestGridBoxDiagonal() const; }; From 1a9782a799cc8f552e20f2e195793e0c99d19702 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 29 Sep 2020 09:21:11 +0000 Subject: [PATCH 135/161] Fix compilation with gridtools backend --- src/atlas/interpolation/method/knn/KNearestNeighbours.cc | 2 +- src/atlas/interpolation/method/knn/NearestNeighbour.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc index d8689c539..ac2856d2f 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc @@ -101,7 +101,7 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp } // find the closest input points to the output point - auto nn = pTree_.closestPoints( PointLonLat{lonlat( ip, LON ), lonlat( ip, LAT )}, k_ ); + auto nn = pTree_.closestPoints( PointLonLat{lonlat( ip, size_t(LON) ), lonlat( ip, size_t(LAT) )}, k_ ); // calculate weights (individual and total, to normalise) using distance // squared diff --git a/src/atlas/interpolation/method/knn/NearestNeighbour.cc b/src/atlas/interpolation/method/knn/NearestNeighbour.cc index 3648b31b1..2491d303c 100644 --- a/src/atlas/interpolation/method/knn/NearestNeighbour.cc +++ b/src/atlas/interpolation/method/knn/NearestNeighbour.cc @@ -91,7 +91,7 @@ void NearestNeighbour::do_setup( const FunctionSpace& source, const FunctionSpac } // find the closest input point to the output point - auto nn = pTree_.closestPoint( PointLonLat{lonlat( ip, LON ), lonlat( ip, LAT )} ); + auto nn = pTree_.closestPoint( PointLonLat{lonlat( ip, size_t(LON) ), lonlat( ip, size_t(LAT) )} ); size_t jp = nn.payload(); // insert the weights into the interpolant matrix From 1b92cc1236ea132837459630c8463e6079bcd323 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 5 Oct 2020 11:22:01 +0100 Subject: [PATCH 136/161] Add ecbuild search hint parallel to source dir --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ce5aeb401..b471b37cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) -find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR}) +find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild ) ################################################################################ # Initialise project Atlas From 3940c995ece0413c165025619a4fc4a4da27a4c6 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 5 Oct 2020 12:26:00 +0100 Subject: [PATCH 137/161] Formatting --- .../test_structuredcolumns_biperiodic.cc | 112 +++++++++--------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/src/tests/functionspace/test_structuredcolumns_biperiodic.cc b/src/tests/functionspace/test_structuredcolumns_biperiodic.cc index 51e376c6f..3831eba99 100644 --- a/src/tests/functionspace/test_structuredcolumns_biperiodic.cc +++ b/src/tests/functionspace/test_structuredcolumns_biperiodic.cc @@ -37,75 +37,75 @@ namespace test { //----------------------------------------------------------------------------- -CASE( "biperiodic_latlon") { +CASE( "biperiodic_latlon" ) { + auto& comm = atlas::mpi::comm(); + const int myproc = comm.rank(); - auto & comm = atlas::mpi::comm (); - const int myproc = comm.rank (); + using namespace atlas::util; + using namespace atlas; + using namespace atlas::grid; - using namespace atlas::util; - using namespace atlas; - using namespace atlas::grid; + const int Nx = 80, Ny = 80; + const double xmin = +20, xmax = +60, ymin = +20, ymax = +60; - const int Nx = 80, Ny = 80; - const double xmin = +20, xmax = +60, ymin = +20, ymax = +60; + std::vector spacings( Ny ); - std::vector spacings (Ny); - - for (int i = 0; i < Ny; i++) - spacings[i] = Spacing (Config ("type", "linear") | Config ("N", Nx) - | Config ("start", xmin) | Config ("end", xmax)); - - StructuredGrid::XSpace xspace (spacings); - StructuredGrid::YSpace yspace (Config ("type", "linear") | Config ("N", Ny) | Config ("start", ymin) | Config ("end", ymax)); - Projection proj (Config ("type", "lonlat")); - - atlas::StructuredGrid grid (xspace, yspace, proj, Domain ()); - - atlas::grid::Distribution dist (grid, atlas::util::Config ("type", "checkerboard")); - atlas::functionspace::StructuredColumns fs (grid, dist, atlas::util::Config ("halo", 3) - | atlas::util::Config ("periodic_x", true) - | atlas::util::Config ("periodic_y", true)); - - - auto f = atlas::Field ("f", - atlas::array::DataType::kind (), - atlas::array::make_shape (fs.size ())); + for ( int i = 0; i < Ny; i++ ) { + spacings[i] = + Spacing( Config( "type", "linear" ) | Config( "N", Nx ) | Config( "start", xmin ) | Config( "end", xmax ) ); + } + StructuredGrid::XSpace xspace( spacings ); + StructuredGrid::YSpace yspace( Config( "type", "linear" ) | Config( "N", Ny ) | Config( "start", ymin ) | + Config( "end", ymax ) ); + Projection proj( Config( "type", "lonlat" ) ); - auto v = atlas::array::make_view (f); + atlas::StructuredGrid grid( xspace, yspace, proj, Domain() ); - for (int i = 0; i < f.size (); i++) - v (i) = static_cast (myproc); + atlas::grid::Distribution dist( grid, atlas::util::Config( "type", "checkerboard" ) ); + atlas::functionspace::StructuredColumns fs( grid, dist, + atlas::util::Config( "halo", 3 ) | + atlas::util::Config( "periodic_x", true ) | + atlas::util::Config( "periodic_y", true ) ); - fs.haloExchange (f); - - auto clamp = [] (int i, int n) - { - while (i < 0) - i += n; - while (i >= n) - i -= n; - return i; - }; - - auto gv = atlas::array::make_view( fs.global_index() ); - auto pv = atlas::array::make_view( fs.partition() ); - for (int j = fs.j_begin_halo (); j < fs.j_end_halo (); j++) - for (int i = fs.i_begin_halo (j); i < fs.i_end_halo (j); i++) - { - int k = fs.index (i, j); - int jj = clamp (j, grid.ny ()); - int ii = clamp (i, grid.nx (jj)); + auto f = atlas::Field( "f", atlas::array::DataType::kind(), atlas::array::make_shape( fs.size() ) ); - int g = grid.index (ii, jj); - int p = dist.partition (g); + auto v = atlas::array::make_view( f ); - EXPECT_EQ (v (k), p); - EXPECT_EQ (p, pv (k)); - EXPECT_EQ (gv (k)-1, g); + for ( int i = 0; i < f.size(); i++ ) { + v( i ) = static_cast( myproc ); } + fs.haloExchange( f ); + + auto clamp = []( int i, int n ) { + while ( i < 0 ) { + i += n; + } + while ( i >= n ) { + i -= n; + } + return i; + }; + + auto gv = atlas::array::make_view( fs.global_index() ); + auto pv = atlas::array::make_view( fs.partition() ); + + for ( int j = fs.j_begin_halo(); j < fs.j_end_halo(); j++ ) { + for ( int i = fs.i_begin_halo( j ); i < fs.i_end_halo( j ); i++ ) { + int k = fs.index( i, j ); + int jj = clamp( j, grid.ny() ); + int ii = clamp( i, grid.nx( jj ) ); + + int g = grid.index( ii, jj ); + int p = dist.partition( g ); + + EXPECT_EQ( v( k ), p ); + EXPECT_EQ( p, pv( k ) ); + EXPECT_EQ( gv( k ) - 1, g ); + } + } } //----------------------------------------------------------------------------- From d0c5c236dd1b0ebc9a0fe27779490aff1c3a701c Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 5 Oct 2020 12:43:52 +0100 Subject: [PATCH 138/161] ATLAS-312 Support Array sizes up to size_t limit --- src/atlas/array/Array.h | 2 +- src/atlas/array/gridtools/GridToolsArrayView.h | 6 +++--- src/atlas/array/native/NativeArray.cc | 4 ++-- src/atlas/array/native/NativeArrayView.h | 8 ++++---- src/atlas/field/Field.cc | 2 +- src/atlas/field/Field.h | 2 +- src/atlas/field/detail/FieldImpl.h | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/atlas/array/Array.h b/src/atlas/array/Array.h index 7385137d1..47dcc614f 100644 --- a/src/atlas/array/Array.h +++ b/src/atlas/array/Array.h @@ -72,7 +72,7 @@ class Array : public util::Object { idx_t bytes() const { return sizeof_data() * spec().allocatedSize(); } - idx_t size() const { return spec_.size(); } + size_t size() const { return spec_.size(); } idx_t rank() const { return spec_.rank(); } diff --git a/src/atlas/array/gridtools/GridToolsArrayView.h b/src/atlas/array/gridtools/GridToolsArrayView.h index 9e101eb2c..a61785dc5 100644 --- a/src/atlas/array/gridtools/GridToolsArrayView.h +++ b/src/atlas/array/gridtools/GridToolsArrayView.h @@ -132,10 +132,10 @@ class ArrayView { } static constexpr idx_t rank() { return Rank; } - idx_t size() const { return size_; } + size_t size() const { return size_; } bool valid() const; - bool contiguous() const { return ( size_ == shape_[0] * strides_[0] ? true : false ); } + bool contiguous() const { return ( size_ == size_t( shape_[0] ) * size_t( strides_[0] ) ? true : false ); } void dump( std::ostream& os ) const; @@ -179,7 +179,7 @@ class ArrayView { data_view_t gt_data_view_; idx_t shape_[Rank]; idx_t strides_[Rank]; - idx_t size_; + size_t size_; ArrayDataStore const* data_store_orig_; Array const* array_; bool is_device_view_{false}; diff --git a/src/atlas/array/native/NativeArray.cc b/src/atlas/array/native/NativeArray.cc index 0a4d9d0db..628639217 100644 --- a/src/atlas/array/native/NativeArray.cc +++ b/src/atlas/array/native/NativeArray.cc @@ -137,9 +137,9 @@ ArrayT::ArrayT( idx_t dim0, idx_t dim1, idx_t dim2, idx_t dim3, idx_t dim template ArrayT::ArrayT( const ArrayShape& shape ) { ATLAS_ASSERT( shape.size() > 0 ); - idx_t size = 1; + size_t size = 1; for ( size_t j = 0; j < shape.size(); ++j ) { - size *= shape[j]; + size *= size_t( shape[j] ); } data_store_ = std::unique_ptr( new native::DataStore( size ) ); spec_ = ArraySpec( shape ); diff --git a/src/atlas/array/native/NativeArrayView.h b/src/atlas/array/native/NativeArrayView.h index 4d531b5d0..e676913b9 100644 --- a/src/atlas/array/native/NativeArrayView.h +++ b/src/atlas/array/native/NativeArrayView.h @@ -151,7 +151,7 @@ class ArrayView { for ( int j = 0; j < Rank; ++j ) { shape_[j] = shape[j]; strides_[j] = strides[j]; - size_ *= shape_[j]; + size_ *= size_t( shape_[j] ); } } #endif @@ -227,7 +227,7 @@ class ArrayView { } /// @brief Return total number of values (accumulated over all dimensions) - idx_t size() const { return size_; } + size_t size() const { return size_; } /// @brief Return the number of dimensions static constexpr idx_t rank() { return Rank; } @@ -260,7 +260,7 @@ class ArrayView { /// /// This means that if there is e.g. padding in the fastest dimension, or if /// the ArrayView represents a slice, the returned value will be false. - bool contiguous() const { return ( size_ == shape_[0] * strides_[0] ? true : false ); } + bool contiguous() const { return ( size_ == size_t( shape_[0] ) * size_t( strides_[0] ) ? true : false ); } ENABLE_IF_NON_CONST void assign( const value_type& value ); @@ -365,7 +365,7 @@ class ArrayView { // -- Private data value_type* data_; - idx_t size_; + size_t size_; std::array shape_; std::array strides_; }; diff --git a/src/atlas/field/Field.cc b/src/atlas/field/Field.cc index 1ab3a905f..c9ba7cd53 100644 --- a/src/atlas/field/Field.cc +++ b/src/atlas/field/Field.cc @@ -127,7 +127,7 @@ idx_t Field::stride( idx_t i ) const { } /// @brief Number of values stored in this field -idx_t Field::size() const { +size_t Field::size() const { return get()->size(); } diff --git a/src/atlas/field/Field.h b/src/atlas/field/Field.h index e9f9d3852..40334c8f0 100644 --- a/src/atlas/field/Field.h +++ b/src/atlas/field/Field.h @@ -137,7 +137,7 @@ class Field : DOXYGEN_HIDE( public util::ObjectHandle ) { idx_t stride( idx_t i ) const; /// @brief Number of values stored in this field - idx_t size() const; + size_t size() const; /// @brief Rank of field idx_t rank() const; diff --git a/src/atlas/field/detail/FieldImpl.h b/src/atlas/field/detail/FieldImpl.h index 4ed8ce39d..b67c426cf 100644 --- a/src/atlas/field/detail/FieldImpl.h +++ b/src/atlas/field/detail/FieldImpl.h @@ -130,7 +130,7 @@ class FieldImpl : public util::Object { idx_t stride( idx_t i ) const { return array_->stride( i ); } /// @brief Number of values stored in this field - idx_t size() const { return array_->size(); } + size_t size() const { return array_->size(); } /// @brief Rank of field idx_t rank() const { return array_->rank(); } From 4fe2f55f686af7011dab8ca97468eec2651e12f5 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 10:41:00 +0100 Subject: [PATCH 139/161] Use only 1 MPI task for signal handling logging --- src/atlas/library/FloatingPointExceptions.cc | 54 +++++++++++++++----- src/atlas/library/FloatingPointExceptions.h | 1 + src/atlas/library/Library.cc | 11 +++- 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/atlas/library/FloatingPointExceptions.cc b/src/atlas/library/FloatingPointExceptions.cc index 9202103dd..312444cf3 100644 --- a/src/atlas/library/FloatingPointExceptions.cc +++ b/src/atlas/library/FloatingPointExceptions.cc @@ -20,10 +20,20 @@ #include "eckit/utils/Translator.h" #include "atlas/library/config.h" +#include "atlas/parallel/mpi/mpi.h" #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "atlas/runtime/Trace.h" +namespace { +int getEnv( const std::string& env, int default_value ) { + if ( ::getenv( env.c_str() ) ) { + return eckit::Translator()( ::getenv( env.c_str() ) ); + } + return default_value; +} +} // namespace + static int atlas_feenableexcept( int excepts ) { #if ATLAS_HAVE_FEENABLEEXCEPT return ::feenableexcept( excepts ); @@ -96,7 +106,7 @@ class Signals { // Not sure if this should be made public (in header file) just yet private: - Signals() {} + Signals(); public: static Signals& instance(); @@ -109,6 +119,7 @@ class Signals { private: typedef std::map registered_signals_t; registered_signals_t registered_signals_; + eckit::Channel& out_; }; // ------------------------------------------------------------------------------------ @@ -161,6 +172,15 @@ class Signals { //---------------------------------------------------------------------------------------------------------------------- +Signals::Signals() : + out_( [&]() -> eckit::Channel& { + if ( getEnv( "ATLAS_LOG_RANK", 0 ) == int( mpi::rank() ) ) { + return Log::debug(); + } + static eckit::Channel sink; + return sink; + }() ) {} + Signals& Signals::instance() { static Signals signals; return signals; @@ -168,24 +188,24 @@ Signals& Signals::instance() { void Signals::restoreSignalHandler( int signum ) { if ( registered_signals_.find( signum ) != registered_signals_.end() ) { - Log::debug() << "\n"; + out_ << "\n"; std::signal( signum, SIG_DFL ); - Log::debug() << "Atlas restored default signal handler for signal " << std::setw( 7 ) << std::left - << registered_signals_[signum].code() << " [" << registered_signals_[signum] << "]\n"; - Log::debug() << std::endl; + out_ << "Atlas restored default signal handler for signal " << std::setw( 7 ) << std::left + << registered_signals_[signum].code() << " [" << registered_signals_[signum] << "]\n"; + out_ << std::endl; registered_signals_.erase( signum ); } } void Signals::restoreAllSignalHandlers() { - Log::debug() << "\n"; + out_ << "\n"; for ( registered_signals_t::const_iterator it = registered_signals_.begin(); it != registered_signals_.end(); ++it ) { std::signal( it->first, SIG_DFL ); - Log::debug() << "Atlas restored default signal handler for signal " << std::setw( 7 ) << std::left - << it->second.code() << " [" << it->second.str() << "]\n"; + out_ << "Atlas restored default signal handler for signal " << std::setw( 7 ) << std::left << it->second.code() + << " [" << it->second.str() << "]\n"; } - Log::debug() << std::endl; + out_ << std::endl; registered_signals_.clear(); } @@ -199,7 +219,6 @@ std::ostream& operator<<( std::ostream& out, const Signal& signal ) { } void Signals::setSignalHandlers() { - Log::debug() << "\n"; setSignalHandler( SIGINT ); setSignalHandler( SIGILL ); setSignalHandler( SIGABRT ); @@ -211,8 +230,8 @@ void Signals::setSignalHandlers() { void Signals::setSignalHandler( const Signal& signal ) { registered_signals_[signal] = signal; sigaction( signal, signal.action(), nullptr ); - Log::debug() << "Atlas registered signal handler for signal " << std::setw( 7 ) << std::left << signal.code() - << " [" << signal << "]" << std::endl; + out_ << "Atlas registered signal handler for signal " << std::setw( 7 ) << std::left << signal.code() << " [" + << signal << "]" << std::endl; } @@ -236,7 +255,16 @@ Signal::Signal( int signum, signal_action_t signal_action ) : signum_( signum ), signal_action_.sa_flags = SA_SIGINFO; } + void enable_floating_point_exceptions() { + auto& out = [&]() -> eckit::Channel& { + if ( getEnv( "ATLAS_LOG_RANK", 0 ) == int( mpi::rank() ) ) { + return Log::debug(); + } + static eckit::Channel sink; + return sink; + }(); + // Following line gives runtime errors with Cray 8.6 due to compiler bug (but works with Cray 8.5 and Cray 8.7) // std::vector floating_point_exceptions = eckit::Resource>( "atlasFPE;$ATLAS_FPE", {"false"} ); // Instead, manually access environment @@ -256,7 +284,7 @@ void enable_floating_point_exceptions() { auto enable = [&]( int except ) { _excepts |= except; _enable = true; - Log::debug() << "Atlas enabled floating point exception " << except_to_str[except] << std::endl; + out << "Atlas enabled floating point exception " << except_to_str[except] << std::endl; }; bool skip_map = false; if ( floating_point_exceptions.size() == 1 ) { diff --git a/src/atlas/library/FloatingPointExceptions.h b/src/atlas/library/FloatingPointExceptions.h index 87c97d604..68fccf57f 100644 --- a/src/atlas/library/FloatingPointExceptions.h +++ b/src/atlas/library/FloatingPointExceptions.h @@ -9,6 +9,7 @@ */ #include +#include #include #include diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index 8368aee27..91e88646b 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -230,6 +230,14 @@ void Library::initialise( const eckit::Parametrisation& config ) { warning_channel_.reset(); } + auto& out = [&]() -> eckit::Channel& { + if ( getEnv( "ATLAS_LOG_RANK", 0 ) == int( mpi::rank() ) ) { + return Log::debug(); + } + static eckit::Channel sink; + return sink; + }(); + std::vector plugin_names = eckit::Resource>( "atlasPlugins;$ATLAS_PLUGINS", {} ); @@ -264,7 +272,7 @@ void Library::initialise( const eckit::Parametrisation& config ) { bool library_loaded = eckit::system::Library::exists( library_name ); if ( not library_loaded ) { ATLAS_ASSERT( plugin_library_exists( plugin_dir, library_name, library_path ) ); - Log::debug() << "Loading plugin [" << plugin_name << "] library " << library_path << std::endl; + out << "Loading plugin [" << plugin_name << "] library " << library_path << std::endl; load_library( library_path, library_name ); library_loaded = true; } @@ -285,7 +293,6 @@ void Library::initialise( const eckit::Parametrisation& config ) { // Summary if ( getEnv( "ATLAS_LOG_RANK", 0 ) == int( mpi::rank() ) ) { - std::ostream& out = Log::debug(); out << "Executable [" << Main::instance().name() << "]\n"; out << " \n"; out << " current dir [" << PathName( LocalPathName::cwd() ).fullName() << "]\n"; From 840754658c4c8a48865bff9e9da02908a3a36a54 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 10:53:49 +0100 Subject: [PATCH 140/161] Added "regular" option to Checkerboard partitioner to generate regular subdomains at cost of load balance --- .../detail/partitioner/BandsPartitioner.cc | 2 +- .../partitioner/CheckerboardPartitioner.cc | 50 +++++++++++-------- .../partitioner/CheckerboardPartitioner.h | 3 +- .../partitioner/EqualRegionsPartitioner.cc | 2 +- 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/atlas/grid/detail/partitioner/BandsPartitioner.cc b/src/atlas/grid/detail/partitioner/BandsPartitioner.cc index 498dbed9a..2e47a7ea9 100644 --- a/src/atlas/grid/detail/partitioner/BandsPartitioner.cc +++ b/src/atlas/grid/detail/partitioner/BandsPartitioner.cc @@ -22,7 +22,7 @@ namespace grid { namespace detail { namespace partitioner { -size_t BandsPartitioner::blocksize( const atlas::grid::detail::partitioner::Partitioner::Grid& grid ) const { +size_t BandsPartitioner::blocksize( const Grid& grid ) const { if ( blocksize_ == BLOCKSIZE_NX ) { if ( auto regular = RegularGrid( grid ) ) { return regular.nx(); diff --git a/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.cc b/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.cc index 0d6fac93f..181b3233c 100644 --- a/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.cc +++ b/src/atlas/grid/detail/partitioner/CheckerboardPartitioner.cc @@ -37,6 +37,7 @@ CheckerboardPartitioner::CheckerboardPartitioner( int N ) : Partitioner( N ) {} CheckerboardPartitioner::CheckerboardPartitioner( int N, const eckit::Parametrisation& config ) : Partitioner( N ) { config.get( "bands", nbands_ ); + config.get( "regular", regular_ ); } CheckerboardPartitioner::CheckerboardPartitioner( int N, int nbands ) : Partitioner( N ), nbands_{nbands} {} @@ -114,7 +115,7 @@ void CheckerboardPartitioner::partition( const Checkerboard& cb, int nb_nodes, N size_t nbands = cb.nbands; size_t nx = cb.nx; size_t ny = cb.ny; - size_t remainder; + long remainder; /* Sort nodes from south to north (increasing y), and west to east (increasing x). @@ -137,12 +138,14 @@ Number of procs per band ++npartsb[iband]; } + bool split_lons = not regular_; + bool split_lats = not regular_; /* Number of gridpoints per band */ std::vector ngpb( nbands, 0 ); // split latitudes? - if ( true ) { + if ( split_lats ) { remainder = nb_nodes; for ( size_t iband = 0; iband < nbands; iband++ ) { ngpb[iband] = ( nb_nodes * npartsb[iband] ) / nparts; @@ -165,14 +168,9 @@ Number of gridpoints per band } } - // for (int iband=0;iband ngpp( npartsb[iband], 0 ); remainder = ngpb[iband]; - // std::cout << "remainder = " << remainder << std::endl; + + int part_ny = ngpb[iband] / cb.nx; + int part_nx = ngpb[iband] / npartsb[iband] / part_ny; + for ( size_t ipart = 0; ipart < npartsb[iband]; ipart++ ) { - ngpp[ipart] = ngpb[iband] / npartsb[iband]; + if ( split_lons ) { + ngpp[ipart] = ngpb[iband] / npartsb[iband]; + } + else { + ngpp[ipart] = part_nx * part_ny; + } remainder -= ngpp[ipart]; - // std::cout << "remainder = " << remainder << std::endl; } - // distribute remaining gridpoints over first parts - for ( size_t ipart = 0; ipart < remainder; ipart++ ) { - ++ngpp[ipart]; + if ( split_lons ) { + // distribute remaining gridpoints over first parts + for ( size_t ipart = 0; ipart < remainder; ipart++ ) { + ++ngpp[ipart]; + } + } + else { + size_t ipart = 0; + while ( remainder > part_ny ) { + ngpp[ipart++] += part_ny; + remainder -= part_ny; + } + ngpp[npartsb[iband] - 1] += remainder; } - - /* -std::cout << "ngpb = " << ngpb[iband] << "\n"; -for (int ipart=0;ipart structured_grid.x( 0, 0 ) ); ATLAS_TRACE( "Take shortcut" ); From b067e7aa896bb11811513eb12452d85ff84fbb37 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 10:55:32 +0100 Subject: [PATCH 141/161] Grid can recommend a partitioner or meshgenerator via config --- src/atlas/grid/Grid.cc | 8 ++++++++ src/atlas/grid/Grid.h | 3 +++ src/atlas/grid/detail/grid/Grid.cc | 8 ++++++++ src/atlas/grid/detail/grid/Grid.h | 3 +++ src/atlas/grid/detail/grid/Structured.cc | 19 +++++++++++++++++++ src/atlas/grid/detail/grid/Structured.h | 3 +++ src/atlas/grid/detail/grid/Unstructured.cc | 7 +++++++ src/atlas/grid/detail/grid/Unstructured.h | 3 +++ 8 files changed, 54 insertions(+) diff --git a/src/atlas/grid/Grid.cc b/src/atlas/grid/Grid.cc index 2580f2560..6b19bc597 100644 --- a/src/atlas/grid/Grid.cc +++ b/src/atlas/grid/Grid.cc @@ -85,4 +85,12 @@ Grid::Spec Grid::spec() const { return get()->spec(); } +Grid::Config Grid::meshgenerator() const { + return get()->meshgenerator(); +} + +Grid::Config Grid::partitioner() const { + return get()->partitioner(); +} + } // namespace atlas diff --git a/src/atlas/grid/Grid.h b/src/atlas/grid/Grid.h index 62fb0fb48..224c243a2 100644 --- a/src/atlas/grid/Grid.h +++ b/src/atlas/grid/Grid.h @@ -99,6 +99,9 @@ class Grid : DOXYGEN_HIDE( public util::ObjectHandle ) size_t footprint() const; Spec spec() const; + + Config meshgenerator() const; + Config partitioner() const; }; //--------------------------------------------------------------------------------------------------------------------- diff --git a/src/atlas/grid/detail/grid/Grid.cc b/src/atlas/grid/detail/grid/Grid.cc index 2c132a79e..e4ac07319 100644 --- a/src/atlas/grid/detail/grid/Grid.cc +++ b/src/atlas/grid/detail/grid/Grid.cc @@ -139,6 +139,14 @@ void Grid::detachObserver( GridObserver& observer ) const { grid_observers_.end() ); } +Grid::Config Grid::meshgenerator() const { + ATLAS_NOTIMPLEMENTED; +} + +Grid::Config Grid::partitioner() const { + ATLAS_NOTIMPLEMENTED; +} + idx_t atlas__grid__Grid__size( Grid* This ) { ATLAS_ASSERT( This != nullptr, "Cannot access uninitialised atlas_Grid" ); return This->size(); diff --git a/src/atlas/grid/detail/grid/Grid.h b/src/atlas/grid/detail/grid/Grid.h index d79a76364..3f58312be 100644 --- a/src/atlas/grid/detail/grid/Grid.h +++ b/src/atlas/grid/detail/grid/Grid.h @@ -127,6 +127,9 @@ class Grid : public util::Object { void attachObserver( GridObserver& ) const; void detachObserver( GridObserver& ) const; + virtual Config meshgenerator() const; + virtual Config partitioner() const; + protected: // methods /// Fill provided me virtual void print( std::ostream& ) const = 0; diff --git a/src/atlas/grid/detail/grid/Structured.cc b/src/atlas/grid/detail/grid/Structured.cc index 52fcc25b6..b42863a73 100644 --- a/src/atlas/grid/detail/grid/Structured.cc +++ b/src/atlas/grid/detail/grid/Structured.cc @@ -24,6 +24,7 @@ #include "atlas/grid/detail/grid/GridFactory.h" #include "atlas/grid/detail/spacing/CustomSpacing.h" #include "atlas/grid/detail/spacing/LinearSpacing.h" +#include "atlas/parallel/mpi/mpi.h" #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" #include "atlas/runtime/Trace.h" @@ -649,6 +650,24 @@ std::string Structured::type() const { return static_type(); } +Grid::Config Structured::meshgenerator() const { + return Config( "type", "structured" ); +} + +Grid::Config Structured::partitioner() const { + Config config; + if ( mpi::size() == 1 ) { + config.set( "type", "serial" ); + } + else if ( reduced() ) { + config.set( "type", "equal_regions" ); + } + else { + config.set( "type", "checkerboard" ); + } + return config; +} + void Structured::hash( eckit::Hash& h ) const { double multiplier = projection().units() == "meters" ? 1e2 : 1e8; auto add_double = [&]( const double& x ) { h.add( std::round( x * multiplier ) ); }; diff --git a/src/atlas/grid/detail/grid/Structured.h b/src/atlas/grid/detail/grid/Structured.h index b5c5431a7..9e95c2809 100644 --- a/src/atlas/grid/detail/grid/Structured.h +++ b/src/atlas/grid/detail/grid/Structured.h @@ -370,6 +370,9 @@ class Structured : public Grid { j = ja; } + Config meshgenerator() const override; + Config partitioner() const override; + protected: // methods virtual void print( std::ostream& ) const override; diff --git a/src/atlas/grid/detail/grid/Unstructured.cc b/src/atlas/grid/detail/grid/Unstructured.cc index 643ced53a..0377e25e3 100644 --- a/src/atlas/grid/detail/grid/Unstructured.cc +++ b/src/atlas/grid/detail/grid/Unstructured.cc @@ -220,6 +220,13 @@ Grid::Spec Unstructured::spec() const { return *cached_spec_; } +Grid::Config Unstructured::meshgenerator() const { + return Config( "type", "delaunay" ); +} +Grid::Config Unstructured::partitioner() const { + return Config( "type", "serial" ); +} + void Unstructured::print( std::ostream& os ) const { os << "Unstructured(Npts:" << size() << ")"; } diff --git a/src/atlas/grid/detail/grid/Unstructured.h b/src/atlas/grid/detail/grid/Unstructured.h index 160e0dd25..f7879ade4 100644 --- a/src/atlas/grid/detail/grid/Unstructured.h +++ b/src/atlas/grid/detail/grid/Unstructured.h @@ -187,6 +187,9 @@ class Unstructured : public Grid { return std::unique_ptr( new IteratorLonLat( *this, false ) ); } + Config meshgenerator() const override; + Config partitioner() const override; + private: // methods virtual void print( std::ostream& ) const override; From 212b2c76231ffc362561354c2931f5d6a45cec2b Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 11:08:58 +0100 Subject: [PATCH 142/161] Polygon can return lonlat points in addition to xy points --- src/atlas/grid/StructuredPartitionPolygon.cc | 4 +++ src/atlas/grid/StructuredPartitionPolygon.h | 1 + src/atlas/mesh/PartitionPolygon.cc | 32 +++++++++++++++----- src/atlas/mesh/PartitionPolygon.h | 1 + src/atlas/util/Polygon.h | 4 +++ 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/atlas/grid/StructuredPartitionPolygon.cc b/src/atlas/grid/StructuredPartitionPolygon.cc index 5d719be6c..88ed89250 100644 --- a/src/atlas/grid/StructuredPartitionPolygon.cc +++ b/src/atlas/grid/StructuredPartitionPolygon.cc @@ -414,6 +414,10 @@ util::PartitionPolygon::PointsXY StructuredPartitionPolygon::xy() const { return points_; } +util::PartitionPolygon::PointsXY StructuredPartitionPolygon::lonlat() const { + return points_; +} + void StructuredPartitionPolygon::allGather( util::PartitionPolygons& polygons_ ) const { ATLAS_TRACE(); diff --git a/src/atlas/grid/StructuredPartitionPolygon.h b/src/atlas/grid/StructuredPartitionPolygon.h index 9fb29e286..718a586fb 100644 --- a/src/atlas/grid/StructuredPartitionPolygon.h +++ b/src/atlas/grid/StructuredPartitionPolygon.h @@ -44,6 +44,7 @@ class StructuredPartitionPolygon : public util::PartitionPolygon { void outputPythonScript( const eckit::PathName&, const eckit::Configuration& = util::NoConfig() ) const override; PointsXY xy() const override; + PointsLonLat lonlat() const override; const RectangularDomain& inscribedDomain() const override { return inscribed_domain_; } diff --git a/src/atlas/mesh/PartitionPolygon.cc b/src/atlas/mesh/PartitionPolygon.cc index aacd1d957..fe7a04064 100644 --- a/src/atlas/mesh/PartitionPolygon.cc +++ b/src/atlas/mesh/PartitionPolygon.cc @@ -250,11 +250,13 @@ void PartitionPolygon::print( std::ostream& out ) const { << "halo:" << halo_ << ",size:" << size() << ",nodes:" << static_cast( *this ) << "}"; } -PartitionPolygon::PointsXY PartitionPolygon::xy() const { - PointsXY points_xy; - points_xy.reserve( size() ); +namespace { +PartitionPolygon::PointsXY points( const Mesh::Implementation& mesh_, const Field& coords, + const std::vector& polygon ) { + PartitionPolygon::PointsXY points_xy; + points_xy.reserve( polygon.size() ); - auto xy_view = array::make_view( mesh_.nodes().xy() ); + auto xy_view = array::make_view( coords ); auto flags = array::make_view( mesh_.nodes().flags() ); bool domain_includes_north_pole = false; @@ -280,12 +282,28 @@ PartitionPolygon::PointsXY PartitionPolygon::xy() const { return false; }; - for ( idx_t i : static_cast( *this ) ) { - double y = bc_north( i ) ? 90. : bc_south( i ) ? -90. : xy_view( i, idx_t( YY ) ); - points_xy.emplace_back( xy_view( i, idx_t( XX ) ), y ); + if ( coords.name() == "xy" ) { + for ( idx_t i : polygon ) { + double y = bc_north( i ) ? 90. : bc_south( i ) ? -90. : xy_view( i, idx_t( YY ) ); + points_xy.emplace_back( xy_view( i, idx_t( XX ) ), y ); + } + } + else { + for ( idx_t i : polygon ) { + points_xy.emplace_back( xy_view( i, idx_t( XX ) ), xy_view( i, idx_t( YY ) ) ); + } } return points_xy; } +} // namespace + +PartitionPolygon::PointsXY PartitionPolygon::xy() const { + return points( mesh_, mesh_.nodes().xy(), *this ); +} + +PartitionPolygon::PointsLonLat PartitionPolygon::lonlat() const { + return points( mesh_, mesh_.nodes().lonlat(), *this ); +} } // namespace mesh } // namespace atlas diff --git a/src/atlas/mesh/PartitionPolygon.h b/src/atlas/mesh/PartitionPolygon.h index 14b32b538..37e0ac4bf 100644 --- a/src/atlas/mesh/PartitionPolygon.h +++ b/src/atlas/mesh/PartitionPolygon.h @@ -55,6 +55,7 @@ class PartitionPolygon : public util::PartitionPolygon { void outputPythonScript( const eckit::PathName&, const eckit::Configuration& = util::NoConfig() ) const override; PointsXY xy() const override; + PointsXY lonlat() const override; void allGather( util::PartitionPolygons& ) const override; diff --git a/src/atlas/util/Polygon.h b/src/atlas/util/Polygon.h index c01d7de8b..d2be76a83 100644 --- a/src/atlas/util/Polygon.h +++ b/src/atlas/util/Polygon.h @@ -116,6 +116,9 @@ class PartitionPolygon : public Polygon, util::Object { /// @brief All (x,y) coordinates defining a polygon. Last point should match first. virtual PointsXY xy() const = 0; + /// @brief All (lon,lat) coordinates defining a polygon. Last point should match first. + virtual PointsLonLat lonlat() const = 0; + virtual void allGather( PartitionPolygons& ) const = 0; }; @@ -132,6 +135,7 @@ class ExplicitPartitionPolygon : public util::PartitionPolygon { } PointsXY xy() const override { return points_; } + PointsLonLat lonlat() const override { return points_; } void allGather( util::PartitionPolygons& ) const override; From 0c1dd26af83a2e705561756b7c026280edd1d66a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 11:11:52 +0100 Subject: [PATCH 143/161] Added function `string MeshGenerator::type()` --- src/atlas/meshgenerator/MeshGenerator.cc | 4 ++++ src/atlas/meshgenerator/MeshGenerator.h | 2 ++ src/atlas/meshgenerator/detail/DelaunayMeshGenerator.h | 2 ++ src/atlas/meshgenerator/detail/MeshGeneratorImpl.h | 2 ++ src/atlas/meshgenerator/detail/RegularMeshGenerator.h | 2 ++ src/atlas/meshgenerator/detail/StructuredMeshGenerator.h | 2 ++ 6 files changed, 14 insertions(+) diff --git a/src/atlas/meshgenerator/MeshGenerator.cc b/src/atlas/meshgenerator/MeshGenerator.cc index 6288b086c..d293056ff 100644 --- a/src/atlas/meshgenerator/MeshGenerator.cc +++ b/src/atlas/meshgenerator/MeshGenerator.cc @@ -57,6 +57,10 @@ Mesh MeshGenerator::operator()( const Grid& g ) const { return get()->operator()( g ); } +std::string MeshGenerator::type() const { + return get()->type(); +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace atlas diff --git a/src/atlas/meshgenerator/MeshGenerator.h b/src/atlas/meshgenerator/MeshGenerator.h index a0c552b0a..fabb8345d 100644 --- a/src/atlas/meshgenerator/MeshGenerator.h +++ b/src/atlas/meshgenerator/MeshGenerator.h @@ -56,6 +56,8 @@ class MeshGenerator : DOXYGEN_HIDE( public util::ObjectHandle Date: Wed, 7 Oct 2020 11:32:54 +0100 Subject: [PATCH 144/161] Added new function `MeshGenerator::generate(Mesh,Partitioner)` --- src/atlas/meshgenerator/MeshGenerator.cc | 8 ++++++-- src/atlas/meshgenerator/MeshGenerator.h | 3 +++ .../meshgenerator/detail/MeshGeneratorImpl.cc | 16 ++++++++++++++++ .../meshgenerator/detail/MeshGeneratorImpl.h | 4 ++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/atlas/meshgenerator/MeshGenerator.cc b/src/atlas/meshgenerator/MeshGenerator.cc index d293056ff..d6734044f 100644 --- a/src/atlas/meshgenerator/MeshGenerator.cc +++ b/src/atlas/meshgenerator/MeshGenerator.cc @@ -44,7 +44,9 @@ void MeshGenerator::hash( eckit::Hash& h ) const { Mesh MeshGenerator::generate( const Grid& g, const grid::Distribution& d ) const { return get()->generate( g, d ); } - +Mesh MeshGenerator::generate( const Grid& g, const grid::Partitioner& p ) const { + return get()->generate( g, p ); +} Mesh MeshGenerator::generate( const Grid& g ) const { return get()->generate( g ); } @@ -52,7 +54,9 @@ Mesh MeshGenerator::generate( const Grid& g ) const { Mesh MeshGenerator::operator()( const Grid& g, const grid::Distribution& d ) const { return get()->operator()( g, d ); } - +Mesh MeshGenerator::operator()( const Grid& g, const grid::Partitioner& p ) const { + return get()->operator()( g, p ); +} Mesh MeshGenerator::operator()( const Grid& g ) const { return get()->operator()( g ); } diff --git a/src/atlas/meshgenerator/MeshGenerator.h b/src/atlas/meshgenerator/MeshGenerator.h index fabb8345d..df1684b2f 100644 --- a/src/atlas/meshgenerator/MeshGenerator.h +++ b/src/atlas/meshgenerator/MeshGenerator.h @@ -30,6 +30,7 @@ class Projection; namespace atlas { namespace grid { class Distribution; +class Partitioner; } // namespace grid } // namespace atlas @@ -52,9 +53,11 @@ class MeshGenerator : DOXYGEN_HIDE( public util::ObjectHandle Date: Wed, 7 Oct 2020 11:40:07 +0100 Subject: [PATCH 145/161] Added constructor `Mesh::Mesh( const Grid& )` This uses Grid's recommended MeshGenerator and Partitioner. --- src/atlas/mesh/Mesh.cc | 13 +++++++++++++ src/atlas/mesh/Mesh.h | 10 +++++++--- src/tests/mesh/test_meshgen3d.cc | 6 ++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/atlas/mesh/Mesh.cc b/src/atlas/mesh/Mesh.cc index ecac60573..8f51ded42 100644 --- a/src/atlas/mesh/Mesh.cc +++ b/src/atlas/mesh/Mesh.cc @@ -9,6 +9,9 @@ */ #include "atlas/mesh/Mesh.h" +#include "atlas/grid/Grid.h" +#include "atlas/grid/Partitioner.h" +#include "atlas/meshgenerator/MeshGenerator.h" namespace atlas { @@ -16,6 +19,16 @@ namespace atlas { Mesh::Mesh() : Handle( new Implementation() ) {} +Mesh::Mesh( const Grid& grid ) : + Handle( [&]() { + auto meshgenerator = MeshGenerator{grid.meshgenerator()}; + auto mesh = meshgenerator.generate( grid, grid::Partitioner( grid.partitioner() ) ); + mesh.get()->attach(); + return mesh.get(); + }() ) { + get()->detach(); +} + Mesh::Mesh( eckit::Stream& stream ) : Handle( new Implementation( stream ) ) {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/atlas/mesh/Mesh.h b/src/atlas/mesh/Mesh.h index 7cabf3a00..20417d7b5 100644 --- a/src/atlas/mesh/Mesh.h +++ b/src/atlas/mesh/Mesh.h @@ -21,7 +21,8 @@ namespace atlas { class Projection; -} +class Grid; +} // namespace atlas namespace atlas { namespace util { @@ -65,14 +66,15 @@ class Mesh : DOXYGEN_HIDE( public util::ObjectHandle ) { using Handle::Handle; Mesh(); + /// @brief Generate a mesh from a Grid with recommended mesh generator and partitioner strategy + Mesh( const Grid& ); + /// @brief Construct a mesh from a Stream (serialization) explicit Mesh( eckit::Stream& ); /// @brief Serialization to Stream void encode( eckit::Stream& s ) const { return get()->encode( s ); } - void print( std::ostream& out ) const { get()->print( out ); } - const util::Metadata& metadata() const { return get()->metadata(); } util::Metadata& metadata() { return get()->metadata(); } @@ -124,6 +126,8 @@ class Mesh : DOXYGEN_HIDE( public util::ObjectHandle ) { const Grid grid() const { return get()->grid(); } private: // methods + void print( std::ostream& out ) const { get()->print( out ); } + friend std::ostream& operator<<( std::ostream& s, const Mesh& p ) { p.print( s ); return s; diff --git a/src/tests/mesh/test_meshgen3d.cc b/src/tests/mesh/test_meshgen3d.cc index d3a3dfec6..51e40dd9b 100644 --- a/src/tests/mesh/test_meshgen3d.cc +++ b/src/tests/mesh/test_meshgen3d.cc @@ -25,6 +25,12 @@ namespace test { //----------------------------------------------------------------------------- +CASE( "test_create_mesh_simple" ) { + auto mesh = Mesh{Grid{"O32"}}; + Gmsh{"O32.msh"}.write( mesh ); +} + + CASE( "test_create_mesh" ) { Mesh m; From afeaeda838763bd25af185d3aef28d9fd79c9ebf Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 11:44:08 +0100 Subject: [PATCH 146/161] Tool atlas-meshgen now uses by default the Grid's recommended MeshGenerator and Partitioner --- src/apps/atlas-meshgen.cc | 95 ++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/src/apps/atlas-meshgen.cc b/src/apps/atlas-meshgen.cc index c056e9d7e..cd790b46f 100644 --- a/src/apps/atlas-meshgen.cc +++ b/src/apps/atlas-meshgen.cc @@ -56,6 +56,35 @@ using eckit::PathName; //------------------------------------------------------------------------------ +MeshGenerator make_meshgenerator( const Grid& grid, const AtlasTool::Args& args ) { + auto config = grid.meshgenerator(); // recommended by the grid itself + if ( args.has( "generator" ) ) { + config.set( "type", args.getString( "generator" ) ); + } + + if ( mpi::comm().size() > 1 || args.getBool( "edges", false ) ) { + config.set( "3d", false ); + } + + config.set( "patch_pole", args.getBool( "patch_pole", false ) ); + + return MeshGenerator{config}; +} + +Partitioner make_partitioner( const Grid& grid, const AtlasTool::Args& args ) { + auto config = grid.partitioner(); // recommended by the grid itself + if ( args.has( "partitioner" ) ) { + config.set( "type", args.getString( "partitioner" ) ); + } + if ( args.has( "regular" ) ) { + config.set( "regular", args.getBool( "regular" ) ); + } + + return Partitioner{config}; +} + +//------------------------------------------------------------------------------ + class Meshgen2Gmsh : public AtlasTool { int execute( const Args& args ) override; std::string briefDescription() override { return "Mesh generator and output to Gmsh format"; } @@ -81,7 +110,6 @@ class Meshgen2Gmsh : public AtlasTool { bool stats; bool info; bool ghost; - bool binary; std::string identifier; PathName path_in; PathName path_out; @@ -90,38 +118,39 @@ class Meshgen2Gmsh : public AtlasTool { //----------------------------------------------------------------------------- Meshgen2Gmsh::Meshgen2Gmsh( int argc, char** argv ) : AtlasTool( argc, argv ) { + add_option( new SimpleOption( "lonlat", "Output mesh in lon-lat coordinates" ) ); + add_option( new SimpleOption( "3d", + "Output mesh as sphere, and generate " + "mesh connecting East and West in " + "case serial" ) ); + add_option( new SimpleOption( "ghost", "Output ghost elements" ) ); add_option( new SimpleOption( - "grid.name", "Grid unique identifier\n" + indent() + " Example values: N80, F40, O24, L32" ) ); - add_option( new SimpleOption( "grid.json", "Grid described by json file" ) ); - add_option( new SimpleOption( "angle", "Maximum element-edge slant deviation from meridian in degrees. \n" + - indent() + " Value range between 0 and 30\n" + indent() + - " 0: Mostly triangular, with only perfect quads\n" + - indent() + - " 30: Mostly skewed quads with only triags when " - "skewness becomes too large\n" + - indent() + " -1: Only triangles" ) ); + "generator", "Mesh generator [structured,regular,delaunay] (default = structured)" ) ); + add_option( new SimpleOption( + "partitioner", "Mesh partitioner [equal_regions,checkerboard,equal_bands,regular_bands" ) ); + add_option( new Separator( "Options for `--generator=structured`" ) ); add_option( new SimpleOption( "include_pole", "Include pole point" ) ); add_option( new SimpleOption( "patch_pole", "Patch poles with elements." ) ); - add_option( new SimpleOption( "ghost", "Output ghost elements" ) ); + add_option( new SimpleOption( + "angle", "Maximum element-edge slant deviation from meridian in degrees. \n" + indent() + + " Value range between 0 and 30\n" + indent() + + " 0: Mostly triangular, with only perfect quads (=default)\n" + indent() + + " 30: Mostly skewed quads with only triags when skewness becomes too large\n" + indent() + + " -1: Only triangles" ) ); + + add_option( new Separator( "Options for `--partitioner=checkerboard`" ) ); + add_option( new SimpleOption( "regular", "regular checkerboard partitioner" ) ); + add_option( new Separator( "Advanced" ) ); add_option( new SimpleOption( "halo", "Halo size" ) ); add_option( new SimpleOption( "edges", "Build edge datastructure" ) ); add_option( new SimpleOption( "brick", "Build brick dual mesh" ) ); add_option( new SimpleOption( "stats", "Write statistics file" ) ); add_option( new SimpleOption( "info", "Write Info" ) ); - add_option( new SimpleOption( "binary", "Write binary file" ) ); - add_option( new SimpleOption( - "generator", "Mesh generator [structured,regular,delaunay] (default = structured)" ) ); - add_option( new SimpleOption( "partitioner", "Mesh partitioner" ) ); add_option( new SimpleOption( "periodic_x", "periodic mesh in x-direction" ) ); add_option( new SimpleOption( "periodic_y", "periodic mesh in y-direction" ) ); add_option( new SimpleOption( "torus", "Output mesh as torus" ) ); - add_option( new SimpleOption( "lonlat", "Output mesh in lon-lat coordinates" ) ); - add_option( new SimpleOption( "3d", - "Output mesh as sphere, and generate " - "mesh connecting East and West in " - "case serial" ) ); } //----------------------------------------------------------------------------- @@ -173,13 +202,6 @@ int Meshgen2Gmsh::execute( const Args& args ) { args.get( "brick", brick ); ghost = false; args.get( "ghost", ghost ); - binary = false; - args.get( "binary", binary ); - - std::string path_in_str = ""; - if ( args.get( "grid.json", path_in_str ) ) { - path_in = path_in_str; - } key = get_positional_arg( args, 0 ); path_out = get_arg( args, "-o", args.count() > 1 ? get_positional_arg( args, 1 ) : "mesh.msh" ); @@ -231,22 +253,13 @@ int Meshgen2Gmsh::execute( const Args& args ) { } Log::debug() << "Spec: " << grid.spec() << std::endl; - std::string generator = ( StructuredGrid( grid ) ? "structured" : "delaunay" ); - args.get( "generator", generator ); - eckit::LocalConfiguration meshgenerator_config( args ); - if ( mpi::comm().size() > 1 || edges ) { - meshgenerator_config.set( "3d", false ); - } - bool patch_pole = false; - args.get( "patch_pole", patch_pole ); - meshgenerator_config.set( "patch_pole", patch_pole ); - - MeshGenerator meshgenerator( generator, meshgenerator_config ); + auto meshgenerator = make_meshgenerator( grid, args ); + auto partitioner = make_partitioner( grid, args ); Mesh mesh; try { - Log::info() << "Generating mesh using " << generator << " generator" << std::endl; - mesh = meshgenerator.generate( grid ); + Log::info() << "Generating mesh using " << meshgenerator.type() << " generator" << std::endl; + mesh = meshgenerator.generate( grid, partitioner ); } catch ( eckit::Exception& e ) { Log::error() << e.what() << std::endl; @@ -295,7 +308,7 @@ int Meshgen2Gmsh::execute( const Args& args ) { atlas::output::Gmsh gmsh( path_out, Config( "info", info )( "ghost", ghost )( "coordinates", dim_3d ? "xyz" : lonlat ? "lonlat" : "xy" )( - "edges", edges )( "binary", binary ) ); + "edges", edges ) ); Log::info() << "Writing mesh to gmsh file \"" << path_out << "\" generated from grid \"" << grid.name() << "\"" << std::endl; gmsh.write( mesh ); From f39d52b58537807e4eafb1792fdb8a30c0dad9d7 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 11:45:15 +0100 Subject: [PATCH 147/161] Support mesh::PartitionPolygon::outputPythonScript to output polygon in "xy" or "lonlat" coordinates (default="xy") --- src/apps/atlas-meshgen.cc | 5 ++-- src/atlas/mesh/PartitionPolygon.cc | 42 ++++++++++++++++++------------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/apps/atlas-meshgen.cc b/src/apps/atlas-meshgen.cc index cd790b46f..16fdbe757 100644 --- a/src/apps/atlas-meshgen.cc +++ b/src/apps/atlas-meshgen.cc @@ -317,8 +317,9 @@ int Meshgen2Gmsh::execute( const Args& args ) { Log::info() << "Partitioning graph: \n" << mesh.partitionGraph() << std::endl; Log::info() << "Mesh partition footprint: " << eckit::Bytes( mesh.footprint() ) << std::endl; for ( idx_t jhalo = 0; jhalo <= halo; ++jhalo ) { - mesh.polygon( jhalo ).outputPythonScript( "polygon_halo" + std::to_string( jhalo ) + ".py", - Config( "nodes", false ) ); + mesh.polygon( jhalo ).outputPythonScript( + "polygon_halo" + std::to_string( jhalo ) + ".py", + Config( "nodes", false )( "coordinates", dim_3d ? "xy" : lonlat ? "lonlat" : "xy" ) ); } } return success(); diff --git a/src/atlas/mesh/PartitionPolygon.cc b/src/atlas/mesh/PartitionPolygon.cc index fe7a04064..f2b21e587 100644 --- a/src/atlas/mesh/PartitionPolygon.cc +++ b/src/atlas/mesh/PartitionPolygon.cc @@ -9,6 +9,7 @@ */ #include +#include #include "atlas/array/MakeView.h" #include "atlas/field/Field.h" @@ -70,9 +71,11 @@ void PartitionPolygon::outputPythonScript( const eckit::PathName& filepath, cons int mpi_rank = int( comm.rank() ); int mpi_size = int( comm.size() ); - auto points = this->xy(); + std::string coordinates = config.getString( "coordinates", "xy" ); + auto points = coordinates == "xy" ? this->xy() : this->lonlat(); + ATLAS_ASSERT( points.size() == size() ); - const auto nodes_xy = array::make_view( mesh_.nodes().xy() ); + const auto nodes_xy = array::make_view( mesh_.nodes().field( coordinates ) ); double xmin = std::numeric_limits::max(); double xmax = -std::numeric_limits::max(); for ( const auto& p : points ) { @@ -187,20 +190,27 @@ void PartitionPolygon::outputPythonScript( const eckit::PathName& filepath, cons f << "\n" ""; if ( mpi_rank == mpi_size - 1 ) { - f << "\n" - "ax.set_xlim( " - << xmin << "-5, " << xmax - << "+5)" - "\n" - "ax.set_ylim(-90-5, 90+5)" - "\n" - "ax.set_xticks([0,45,90,135,180,225,270,315,360])" - "\n" - "ax.set_yticks([-90,-45,0,45,90])" - "\n" - "plt.grid()" - "\n" - "plt.show()"; + auto xticks = [&]() -> std::string { + std::vector xticks; + xticks.push_back( std::floor( xmin - int( xmin ) % 45 ) ); + while ( xticks.back() < int( std::round( xmax ) ) ) { + xticks.push_back( xticks.back() + 45 ); + } + std::stringstream xticks_ss; + for ( int i = 0; i < xticks.size(); ++i ) { + xticks_ss << ( i == 0 ? "[" : "," ) << xticks[i]; + } + xticks_ss << "]"; + return xticks_ss.str(); + }; + // clang-format off + f << "\n" "ax.set_xlim( " << xmin << "-5, " << xmax << "+5)" + "\n" "ax.set_ylim(-90-5, 90+5)" + "\n" "ax.set_xticks(" << xticks() << ")" + "\n" "ax.set_yticks([-90,-45,0,45,90])" + "\n" "plt.grid()" + "\n" "plt.show()"; + // clang-format on } } comm.barrier(); From c19b42ec33db1da4a8f34584500c8eefdd968337 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 5 Oct 2020 14:23:00 +0100 Subject: [PATCH 148/161] Gmsh output of idx_t coordinate fields --- src/apps/atlas-meshgen.cc | 11 ++++++----- src/atlas/output/detail/GmshIO.cc | 24 +++++++++++++++++++----- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/apps/atlas-meshgen.cc b/src/apps/atlas-meshgen.cc index 16fdbe757..9b463eec4 100644 --- a/src/apps/atlas-meshgen.cc +++ b/src/apps/atlas-meshgen.cc @@ -118,7 +118,8 @@ class Meshgen2Gmsh : public AtlasTool { //----------------------------------------------------------------------------- Meshgen2Gmsh::Meshgen2Gmsh( int argc, char** argv ) : AtlasTool( argc, argv ) { - add_option( new SimpleOption( "lonlat", "Output mesh in lon-lat coordinates" ) ); + add_option( new SimpleOption( "lonlat", "Output mesh in lon,lat coordinates" ) ); + add_option( new SimpleOption( "ij", "Output mesh in i,j coordinates" ) ); add_option( new SimpleOption( "3d", "Output mesh as sphere, and generate " "mesh connecting East and West in " @@ -303,12 +304,12 @@ int Meshgen2Gmsh::execute( const Args& args ) { } } - bool lonlat = false; - args.get( "lonlat", lonlat ); + bool lonlat = args.getBool( "lonlat", false ); + bool ij = args.getBool( "ij", false ); + std::string coordinates = dim_3d ? "xyz" : lonlat ? "lonlat" : ij ? "ij" : "xy"; atlas::output::Gmsh gmsh( - path_out, Config( "info", info )( "ghost", ghost )( "coordinates", dim_3d ? "xyz" : lonlat ? "lonlat" : "xy" )( - "edges", edges ) ); + path_out, Config( "info", info )( "ghost", ghost )( "coordinates", coordinates )( "edges", edges ) ); Log::info() << "Writing mesh to gmsh file \"" << path_out << "\" generated from grid \"" << grid.name() << "\"" << std::endl; gmsh.write( mesh ); diff --git a/src/atlas/output/detail/GmshIO.cc b/src/atlas/output/detail/GmshIO.cc index 3ca17bdee..6a9ed527e 100644 --- a/src/atlas/output/detail/GmshIO.cc +++ b/src/atlas/output/detail/GmshIO.cc @@ -815,10 +815,17 @@ void GmshIO::write( const Mesh& mesh, const PathName& file_path ) const { std::string nodes_field = options.get( "nodes" ); const mesh::Nodes& nodes = mesh.nodes(); - auto coords = array::make_view( nodes.field( nodes_field ) ); - auto glb_idx = array::make_view( nodes.global_index() ); - const idx_t surfdim = coords.shape( 1 ); // nb of variables in coords + const Field coords_field = nodes.field( nodes_field ); + array::ArrayT dummy_double( 1, 1 ); + array::ArrayT dummy_idx( 1, 1 ); + bool coords_is_idx = coords_field.datatype().kind() == array::make_datatype().kind(); + auto coords = array::make_view( coords_is_idx ? dummy_double : coords_field.array() ); + auto coords_idx = array::make_view( coords_is_idx ? coords_field.array() : dummy_idx ); + + auto glb_idx = array::make_view( nodes.global_index() ); + + const idx_t surfdim = nodes.field( nodes_field ).shape( 1 ); // nb of variables in coords bool include_patch = ( surfdim == 3 ); @@ -851,8 +858,15 @@ void GmshIO::write( const Mesh& mesh, const PathName& file_path ) const { for ( idx_t n = 0; n < nb_nodes; ++n ) { gidx_t g = glb_idx( n ); - for ( idx_t d = 0; d < surfdim; ++d ) { - xyz[d] = coords( n, d ); + if ( coords_is_idx ) { + for ( idx_t d = 0; d < surfdim; ++d ) { + xyz[d] = coords_idx( n, d ); + } + } + else { + for ( idx_t d = 0; d < surfdim; ++d ) { + xyz[d] = coords( n, d ); + } } if ( binary ) { From 7a287f8c8b787c5f7dd24004e38e97579e1e5dfb Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 09:42:32 +0100 Subject: [PATCH 149/161] atlas-grids outputs footprint of grid --- src/apps/atlas-grids.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/apps/atlas-grids.cc b/src/apps/atlas-grids.cc index 17441040d..8d48429c9 100644 --- a/src/apps/atlas-grids.cc +++ b/src/apps/atlas-grids.cc @@ -195,6 +195,8 @@ int AtlasGrids::execute( const Args& args ) { } Log::info() << " number of points: " << grid.size() << std::endl; + Log::info() << " memory footprint of grid: " << eckit::Bytes( grid.footprint() ) << std::endl; + size_t memsize = grid.size() * sizeof( double ); From e116dc7ae7d6fcbfa2583073cddf774f8fa5428f Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 10:04:25 +0100 Subject: [PATCH 150/161] Move Topology to util namespace, created BitflagsView --- src/atlas/CMakeLists.txt | 1 + src/atlas/mesh/Nodes.h | 19 ++--------- src/atlas/util/Bitflags.h | 70 +++++++++++++++++++++++++++++++++++++++ src/atlas/util/Topology.h | 38 +++++++++++++++++++++ 4 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 src/atlas/util/Topology.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 44d3a932c..c08e3bb2a 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -669,6 +669,7 @@ mesh/IsGhostNode.h util/LonLatMicroDeg.h util/CoordinateEnums.h util/PeriodicTransform.h +util/Topology.h util/Unique.h util/Unique.cc util/Allocate.h diff --git a/src/atlas/mesh/Nodes.h b/src/atlas/mesh/Nodes.h index de436d4ed..5ac458db3 100644 --- a/src/atlas/mesh/Nodes.h +++ b/src/atlas/mesh/Nodes.h @@ -21,8 +21,8 @@ #include "atlas/util/ObjectHandle.h" #include "atlas/field/Field.h" -#include "atlas/util/Bitflags.h" #include "atlas/util/Metadata.h" +#include "atlas/util/Topology.h" namespace atlas { namespace mesh { @@ -44,22 +44,7 @@ class Nodes : public util::Object { public: using Connectivity = IrregularConnectivity; - class Topology : public util::Bitflags { - public: - enum - { - NONE = 0, - GHOST = ( 1 << 1 ), - PERIODIC = ( 1 << 2 ), - BC = ( 1 << 3 ), - WEST = ( 1 << 4 ), - EAST = ( 1 << 5 ), - NORTH = ( 1 << 6 ), - SOUTH = ( 1 << 7 ), - PATCH = ( 1 << 8 ), - POLE = ( 1 << 9 ) - }; - }; + using Topology = util::Topology; public: // methods //-- Constructors diff --git a/src/atlas/util/Bitflags.h b/src/atlas/util/Bitflags.h index 4812fee6c..b5b26d845 100644 --- a/src/atlas/util/Bitflags.h +++ b/src/atlas/util/Bitflags.h @@ -15,6 +15,17 @@ namespace atlas { namespace util { +//---------------------------------------------------------------------------------------------------------------------- + +// Forward declaration of BitflagsView, defined further down in this file +namespace detail { +template +class BitflagsView {}; +} // namespace detail + +//---------------------------------------------------------------------------------------------------------------------- + +/// @brief Convenience class to modify and interpret bitflags class Bitflags { public: static void reset( int& flags, int bit = 0 ) { flags = bit; } @@ -40,7 +51,66 @@ class Bitflags { } return std::string( str, 9 ); } + + /// @brief Create convenience accessor to modify flags + /// @note Use `auto` for return type! (will be more clear with C++14) + static detail::BitflagsView view( int& flags ); + + /// @brief Create convenience accessor to modify flags + /// @note Use `auto` for return type! (will be more clear with C++14) + static detail::BitflagsView view( const int& flags ); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +namespace detail { + +//---------------------------------------------------------------------------------------------------------------------- + +// Template specialication for constant flags. There are no functions to edit the flags +template <> +class BitflagsView { + const int flags_; + +public: + BitflagsView( const int flags ) : flags_( flags ) {} + bool check( int bit ) const { return Bitflags::check( flags_, bit ); } + bool check_all( int bit ) const { return Bitflags::check_all( flags_, bit ); } + bool check_any( int bit ) const { return Bitflags::check_any( flags_, bit ); } +}; + +//---------------------------------------------------------------------------------------------------------------------- + +// Template specialication for nonconst flags. There are functions to edit the flags +template <> +class BitflagsView { + int& flags_; + +public: + BitflagsView( int& flags ) : flags_( flags ) {} + void reset( int bit = 0 ) { Bitflags::reset( flags_, bit ); } + void set( int bit ) { Bitflags::set( flags_, bit ); } + void unset( int bit ) { Bitflags::unset( flags_, bit ); } + void toggle( int bit ) { Bitflags::toggle( flags_, bit ); } + bool check( int bit ) const { return Bitflags::check( flags_, bit ); } + bool check_all( int bit ) const { return Bitflags::check_all( flags_, bit ); } + bool check_any( int bit ) const { return Bitflags::check_any( flags_, bit ); } }; +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace detail + +//---------------------------------------------------------------------------------------------------------------------- + +inline detail::BitflagsView Bitflags::view( const int& flags ) { + return detail::BitflagsView( flags ); +} +inline detail::BitflagsView Bitflags::view( int& flags ) { + return detail::BitflagsView( flags ); +} + +//---------------------------------------------------------------------------------------------------------------------- + } // namespace util } // namespace atlas diff --git a/src/atlas/util/Topology.h b/src/atlas/util/Topology.h new file mode 100644 index 000000000..ad17a8414 --- /dev/null +++ b/src/atlas/util/Topology.h @@ -0,0 +1,38 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include "atlas/util/Bitflags.h" + +namespace atlas { +namespace util { + +class Topology : public util::Bitflags { +public: + enum + { + NONE = 0, + GHOST = ( 1 << 1 ), + PERIODIC = ( 1 << 2 ), + BC = ( 1 << 3 ), + WEST = ( 1 << 4 ), + EAST = ( 1 << 5 ), + NORTH = ( 1 << 6 ), + SOUTH = ( 1 << 7 ), + PATCH = ( 1 << 8 ), + POLE = ( 1 << 9 ), + LAND = ( 1 << 10 ), + WATER = ( 1 << 11 ), + }; +}; + +} // namespace util +} // namespace atlas From 58572e385aa71a6b8b2df2780b0968a11caaa696 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 10:05:18 +0100 Subject: [PATCH 151/161] Gmsh control output of LAND/WATER elements --- src/apps/atlas-meshgen.cc | 22 +++++++-- src/atlas/mesh/PartitionPolygon.cc | 5 +- src/atlas/mesh/actions/BuildHalo.cc | 13 ++++- src/atlas/output/detail/GmshIO.cc | 75 +++++++++++++++++++++++++---- src/atlas/output/detail/GmshImpl.cc | 13 +++++ src/atlas/output/detail/GmshImpl.h | 5 ++ 6 files changed, 115 insertions(+), 18 deletions(-) diff --git a/src/apps/atlas-meshgen.cc b/src/apps/atlas-meshgen.cc index 9b463eec4..95223ea70 100644 --- a/src/apps/atlas-meshgen.cc +++ b/src/apps/atlas-meshgen.cc @@ -152,6 +152,10 @@ Meshgen2Gmsh::Meshgen2Gmsh( int argc, char** argv ) : AtlasTool( argc, argv ) { add_option( new SimpleOption( "periodic_x", "periodic mesh in x-direction" ) ); add_option( new SimpleOption( "periodic_y", "periodic mesh in y-direction" ) ); add_option( new SimpleOption( "torus", "Output mesh as torus" ) ); + add_option( new SimpleOption( + "water", "Output elements containing water points (not specifying --water or --land enables both)" ) ); + add_option( new SimpleOption( + "land", "Output elements containing land points (not specifying --water or --land enables both)" ) ); } //----------------------------------------------------------------------------- @@ -304,12 +308,21 @@ int Meshgen2Gmsh::execute( const Args& args ) { } } + bool lonlat = args.getBool( "lonlat", false ); bool ij = args.getBool( "ij", false ); std::string coordinates = dim_3d ? "xyz" : lonlat ? "lonlat" : ij ? "ij" : "xy"; - atlas::output::Gmsh gmsh( - path_out, Config( "info", info )( "ghost", ghost )( "coordinates", coordinates )( "edges", edges ) ); + Config gmsh_config; + gmsh_config.set( "coordinates", coordinates ); + gmsh_config.set( "edges", edges ); + gmsh_config.set( "ghost", ghost ); + gmsh_config.set( "info", info ); + if ( args.has( "land" ) || args.has( "water" ) ) { + gmsh_config.set( "land", args.getBool( "land", false ) ); + gmsh_config.set( "water", args.getBool( "water", false ) ); + } + atlas::output::Gmsh gmsh( path_out, gmsh_config ); Log::info() << "Writing mesh to gmsh file \"" << path_out << "\" generated from grid \"" << grid.name() << "\"" << std::endl; gmsh.write( mesh ); @@ -318,9 +331,8 @@ int Meshgen2Gmsh::execute( const Args& args ) { Log::info() << "Partitioning graph: \n" << mesh.partitionGraph() << std::endl; Log::info() << "Mesh partition footprint: " << eckit::Bytes( mesh.footprint() ) << std::endl; for ( idx_t jhalo = 0; jhalo <= halo; ++jhalo ) { - mesh.polygon( jhalo ).outputPythonScript( - "polygon_halo" + std::to_string( jhalo ) + ".py", - Config( "nodes", false )( "coordinates", dim_3d ? "xy" : lonlat ? "lonlat" : "xy" ) ); + mesh.polygon( jhalo ).outputPythonScript( "polygon_halo" + std::to_string( jhalo ) + ".py", + Config( "nodes", false )( "coordinates", coordinates ) ); } } return success(); diff --git a/src/atlas/mesh/PartitionPolygon.cc b/src/atlas/mesh/PartitionPolygon.cc index f2b21e587..4077bd3d1 100644 --- a/src/atlas/mesh/PartitionPolygon.cc +++ b/src/atlas/mesh/PartitionPolygon.cc @@ -72,7 +72,10 @@ void PartitionPolygon::outputPythonScript( const eckit::PathName& filepath, cons int mpi_size = int( comm.size() ); std::string coordinates = config.getString( "coordinates", "xy" ); - auto points = coordinates == "xy" ? this->xy() : this->lonlat(); + if ( coordinates == "ij" ) { + coordinates = "xy"; + } + auto points = coordinates == "xy" ? this->xy() : this->lonlat(); ATLAS_ASSERT( points.size() == size() ); const auto nodes_xy = array::make_view( mesh_.nodes().field( coordinates ) ); diff --git a/src/atlas/mesh/actions/BuildHalo.cc b/src/atlas/mesh/actions/BuildHalo.cc index d4e92b162..99f38fa3b 100644 --- a/src/atlas/mesh/actions/BuildHalo.cc +++ b/src/atlas/mesh/actions/BuildHalo.cc @@ -446,9 +446,18 @@ void build_lookup_uid2node( Mesh& mesh, Uid2Node& uid2node ) { if ( not inserted ) { int other = uid2node[uid]; std::stringstream msg; - msg << "Node uid: " << uid << " " << glb_idx( jnode ) << " (" << xy( jnode, XX ) << "," << xy( jnode, YY ) - << ") has already been added as node " << glb_idx( other ) << " (" << xy( other, XX ) << "," + msg << std::setprecision( 10 ) << std::fixed << "Node uid: " << uid << " " << glb_idx( jnode ) << " xy(" + << xy( jnode, XX ) << "," << xy( jnode, YY ) << ")"; + if ( nodes.has_field( "ij" ) ) { + auto ij = array::make_view( nodes.field( "ij" ) ); + msg << " ij(" << ij( jnode, XX ) << "," << ij( jnode, YY ) << ")"; + } + msg << " has already been added as node " << glb_idx( other ) << " (" << xy( other, XX ) << "," << xy( other, YY ) << ")"; + if ( nodes.has_field( "ij" ) ) { + auto ij = array::make_view( nodes.field( "ij" ) ); + msg << " ij(" << ij( other, XX ) << "," << ij( other, YY ) << ")"; + } notes.add_error( msg.str() ); } } diff --git a/src/atlas/output/detail/GmshIO.cc b/src/atlas/output/detail/GmshIO.cc index 6a9ed527e..a04c5a31b 100644 --- a/src/atlas/output/detail/GmshIO.cc +++ b/src/atlas/output/detail/GmshIO.cc @@ -38,6 +38,7 @@ using atlas::functionspace::NodeColumns; using atlas::util::Metadata; +using atlas::util::Topology; using eckit::PathName; namespace atlas { @@ -812,6 +813,10 @@ void GmshIO::write( const Mesh& mesh, const PathName& file_path ) const { int part = mesh.metadata().has( "part" ) ? mesh.metadata().get( "part" ) : mpi::rank(); bool include_ghost = options.get( "ghost" ) && options.get( "elements" ); + bool land_water_flag = options.has( "water" ) || options.has( "land" ); + bool include_water = options.getBool( "water", false ); + bool include_land = options.getBool( "land", false ); + std::string nodes_field = options.get( "nodes" ); const mesh::Nodes& nodes = mesh.nodes(); @@ -898,20 +903,46 @@ void GmshIO::write( const Mesh& mesh, const PathName& file_path ) const { nb_elements += hybrid->size(); const auto hybrid_halo = array::make_view( hybrid->halo() ); const auto hybrid_flags = array::make_view( hybrid->flags() ); - auto hybrid_patch = [&]( idx_t e ) { - return mesh::Nodes::Topology::check( hybrid_flags( e ), mesh::Nodes::Topology::PATCH ); + auto hybrid_patch = [&]( idx_t e ) { return Topology::check( hybrid_flags( e ), Topology::PATCH ); }; + auto hybrid_ghost_flag = [&]( idx_t e ) { return Topology::check( hybrid_flags( e ), Topology::GHOST ); }; + + auto include = [&]( idx_t e ) { + auto topology = Topology::view( hybrid_flags( e ) ); + if ( land_water_flag && not( include_water && include_land ) ) { + if ( include_water && !topology.check( Topology::WATER ) ) { + return false; + } + if ( include_land && !topology.check( Topology::LAND ) ) { + return false; + } + } + if ( not include_ghost ) { + if ( topology.check( Topology::GHOST ) || hybrid_halo( e ) ) { + return false; + } + } + if ( not include_patch ) { + if ( topology.check( Topology::PATCH ) ) { + return false; + } + } + return true; }; + auto exclude = [&]( idx_t e ) { if ( !include_ghost && hybrid_halo( e ) ) { return true; } + if ( !include_ghost && hybrid_ghost_flag( e ) ) { + return true; + } if ( !include_patch && hybrid_patch( e ) ) { return true; } return false; }; for ( idx_t e = 0; e < hybrid->size(); ++e ) { - if ( exclude( e ) ) { + if ( not include( e ) ) { --nb_elements; } } @@ -941,8 +972,32 @@ void GmshIO::write( const Mesh& mesh, const PathName& file_path ) const { auto elems_glb_idx = elements.view( elements.global_index() ); auto elems_partition = elements.view( elements.partition() ); - auto elems_halo = elements.view( elements.halo() ); + auto elems_halo = elements.view( elements.halo() ); + auto elems_flags = elements.view( elements.flags() ); + + auto include = [&]( idx_t e ) { + auto topology = Topology::view( elems_flags( e ) ); + if ( land_water_flag && not( include_water && include_land ) ) { + if ( include_water && !topology.check( Topology::WATER ) ) { + return false; + } + if ( include_land && !topology.check( Topology::LAND ) ) { + return false; + } + } + if ( not include_ghost ) { + if ( topology.check( Topology::GHOST ) || elems_halo( e ) ) { + return false; + } + } + if ( not include_patch ) { + if ( topology.check( Topology::PATCH ) ) { + return false; + } + } + return true; + }; if ( binary ) { idx_t nb_elems = elements.size(); if ( !include_ghost ) { @@ -979,7 +1034,7 @@ void GmshIO::write( const Mesh& mesh, const PathName& file_path ) const { ss_elem_info << " " << gmsh_elem_type << " 4 1 1 1 "; std::string elem_info = ss_elem_info.str(); for ( idx_t elem = 0; elem < elements.size(); ++elem ) { - if ( include_ghost || !elems_halo( elem ) ) { + if ( include( elem ) ) { file << elems_glb_idx( elem ) << elem_info << elems_partition( elem ); for ( idx_t n = 0; n < node_connectivity.cols(); ++n ) { file << " " << glb_idx( node_connectivity( elem, n ) ); @@ -1007,13 +1062,13 @@ void GmshIO::write( const Mesh& mesh, const PathName& file_path ) const { write( nodes.partition(), function_space, mesh_info, std::ios_base::out ); - if ( nodes.has_field( "dual_volumes" ) ) { - write( nodes.field( "dual_volumes" ), function_space, mesh_info, std::ios_base::app ); + std::vector extra_fields = {"dual_volumes", "dual_delta_sph", "lsm", "core", "ghost", "halo"}; + for ( auto& f : extra_fields ) { + if ( nodes.has_field( f ) ) { + write( nodes.field( f ), function_space, mesh_info, std::ios_base::app ); + } } - if ( nodes.has_field( "dual_delta_sph" ) ) { - write( nodes.field( "dual_delta_sph" ), function_space, mesh_info, std::ios_base::app ); - } //[next] if( mesh.has_function_space("edges") ) //[next] { diff --git a/src/atlas/output/detail/GmshImpl.cc b/src/atlas/output/detail/GmshImpl.cc index c39fc4167..e420be0e2 100644 --- a/src/atlas/output/detail/GmshImpl.cc +++ b/src/atlas/output/detail/GmshImpl.cc @@ -40,6 +40,10 @@ void GmshImpl::defaults() { config_.info = false; config_.openmode = "w"; config_.coordinates = "xy"; + + config_.configured_land_water = false; + config_.land = false; + config_.water = false; } // ----------------------------------------------------------------------------- @@ -60,6 +64,11 @@ void merge( GmshImpl::Configuration& present, const eckit::Parametrisation& upda update.get( "info", present.info ); update.get( "openmode", present.openmode ); update.get( "coordinates", present.coordinates ); + if ( update.has( "water" ) || update.has( "land" ) ) { + update.get( "land", present.land ); + update.get( "water", present.water ); + present.configured_land_water = true; + } } // ----------------------------------------------------------------------------- @@ -102,6 +111,10 @@ void GmshImpl::setGmshConfiguration( detail::GmshIO& gmsh, const GmshImpl::Confi gmsh.options.set( "levels", c.levels ); gmsh.options.set( "info", c.info ); gmsh.options.set( "nodes", c.coordinates ); + if ( c.configured_land_water ) { + gmsh.options.set( "land", c.land ); + gmsh.options.set( "water", c.water ); + } } // ----------------------------------------------------------------------------- diff --git a/src/atlas/output/detail/GmshImpl.h b/src/atlas/output/detail/GmshImpl.h index c073b5850..2ceffcb4b 100644 --- a/src/atlas/output/detail/GmshImpl.h +++ b/src/atlas/output/detail/GmshImpl.h @@ -63,6 +63,11 @@ class GmshImpl : public OutputImpl { bool gather; bool ghost; bool info; + + bool configured_land_water; + bool land; + bool water; + std::vector levels; std::string nodes; std::string file; From 87ab1b31db44a7a77fd782377c305052c13084db Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 14:41:31 +0100 Subject: [PATCH 152/161] Simplify version C++ file --- src/atlas/library/version.h.in | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/src/atlas/library/version.h.in b/src/atlas/library/version.h.in index 21c5da70e..b6a87d73b 100644 --- a/src/atlas/library/version.h.in +++ b/src/atlas/library/version.h.in @@ -10,35 +10,27 @@ #pragma once -#define ATLAS_VERSION_STR "@atlas_VERSION_STR@" -#define ATLAS_VERSION "@atlas_VERSION@" - -#define ATLAS_VERSION_STRIDE_MAJOR 10000000000 -#define ATLAS_VERSION_STRIDE_MINOR 100000000 -#define ATLAS_VERSION_STRIDE_PATCH 1 - -#define ATLAS_VERSION_INT ( \ - ATLAS_VERSION_STRIDE_MAJOR*@atlas_VERSION_MAJOR@ + \ - ATLAS_VERSION_STRIDE_MINOR*@atlas_VERSION_MINOR@ + \ - ATLAS_VERSION_STRIDE_PATCH*@atlas_VERSION_PATCH@ ) - -#define ATLAS_VERSION_MAJOR @atlas_VERSION_MAJOR@ -#define ATLAS_VERSION_MINOR @atlas_VERSION_MINOR@ -#define ATLAS_VERSION_PATCH @atlas_VERSION_PATCH@ - namespace atlas { namespace library { constexpr const char* version() { - return ATLAS_VERSION_STR; + return "@atlas_VERSION_STR@"; } constexpr const char* semantic_version() { - return ATLAS_VERSION; + return "@atlas_VERSION@"; +} + +constexpr long semantic_version_major() { + return @atlas_VERSION_MAJOR@; +} + +constexpr long semantic_version_minor() { + return @atlas_VERSION_MINOR@; } -constexpr long semantic_version_int() { - return ATLAS_VERSION_INT; +constexpr long semantic_version_patch() { + return @atlas_VERSION_PATCH@; } } From b8a07743272e7085f0e0fba2cb0ad866c91d3114 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 9 Oct 2020 15:48:32 +0100 Subject: [PATCH 153/161] Formatting --- .../detail/StructuredColumns_setup.cc | 53 +++++++++---------- .../method/knn/KNearestNeighbours.cc | 2 +- .../method/knn/NearestNeighbour.cc | 2 +- src/tests/parallel/test_setcomm.cc | 52 +++++++++--------- 4 files changed, 52 insertions(+), 57 deletions(-) diff --git a/src/atlas/functionspace/detail/StructuredColumns_setup.cc b/src/atlas/functionspace/detail/StructuredColumns_setup.cc index 993702c87..4cd9d5cbd 100644 --- a/src/atlas/functionspace/detail/StructuredColumns_setup.cc +++ b/src/atlas/functionspace/detail/StructuredColumns_setup.cc @@ -91,8 +91,8 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck } bool periodic_x = false, periodic_y = false; - config.get ("periodic_x", periodic_x); - config.get ("periodic_y", periodic_y); + config.get( "periodic_x", periodic_x ); + config.get( "periodic_y", periodic_y ); const double eps = 1.e-12; @@ -249,27 +249,25 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck std::function compute_j; compute_j = [this, &compute_j, &periodic_y]( idx_t j ) -> idx_t { - if (periodic_y) - { + if ( periodic_y ) { const idx_t ny = grid_->ny(); - while (j < 0) - j += ny; - while (j >= ny) - j -= ny; - } - else - { + while ( j < 0 ) + j += ny; + while ( j >= ny ) + j -= ny; + } + else { if ( j < 0 ) { j = ( grid_->y( 0 ) == 90. ) ? -j : -j - 1; } else if ( j >= grid_->ny() ) { idx_t jlast = grid_->ny() - 1; - j = ( grid_->y( jlast ) == -90. ) ? jlast - 1 - ( j - grid_->ny() ) : jlast - ( j - grid_->ny() ); + j = ( grid_->y( jlast ) == -90. ) ? jlast - 1 - ( j - grid_->ny() ) : jlast - ( j - grid_->ny() ); } if ( j < 0 or j >= grid_->ny() ) { j = compute_j( j ); } - } + } return j; }; @@ -299,15 +297,13 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck double y; idx_t jj; jj = compute_j( j ); - if (periodic_y) - { + if ( periodic_y ) { y = grid_->y( jj ); - } - else - { - y = ( j < 0 ) ? 90. + ( 90. - grid_->y( jj ) ) + } + else { + y = ( j < 0 ) ? 90. + ( 90. - grid_->y( jj ) ) : ( j >= grid_->ny() ) ? -90. + ( -90. - grid_->y( jj ) ) : grid_->y( jj ); - } + } return y; }; @@ -318,24 +314,23 @@ void StructuredColumns::setup( const grid::Distribution& distribution, const eck grid_idx += grid_->nx( j ); } - auto compute_g = [this, &global_offsets, &compute_i, - &compute_j, &periodic_y]( idx_t i, idx_t j ) -> gidx_t { + auto compute_g = [this, &global_offsets, &compute_i, &compute_j, &periodic_y]( idx_t i, idx_t j ) -> gidx_t { idx_t ii, jj; gidx_t g; jj = compute_j( j ); ii = compute_i( i, jj ); - if (! periodic_y) - if (jj != j) { - ATLAS_ASSERT( grid_->nx( jj ) % 2 == 0 ); // assert even number of points - ii = ( ii < grid_->nx( jj ) / 2 ) ? ii + grid_->nx( jj ) / 2 - : ( ii >= grid_->nx( jj ) / 2 ) ? ii - grid_->nx( jj ) / 2 : ii; - } + if ( !periodic_y ) + if ( jj != j ) { + ATLAS_ASSERT( grid_->nx( jj ) % 2 == 0 ); // assert even number of points + ii = ( ii < grid_->nx( jj ) / 2 ) ? ii + grid_->nx( jj ) / 2 + : ( ii >= grid_->nx( jj ) / 2 ) ? ii - grid_->nx( jj ) / 2 : ii; + } g = global_offsets[jj] + ii + 1; return g; }; auto compute_p = [&compute_g, &distribution]( idx_t i, idx_t j ) -> int { - return distribution.partition (compute_g (i, j) - 1); + return distribution.partition( compute_g( i, j ) - 1 ); }; GridPointSet gridpoints; diff --git a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc index ac2856d2f..5ec1a9efc 100644 --- a/src/atlas/interpolation/method/knn/KNearestNeighbours.cc +++ b/src/atlas/interpolation/method/knn/KNearestNeighbours.cc @@ -101,7 +101,7 @@ void KNearestNeighbours::do_setup( const FunctionSpace& source, const FunctionSp } // find the closest input points to the output point - auto nn = pTree_.closestPoints( PointLonLat{lonlat( ip, size_t(LON) ), lonlat( ip, size_t(LAT) )}, k_ ); + auto nn = pTree_.closestPoints( PointLonLat{lonlat( ip, size_t( LON ) ), lonlat( ip, size_t( LAT ) )}, k_ ); // calculate weights (individual and total, to normalise) using distance // squared diff --git a/src/atlas/interpolation/method/knn/NearestNeighbour.cc b/src/atlas/interpolation/method/knn/NearestNeighbour.cc index 2491d303c..d19241f78 100644 --- a/src/atlas/interpolation/method/knn/NearestNeighbour.cc +++ b/src/atlas/interpolation/method/knn/NearestNeighbour.cc @@ -91,7 +91,7 @@ void NearestNeighbour::do_setup( const FunctionSpace& source, const FunctionSpac } // find the closest input point to the output point - auto nn = pTree_.closestPoint( PointLonLat{lonlat( ip, size_t(LON) ), lonlat( ip, size_t(LAT) )} ); + auto nn = pTree_.closestPoint( PointLonLat{lonlat( ip, size_t( LON ) ), lonlat( ip, size_t( LAT ) )} ); size_t jp = nn.payload(); // insert the weights into the interpolant matrix diff --git a/src/tests/parallel/test_setcomm.cc b/src/tests/parallel/test_setcomm.cc index c0e564443..381845197 100644 --- a/src/tests/parallel/test_setcomm.cc +++ b/src/tests/parallel/test_setcomm.cc @@ -1,46 +1,46 @@ -#include "eckit/mpi/Comm.h" #include "atlas/library/Library.h" -#include "atlas/util/Config.h" #include "atlas/parallel/mpi/mpi.h" +#include "atlas/util/Config.h" +#include "eckit/mpi/Comm.h" #include "tests/AtlasTestEnvironment.h" -#include #include +#include namespace atlas { namespace test { -CASE ("test_setcomm") -{ - - printf (" eckit::mpi::comm () = 0x%llx, eckit::mpi::comm ().size () = %8d\n", &eckit::mpi::comm (), eckit::mpi::comm ().size ()); - std::cout << eckit::mpi::comm ().name () << std::endl; - printf (" atlas::mpi::comm () = 0x%llx, atlas::mpi::comm ().size () = %8d\n", &atlas::mpi::comm (), atlas::mpi::comm ().size ()); - std::cout << atlas::mpi::comm ().name () << std::endl; - - EXPECT_EQ (eckit::mpi::comm ().size (), atlas::mpi::comm ().size ()); +CASE( "test_setcomm" ) { + printf( " eckit::mpi::comm () = 0x%llx, eckit::mpi::comm ().size () = %8d\n", &eckit::mpi::comm(), + eckit::mpi::comm().size() ); + std::cout << eckit::mpi::comm().name() << std::endl; + printf( " atlas::mpi::comm () = 0x%llx, atlas::mpi::comm ().size () = %8d\n", &atlas::mpi::comm(), + atlas::mpi::comm().size() ); + std::cout << atlas::mpi::comm().name() << std::endl; - int irank = eckit::mpi::comm ().rank (); + EXPECT_EQ( eckit::mpi::comm().size(), atlas::mpi::comm().size() ); - auto & comm = eckit::mpi::comm ().split (irank, "SPLIT"); + int irank = eckit::mpi::comm().rank(); - eckit::mpi::setCommDefault ("SPLIT"); + auto& comm = eckit::mpi::comm().split( irank, "SPLIT" ); - printf (" eckit::mpi::comm () = 0x%llx, eckit::mpi::comm ().size () = %8d\n", &eckit::mpi::comm (), eckit::mpi::comm ().size ()); - std::cout << eckit::mpi::comm ().name () << std::endl; - printf (" atlas::mpi::comm () = 0x%llx, atlas::mpi::comm ().size () = %8d\n", &atlas::mpi::comm (), atlas::mpi::comm ().size ()); - std::cout << atlas::mpi::comm ().name () << std::endl; + eckit::mpi::setCommDefault( "SPLIT" ); - EXPECT_EQ (eckit::mpi::comm ().size (), atlas::mpi::comm ().size ()); + printf( " eckit::mpi::comm () = 0x%llx, eckit::mpi::comm ().size () = %8d\n", &eckit::mpi::comm(), + eckit::mpi::comm().size() ); + std::cout << eckit::mpi::comm().name() << std::endl; + printf( " atlas::mpi::comm () = 0x%llx, atlas::mpi::comm ().size () = %8d\n", &atlas::mpi::comm(), + atlas::mpi::comm().size() ); + std::cout << atlas::mpi::comm().name() << std::endl; - std::cout << "----- STOP -----" << std::endl; + EXPECT_EQ( eckit::mpi::comm().size(), atlas::mpi::comm().size() ); + std::cout << "----- STOP -----" << std::endl; } -} -} +} // namespace test +} // namespace atlas -int main (int argc, char* argv[]) -{ - return atlas::test::run (argc, argv); +int main( int argc, char* argv[] ) { + return atlas::test::run( argc, argv ); } From 74cd49cd7b387ee3829042d3e455a910eb932109 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 9 Oct 2020 15:57:34 +0100 Subject: [PATCH 154/161] Grid convenience constructors taking Projection --- src/atlas/grid/Grid.cc | 12 ++++++++++++ src/atlas/grid/Grid.h | 1 + src/atlas/grid/StructuredGrid.cc | 3 +++ src/atlas/grid/StructuredGrid.h | 1 + 4 files changed, 17 insertions(+) diff --git a/src/atlas/grid/Grid.cc b/src/atlas/grid/Grid.cc index 6b19bc597..7f09d4278 100644 --- a/src/atlas/grid/Grid.cc +++ b/src/atlas/grid/Grid.cc @@ -41,6 +41,18 @@ Grid::Grid( const std::string& shortname, const Domain& domain ) : return Grid::Implementation::create( shortname, config ); }() ) {} +Grid::Grid( const std::string& shortname, const Projection& projection, const Domain& domain ) : + Handle( [&] { + Config config; + if ( projection ) { + config.set( "projection", projection.spec() ); + } + if ( domain ) { + config.set( "domain", domain.spec() ); + } + return Grid::Implementation::create( shortname, config ); + }() ) {} + Grid::Grid( const Grid& grid, const Grid::Domain& domain ) : Handle( [&] { ATLAS_ASSERT( grid ); diff --git a/src/atlas/grid/Grid.h b/src/atlas/grid/Grid.h index 224c243a2..3d7492ce5 100644 --- a/src/atlas/grid/Grid.h +++ b/src/atlas/grid/Grid.h @@ -79,6 +79,7 @@ class Grid : DOXYGEN_HIDE( public util::ObjectHandle ) using Handle::Handle; Grid() = default; Grid( const std::string& name, const Domain& = Domain() ); + Grid( const std::string& name, const Projection&, const Domain& = Domain() ); Grid( const Grid&, const Domain& ); Grid( const Config& ); diff --git a/src/atlas/grid/StructuredGrid.cc b/src/atlas/grid/StructuredGrid.cc index 57ae502b7..6acdf966a 100644 --- a/src/atlas/grid/StructuredGrid.cc +++ b/src/atlas/grid/StructuredGrid.cc @@ -39,6 +39,9 @@ StructuredGrid::StructuredGrid( const Grid::Implementation* grid ) : Grid( grid StructuredGrid::StructuredGrid( const std::string& grid, const Domain& domain ) : Grid( grid, domain ), grid_( structured_grid( get() ) ) {} +StructuredGrid::StructuredGrid( const std::string& grid, const Projection& projection, const Domain& domain ) : + Grid( grid, projection, domain ), grid_( structured_grid( get() ) ) {} + StructuredGrid::StructuredGrid( const Config& grid ) : Grid( grid ), grid_( structured_grid( get() ) ) {} StructuredGrid::StructuredGrid( const XSpace& xspace, const YSpace& yspace, const Projection& projection, diff --git a/src/atlas/grid/StructuredGrid.h b/src/atlas/grid/StructuredGrid.h index 43c0b0afe..9bb099d17 100644 --- a/src/atlas/grid/StructuredGrid.h +++ b/src/atlas/grid/StructuredGrid.h @@ -64,6 +64,7 @@ class StructuredGrid : public Grid { StructuredGrid( const Grid& ); StructuredGrid( const Grid::Implementation* ); StructuredGrid( const std::string& name, const Domain& = Domain() ); + StructuredGrid( const std::string& name, const Projection&, const Domain& = Domain() ); StructuredGrid( const Config& ); StructuredGrid( const XSpace&, const YSpace&, const Projection& = Projection(), const Domain& = Domain() ); StructuredGrid( const Grid&, const Domain& ); From dfd909066b55cdb9c46e473c1e628a1ef03d853f Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 9 Oct 2020 10:25:40 +0100 Subject: [PATCH 155/161] Fix tools/install-pgi.sh download link --- .travis.yml | 4 ++-- tools/install-pgi.sh | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 22062de59..d8b076204 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,8 +84,8 @@ matrix: - CXX_COMPILER='pgc++' C_COMPILER='pgcc' Fortran_COMPILER='pgfortran' - MPI='openmpi' - PGI_VERSION="CommunityEdition" - - ECKIT_CMAKE_OPTIONS="-DENABLE_ECKIT-395=OFF -DRT_LIB=/usr/lib/x86_64-linux-gnu/librt.so -DCURSES_LIBRARY=/usr/lib/x86_64-linux-gnu/libcurses.so" - - ATLAS_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG" + - ECKIT_CMAKE_OPTIONS="-DENABLE_ECKIT-395=OFF -DRT_LIB=/usr/lib/x86_64-linux-gnu/librt.so -DCURSES_LIBRARY=/usr/lib/x86_64-linux-gnu/libcurses.so -DMPI_ARGS=--oversubscribe" + - ATLAS_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DMPI_ARGS=--oversubscribe" before_install: diff --git a/tools/install-pgi.sh b/tools/install-pgi.sh index d416a12ec..a9d8a9c90 100755 --- a/tools/install-pgi.sh +++ b/tools/install-pgi.sh @@ -43,7 +43,7 @@ case "$(uname -m)" in ;; esac -URL="$(curl -s 'https://developer.nvidia.com/nvidia-hpc-sdk-download' | grep -oP "https://developer.download.nvidia.com/hpc-sdk/nvhpc_([0-9]{4})_([0-9]+)_Linux_$(uname -m)_cuda_([0-9\.]+).tar.gz")" +URL="$(curl -s 'https://developer.nvidia.com/nvidia-hpc-sdk-download' | grep -oP "https://developer.download.nvidia.com/hpc-sdk/([0-9]{2}\.[0-9]+)/nvhpc_([0-9]{4})_([0-9]+)_Linux_$(uname -m)_cuda_([0-9\.]+).tar.gz" | sort | tail -1)" FOLDER="$(basename "$(echo "${URL}" | grep -oP '[^/]+$')" .tar.gz)" if [ ! -z "${TRAVIS_REPO_SLUG}" ]; then @@ -74,11 +74,11 @@ echo "+ ${TEMPORARY_FILES}/${FOLDER}/install" #comment out to cleanup #rm -rf "${TEMPORARY_FILES}/${FOLDER}" -PGI_VERSION=$(basename "${NVHPC_INSTALL_DIR}"/Linux_x86_64/*.*/) +PGI_VERSION=$(basename "${NVHPC_INSTALL_DIR}"/Linux_$(uname -m)/*.*/) # Use gcc which is available in PATH -${NVHPC_INSTALL_DIR}/Linux_x86_64/${PGI_VERSION}/compilers/bin/makelocalrc \ - -x ${NVHPC_INSTALL_DIR}/Linux_x86_64/${PGI_VERSION}/compilers/bin \ +${NVHPC_INSTALL_DIR}/Linux_$(uname -m)/${PGI_VERSION}/compilers/bin/makelocalrc \ + -x ${NVHPC_INSTALL_DIR}/Linux_$(uname -m)/${PGI_VERSION}/compilers/bin \ -gcc $(which gcc) \ -gpp $(which g++) \ -g77 $(which gfortran) @@ -89,7 +89,7 @@ PGI_INSTALL_DIR=${NVHPC_INSTALL_DIR} PGI_VERSION=${PGI_VERSION} ### Compilers -PGI_DIR=\${PGI_INSTALL_DIR}/Linux_x86_64/\${PGI_VERSION} +PGI_DIR=\${PGI_INSTALL_DIR}/Linux_$(uname -m)/\${PGI_VERSION} export PATH=\${PGI_DIR}/compilers/bin:\${PATH} EOF @@ -98,5 +98,6 @@ cat >> ${NVHPC_INSTALL_DIR}/env.sh << EOF ### MPI export MPI_HOME=\${PGI_DIR}/comm_libs/mpi export PATH=\${MPI_HOME}/bin:\${PATH} +export LD_LIBRARY_PATH=\${PGI_DIR}/compilers/lib:\${LD_LIBRARY_PATH} EOF From f37ea364d3d59f10f9c1481e055530e15b9d9649 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 9 Oct 2020 10:32:19 +0100 Subject: [PATCH 156/161] travis: Update OSX version --- .travis.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index d8b076204..df17d8142 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,6 @@ -sudo: false language: cpp - -# Workaround for https://github.com/travis-ci/travis-ci/issues/4681 -matrix: - - TRAVIS_EMPTY_JOB_WORKAROUND=true - - cache: directories: - ${HOME}/deps/cmake @@ -17,11 +10,7 @@ cache: - ${HOME}/deps/fckit -matrix: - exclude: - - - env: TRAVIS_EMPTY_JOB_WORKAROUND - +jobs: include: - os: linux @@ -68,14 +57,14 @@ matrix: - MPI=openmpi - ATLAS_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG" - ATLAS_CTEST_OPTIONS="-E atlas_test_stencil_parallel_mpi16" - osx_image: xcode10.1 + osx_image: xcode12 - os: osx env: - CACHE_NAME=osx-clang-mpich - CXX_COMPILER='clang++' C_COMPILER='clang' Fortran_COMPILER='gfortran' - MPI=mpich - osx_image: xcode9 + osx_image: xcode12 - os: linux compiler: gcc @@ -154,6 +143,10 @@ install: mkdir -p ${DEPS_DIR}/cmake && travis_retry wget --no-check-certificate --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}/cmake fi export PATH=${DEPS_DIR}/cmake/bin:${PATH} + else + brew upgrade cmake || brew install cmake + brew install openssl + export OPENSSL_ROOT_DIR=/usr/local/opt/openssl fi cmake --version From 4c5a0c8e9acfcf35372938eb78a982066c053e41 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 9 Oct 2020 12:26:34 +0100 Subject: [PATCH 157/161] travis: Update CMake to 3.18.4 --- .travis.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index df17d8142..8c30d00ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -138,9 +138,24 @@ install: - | ### Install CMake if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then + CMAKE_VERSION=3.18.4 if [[ -z "$(ls -A ${DEPS_DIR}/cmake)" ]]; then - CMAKE_URL="https://cmake.org/files/v3.15.2/cmake-3.15.2-Linux-x86_64.tar.gz" - mkdir -p ${DEPS_DIR}/cmake && travis_retry wget --no-check-certificate --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}/cmake + if [[ "${TRAVIS_ARCH}" == "amd64" ]]; then + CMAKE_URL="https://cmake.org/files/v${CMAKE_VERSION%.*}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" + mkdir -p ${DEPS_DIR}/cmake && travis_retry wget --no-check-certificate --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}/cmake + fi + if [[ "${TRAVIS_ARCH}" = "ppc64le" ]]; then + # Build from source + CMAKE_URL="https://cmake.org/files/v${CMAKE_VERSION%.*}/cmake-${CMAKE_VERSION}.tar.gz" + mkdir -p ${DEPS_DIR}/cmake-source && travis_retry wget --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}/cmake-source + ( + cd ${DEPS_DIR}/cmake-source + mkdir -p build && cd build + ../bootstrap --prefix=${DEPS_DIR}/cmake && make -j4 install + ) + fi + else + echo "CMake already found in cache" fi export PATH=${DEPS_DIR}/cmake/bin:${PATH} else From 739ef2c42a5b2bb3636519d7decea4f04598fcf3 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 9 Oct 2020 16:42:03 +0100 Subject: [PATCH 158/161] travis: Use nproc for faster compilation --- .travis.yml | 2 +- tools/install-dep.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c30d00ed..8fc1b7d6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -229,7 +229,7 @@ script: ################################################################# - mkdir -p ${ATLAS_BUILD_DIR} && cd ${ATLAS_BUILD_DIR} - cmake -DCMAKE_MODULE_PATH=${ECBUILD_MODULE_PATH} ${ATLAS_CMAKE_OPTIONS} ${ATLAS_SOURCE_DIR} - - make -j4 + - make -j$(nproc) - bin/atlas --info ################################################################# diff --git a/tools/install-dep.sh b/tools/install-dep.sh index 1482cfadc..21370ac31 100755 --- a/tools/install-dep.sh +++ b/tools/install-dep.sh @@ -140,7 +140,9 @@ if ! ${skip_install} ; then fi echo "+ cmake ${SET_CMAKE_MODULE_PATH} -DCMAKE_INSTALL_PREFIX=${PREFIX} ${CMAKE_OPTIONS} ${SOURCE_DIR}" cmake ${SET_CMAKE_MODULE_PATH} -DCMAKE_INSTALL_PREFIX=${PREFIX} ${CMAKE_OPTIONS} ${SOURCE_DIR} - make -j4 install + core_count=$(nproc || echo 4) + echo "+ make -j${core_count} install" + make -j${core_count} install # Store version information mkdir -p ${PREFIX}/sha From fc2c03d6eda25d06f3bd4a9e8c049d027ec255f1 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 17 Apr 2020 11:57:26 +0000 Subject: [PATCH 159/161] Add a jacobian method to atlas::Projection object --- src/atlas/projection/Projection.cc | 4 + src/atlas/projection/Projection.h | 6 +- .../LambertAzimuthalEqualAreaProjection.cc | 4 + .../LambertAzimuthalEqualAreaProjection.h | 2 + .../detail/LambertConformalConicProjection.cc | 32 ++++ .../detail/LambertConformalConicProjection.h | 2 + .../projection/detail/LonLatProjection.cc | 13 ++ .../projection/detail/LonLatProjection.h | 2 + .../projection/detail/MercatorProjection.cc | 5 + .../projection/detail/MercatorProjection.h | 2 + src/atlas/projection/detail/ProjProjection.cc | 4 + src/atlas/projection/detail/ProjProjection.h | 3 + src/atlas/projection/detail/ProjectionImpl.h | 110 +++++++++++ .../projection/detail/SchmidtProjection.cc | 48 +++++ .../projection/detail/SchmidtProjection.h | 2 + src/tests/projection/CMakeLists.txt | 1 + src/tests/projection/test_jacobian.cc | 176 ++++++++++++++++++ 17 files changed, 415 insertions(+), 1 deletion(-) create mode 100644 src/tests/projection/test_jacobian.cc diff --git a/src/atlas/projection/Projection.cc b/src/atlas/projection/Projection.cc index da45f883b..88d64e147 100644 --- a/src/atlas/projection/Projection.cc +++ b/src/atlas/projection/Projection.cc @@ -50,6 +50,10 @@ void atlas::Projection::lonlat2xy( Point2& point ) const { return get()->lonlat2xy( point ); } +atlas::Projection::Jacobian atlas::Projection::jacobian( const PointLonLat& p ) const { + return get()->jacobian( p ); +} + PointLonLat atlas::Projection::lonlat( const PointXY& xy ) const { return get()->lonlat( xy ); } diff --git a/src/atlas/projection/Projection.h b/src/atlas/projection/Projection.h index cae33068a..998303602 100644 --- a/src/atlas/projection/Projection.h +++ b/src/atlas/projection/Projection.h @@ -14,6 +14,7 @@ #include "atlas/domain/Domain.h" #include "atlas/library/config.h" +#include "atlas/projection/detail/ProjectionImpl.h" #include "atlas/util/ObjectHandle.h" //--------------------------------------------------------------------------------------------------------------------- @@ -47,7 +48,8 @@ class ProjectionImpl; class Projection : DOXYGEN_HIDE( public util::ObjectHandle ) { public: - using Spec = util::Config; + using Spec = util::Config; + using Jacobian = projection::detail::ProjectionImpl::Jacobian; public: using Handle::Handle; @@ -62,6 +64,8 @@ class Projection : DOXYGEN_HIDE( public util::ObjectHandle::xy2lonlat( double[] ) const {} template <> void LonLatProjectionT::lonlat2xy( double[] ) const {} +template <> +ProjectionImpl::Jacobian LonLatProjectionT::jacobian( const PointLonLat& ) const { + Jacobian jac; + jac[0] = {1.0, 0.0}; + jac[1] = {0.0, 1.0}; + return jac; +} + +template +ProjectionImpl::Jacobian LonLatProjectionT::jacobian( const PointLonLat& ) const { + throw_NotImplemented( "LonLatProjectionT::jacobian", Here() ); +} + template <> RectangularLonLatDomain LonLatProjectionT::lonlatBoundingBox( const Domain& domain ) const { return domain; diff --git a/src/atlas/projection/detail/LonLatProjection.h b/src/atlas/projection/detail/LonLatProjection.h index 204e04165..eab187973 100644 --- a/src/atlas/projection/detail/LonLatProjection.h +++ b/src/atlas/projection/detail/LonLatProjection.h @@ -41,6 +41,8 @@ class LonLatProjectionT final : public ProjectionImpl { void xy2lonlat( double crd[] ) const override { rotation_.rotate( crd ); } void lonlat2xy( double crd[] ) const override { rotation_.unrotate( crd ); } + Jacobian jacobian( const PointLonLat& ) const override; + bool strictlyRegional() const override { return false; } RectangularLonLatDomain lonlatBoundingBox( const Domain& ) const override; diff --git a/src/atlas/projection/detail/MercatorProjection.cc b/src/atlas/projection/detail/MercatorProjection.cc index 4dd4c6f0a..bbac071c3 100644 --- a/src/atlas/projection/detail/MercatorProjection.cc +++ b/src/atlas/projection/detail/MercatorProjection.cc @@ -153,6 +153,11 @@ void MercatorProjectionT::xy2lonlat( double crd[] ) const { normalise_( crd ); } +template +ProjectionImpl::Jacobian MercatorProjectionT::jacobian( const PointLonLat& ) const { + throw_NotImplemented( "MercatorProjectionT::jacobian", Here() ); +} + // specification template typename MercatorProjectionT::Spec MercatorProjectionT::spec() const { diff --git a/src/atlas/projection/detail/MercatorProjection.h b/src/atlas/projection/detail/MercatorProjection.h index 503dc1797..032ac3161 100644 --- a/src/atlas/projection/detail/MercatorProjection.h +++ b/src/atlas/projection/detail/MercatorProjection.h @@ -32,6 +32,8 @@ class MercatorProjectionT final : public ProjectionImpl { void xy2lonlat( double crd[] ) const override; void lonlat2xy( double crd[] ) const override; + Jacobian jacobian( const PointLonLat& ) const override; + bool strictlyRegional() const override { return true; } // Mercator projection cannot be used for global grids RectangularLonLatDomain lonlatBoundingBox( const Domain& domain ) const override { return ProjectionImpl::lonlatBoundingBox( domain ); diff --git a/src/atlas/projection/detail/ProjProjection.cc b/src/atlas/projection/detail/ProjProjection.cc index c326e5730..b5937565b 100644 --- a/src/atlas/projection/detail/ProjProjection.cc +++ b/src/atlas/projection/detail/ProjProjection.cc @@ -144,6 +144,10 @@ void ProjProjection::lonlat2xy( double crd[] ) const { } +ProjectionImpl::Jacobian ProjProjection::jacobian( const PointLonLat& ) const { + throw_NotImplemented( "ProjProjection::jacobian", Here() ); +} + PointXYZ ProjProjection::xyz( const PointLonLat& lonlat ) const { PJ_COORD P = proj_coord( lonlat.lon(), lonlat.lat(), 0, 0 ); P = proj_trans( sourceToGeocentric_, PJ_FWD, P ); diff --git a/src/atlas/projection/detail/ProjProjection.h b/src/atlas/projection/detail/ProjProjection.h index 4e61884da..230eed30d 100644 --- a/src/atlas/projection/detail/ProjProjection.h +++ b/src/atlas/projection/detail/ProjProjection.h @@ -55,6 +55,9 @@ class ProjProjection final : public ProjectionImpl { void xy2lonlat( double[] ) const override; void lonlat2xy( double[] ) const override; + + Jacobian jacobian( const PointLonLat& ) const override; + PointXYZ xyz( const PointLonLat& ) const override; bool strictlyRegional() const override { return false; } diff --git a/src/atlas/projection/detail/ProjectionImpl.h b/src/atlas/projection/detail/ProjectionImpl.h index 45b3fa38f..af194e784 100644 --- a/src/atlas/projection/detail/ProjectionImpl.h +++ b/src/atlas/projection/detail/ProjectionImpl.h @@ -39,6 +39,114 @@ namespace detail { class ProjectionImpl : public util::Object { public: using Spec = atlas::util::Config; + class Jacobian : public std::array, 2> { + public: + static Jacobian identity() { + Jacobian id; + id[0] = {1.0, 0.0}; + id[1] = {0.0, 1.0}; + return id; + } + + Jacobian inverse() const { + const Jacobian& jac = *this; + Jacobian inv; + double det = jac[0][0] * jac[1][1] - jac[0][1] * jac[1][0]; + inv[0][0] = +jac[1][1] / det; + inv[0][1] = -jac[0][1] / det; + inv[1][0] = -jac[1][0] / det; + inv[1][1] = +jac[0][0] / det; + return inv; + }; + + Jacobian transpose() const { + Jacobian tra = *this; + std::swap( tra[0][1], tra[1][0] ); + return tra; + }; + + Jacobian operator-( const Jacobian& jac2 ) const { + const Jacobian& jac1 = *this; + Jacobian jac; + jac[0][0] = jac1[0][0] - jac2[0][0]; + jac[0][1] = jac1[0][1] - jac2[0][1]; + jac[1][0] = jac1[1][0] - jac2[1][0]; + jac[1][1] = jac1[1][1] - jac2[1][1]; + return jac; + } + + Jacobian operator+( const Jacobian& jac2 ) const { + const Jacobian& jac1 = *this; + Jacobian jac; + jac[0][0] = jac1[0][0] + jac2[0][0]; + jac[0][1] = jac1[0][1] + jac2[0][1]; + jac[1][0] = jac1[1][0] + jac2[1][0]; + jac[1][1] = jac1[1][1] + jac2[1][1]; + return jac; + } + + double norm() const { + const Jacobian& jac = *this; + return sqrt( jac[0][0] * jac[0][0] + jac[0][1] * jac[0][1] + jac[1][0] * jac[1][0] + + jac[1][1] * jac[1][1] ); + } + + Jacobian operator*( const Jacobian& jac2 ) const { + const Jacobian& jac1 = *this; + Jacobian jac; + jac[0][0] = jac1[0][0] * jac2[0][0] + jac1[0][1] * jac2[1][0]; + jac[0][1] = jac1[0][0] * jac2[0][1] + jac1[0][1] * jac2[1][1]; + jac[1][0] = jac1[1][0] * jac2[0][0] + jac1[1][1] * jac2[1][0]; + jac[1][1] = jac1[1][0] * jac2[0][1] + jac1[1][1] * jac2[1][1]; + return jac; + } + + double dx_dlon() const { + const Jacobian& jac = *this; + return jac[JDX][JDLON]; + } + double dy_dlon() const { + const Jacobian& jac = *this; + return jac[JDY][JDLON]; + } + double dx_dlat() const { + const Jacobian& jac = *this; + return jac[JDX][JDLAT]; + } + double dy_dlat() const { + const Jacobian& jac = *this; + return jac[JDY][JDLAT]; + } + + double dlon_dx() const { + const Jacobian& jac = *this; + return jac[JDLON][JDX]; + } + double dlon_dy() const { + const Jacobian& jac = *this; + return jac[JDLON][JDY]; + } + double dlat_dx() const { + const Jacobian& jac = *this; + return jac[JDLAT][JDX]; + } + double dlat_dy() const { + const Jacobian& jac = *this; + return jac[JDLAT][JDY]; + } + + private: + enum + { + JDX = 0, + JDY = 1 + }; + enum + { + JDLON = 0, + JDLAT = 1 + }; + }; public: static const ProjectionImpl* create( const eckit::Parametrisation& p ); @@ -52,6 +160,8 @@ class ProjectionImpl : public util::Object { virtual void xy2lonlat( double crd[] ) const = 0; virtual void lonlat2xy( double crd[] ) const = 0; + virtual Jacobian jacobian( const PointLonLat& ) const = 0; + void xy2lonlat( Point2& ) const; void lonlat2xy( Point2& ) const; diff --git a/src/atlas/projection/detail/SchmidtProjection.cc b/src/atlas/projection/detail/SchmidtProjection.cc index 0bcea5e4d..3a593cb36 100644 --- a/src/atlas/projection/detail/SchmidtProjection.cc +++ b/src/atlas/projection/detail/SchmidtProjection.cc @@ -18,6 +18,7 @@ #include "atlas/runtime/Exception.h" #include "atlas/util/Config.h" #include "atlas/util/Constants.h" +#include "atlas/util/UnitSphere.h" namespace { static double D2R( const double x ) { @@ -65,6 +66,53 @@ void SchmidtProjectionT::lonlat2xy( double crd[] ) const { R2D( std::asin( std::cos( 2. * std::atan( c_ * std::tan( std::acos( std::sin( D2R( crd[1] ) ) ) * 0.5 ) ) ) ) ); } +template <> +ProjectionImpl::Jacobian SchmidtProjectionT::jacobian( const PointLonLat& ) const { + throw_NotImplemented( "SchmidtProjectionT::jacobian", Here() ); +} + +template +ProjectionImpl::Jacobian SchmidtProjectionT::jacobian( const PointLonLat& lonlat ) const { + double xy[2] = {lonlat.lon(), lonlat.lat()}; + + lonlat2xy( xy ); + + PointXYZ xyz, north1, north0( 0.0, 0.0, 1.0 ); + atlas::util::UnitSphere::convertSphericalToCartesian( rotation_.northPole(), north1 ); + atlas::util::UnitSphere::convertSphericalToCartesian( lonlat, xyz ); + + north1 = PointXYZ::normalize( north1 ); + + // Base vectors in unrotated frame + auto u0 = PointXYZ::normalize( PointXYZ::cross( north0, xyz ) ); + auto v0 = PointXYZ::normalize( PointXYZ::cross( xyz, u0 ) ); + + // Base vectors in rotated frame + auto u1 = PointXYZ::normalize( PointXYZ::cross( north1, xyz ) ); + auto v1 = PointXYZ::normalize( PointXYZ::cross( xyz, u1 ) ); + + double u0u1 = PointXYZ::dot( u0, u1 ); + double v0u1 = PointXYZ::dot( v0, u1 ); + double u0v1 = PointXYZ::dot( u0, v1 ); + double v0v1 = PointXYZ::dot( v0, v1 ); + + Jacobian jac; + + double zomc2 = 1.0 - 1.0 / ( c_ * c_ ); + double zopc2 = 1.0 + 1.0 / ( c_ * c_ ); + + double zcosy = cos( D2R( xy[1] ) ), zsiny = sin( D2R( xy[1] ) ); + double zcoslat = cos( D2R( lonlat.lat() ) ); + + double zfactor = sqrt( ( zopc2 + zsiny * zomc2 ) * ( zopc2 + zsiny * zomc2 ) / ( zopc2 * zopc2 - zomc2 * zomc2 ) ); + + + jac[0] = {zcoslat * u0u1 * zfactor / zcosy, v0u1 * zfactor / zcosy}; + jac[1] = {zcoslat * u0v1 * zfactor, v0v1 * zfactor}; + + return jac; +} + // specification template typename SchmidtProjectionT::Spec SchmidtProjectionT::spec() const { diff --git a/src/atlas/projection/detail/SchmidtProjection.h b/src/atlas/projection/detail/SchmidtProjection.h index 3e5e93932..19f976904 100644 --- a/src/atlas/projection/detail/SchmidtProjection.h +++ b/src/atlas/projection/detail/SchmidtProjection.h @@ -32,6 +32,8 @@ class SchmidtProjectionT final : public ProjectionImpl { void xy2lonlat( double crd[] ) const override; void lonlat2xy( double crd[] ) const override; + Jacobian jacobian( const PointLonLat& ) const override; + bool strictlyRegional() const override { return false; } // schmidt is global grid RectangularLonLatDomain lonlatBoundingBox( const Domain& domain ) const override { return ProjectionImpl::lonlatBoundingBox( domain ); diff --git a/src/tests/projection/CMakeLists.txt b/src/tests/projection/CMakeLists.txt index 4557c11c2..2ee8aa196 100644 --- a/src/tests/projection/CMakeLists.txt +++ b/src/tests/projection/CMakeLists.txt @@ -9,6 +9,7 @@ foreach(test test_bounding_box test_projection_LAEA + test_jacobian test_rotation ) ecbuild_add_test( TARGET atlas_${test} SOURCES ${test}.cc LIBS atlas ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) diff --git a/src/tests/projection/test_jacobian.cc b/src/tests/projection/test_jacobian.cc new file mode 100644 index 000000000..49e5290cb --- /dev/null +++ b/src/tests/projection/test_jacobian.cc @@ -0,0 +1,176 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "eckit/utils/MD5.h" + +#include "atlas/grid.h" +#include "atlas/projection.h" +#include "atlas/util/Config.h" +#include "atlas/util/Point.h" + +#include "tests/AtlasTestEnvironment.h" + + +namespace atlas { +namespace test { + +//----------------------------------------------------------------------------- + + +Projection::Jacobian getJacobian( const Projection& proj, const PointLonLat& lonlat0 ) { + double dlon = 0.0001, dlat = 0.0001; + + double xy0[2] = {lonlat0.lon() + 0.00, lonlat0.lat() + 0.00}; + double xy1[2] = {lonlat0.lon() + dlon, lonlat0.lat() + 0.00}; + double xy2[2] = {lonlat0.lon() + 0.00, lonlat0.lat() + dlat}; + + proj.lonlat2xy( xy0 ); + proj.lonlat2xy( xy1 ); + proj.lonlat2xy( xy2 ); + + Projection::Jacobian jac; + + jac[0] = {( xy1[0] - xy0[0] ) / dlon, ( xy2[0] - xy0[0] ) / dlat}; + jac[1] = {( xy1[1] - xy0[1] ) / dlon, ( xy2[1] - xy0[1] ) / dlat}; + + return jac; +} + +void printJacobian( const Projection::Jacobian& jac, const std::string& name ) { + printf( " %s = \n", name.c_str() ); + printf( " %12.4f, %12.4f | %12.4f\n", jac[0][0], jac[0][1], + sqrt( jac[0][0] * jac[0][0] + jac[0][1] * jac[0][1] ) ); + printf( " %12.4f, %12.4f | %12.4f\n", jac[1][0], jac[1][1], + sqrt( jac[1][0] * jac[1][0] + jac[1][1] * jac[1][1] ) ); + printf( " %12.4f, %12.4f\n", sqrt( jac[0][0] * jac[0][0] + jac[1][0] * jac[1][0] ), + sqrt( jac[0][1] * jac[0][1] + jac[1][1] * jac[1][1] ) ); +} + +template +void doTaylorTest( const StructuredGrid& grid, double dmax, SKIP skip ) { + const auto& proj = grid.projection(); + + eckit::MD5 hash; + + for ( int j = 0, jglo = 0; j < grid.ny(); j++ ) { + for ( int i = 0; i < grid.nx( j ); i++, jglo++ ) { + if ( !skip( grid, i, j ) ) { + double lonlat[2]; + grid.lonlat( i, j, lonlat ); + + auto jacA = proj.jacobian( PointLonLat( lonlat ) ); + auto jacB = getJacobian( proj, PointLonLat( lonlat ) ); + auto jacC = jacB * jacA.inverse() - Projection::Jacobian::identity(); + + hash.add( jacA[0][0] ); + hash.add( jacA[0][1] ); + hash.add( jacA[1][0] ); + hash.add( jacA[1][1] ); + + double diff = jacC.norm(); + + if ( diff > dmax ) { + printf( "------------------\n" ); + printf( " i, j = %8d, %8d\n", i, j ); + printJacobian( jacA, "jacA" ); + printf( "\n" ); + printJacobian( jacB, "jacB" ); + printf( "\n" ); + printf( "%12.4e\n", diff ); + printf( "\n" ); + } + + EXPECT( diff < dmax ); + } + } + } + Log::info() << "Jacobian checksum (not crossplatform) : " << hash.digest() << std::endl; +} + + +void doTaylorTest( const StructuredGrid& grid, double dmax ) { + auto noskip = []( const StructuredGrid& grid, int i, int j ) { return false; }; + doTaylorTest( grid, dmax, noskip ); +} + +CASE( "test_rotated_schmidt" ) { + auto projection = [] { + util::Config config; + config.set( "type", "rotated_schmidt" ); + config.set( "stretching_factor", 2.4 ); + config.set( "rotation_angle", 0.0 ); + config.set( "north_pole", {2.0, 46.7} ); + return Projection{config}; + }(); + + auto jacobian = projection.jacobian( PointLonLat{0., 0.} ); + double tol = 1.e-10; + EXPECT_APPROX_EQ( jacobian[0][0], 1.372570713035, tol ); + EXPECT_APPROX_EQ( jacobian[0][1], -0.045140586761, tol ); + EXPECT_APPROX_EQ( jacobian[1][0], 0.045110961287, tol ); + EXPECT_APPROX_EQ( jacobian[1][1], 1.371669903783, tol ); + + doTaylorTest( Grid{"N16", projection}, 1e-3, + // Avoid large jumps of pseudo-longitude (ie 359.5 -> 0) + []( const StructuredGrid& grid, int i, int j ) { + return ( i == 0 ) || ( i == grid.nx( j ) - 1 ) || ( i == grid.nx( j ) / 2 ); + } ); +} + + +CASE( "test_lambert_conformal_conic" ) { + const int Nx = 64, Ny = 64, Nux = 53, Nuy = 53; + const double DxInMetres = 50000., DyInMetres = 50000.; + const double LaDInDegrees = 46.2, Latin1InDegrees = 46.2, Latin2InDegrees = 46.2, LoVInDegrees = 2.0; + const double XMinInMetres = -Nux / 2 * DxInMetres, YMinInMetres = -Nuy / 2 * DyInMetres; + + auto projection = [&] { + util::Config config; + config.set( "type", "lambert_conformal_conic" ); + config.set( "longitude0", LoVInDegrees ); + config.set( "latitude0", LaDInDegrees ); + config.set( "latitude1", Latin1InDegrees ); + config.set( "latitude2", Latin2InDegrees ); + return Projection{config}; + }(); + + auto jacobian = projection.jacobian( PointLonLat{0., 0.} ); + double tol = 1.e-5; + EXPECT_APPROX_EQ( jacobian[0][0], 148530.135993682081, tol ); + EXPECT_APPROX_EQ( jacobian[0][1], 3742.887654051571, tol ); + EXPECT_APPROX_EQ( jacobian[1][0], -3742.887654051572, tol ); + EXPECT_APPROX_EQ( jacobian[1][1], 148530.135993682081, tol ); + + util::Config grid; + grid.set( "type", "regional" ); + grid.set( "dx", DxInMetres ); + grid.set( "dy", DyInMetres ); + grid.set( "xmin", XMinInMetres ); + grid.set( "ymin", YMinInMetres ); + grid.set( "nx", Nx ); + grid.set( "ny", Ny ); + grid.set( "projection", projection.spec() ); + + doTaylorTest( Grid( grid ), 1e-6 ); +} + +CASE( "test_lonlat" ) { + doTaylorTest( StructuredGrid( "N16" ), 1e-9 ); +} + + +//----------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas + +int main( int argc, char** argv ) { + return atlas::test::run( argc, argv ); +} From 6dbdc5bb11a183259367b85501b0452ed6e7ada4 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 7 Oct 2020 13:52:22 +0100 Subject: [PATCH 160/161] Version 0.22.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 885415662..215740905 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.21.0 +0.22.0 From 5465839b346530b8b2924fab1dbdb4963c8d0cb4 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 14 Oct 2020 12:07:43 +0100 Subject: [PATCH 161/161] Update Changelog --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba31ff033..299a739f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ## [Unreleased] +## [0.22.0] - 2020-10-14 +### Fixed +- Feature INIT_SNAN was not working +- Support KNearestNeighbour interpolation for functionspace with + smaller halo than the mesh contains +- Support array size up to size_t limit + +### Changed +- Migration to use ecbuild 3.4 +- ATLAS_BITS_LOCAL can be configured to 32 or 64 + +### Added +- Fields can be created with given alignment, which adds padding in innermost dimension +- Added conservative interpolation with "grid-box average" and "grid-box maximum" +- Missing value definition for fields +- Add support for missing values in matrix-based interpolation methods +- Floating point trapping and signal handling mechanism +- Fortran: GridDistribution constructors +- Fortran: Domain access +- Fortran: Get lonlat_bounding_box via domain +- Possibility to access Jacobian of projections (with only some projections implemented) + ## [0.21.0] - 2020-06-23 ### Fixed - Fixed Rotation order of applying the rotation angle @@ -223,6 +245,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ## 0.13.0 - 2018-02-16 [Unreleased]: https://github.com/ecmwf/atlas/compare/master...develop +[0.22.0]: https://github.com/ecmwf/atlas/compare/0.21.0...0.22.0 [0.21.0]: https://github.com/ecmwf/atlas/compare/0.20.2...0.21.0 [0.20.2]: https://github.com/ecmwf/atlas/compare/0.20.1...0.20.2 [0.20.1]: https://github.com/ecmwf/atlas/compare/0.20.0...0.20.1