From 690649a8c203abe745d323ee6f134b4775c3afb1 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 31 Jul 2024 11:24:01 -0400 Subject: [PATCH 1/9] Fixes #2926. Update quantize output --- Apps/Regrid_Util.F90 | 2 +- CHANGELOG.md | 3 +++ gridcomps/History/MAPL_HistoryGridComp.F90 | 8 +++++--- griddedio/GriddedIO.F90 | 24 ++++++++++++++++++++-- pfio/Variable.F90 | 4 ++-- shared/Constants/InternalConstants.F90 | 2 +- 6 files changed, 34 insertions(+), 9 deletions(-) diff --git a/Apps/Regrid_Util.F90 b/Apps/Regrid_Util.F90 index 8b4810c4d073..3e472d62d345 100644 --- a/Apps/Regrid_Util.F90 +++ b/Apps/Regrid_Util.F90 @@ -96,7 +96,7 @@ subroutine process_command_line(this,rc) this%lat_range=uninit this%shave=64 this%deflate=0 - this%quantize_algorithm=1 + this%quantize_algorithm=0 this%quantize_level=0 this%use_weights = .false. nargs = command_argument_count() diff --git a/CHANGELOG.md b/CHANGELOG.md index 02d6aeed0260..8d5e32194827 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +- Deprecate `GranularBR` as a quantization method keyword in History. We will prefer `granular_bitround` in the future to match + draft CF conventions. This will be removed in MAPL 3. + ## [2.47.1] - 2024-07-17 ### Fixed diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index 4b048912bd70..556d12f9e5f4 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -839,18 +839,20 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) label=trim(string) // 'quantize_algorithm:' ,_RC ) ! Uppercase the algorithm string just to allow for any case + ! CF Conventions will prefer 'bitgroom', 'bitround', and 'granular_bitround' + ! but we will allow 'GranularBR' in MAPL2, deprecate it, and remove it in MAPL3 uppercase_algorithm = ESMF_UtilStringUpperCase(list(n)%quantize_algorithm_string,_RC) select case (trim(uppercase_algorithm)) case ('NONE') list(n)%quantize_algorithm = MAPL_Quantize_Disabled case ('BITGROOM') list(n)%quantize_algorithm = MAPL_Quantize_BitGroom - case ('GRANULARBR') - list(n)%quantize_algorithm = MAPL_Quantize_GranularBR + case ('GRANULARBR', 'GRANULAR_BITROUND') + list(n)%quantize_algorithm = MAPL_Quantize_Granular_BitRound case ('BITROUND') list(n)%quantize_algorithm = MAPL_Quantize_BitRound case default - _FAIL('Invalid quantize_algorithm. Allowed values are NONE, BitGroom, GranularBR, BitRound') + _FAIL('Invalid quantize_algorithm. Allowed values are NONE, bitgroom, granular_bitround, granularbr (deprecated), and bitround') end select call ESMF_ConfigGetAttribute ( cfg, list(n)%quantize_level, default=0, & diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index fe2e6fb45be5..f1817666edfb 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -190,7 +190,6 @@ subroutine CreateFileMetaData(this,items,bundle,timeInfo,vdata,ogrid,global_attr call this%timeInfo%add_time_to_metadata(this%metadata,rc=status) _VERIFY(status) - iter = this%items%begin() if (.not.allocated(this%chunking)) then call this%set_default_chunking(rc=status) _VERIFY(status) @@ -198,11 +197,11 @@ subroutine CreateFileMetaData(this,items,bundle,timeInfo,vdata,ogrid,global_attr call this%check_chunking(this%vdata%lm,_RC) end if - order = this%metadata%get_order(rc=status) _VERIFY(status) metadataVarsSize = order%size() + iter = this%items%begin() do while (iter /= this%items%end()) item => iter%get() if (item%itemType == ItemTypeScalar) then @@ -423,6 +422,27 @@ subroutine CreateVariable(this,itemName,rc) #else call v%add_attribute('regrid_method', regrid_method_int_to_string(this%regrid_method)) #endif + ! The CF Convention will soon support quantization. This requires three new attributes + ! if enabled: + ! 1. quantization --> Will point to a quantization_info containter with the quantization algorithm + ! 2a. quantization_nsb --> Number of significant bits (only for bitround) + ! 2b. quantization_nsd --> Number of significant digits (only for bitgroom and granular_bitgroom) + ! 3. quantization_maximum_relative_error --> Maximum relative error (defined as 2^(-nsb) for bitround, and UNDEFINED? for bitgroom and granular_bitgroom) + + ! Bitround ==> 1 + if (this%quantizeAlgorithm == 1) then + call v%add_attribute('quantization', 'quantization_info') + call v%add_attribute('quantization_nsb', this%quantizeLevel) + call v%add_attribute('quantization_maximum_relative_error', 0.5 * 2.0**(-this%quantizeLevel)) + end if + ! granular_bitgroom ==> 2, bitgroom ==> 3 + if (this%quantizeAlgorithm == 2 .or. this%quantizeAlgorithm == 3) then + call v%add_attribute('quantization', 'quantization_info') + call v%add_attribute('quantization_nsd', this%quantizeLevel) + ! For now, don't add this until we know what to do + !call v%add_attribute('quantization_maximum_relative_error', 0.5 * 2.0**(-this%quantizeLevel)) + end if + call factory%append_variable_metadata(v) call this%metadata%add_variable(trim(varName),v,rc=status) _VERIFY(status) diff --git a/pfio/Variable.F90 b/pfio/Variable.F90 index 9d42cf97f7f2..67344a40fa5b 100644 --- a/pfio/Variable.F90 +++ b/pfio/Variable.F90 @@ -28,7 +28,7 @@ module pFIO_VariableMod type (StringAttributeMap) :: attributes type (UnlimitedEntity) :: const_value integer :: deflation = 0 ! default no compression - integer :: quantize_algorithm = 1 ! default bitgroom + integer :: quantize_algorithm = 0 ! default no quantize_algorithm integer :: quantize_level = 0 ! default no quantize_level integer, allocatable :: chunksizes(:) contains @@ -85,7 +85,7 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat var%type = -1 var%deflation = 0 - var%quantize_algorithm = 1 + var%quantize_algorithm = 0 var%quantize_level = 0 var%chunksizes = empty var%dimensions = StringVector() diff --git a/shared/Constants/InternalConstants.F90 b/shared/Constants/InternalConstants.F90 index ac2935ea9911..57a8c8eda94b 100644 --- a/shared/Constants/InternalConstants.F90 +++ b/shared/Constants/InternalConstants.F90 @@ -170,7 +170,7 @@ module MAPL_InternalConstantsMod enum, bind(c) enumerator MAPL_Quantize_Disabled enumerator MAPL_Quantize_BitGroom - enumerator MAPL_Quantize_GranularBR + enumerator MAPL_Quantize_Granular_BitRound enumerator MAPL_Quantize_BitRound endenum ! Constant masking From 58f9f7e84126bd66aa1549fa133a2b3acf7aae79 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 31 Jul 2024 15:44:56 -0400 Subject: [PATCH 2/9] Add new quantization_info --- CHANGELOG.md | 3 + gridcomps/History/MAPL_HistoryGridComp.F90 | 28 ++++---- griddedio/GriddedIO.F90 | 75 +++++++++++++++++++--- pfio/Variable.F90 | 2 +- shared/Constants/InternalConstants.F90 | 10 +-- 5 files changed, 87 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d5e32194827..a6612b17748a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ESMA_cmake v3.48.0 - Update `esma_add_fortran_submodules` function - Move MPI detection out of FindBaselibs +- Add support for preliminary CF Conventions quantization properties + - Add new quantization keyword `granular_bitround` to History. This will be the preferred keyword for quantization in the future + replacing `GranularBR` ### Fixed diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index 556d12f9e5f4..f86135d6880e 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -838,38 +838,34 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute ( cfg, list(n)%quantize_algorithm_string, default='NONE', & label=trim(string) // 'quantize_algorithm:' ,_RC ) + call ESMF_ConfigGetAttribute ( cfg, list(n)%quantize_level, default=0, & + label=trim(string) // 'quantize_level:' ,_RC ) + ! Uppercase the algorithm string just to allow for any case ! CF Conventions will prefer 'bitgroom', 'bitround', and 'granular_bitround' ! but we will allow 'GranularBR' in MAPL2, deprecate it, and remove it in MAPL3 uppercase_algorithm = ESMF_UtilStringUpperCase(list(n)%quantize_algorithm_string,_RC) select case (trim(uppercase_algorithm)) case ('NONE') - list(n)%quantize_algorithm = MAPL_Quantize_Disabled + list(n)%quantize_algorithm = MAPL_NOQUANTIZE + ! If quantize_algorithm is 0, then quantize_level must be 0 + _ASSERT( list(n)%quantize_level == 0, 'quantize_algorithm is none, so quantize_level must be "none"') case ('BITGROOM') - list(n)%quantize_algorithm = MAPL_Quantize_BitGroom + list(n)%quantize_algorithm = MAPL_QUANTIZE_BITGROOM case ('GRANULARBR', 'GRANULAR_BITROUND') - list(n)%quantize_algorithm = MAPL_Quantize_Granular_BitRound + list(n)%quantize_algorithm = MAPL_QUANTIZE_GRANULAR_BITROUND case ('BITROUND') - list(n)%quantize_algorithm = MAPL_Quantize_BitRound + list(n)%quantize_algorithm = MAPL_QUANTIZE_BITROUND case default - _FAIL('Invalid quantize_algorithm. Allowed values are NONE, bitgroom, granular_bitround, granularbr (deprecated), and bitround') + _FAIL('Invalid quantize_algorithm. Allowed values are none, bitgroom, granular_bitround, granularbr (deprecated), and bitround') end select - call ESMF_ConfigGetAttribute ( cfg, list(n)%quantize_level, default=0, & - label=trim(string) // 'quantize_level:' ,_RC ) - ! If nbits_to_keep < MAPL_NBITS_UPPER_LIMIT (24) and quantize_algorithm greater than 0, then a user might be doing different ! shaving algorithms. We do not allow this - _ASSERT( .not. ( (list(n)%nbits_to_keep < MAPL_NBITS_UPPER_LIMIT) .and. (list(n)%quantize_algorithm > 0) ), 'nbits < 24 and quantize_algorithm > 0 is not allowed. Choose one bit grooming method.') - - ! quantize_algorithm must be between 0 and 3 where 0 means not enabled - _ASSERT( (list(n)%quantize_algorithm >= 0) .and. (list(n)%quantize_algorithm <= 3), 'quantize_algorithm must be between 0 and 3, where 0 means not enabled') + _ASSERT( .not. ( (list(n)%nbits_to_keep < MAPL_NBITS_UPPER_LIMIT) .and. (list(n)%quantize_algorithm > MAPL_NOQUANTIZE) ), 'nbits < 24 and quantize_algorithm not "none" is not allowed. Choose a supported quantization method.') ! Now we test in the case that a valid quantize algorithm is chosen - if (list(n)%quantize_algorithm == 0) then - ! If quantize_algorithm is 0, then quantize_level must be 0 - _ASSERT( list(n)%quantize_level == 0, 'quantize_algorithm is 0, so quantize_level must be 0') - else + if (list(n)%quantize_algorithm /= MAPL_NOQUANTIZE) then ! If quantize_algorithm is greater than 0, then quantize_level must be greater than or equal to 0 _ASSERT( list(n)%quantize_level >= 0, 'netCDF quantize has been enabled, so quantize_level must be greater than or equal to 0') end if diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index f1817666edfb..3b7e0f67dd5c 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -51,7 +51,7 @@ module MAPL_GriddedIOMod type(VerticalData) :: vdata type(GriddedIOitemVector) :: items integer :: deflateLevel = 0 - integer :: quantizeAlgorithm = 1 + integer :: quantizeAlgorithm = MAPL_NOQUANTIZE integer :: quantizeLevel = 0 integer, allocatable :: chunking(:) logical :: itemOrderAlphabetical = .true. @@ -60,6 +60,7 @@ module MAPL_GriddedIOMod contains procedure :: CreateFileMetaData procedure :: CreateVariable + procedure :: CreateQuantizationInfo procedure :: modifyTime procedure :: modifyTimeIncrement procedure :: bundlePost @@ -201,6 +202,12 @@ subroutine CreateFileMetaData(this,items,bundle,timeInfo,vdata,ogrid,global_attr _VERIFY(status) metadataVarsSize = order%size() + ! If quantize algorithm is set, create a quantization_info variable + if (this%quantizeAlgorithm /= MAPL_NOQUANTIZE) then + call this%CreateQuantizationInfo(rc=status) + _VERIFY(status) + end if + iter = this%items%begin() do while (iter /= this%items%end()) item => iter%get() @@ -424,23 +431,25 @@ subroutine CreateVariable(this,itemName,rc) #endif ! The CF Convention will soon support quantization. This requires three new attributes ! if enabled: - ! 1. quantization --> Will point to a quantization_info containter with the quantization algorithm + ! 1. quantization --> Will point to a quantization_info container with the quantization algorithm + ! (NOTE: this will need to be programmatic when per-variable quantization is enabled) ! 2a. quantization_nsb --> Number of significant bits (only for bitround) - ! 2b. quantization_nsd --> Number of significant digits (only for bitgroom and granular_bitgroom) - ! 3. quantization_maximum_relative_error --> Maximum relative error (defined as 2^(-nsb) for bitround, and UNDEFINED? for bitgroom and granular_bitgroom) + ! 2b. quantization_nsd --> Number of significant digits (only for bitgroom and granular_bitround) + ! 3. quantization_maximum_relative_error --> Maximum relative error (defined as 2^(-nsb) for bitround, and UNDEFINED? for bitgroom and granular_bitround) - ! Bitround ==> 1 - if (this%quantizeAlgorithm == 1) then + ! Bitround + if (this%quantizeAlgorithm == MAPL_QUANTIZE_BITROUND) then call v%add_attribute('quantization', 'quantization_info') call v%add_attribute('quantization_nsb', this%quantizeLevel) call v%add_attribute('quantization_maximum_relative_error', 0.5 * 2.0**(-this%quantizeLevel)) end if - ! granular_bitgroom ==> 2, bitgroom ==> 3 - if (this%quantizeAlgorithm == 2 .or. this%quantizeAlgorithm == 3) then + ! granular_bitround and bitgroom + if (this%quantizeAlgorithm == MAPL_QUANTIZE_BITGROOM .or. this%quantizeAlgorithm == MAPL_QUANTIZE_GRANULAR_BITROUND) then call v%add_attribute('quantization', 'quantization_info') call v%add_attribute('quantization_nsd', this%quantizeLevel) ! For now, don't add this until we know what to do - !call v%add_attribute('quantization_maximum_relative_error', 0.5 * 2.0**(-this%quantizeLevel)) + ! Add something for testing + call v%add_attribute('quantization_maximum_relative_error', 0.5 * 10.0**(-this%quantizeLevel)) end if call factory%append_variable_metadata(v) @@ -462,6 +471,54 @@ subroutine CreateVariable(this,itemName,rc) end subroutine CreateVariable + subroutine CreateQuantizationInfo(this,rc) + class (MAPL_GriddedIO), intent(inout) :: this + integer, optional, intent(out) :: rc + + integer :: status + + class (AbstractGridFactory), pointer :: factory + character(len=:), allocatable :: varName + type(Variable) :: v + + factory => get_factory(this%output_grid,rc=status) + _VERIFY(status) + + v = Variable(type=PFIO_CHAR) + + ! In the future when we can do per variable quantization, we will need + ! to do things like quantization_info1, quantization_info2, etc. + ! For now, we will just use quantization_info as it is per collection + varName = "quantization_info" + + ! We need to convert the quantization algorithm to a string + select case (this%quantizeAlgorithm) + case (MAPL_QUANTIZE_BITGROOM) + call v%add_attribute('algorithm', 'bitgroom') + case (MAPL_QUANTIZE_BITROUND) + call v%add_attribute('algorithm', 'bitround') + case (MAPL_QUANTIZE_GRANULAR_BITROUND) + call v%add_attribute('algorithm', 'granular_bitround') + case default + _FAIL('Unknown quantization algorithm') + end select + + ! Next add the implementation details + ! 3. implementation: This property contains free-form text + ! that concisely conveys the algorithm provenance, including the + ! name of the library or client that performed the quantization, + ! the software version, and the name of the author(s) if deemed + ! relevant. + call v%add_attribute('implementation', 'MAPL') + + call factory%append_variable_metadata(v) + call this%metadata%add_variable(trim(varName),v,rc=status) + _VERIFY(status) + + _RETURN(_SUCCESS) + + end subroutine CreateQuantizationInfo + subroutine modifyTime(this, oClients, rc) class(MAPL_GriddedIO), intent(inout) :: this type (ClientManager), optional, intent(inout) :: oClients diff --git a/pfio/Variable.F90 b/pfio/Variable.F90 index 67344a40fa5b..624954bfd162 100644 --- a/pfio/Variable.F90 +++ b/pfio/Variable.F90 @@ -28,7 +28,7 @@ module pFIO_VariableMod type (StringAttributeMap) :: attributes type (UnlimitedEntity) :: const_value integer :: deflation = 0 ! default no compression - integer :: quantize_algorithm = 0 ! default no quantize_algorithm + integer :: quantize_algorithm = 0 ! default no quantization integer :: quantize_level = 0 ! default no quantize_level integer, allocatable :: chunksizes(:) contains diff --git a/shared/Constants/InternalConstants.F90 b/shared/Constants/InternalConstants.F90 index 57a8c8eda94b..f2cae5cf9cb6 100644 --- a/shared/Constants/InternalConstants.F90 +++ b/shared/Constants/InternalConstants.F90 @@ -166,12 +166,12 @@ module MAPL_InternalConstantsMod integer, parameter :: MAPL_NBITS_NOT_SET = 1000 integer, parameter :: MAPL_NBITS_UPPER_LIMIT = 24 - ! Constants for netCDF quantize + ! Constants for netCDF quantize (these echo the values in the netcdf-fortran library) enum, bind(c) - enumerator MAPL_Quantize_Disabled - enumerator MAPL_Quantize_BitGroom - enumerator MAPL_Quantize_Granular_BitRound - enumerator MAPL_Quantize_BitRound + enumerator MAPL_NOQUANTIZE + enumerator MAPL_QUANTIZE_BITGROOM + enumerator MAPL_QUANTIZE_GRANULAR_BITROUND + enumerator MAPL_QUANTIZE_BITROUND endenum ! Constant masking enum, bind(c) From 2345b5b5c7005e3858a486dbdf2a531a4269387a Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 1 Aug 2024 11:46:26 -0400 Subject: [PATCH 3/9] More updates --- gridcomps/History/MAPL_HistoryGridComp.F90 | 11 ++++++ griddedio/GriddedIO.F90 | 46 +++++++++++++++++++--- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index f86135d6880e..da35528dd3d4 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -870,6 +870,17 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) _ASSERT( list(n)%quantize_level >= 0, 'netCDF quantize has been enabled, so quantize_level must be greater than or equal to 0') end if + ! If a user has chosen MAPL_QUANTIZE_BITROUND, then we allow a maximum of 23 bits to be kept + if (list(n)%quantize_algorithm == MAPL_QUANTIZE_BITROUND) then + _ASSERT( list(n)%quantize_level <= 23, 'netCDF bitround has been enabled, so number of significant bits (quantize_level) must be less than or equal to 23') + end if + + ! For MAPL_QUANTIZE_GRANULAR_BITROUND and MAPL_QUANTIZE_BITGROOM, these use number of + ! significant digits, so for single precision, we allow a maximum of 7 digits to be kept + if (list(n)%quantize_algorithm == MAPL_QUANTIZE_GRANULAR_BITROUND .or. list(n)%quantize_algorithm == MAPL_QUANTIZE_BITGROOM) then + _ASSERT( list(n)%quantize_level <= 7, 'netCDF granular bitround or bitgroom has been enabled, so number of significant digits (quantize_level) must be less than or equal to 7') + end if + tm_default = -1 call ESMF_ConfigGetAttribute ( cfg, list(n)%tm, default=tm_default, & label=trim(string) // 'tm:', _RC ) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index 3b7e0f67dd5c..3239fa367e19 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -26,6 +26,7 @@ module MAPL_GriddedIOMod use, intrinsic :: ISO_C_BINDING use, intrinsic :: iso_fortran_env, only: REAL64 use ieee_arithmetic, only: isnan => ieee_is_nan + use netcdf, only: nf90_inq_libvers implicit none private @@ -447,9 +448,11 @@ subroutine CreateVariable(this,itemName,rc) if (this%quantizeAlgorithm == MAPL_QUANTIZE_BITGROOM .or. this%quantizeAlgorithm == MAPL_QUANTIZE_GRANULAR_BITROUND) then call v%add_attribute('quantization', 'quantization_info') call v%add_attribute('quantization_nsd', this%quantizeLevel) - ! For now, don't add this until we know what to do - ! Add something for testing - call v%add_attribute('quantization_maximum_relative_error', 0.5 * 10.0**(-this%quantizeLevel)) + ! Per czender, these have maximum_absolute_error. We use the calculate_mae function below + ! which replicates a table in doi:10.5194/gmd-12-4099-2019 + ! NOTE: This might not be the right formula. As the CF Convention draft is updated, + ! we will update this code. + call v%add_attribute('quantization_maximum_absolute_error', calculate_mae(this%quantizeLevel)) end if call factory%append_variable_metadata(v) @@ -471,6 +474,30 @@ subroutine CreateVariable(this,itemName,rc) end subroutine CreateVariable + function calculate_mae(nsd) result(mae) + + ! This function is based on Table 3 of doi:10.5194/gmd-12-4099-2019 + ! The algorithm is weird, but it does duplicate the table + + implicit none + integer, intent(in) :: nsd + real(kind=REAL32) :: mae + real(kind=REAL32) :: mae_base + integer :: correction + + mae_base = 4.0 * (1.0/16.0)**floor(real(nsd)/2.0) * (1.0/8.0)**ceiling(real(nsd)/2.0) + + if (nsd > 2 .and. mod(nsd, 2) == 0) then + correction = 2 + else if (nsd == 7) then + correction = 2 + else + correction = 1 + end if + + mae = mae_base * correction + end function calculate_mae + subroutine CreateQuantizationInfo(this,rc) class (MAPL_GriddedIO), intent(inout) :: this integer, optional, intent(out) :: rc @@ -478,7 +505,7 @@ subroutine CreateQuantizationInfo(this,rc) integer :: status class (AbstractGridFactory), pointer :: factory - character(len=:), allocatable :: varName + character(len=:), allocatable :: varName, netcdf_version type(Variable) :: v factory => get_factory(this%output_grid,rc=status) @@ -509,7 +536,16 @@ subroutine CreateQuantizationInfo(this,rc) ! name of the library or client that performed the quantization, ! the software version, and the name of the author(s) if deemed ! relevant. - call v%add_attribute('implementation', 'MAPL') + ! + ! In the current case, all algorithms are from libnetcdf + ! we make a string using nf90_inq_libvers() + + netcdf_version = 'libnetcdf ' // nf90_inq_libvers() + call v%add_attribute('implementation', netcdf_version) + + ! NOTE: In the future if we add the MAPL bit-shaving + ! to use the quantization parts of the code, it will + ! need a different implementation string call factory%append_variable_metadata(v) call this%metadata%add_variable(trim(varName),v,rc=status) From e47dc197830b58ae08e9db2fb535d7d802b0e1ec Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 1 Aug 2024 11:51:14 -0400 Subject: [PATCH 4/9] Make the DOI's less URL like --- griddedio/GriddedIO.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index 3239fa367e19..4a27dbe113c7 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -449,7 +449,7 @@ subroutine CreateVariable(this,itemName,rc) call v%add_attribute('quantization', 'quantization_info') call v%add_attribute('quantization_nsd', this%quantizeLevel) ! Per czender, these have maximum_absolute_error. We use the calculate_mae function below - ! which replicates a table in doi:10.5194/gmd-12-4099-2019 + ! which replicates a table in doi 10.5194/gmd-12-4099-2019 ! NOTE: This might not be the right formula. As the CF Convention draft is updated, ! we will update this code. call v%add_attribute('quantization_maximum_absolute_error', calculate_mae(this%quantizeLevel)) @@ -476,7 +476,7 @@ end subroutine CreateVariable function calculate_mae(nsd) result(mae) - ! This function is based on Table 3 of doi:10.5194/gmd-12-4099-2019 + ! This function is based on Table 3 of doi 10.5194/gmd-12-4099-2019 ! The algorithm is weird, but it does duplicate the table implicit none From 84fb0ff6e34bbd68300a1039208f9fc52eca8479 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 1 Aug 2024 12:11:12 -0400 Subject: [PATCH 5/9] Update griddedio/GriddedIO.F90 Co-authored-by: Tom Clune --- griddedio/GriddedIO.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index 4a27dbe113c7..f39906f6c23b 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -479,7 +479,6 @@ function calculate_mae(nsd) result(mae) ! This function is based on Table 3 of doi 10.5194/gmd-12-4099-2019 ! The algorithm is weird, but it does duplicate the table - implicit none integer, intent(in) :: nsd real(kind=REAL32) :: mae real(kind=REAL32) :: mae_base From 175b0e676b45eec098c374bf43e5f18c747d492a Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 1 Aug 2024 12:11:33 -0400 Subject: [PATCH 6/9] Update griddedio/GriddedIO.F90 Co-authored-by: Tom Clune --- griddedio/GriddedIO.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index f39906f6c23b..fb739083427c 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -547,8 +547,7 @@ subroutine CreateQuantizationInfo(this,rc) ! need a different implementation string call factory%append_variable_metadata(v) - call this%metadata%add_variable(trim(varName),v,rc=status) - _VERIFY(status) + call this%metadata%add_variable(trim(varName),v,_RC) _RETURN(_SUCCESS) From fed3655dfac38f9ecb874aa094915897254c17d2 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 1 Aug 2024 12:11:44 -0400 Subject: [PATCH 7/9] Update griddedio/GriddedIO.F90 Co-authored-by: Tom Clune --- griddedio/GriddedIO.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index fb739083427c..e34d8a00d9a1 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -507,8 +507,7 @@ subroutine CreateQuantizationInfo(this,rc) character(len=:), allocatable :: varName, netcdf_version type(Variable) :: v - factory => get_factory(this%output_grid,rc=status) - _VERIFY(status) + factory => get_factory(this%output_grid,_RC) v = Variable(type=PFIO_CHAR) From 54aea8745663300275706e26034928c0592d353b Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 1 Aug 2024 12:11:51 -0400 Subject: [PATCH 8/9] Update griddedio/GriddedIO.F90 Co-authored-by: Tom Clune --- griddedio/GriddedIO.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index e34d8a00d9a1..bc6351daee10 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -205,8 +205,7 @@ subroutine CreateFileMetaData(this,items,bundle,timeInfo,vdata,ogrid,global_attr ! If quantize algorithm is set, create a quantization_info variable if (this%quantizeAlgorithm /= MAPL_NOQUANTIZE) then - call this%CreateQuantizationInfo(rc=status) - _VERIFY(status) + call this%CreateQuantizationInfo(_RC) end if iter = this%items%begin() From d26d17f5355f23f2cdbf76a7da2231f0efc5d5bb Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 1 Aug 2024 13:14:28 -0400 Subject: [PATCH 9/9] Fix typo --- gridcomps/History/MAPL_HistoryGridComp.F90 | 7 +++++-- griddedio/GriddedIO.F90 | 7 ++----- shared/Constants/InternalConstants.F90 | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index da35528dd3d4..7aa0afff2a1c 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -425,6 +425,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) logical :: has_conservative_keyword, has_regrid_keyword integer :: create_mode character(len=:), allocatable :: uppercase_algorithm + character(len=2) :: tmpchar ! Begin !------ @@ -872,13 +873,15 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) ! If a user has chosen MAPL_QUANTIZE_BITROUND, then we allow a maximum of 23 bits to be kept if (list(n)%quantize_algorithm == MAPL_QUANTIZE_BITROUND) then - _ASSERT( list(n)%quantize_level <= 23, 'netCDF bitround has been enabled, so number of significant bits (quantize_level) must be less than or equal to 23') + write(tmpchar, '(I2)') MAPL_QUANTIZE_MAX_NSB + _ASSERT( list(n)%quantize_level <= MAPL_QUANTIZE_MAX_NSB, 'netCDF bitround has been enabled, so number of significant bits (quantize_level) must be less than or equal to ' // trim(tmpchar)) end if ! For MAPL_QUANTIZE_GRANULAR_BITROUND and MAPL_QUANTIZE_BITGROOM, these use number of ! significant digits, so for single precision, we allow a maximum of 7 digits to be kept if (list(n)%quantize_algorithm == MAPL_QUANTIZE_GRANULAR_BITROUND .or. list(n)%quantize_algorithm == MAPL_QUANTIZE_BITGROOM) then - _ASSERT( list(n)%quantize_level <= 7, 'netCDF granular bitround or bitgroom has been enabled, so number of significant digits (quantize_level) must be less than or equal to 7') + write(tmpchar, '(I2)') MAPL_QUANTIZE_MAX_NSD + _ASSERT( list(n)%quantize_level <= MAPL_QUANTIZE_MAX_NSD, 'netCDF granular bitround or bitgroom has been enabled, so number of significant digits (quantize_level) must be less than or equal to ' // trim(tmpchar)) end if tm_default = -1 diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index bc6351daee10..7eb3cf53c7f7 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -485,12 +485,9 @@ function calculate_mae(nsd) result(mae) mae_base = 4.0 * (1.0/16.0)**floor(real(nsd)/2.0) * (1.0/8.0)**ceiling(real(nsd)/2.0) - if (nsd > 2 .and. mod(nsd, 2) == 0) then + correction = 1 + if ( (nsd > 2 .and. mod(nsd, 2) == 0) .or. nsd == 7 ) then correction = 2 - else if (nsd == 7) then - correction = 2 - else - correction = 1 end if mae = mae_base * correction diff --git a/shared/Constants/InternalConstants.F90 b/shared/Constants/InternalConstants.F90 index f2cae5cf9cb6..3cad2914e86f 100644 --- a/shared/Constants/InternalConstants.F90 +++ b/shared/Constants/InternalConstants.F90 @@ -173,6 +173,10 @@ module MAPL_InternalConstantsMod enumerator MAPL_QUANTIZE_GRANULAR_BITROUND enumerator MAPL_QUANTIZE_BITROUND endenum + ! Maximum number of significant digits for quantization (bitgroom, granular_bitround) + integer, parameter :: MAPL_QUANTIZE_MAX_NSD = 7 + ! Maximum number of significant bits for quantization (bitround) + integer, parameter :: MAPL_QUANTIZE_MAX_NSB = 23 ! Constant masking enum, bind(c) enumerator MAPL_MASK_OUT