From e913eee81c3176c9fa29618b38fc9fb723ec1e61 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Fri, 15 Mar 2024 12:05:14 -0500 Subject: [PATCH 01/46] defined CMAKE_H5_HAVE_DARWIN (#4146) --- fortran/src/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fortran/src/CMakeLists.txt b/fortran/src/CMakeLists.txt index 99644b83fbb..b2ac81ea0c6 100644 --- a/fortran/src/CMakeLists.txt +++ b/fortran/src/CMakeLists.txt @@ -31,6 +31,12 @@ else () set (CMAKE_H5_HAVE_MPI_F08 0) endif () +if (H5_HAVE_DARWIN) # Used in testing + set (CMAKE_H5_HAVE_DARWIN 1) +else () + set (CMAKE_H5_HAVE_DARWIN 0) +endif () + # configure for Fortran preprocessor # Define Parallel variable for passing to H5config_f.inc.cmake From 26837b4af843eefd8f867091482dd222ebf698e2 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Fri, 15 Mar 2024 10:36:18 -0700 Subject: [PATCH 02/46] Make the newsletter scheme work like HDF4 (#4149) --- release_docs/NEWSLETTER.txt | 25 ------------------------- release_docs/NEWSLETTER_README.txt | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 25 deletions(-) create mode 100644 release_docs/NEWSLETTER_README.txt diff --git a/release_docs/NEWSLETTER.txt b/release_docs/NEWSLETTER.txt index f03f710d717..e69de29bb2d 100644 --- a/release_docs/NEWSLETTER.txt +++ b/release_docs/NEWSLETTER.txt @@ -1,25 +0,0 @@ -INTRODUCTION -============ - -This purpose of this document is to contain entries that can be used to quickly -produce a release newsletter. When something is added to the library that is -"newsletter worthy" (i.e., new feature, CVE fix, etc.) a summary note should -be added here. - -The format should look like this: - -* SUMMARY OF NEWSLETTER-WORTHY THING - - Here is where you describe the summary. Summarize the feature, fix, or - change in general language. Remember, RELEASE.txt is for communicating - technical specifics. Text entered here is more like advertising. - - (GitHub #123, #125) - -The GitHub #s could be relevant issues or PRs. They will probably not appear -in the final newsletter, but are so that the person writing the newsletter -has easy access to context if they have questions. - -Every entry in RELEASE.txt does NOT require an entry here. The newsletter is -for communicating major changes that are of interest to anyone. Minor bugfixes, -memory leak fixes, etc. do not require entries. diff --git a/release_docs/NEWSLETTER_README.txt b/release_docs/NEWSLETTER_README.txt new file mode 100644 index 00000000000..f03f710d717 --- /dev/null +++ b/release_docs/NEWSLETTER_README.txt @@ -0,0 +1,25 @@ +INTRODUCTION +============ + +This purpose of this document is to contain entries that can be used to quickly +produce a release newsletter. When something is added to the library that is +"newsletter worthy" (i.e., new feature, CVE fix, etc.) a summary note should +be added here. + +The format should look like this: + +* SUMMARY OF NEWSLETTER-WORTHY THING + + Here is where you describe the summary. Summarize the feature, fix, or + change in general language. Remember, RELEASE.txt is for communicating + technical specifics. Text entered here is more like advertising. + + (GitHub #123, #125) + +The GitHub #s could be relevant issues or PRs. They will probably not appear +in the final newsletter, but are so that the person writing the newsletter +has easy access to context if they have questions. + +Every entry in RELEASE.txt does NOT require an entry here. The newsletter is +for communicating major changes that are of interest to anyone. Minor bugfixes, +memory leak fixes, etc. do not require entries. From 3861c0c7779c8badfdfef2c43f8148d47733fb83 Mon Sep 17 00:00:00 2001 From: "H. Joe Lee" Date: Fri, 15 Mar 2024 15:00:47 -0500 Subject: [PATCH 03/46] Remove at the end of list item. (#4151) --- release_docs/INSTALL_CMake.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release_docs/INSTALL_CMake.txt b/release_docs/INSTALL_CMake.txt index 4d4af2423c9..94f525be451 100644 --- a/release_docs/INSTALL_CMake.txt +++ b/release_docs/INSTALL_CMake.txt @@ -1104,7 +1104,7 @@ Using individual command presets (where is GNUC or MSVC or Clan cpack --preset ci-StdShar- -Using the workflow preset to configure, build, test and package the standard configuration is: +Using the workflow preset to configure, build, test and package the standard configuration: change directory to the hdf5 source folder execute "cmake --workflow --preset ci-StdShar- --fresh" where is GNUC or MSVC or Clang From 6635198d7f300674c61289cfbeb08699d117f2b5 Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Fri, 15 Mar 2024 21:01:03 +0100 Subject: [PATCH 04/46] Fix buffer size calculation in the deflate filter (#4147) --- src/H5Zdeflate.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c index 032945ae0bf..7d580646d9a 100644 --- a/src/H5Zdeflate.c +++ b/src/H5Zdeflate.c @@ -42,8 +42,6 @@ const H5Z_class2_t H5Z_DEFLATE[1] = {{ H5Z__filter_deflate, /* The actual filter function */ }}; -#define H5Z_DEFLATE_SIZE_ADJUST(s) (ceil(((double)(s)) * 1.001) + 12) - /*------------------------------------------------------------------------- * Function: H5Z__filter_deflate * @@ -149,7 +147,7 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] */ const Bytef *z_src = (const Bytef *)(*buf); Bytef *z_dst; /*destination buffer */ - uLongf z_dst_nbytes = (uLongf)H5Z_DEFLATE_SIZE_ADJUST(nbytes); + uLongf z_dst_nbytes = (uLongf)compressBound(nbytes); uLong z_src_nbytes = (uLong)nbytes; int aggression; /* Compression aggression setting */ From 6d85ea26abea85935af45081e8ef9d6e2a659171 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Fri, 15 Mar 2024 13:01:22 -0700 Subject: [PATCH 05/46] Remove H5O header and friend status from H5A.c (#4150) --- src/H5A.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/H5A.c b/src/H5A.c index 1db7fd3eb48..6728596ab52 100644 --- a/src/H5A.c +++ b/src/H5A.c @@ -15,7 +15,6 @@ /****************/ #include "H5Amodule.h" /* This source code file is part of the H5A module */ -#define H5O_FRIEND /* Suppress error about including H5Opkg */ /***********/ /* Headers */ @@ -26,7 +25,6 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5ESprivate.h" /* Event Sets */ #include "H5Iprivate.h" /* IDs */ -#include "H5Opkg.h" /* Object headers */ #include "H5Sprivate.h" /* Dataspace functions */ #include "H5VLprivate.h" /* Virtual Object Layer */ From 4b0c6291f8f8bb0870000ff23787fd07a9390df4 Mon Sep 17 00:00:00 2001 From: "H. Joe Lee" Date: Sat, 16 Mar 2024 11:40:37 -0500 Subject: [PATCH 06/46] Remove HDF from Fortran 2003 configuration check message. (#4157) --- m4/aclocal_fc.m4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m4/aclocal_fc.m4 b/m4/aclocal_fc.m4 index 49e5732e61d..f7427924ee2 100644 --- a/m4/aclocal_fc.m4 +++ b/m4/aclocal_fc.m4 @@ -159,7 +159,7 @@ dnl disable Fortran 2003 if it does not. AC_DEFUN([PAC_PROG_FC_HAVE_F2003_REQUIREMENTS],[ HAVE_F2003_REQUIREMENTS="no" - AC_MSG_CHECKING([if Fortran compiler version compatible with Fortran 2003 HDF]) + AC_MSG_CHECKING([if Fortran compiler version compatible with Fortran 2003]) TEST_SRC="`sed -n '/PROG_FC_HAVE_F2003_REQUIREMENTS/,/END PROGRAM PROG_FC_HAVE_F2003_REQUIREMENTS/p' $srcdir/m4/aclocal_fc.f90`" AC_COMPILE_IFELSE([$TEST_SRC], [AC_MSG_RESULT([yes]) HAVE_F2003_REQUIREMENTS="yes"], From 1029e34fb95f58b193476291453f80a5ca697a2c Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Sat, 16 Mar 2024 11:42:01 -0500 Subject: [PATCH 07/46] Suppress H5Dmpio debugging output unless HDF5_DEBUG=d is set (#4155) --- src/H5Dmpio.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 7d8ba21f403..338da022f92 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -613,7 +613,8 @@ H5D__mpio_debug_init(void) if (debug_str) H5D__mpio_parse_debug_str(debug_str); - debug_stream = stdout; + if (H5DEBUG(D)) + debug_stream = stdout; H5D_mpio_debug_inited = true; @@ -1410,7 +1411,7 @@ H5D__piece_io(H5D_io_info_t *io_info) fprintf(debug_log_file, "##############\n\n"); if (EOF == fclose(debug_log_file)) HDONE_ERROR(H5E_IO, H5E_CLOSEERROR, FAIL, "couldn't close debugging log file"); - debug_stream = stdout; + debug_stream = H5DEBUG(D) ? stdout : NULL; } #endif From 9a90122ef898c064d9b1c46f5c50d10d33ac5ee4 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Sat, 16 Mar 2024 09:42:32 -0700 Subject: [PATCH 08/46] Header cleanup in C library (#4154) * Ensure H5FL header is included everywhere * Ensure H5SL header is included everywhere * Ensure H5MM header is included everywhere --- src/H5AC.c | 1 + src/H5ACmpio.c | 2 ++ src/H5ACproxy_entry.c | 2 ++ src/H5Adense.c | 1 + src/H5Aint.c | 1 + src/H5B.c | 1 + src/H5B2.c | 1 + src/H5B2hdr.c | 1 + src/H5B2int.c | 1 + src/H5B2internal.c | 1 + src/H5B2leaf.c | 1 + src/H5Bcache.c | 1 + src/H5C.c | 1 + src/H5CS.c | 1 + src/H5Cdbg.c | 1 + src/H5Centry.c | 2 ++ src/H5Cint.c | 1 + src/H5Cmpio.c | 1 + src/H5Dchunk.c | 1 + src/H5Dint.c | 1 + src/H5Dmpio.c | 1 + src/H5EAhdr.c | 1 + src/H5FAhdr.c | 1 + src/H5FO.c | 1 + src/H5FS.c | 2 ++ src/H5FScache.c | 1 + src/H5FSsection.c | 2 ++ src/H5Faccum.c | 1 + src/H5Fefc.c | 2 ++ src/H5Ffake.c | 7 ++++--- src/H5Fint.c | 1 + src/H5Fsuper.c | 1 + src/H5Gcache.c | 1 + src/H5Gint.c | 1 + src/H5Goh.c | 1 + src/H5Groot.c | 1 + src/H5HF.c | 1 + src/H5HFcache.c | 1 + src/H5HFdblock.c | 1 + src/H5HFhdr.c | 1 + src/H5HFiblock.c | 1 + src/H5HFiter.c | 1 + src/H5HFsection.c | 1 + src/H5HG.c | 1 + src/H5HGcache.c | 1 + src/H5MFsection.c | 9 +++++---- src/H5Oattr.c | 1 + src/H5Ochunk.c | 7 ++++--- src/H5Ocopy.c | 1 + src/H5Ocopy_ref.c | 1 + src/H5Oint.c | 1 + src/H5PB.c | 1 + src/H5Pint.c | 1 + src/H5RS.c | 1 + src/H5SMbtree2.c | 9 +++++---- src/H5Tdeprec.c | 1 + src/H5Tinit_float.c | 1 + src/H5VLnative_file.c | 1 + src/H5VLnative_token.c | 1 + 59 files changed, 79 insertions(+), 14 deletions(-) diff --git a/src/H5AC.c b/src/H5AC.c index 6610af88d09..5f9df26aba1 100644 --- a/src/H5AC.c +++ b/src/H5AC.c @@ -40,6 +40,7 @@ #include "H5CXprivate.h" /* API Contexts */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* Files */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Pprivate.h" /* Property lists */ #include "H5SLprivate.h" /* Skip Lists */ diff --git a/src/H5ACmpio.c b/src/H5ACmpio.c index cdebe80a3b6..70d7344bb06 100644 --- a/src/H5ACmpio.c +++ b/src/H5ACmpio.c @@ -35,8 +35,10 @@ #include "H5Cprivate.h" /* Cache */ #include "H5CXprivate.h" /* API Contexts */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Fpkg.h" /* Files */ #include "H5MMprivate.h" /* Memory management */ +#include "H5SLprivate.h" /* Skip Lists */ #ifdef H5_HAVE_PARALLEL diff --git a/src/H5ACproxy_entry.c b/src/H5ACproxy_entry.c index 1a968d69e87..5ad1673dd5c 100644 --- a/src/H5ACproxy_entry.c +++ b/src/H5ACproxy_entry.c @@ -32,7 +32,9 @@ #include "H5private.h" /* Generic Functions */ #include "H5ACpkg.h" /* Metadata cache */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ +#include "H5SLprivate.h" /* Skip Lists */ /****************/ /* Local Macros */ diff --git a/src/H5Adense.c b/src/H5Adense.c index 27ccf918634..80c3c94f733 100644 --- a/src/H5Adense.c +++ b/src/H5Adense.c @@ -33,6 +33,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Apkg.h" /* Attributes */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ #include "H5SMprivate.h" /* Shared object header messages */ diff --git a/src/H5Aint.c b/src/H5Aint.c index 15f025c6050..35044ca09a6 100644 --- a/src/H5Aint.c +++ b/src/H5Aint.c @@ -32,6 +32,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Apkg.h" /* Attributes */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ diff --git a/src/H5B.c b/src/H5B.c index 7b71b5f9719..5a7a23853c5 100644 --- a/src/H5B.c +++ b/src/H5B.c @@ -101,6 +101,7 @@ #include "H5Bpkg.h" /* B-link trees */ #include "H5CXprivate.h" /* API Contexts */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5B2.c b/src/H5B2.c index 7f59ac8fae7..f49689911dc 100644 --- a/src/H5B2.c +++ b/src/H5B2.c @@ -36,6 +36,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5B2pkg.h" /* v2 B-trees */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ /****************/ diff --git a/src/H5B2hdr.c b/src/H5B2hdr.c index 8196ee22a30..f46b1d0820a 100644 --- a/src/H5B2hdr.c +++ b/src/H5B2hdr.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5B2pkg.h" /* v2 B-trees */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ diff --git a/src/H5B2int.c b/src/H5B2int.c index 1b3ecae7232..0174b527879 100644 --- a/src/H5B2int.c +++ b/src/H5B2int.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5B2pkg.h" /* v2 B-trees */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ diff --git a/src/H5B2internal.c b/src/H5B2internal.c index e97e921487d..82e686a9569 100644 --- a/src/H5B2internal.c +++ b/src/H5B2internal.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5B2pkg.h" /* v2 B-trees */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ /****************/ diff --git a/src/H5B2leaf.c b/src/H5B2leaf.c index 48e5f62fe29..3351909484c 100644 --- a/src/H5B2leaf.c +++ b/src/H5B2leaf.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5B2pkg.h" /* v2 B-trees */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5Bcache.c b/src/H5Bcache.c index d9c94f498ce..007912053e9 100644 --- a/src/H5Bcache.c +++ b/src/H5Bcache.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Bpkg.h" /* B-link trees */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ /****************/ diff --git a/src/H5C.c b/src/H5C.c index 1713e83ef90..44d499df964 100644 --- a/src/H5C.c +++ b/src/H5C.c @@ -66,6 +66,7 @@ #include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ +#include "H5SLprivate.h" /* Skip Lists */ /****************/ /* Local Macros */ diff --git a/src/H5CS.c b/src/H5CS.c index 3728273ea2a..0dca211604f 100644 --- a/src/H5CS.c +++ b/src/H5CS.c @@ -27,6 +27,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5CSprivate.h" /* Function stack */ #include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ #ifdef H5_HAVE_CODESTACK diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index 457b29cbc78..978b1709945 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -32,6 +32,7 @@ #include "H5ACprivate.h" /* Metadata Cache */ #include "H5Cpkg.h" /* Cache */ #include "H5Eprivate.h" /* Error Handling */ +#include "H5SLprivate.h" /* Skip Lists */ /****************/ /* Local Macros */ diff --git a/src/H5Centry.c b/src/H5Centry.c index fec1f4ae755..c6892e90e3c 100644 --- a/src/H5Centry.c +++ b/src/H5Centry.c @@ -34,8 +34,10 @@ #include "H5CXprivate.h" /* API Contexts */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* Files */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ +#include "H5SLprivate.h" /* Skip Lists */ /****************/ /* Local Macros */ diff --git a/src/H5Cint.c b/src/H5Cint.c index 2e79a0da057..905cbf951ec 100644 --- a/src/H5Cint.c +++ b/src/H5Cint.c @@ -34,6 +34,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* Files */ #include "H5MFprivate.h" /* File memory management */ +#include "H5SLprivate.h" /* Skip Lists */ /****************/ /* Local Macros */ diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index c8db5352ff6..233e4f92d2d 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -38,6 +38,7 @@ #include "H5Fpkg.h" /* Files */ #include "H5FDprivate.h" /* File drivers */ #include "H5MMprivate.h" /* Memory management */ +#include "H5SLprivate.h" /* Skip Lists */ #ifdef H5_HAVE_PARALLEL /****************/ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 415a57e6851..69987d1946f 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -56,6 +56,7 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5MFprivate.h" /* File memory management */ #include "H5PBprivate.h" /* Page Buffer */ +#include "H5SLprivate.h" /* Skip Lists */ #include "H5VMprivate.h" /* Vector and array functions */ /****************/ diff --git a/src/H5Dint.c b/src/H5Dint.c index 2e835a5df5d..bad2a664205 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -29,6 +29,7 @@ #include "H5Iprivate.h" /* IDs */ #include "H5Lprivate.h" /* Links */ #include "H5MMprivate.h" /* Memory management */ +#include "H5SLprivate.h" /* Skip Lists */ #include "H5VLprivate.h" /* Virtual Object Layer */ #include "H5VMprivate.h" /* Vector Functions */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 338da022f92..307629f7718 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -34,6 +34,7 @@ #include "H5Oprivate.h" /* Object headers */ #include "H5Pprivate.h" /* Property lists */ #include "H5Sprivate.h" /* Dataspaces */ +#include "H5SLprivate.h" /* Skip Lists */ #include "H5VMprivate.h" /* Vector */ #ifdef H5_HAVE_PARALLEL diff --git a/src/H5EAhdr.c b/src/H5EAhdr.c index 348e5083b60..96c027ec581 100644 --- a/src/H5EAhdr.c +++ b/src/H5EAhdr.c @@ -35,6 +35,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5EApkg.h" /* Extensible Arrays */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ diff --git a/src/H5FAhdr.c b/src/H5FAhdr.c index ef3d689fce2..3a3ee2db8a9 100644 --- a/src/H5FAhdr.c +++ b/src/H5FAhdr.c @@ -35,6 +35,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5FApkg.h" /* Fixed Arrays */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5FO.c b/src/H5FO.c index 66e5ba46fbd..534acbe6905 100644 --- a/src/H5FO.c +++ b/src/H5FO.c @@ -25,6 +25,7 @@ #include "H5FLprivate.h" /* Free lists */ #include "H5FOprivate.h" /* File objects */ #include "H5Oprivate.h" /* Object headers */ +#include "H5SLprivate.h" /* Skip Lists */ /* Private typedefs */ diff --git a/src/H5FS.c b/src/H5FS.c index c21ecf298ef..64bf51f5d46 100644 --- a/src/H5FS.c +++ b/src/H5FS.c @@ -28,9 +28,11 @@ #include "H5private.h" /* Generic Functions */ #include "H5ACprivate.h" /* Metadata cache */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ +#include "H5SLprivate.h" /* Skip Lists */ /****************/ /* Local Macros */ diff --git a/src/H5FScache.c b/src/H5FScache.c index 94c414f1eac..3fa31f00042 100644 --- a/src/H5FScache.c +++ b/src/H5FScache.c @@ -35,6 +35,7 @@ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ +#include "H5SLprivate.h" /* Skip Lists */ #include "H5VMprivate.h" /* Vectors and arrays */ /****************/ diff --git a/src/H5FSsection.c b/src/H5FSsection.c index 9bf2af0cff5..57022a2c02c 100644 --- a/src/H5FSsection.c +++ b/src/H5FSsection.c @@ -29,8 +29,10 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ +#include "H5SLprivate.h" /* Skip Lists */ #include "H5VMprivate.h" /* Vectors and arrays */ /****************/ diff --git a/src/H5Faccum.c b/src/H5Faccum.c index 758b0150801..9c4c8cdbbda 100644 --- a/src/H5Faccum.c +++ b/src/H5Faccum.c @@ -33,6 +33,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5FDprivate.h" /* File drivers */ #include "H5MMprivate.h" /* Memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ diff --git a/src/H5Fefc.c b/src/H5Fefc.c index 82de9e6dae0..fa7dd1e6d0b 100644 --- a/src/H5Fefc.c +++ b/src/H5Fefc.c @@ -28,9 +28,11 @@ #include "H5CXprivate.h" /* API Contexts */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#include "H5SLprivate.h" /* Skip Lists */ /* Special values for the "tag" field below */ #define H5F_EFC_TAG_DEFAULT (-1) diff --git a/src/H5Ffake.c b/src/H5Ffake.c index 81aa7627637..ad9fc8fb2ff 100644 --- a/src/H5Ffake.c +++ b/src/H5Ffake.c @@ -13,9 +13,10 @@ #include "H5Fmodule.h" /* This source code file is part of the H5F module */ /* Packages needed by this file... */ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fpkg.h" /* File access */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ /* PRIVATE PROTOTYPES */ diff --git a/src/H5Fint.c b/src/H5Fint.c index 76515bcd8a1..1aa289dd874 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -27,6 +27,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Gprivate.h" /* Groups */ #include "H5Iprivate.h" /* IDs */ #include "H5MFprivate.h" /* File memory management */ diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 2d27579b67e..18492486700 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -24,6 +24,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5Gcache.c b/src/H5Gcache.c index 8a908131482..9e34e77c253 100644 --- a/src/H5Gcache.c +++ b/src/H5Gcache.c @@ -30,6 +30,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Gpkg.h" /* Groups */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5Gint.c b/src/H5Gint.c index 1be54ed3ac8..46559d82291 100644 --- a/src/H5Gint.c +++ b/src/H5Gint.c @@ -30,6 +30,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5FOprivate.h" /* File objects */ #include "H5Gpkg.h" /* Groups */ #include "H5Iprivate.h" /* IDs */ diff --git a/src/H5Goh.c b/src/H5Goh.c index b72406f61b2..ce98960e8aa 100644 --- a/src/H5Goh.c +++ b/src/H5Goh.c @@ -22,6 +22,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Gpkg.h" /* Groups */ #include "H5Iprivate.h" /* IDs */ #include "H5Opkg.h" /* Object headers */ diff --git a/src/H5Groot.c b/src/H5Groot.c index 7d4a2521319..0b524b7b4de 100644 --- a/src/H5Groot.c +++ b/src/H5Groot.c @@ -32,6 +32,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Gpkg.h" /* Groups */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5HF.c b/src/H5HF.c index 856d792c95a..bbc5ce7e455 100644 --- a/src/H5HF.c +++ b/src/H5HF.c @@ -35,6 +35,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HFpkg.h" /* Fractal heaps */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5HFcache.c b/src/H5HFcache.c index b214a1ca1b4..823e2dea28e 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5ACprivate.h" /* Metadata cache */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HFpkg.h" /* Fractal heaps */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5HFdblock.c b/src/H5HFdblock.c index 62f6d8a07ac..8a6df298482 100644 --- a/src/H5HFdblock.c +++ b/src/H5HFdblock.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HFpkg.h" /* Fractal heaps */ #include "H5MFprivate.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ diff --git a/src/H5HFhdr.c b/src/H5HFhdr.c index da9d014b4bc..9f457385c31 100644 --- a/src/H5HFhdr.c +++ b/src/H5HFhdr.c @@ -30,6 +30,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HFpkg.h" /* Fractal heaps */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5HFiblock.c b/src/H5HFiblock.c index 6c273a838e0..6ec2d763711 100644 --- a/src/H5HFiblock.c +++ b/src/H5HFiblock.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HFpkg.h" /* Fractal heaps */ #include "H5MFprivate.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ diff --git a/src/H5HFiter.c b/src/H5HFiter.c index 8a347e71787..9fc564d09cd 100644 --- a/src/H5HFiter.c +++ b/src/H5HFiter.c @@ -30,6 +30,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HFpkg.h" /* Fractal heaps */ #include "H5VMprivate.h" /* Vectors and arrays */ diff --git a/src/H5HFsection.c b/src/H5HFsection.c index df30f8cdec9..2a61ad43a2a 100644 --- a/src/H5HFsection.c +++ b/src/H5HFsection.c @@ -25,6 +25,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HFpkg.h" /* Fractal heaps */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5HG.c b/src/H5HG.c index 26bc59757bb..7037376118d 100644 --- a/src/H5HG.c +++ b/src/H5HG.c @@ -43,6 +43,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HGpkg.h" /* Global heaps */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5HGcache.c b/src/H5HGcache.c index 6e42a7c624d..f885ecf973a 100644 --- a/src/H5HGcache.c +++ b/src/H5HGcache.c @@ -31,6 +31,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5HGpkg.h" /* Global heaps */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5MFsection.c b/src/H5MFsection.c index 3b81960c975..03d1112624e 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -25,10 +25,11 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fpkg.h" /* File access */ -#include "H5MFpkg.h" /* File memory management */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5MFpkg.h" /* File memory management */ /****************/ /* Local Macros */ diff --git a/src/H5Oattr.c b/src/H5Oattr.c index 6852ebc9796..6d1d23740f7 100644 --- a/src/H5Oattr.c +++ b/src/H5Oattr.c @@ -17,6 +17,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Apkg.h" /* Attributes */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ #include "H5Spkg.h" /* Dataspaces */ diff --git a/src/H5Ochunk.c b/src/H5Ochunk.c index e131d679421..37ce88b5759 100644 --- a/src/H5Ochunk.c +++ b/src/H5Ochunk.c @@ -28,9 +28,10 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Opkg.h" /* Object headers */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Opkg.h" /* Object headers */ /****************/ /* Local Macros */ diff --git a/src/H5Ocopy.c b/src/H5Ocopy.c index e87a7701458..f2f307e82db 100644 --- a/src/H5Ocopy.c +++ b/src/H5Ocopy.c @@ -39,6 +39,7 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ #include "H5Pprivate.h" /* Property lists */ +#include "H5SLprivate.h" /* Skip Lists */ /****************/ /* Local Macros */ diff --git a/src/H5Ocopy_ref.c b/src/H5Ocopy_ref.c index 62dc221dcdd..d6beb389ea6 100644 --- a/src/H5Ocopy_ref.c +++ b/src/H5Ocopy_ref.c @@ -32,6 +32,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5Lprivate.h" /* Links */ #include "H5MMprivate.h" /* Memory management */ diff --git a/src/H5Oint.c b/src/H5Oint.c index 39d8c2d976e..537563dbbad 100644 --- a/src/H5Oint.c +++ b/src/H5Oint.c @@ -39,6 +39,7 @@ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ +#include "H5SLprivate.h" /* Skip Lists */ #include "H5VLprivate.h" /* Virtual Object Layer */ #include "H5VLnative_private.h" /* Native VOL connector */ diff --git a/src/H5PB.c b/src/H5PB.c index b941225c7db..fc09cd56e96 100644 --- a/src/H5PB.c +++ b/src/H5PB.c @@ -33,6 +33,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* Files */ #include "H5FDprivate.h" /* File drivers */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5PBpkg.h" /* File access */ #include "H5SLprivate.h" /* Skip List */ diff --git a/src/H5Pint.c b/src/H5Pint.c index 544ad370249..8f9f5125847 100644 --- a/src/H5Pint.c +++ b/src/H5Pint.c @@ -29,6 +29,7 @@ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Ppkg.h" /* Property lists */ +#include "H5SLprivate.h" /* Skip Lists */ /****************/ /* Local Macros */ diff --git a/src/H5RS.c b/src/H5RS.c index b0322be46bc..ad1f1d0c69e 100644 --- a/src/H5RS.c +++ b/src/H5RS.c @@ -30,6 +30,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free lists */ +#include "H5MMprivate.h" /* Memory management */ #include "H5RSprivate.h" /* Reference-counted strings */ /****************/ diff --git a/src/H5SMbtree2.c b/src/H5SMbtree2.c index ef0da51544c..6165d664f38 100644 --- a/src/H5SMbtree2.c +++ b/src/H5SMbtree2.c @@ -20,10 +20,11 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Opkg.h" /* Object Headers */ -#include "H5SMpkg.h" /* Shared object header messages */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Opkg.h" /* Object Headers */ +#include "H5SMpkg.h" /* Shared object header messages */ /****************/ /* Local Macros */ diff --git a/src/H5Tdeprec.c b/src/H5Tdeprec.c index 76cddc7292b..d9f293071d9 100644 --- a/src/H5Tdeprec.c +++ b/src/H5Tdeprec.c @@ -34,6 +34,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5CXprivate.h" /* API Contexts */ #include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5Ppublic.h" /* Property Lists */ #include "H5Tpkg.h" /* Datatypes */ diff --git a/src/H5Tinit_float.c b/src/H5Tinit_float.c index 3213f00fece..8384c3161df 100644 --- a/src/H5Tinit_float.c +++ b/src/H5Tinit_float.c @@ -27,6 +27,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ #include "H5Tpkg.h" /* Datatypes */ /****************/ diff --git a/src/H5VLnative_file.c b/src/H5VLnative_file.c index 0c13c1d9f7c..b0686d0f1df 100644 --- a/src/H5VLnative_file.c +++ b/src/H5VLnative_file.c @@ -29,6 +29,7 @@ #include "H5Cprivate.h" /* Cache */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* Files */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MFprivate.h" /* File memory management */ #include "H5Pprivate.h" /* Property lists */ diff --git a/src/H5VLnative_token.c b/src/H5VLnative_token.c index 353e8cdad86..f7265043f98 100644 --- a/src/H5VLnative_token.c +++ b/src/H5VLnative_token.c @@ -23,6 +23,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory handling */ #include "H5VLnative_private.h" /* Native VOL connector */ From 87970e1d534f148c4218a8657c13a4f591184f73 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Sat, 16 Mar 2024 09:44:59 -0700 Subject: [PATCH 09/46] Add Doxygen to H5FDmirror.h (#4158) --- src/H5FDmirror.h | 77 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/src/H5FDmirror.h b/src/H5FDmirror.h index 6c98e1a8a6f..304f1e5a3f8 100644 --- a/src/H5FDmirror.h +++ b/src/H5FDmirror.h @@ -25,39 +25,43 @@ /** Identifier for the mirror VFD */ #define H5FD_MIRROR_VALUE H5_VFD_MIRROR -#ifdef __cplusplus -extern "C" { -#endif +/** Magic number to identify the H5FD_mirror_fapl_t struct */ +#define H5FD_MIRROR_FAPL_MAGIC 0xF8DD514C -/* ============================================================================ - * Mirror VFD use and operation. - * ============================================================================ +/** + * The version number of the H5FD_mirror_fapl_t configuration + * structure for the #H5FD_MIRROR driver */ +#define H5FD_MIRROR_CURR_FAPL_T_VERSION 1 -/* --------------------------------------------------------------------------- - * Structure: H5FD_mirror_fapl_t +/** Max size of the remote_ip array in H5FD_mirror_fapl_t */ +#define H5FD_MIRROR_MAX_IP_LEN 32 + +/** + *\struct H5FD_mirror_fapl_t + * \brief Configuration structure for H5Pset_fapl_mirror() / H5Pget_fapl_mirror() * - * Used to pass configuration information to the Mirror VFD. - * Populate components as appropriate and pass structure pointer to - * `H5Pset_fapl_mirror()`. + * \details H5FD_mirror_fapl_t is a public structure that is used to pass + * configuration data to the #H5FD_MIRROR driver via a File Access + * Property List. A pointer to an instance of this structure is + * a parameter to H5Pset_fapl_mirror() and H5Pget_fapl_mirror(). * - * `magic` (uint32_t) + * \var uint32_t H5FD_mirror_fapl_t::magic * Semi-unique number to sanity-check pointers to this structure type. - * MUST equal H5FD_MIRROR_FAPL_MAGIC to be considered valid. + * Must equal H5FD_MIRROR_FAPL_MAGIC to be considered valid. * - * `version` (uint32_t) - * Indicates expected components of the structure. + * \var uint32_t H5FD_mirror_fapl_t::version + * Version number of the H5FD_mirror_fapl_t structure. Any instance passed + * to H5Pset_fapl_mirror() / H5Pget_fapl_mirror() must have a recognized version + * number or an error will be raised. Currently, this field should be set + * to #H5FD_MIRROR_CURR_FAPL_T_VERSION. * - * `handshake_port (int) - * Port number to expect to reach the "Mirror Server" on the remote host. + * \var int H5FD_mirror_fapl_t::handshake_port + * Port number on the remote host. * - * `remote_ip` (char[]) - * IP address string of "Mirror Server" remote host. - * --------------------------------------------------------------------------- + * \var char H5FD_mirror_fapl_t::remote_ip[H5FD_MIRROR_MAX_IP_LEN + 1] + * IP address string of the remote host. */ -#define H5FD_MIRROR_FAPL_MAGIC 0xF8DD514C -#define H5FD_MIRROR_CURR_FAPL_T_VERSION 1 -#define H5FD_MIRROR_MAX_IP_LEN 32 typedef struct H5FD_mirror_fapl_t { uint32_t magic; uint32_t version; @@ -65,6 +69,10 @@ typedef struct H5FD_mirror_fapl_t { char remote_ip[H5FD_MIRROR_MAX_IP_LEN + 1]; } H5FD_mirror_fapl_t; +#ifdef __cplusplus +extern "C" { +#endif + /** @private * * \brief Private initializer for the mirror VFD @@ -74,14 +82,33 @@ H5_DLL hid_t H5FD_mirror_init(void); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Queries a File Access Property List for #H5FD_MIRROR file driver properties + * + * \fapl_id + * \param[out] fa_out Pointer to #H5FD_MIRROR driver configuration structure + * \returns \herr_t + * + * \details H5Pget_fapl_mirror() queries the #H5FD_MIRROR driver properties as set + * by H5Pset_fapl_mirror(). + * + * \since 1.10.7 */ H5_DLL herr_t H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Modifies the file access property list to use the #H5FD_MIRROR driver + * + * \fapl_id + * \param[in] fa Pointer to #H5FD_MIRROR driver configuration structure + * + * \returns \herr_t + * + * \details H5Pset_fapl_mirror() modifies the file access property list to use the + * #H5FD_MIRROR driver. + * + * \since 1.10.7 */ H5_DLL herr_t H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa); From 45f7898272394b48faba7aad75eb69d05fdfffde Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Sat, 16 Mar 2024 11:51:02 -0700 Subject: [PATCH 10/46] Remove lseek64 and stat64 symbols from CMake (#4163) We don't use these in the library. --- config/cmake/ConfigureChecks.cmake | 1 - config/cmake/H5pubconf.h.in | 6 ------ 2 files changed, 7 deletions(-) diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 668739e7074..86a96cead0c 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -143,7 +143,6 @@ else () endif () if (CYGWIN) - set (${HDF_PREFIX}_HAVE_LSEEK64 0) set (CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D_GNU_SOURCE") add_definitions ("-D_GNU_SOURCE") endif () diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index f835da103f8..8d866ec89a5 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -216,9 +216,6 @@ /* Define to 1 if you have the `z' library (-lz). */ #cmakedefine H5_HAVE_LIBZ @H5_HAVE_LIBZ@ -/* Define to 1 if you have the `lseek64' function. */ -#cmakedefine H5_HAVE_LSEEK64 @H5_HAVE_LSEEK64@ - /* Define if the map API (H5M) should be compiled */ #cmakedefine H5_HAVE_MAP_API @H5_HAVE_MAP_API@ @@ -283,9 +280,6 @@ compiled */ #cmakedefine H5_HAVE_ROS3_VFD @H5_HAVE_ROS3_VFD@ -/* Define to 1 if you have the `stat64' function. */ -#cmakedefine H5_HAVE_STAT64 @H5_HAVE_STAT64@ - /* Define if struct stat has the st_blocks field */ #cmakedefine H5_HAVE_STAT_ST_BLOCKS @H5_HAVE_STAT_ST_BLOCKS@ From 68e8c0e62724b2a450e30f2becdcc9452ac0abb4 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Sat, 16 Mar 2024 12:01:24 -0700 Subject: [PATCH 11/46] Remove HAVE_IOEO checks from CMake (#4160) This was intended to check for thread-safety functionality on Windows. The required functionality has been standard since Windows Vista, so these checks can be removed. --- CMakeLists.txt | 4 +- config/cmake/ConfigureChecks.cmake | 65 ++++-------------------------- config/cmake/HDFTests.c | 18 --------- 3 files changed, 10 insertions(+), 77 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb8b596b7d7..81a6e96b6fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -919,8 +919,8 @@ if (HDF5_ENABLE_THREADSAFE) message (VERBOSE " **** Allowing unsupported HL and thread-safety options **** ") endif () endif () - if (H5_HAVE_IOEO) - message (VERBOSE " **** Win32 threads requires WINVER>=0x600 (Windows Vista/7/8) **** ") + if (WIN32) + # When Win32 is available, we use those threads set (H5_HAVE_WIN_THREADS 1) else () if (NOT H5_HAVE_PTHREAD_H) diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 86a96cead0c..2905fe9ad08 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -261,6 +261,13 @@ endif () # # https://docs.oracle.com/cd/E23824_01/html/821-1474/lfcompile-5.html +# MinGW and Cygwin +if (MINGW OR CYGWIN) + set (CMAKE_REQUIRED_DEFINITIONS + "${CURRENT_TEST_DEFINITIONS} -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE" + ) +endif () + #----------------------------------------------------------------------------- # Check the size in bytes of all the int and float types #----------------------------------------------------------------------------- @@ -326,7 +333,7 @@ if (MINGW OR NOT WINDOWS) endif () HDF_CHECK_TYPE_SIZE (off_t ${HDF_PREFIX}_SIZEOF_OFF_T) -HDF_CHECK_TYPE_SIZE (time_t ${HDF_PREFIX}_SIZEOF_TIME_T) +HDF_CHECK_TYPE_SIZE (time_t ${HDF_PREFIX}_SIZEOF_TIME_T) #----------------------------------------------------------------------------- # Extra C99 types @@ -439,62 +446,6 @@ if (MINGW OR NOT WINDOWS) endforeach () endif () -#----------------------------------------------------------------------------- -# Check if InitOnceExecuteOnce is available -#----------------------------------------------------------------------------- -if (WINDOWS) - if (NOT HDF_NO_IOEO_TEST) - message (VERBOSE "Checking for InitOnceExecuteOnce:") - if (NOT DEFINED ${HDF_PREFIX}_HAVE_IOEO) - if (LARGEFILE) - set (CMAKE_REQUIRED_DEFINITIONS - "${CURRENT_TEST_DEFINITIONS} -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_LARGEFILE_SOURCE" - ) - endif () - set (MACRO_CHECK_FUNCTION_DEFINITIONS "-DHAVE_IOEO ${CMAKE_REQUIRED_FLAGS}") - if (CMAKE_REQUIRED_INCLUDES) - set (CHECK_C_SOURCE_COMPILES_ADD_INCLUDES "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}") - else () - set (CHECK_C_SOURCE_COMPILES_ADD_INCLUDES) - endif () - - TRY_RUN(HAVE_IOEO_EXITCODE HAVE_IOEO_COMPILED - ${CMAKE_BINARY_DIR} - ${HDF_RESOURCES_DIR}/HDFTests.c - COMPILE_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} ${MACRO_CHECK_FUNCTION_DEFINITIONS}" - LINK_LIBRARIES "${HDF5_REQUIRED_LIBRARIES}" - CMAKE_FLAGS "${CHECK_C_SOURCE_COMPILES_ADD_INCLUDES} -DCMAKE_SKIP_RPATH:BOOL=${CMAKE_SKIP_RPATH}" - COMPILE_OUTPUT_VARIABLE OUTPUT - ) - # if it did not compile make the return value fail code of 1 - if (NOT HAVE_IOEO_COMPILED) - set (HAVE_IOEO_EXITCODE 1) - endif () - # if the return value was 0 then it worked - if ("${HAVE_IOEO_EXITCODE}" EQUAL 0) - set (${HDF_PREFIX}_HAVE_IOEO 1 CACHE INTERNAL "Test InitOnceExecuteOnce") - message (VERBOSE "Performing Test InitOnceExecuteOnce - Success") - file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log - "Performing C SOURCE FILE Test InitOnceExecuteOnce succeeded with the following output:\n" - "${OUTPUT}\n" - "Return value: ${HAVE_IOEO}\n") - else () - if (CMAKE_CROSSCOMPILING AND "${HAVE_IOEO_EXITCODE}" MATCHES "FAILED_TO_RUN") - set (${HDF_PREFIX}_HAVE_IOEO "${HAVE_IOEO_EXITCODE}") - else () - set (${HDF_PREFIX}_HAVE_IOEO "" CACHE INTERNAL "Test InitOnceExecuteOnce") - endif () - - message (VERBOSE "Performing Test InitOnceExecuteOnce - Failed") - file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log - "Performing InitOnceExecuteOnce Test failed with the following output:\n" - "${OUTPUT}\n" - "Return value: ${HAVE_IOEO_EXITCODE}\n") - endif () - endif () - endif () -endif () - # ---------------------------------------------------------------------- # Set the flag to indicate that the machine can handle converting # denormalized floating-point values. diff --git a/config/cmake/HDFTests.c b/config/cmake/HDFTests.c index 3d16721346c..73f7739bac1 100644 --- a/config/cmake/HDFTests.c +++ b/config/cmake/HDFTests.c @@ -126,21 +126,3 @@ main(void) #endif } #endif - -#ifdef HAVE_IOEO - -#include -typedef void (WINAPI *PGNSI)(LPSYSTEM_INFO); -int main () -{ - PGNSI pGNSI; - - pGNSI = (PGNSI) GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "InitOnceExecuteOnce"); - - if (NULL == pGNSI) - return 1; - else - return 0; -} - -#endif /* HAVE_IOEO */ From a56675e12ad1090eb38d4d3cd8cf08a9af3c9f7f Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Sat, 16 Mar 2024 21:43:47 -0500 Subject: [PATCH 12/46] Fix some minor warnings (#4165) --- HDF5Examples/C/H5T/h5ex_t_array.c | 2 +- HDF5Examples/C/H5T/h5ex_t_arrayatt.c | 2 +- HDF5Examples/C/H5T/h5ex_t_cmpd.c | 2 +- HDF5Examples/C/H5T/h5ex_t_cmpdatt.c | 2 +- HDF5Examples/C/H5T/h5ex_t_cpxcmpd.c | 2 +- HDF5Examples/C/H5T/h5ex_t_cpxcmpdatt.c | 2 +- HDF5Examples/C/H5T/h5ex_t_objref.c | 5 ++++- HDF5Examples/C/H5T/h5ex_t_objrefatt.c | 5 ++++- HDF5Examples/C/H5T/h5ex_t_opaqueatt.c | 2 +- HDF5Examples/C/H5T/h5ex_t_regref.c | 2 +- HDF5Examples/C/H5T/h5ex_t_regrefatt.c | 2 +- HDF5Examples/C/H5T/h5ex_t_vlen.c | 2 +- HDF5Examples/C/H5T/h5ex_t_vlenatt.c | 2 +- java/src/jni/h5pImp.c | 4 ++-- src/H5FDsubfiling/H5FDsubfiling.c | 3 --- src/H5Fint.c | 3 ++- test/atomic_writer.c | 18 +++++++++++++++++- test/dsets.c | 6 ++++-- 18 files changed, 44 insertions(+), 22 deletions(-) diff --git a/HDF5Examples/C/H5T/h5ex_t_array.c b/HDF5Examples/C/H5T/h5ex_t_array.c index b63e4e0eb7f..a29ac455c98 100644 --- a/HDF5Examples/C/H5T/h5ex_t_array.c +++ b/HDF5Examples/C/H5T/h5ex_t_array.c @@ -144,7 +144,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n", DATASET, i); + printf("%s[%" PRIuHSIZE "]:\n", DATASET, i); for (j = 0; j < adims[0]; j++) { printf(" ["); for (k = 0; k < adims[1]; k++) diff --git a/HDF5Examples/C/H5T/h5ex_t_arrayatt.c b/HDF5Examples/C/H5T/h5ex_t_arrayatt.c index a89f2b2584a..f0711286160 100644 --- a/HDF5Examples/C/H5T/h5ex_t_arrayatt.c +++ b/HDF5Examples/C/H5T/h5ex_t_arrayatt.c @@ -155,7 +155,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n", ATTRIBUTE, i); + printf("%s[%" PRIuHSIZE "]:\n", ATTRIBUTE, i); for (j = 0; j < adims[0]; j++) { printf(" ["); for (k = 0; k < adims[1]; k++) diff --git a/HDF5Examples/C/H5T/h5ex_t_cmpd.c b/HDF5Examples/C/H5T/h5ex_t_cmpd.c index 739d0616096..44f15523fba 100644 --- a/HDF5Examples/C/H5T/h5ex_t_cmpd.c +++ b/HDF5Examples/C/H5T/h5ex_t_cmpd.c @@ -136,7 +136,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n", DATASET, i); + printf("%s[%" PRIuHSIZE "]:\n", DATASET, i); printf("Serial number : %d\n", rdata[i].serial_no); printf("Location : %s\n", rdata[i].location); printf("Temperature (F) : %f\n", rdata[i].temperature); diff --git a/HDF5Examples/C/H5T/h5ex_t_cmpdatt.c b/HDF5Examples/C/H5T/h5ex_t_cmpdatt.c index 246537b1d92..04c72a510e2 100644 --- a/HDF5Examples/C/H5T/h5ex_t_cmpdatt.c +++ b/HDF5Examples/C/H5T/h5ex_t_cmpdatt.c @@ -146,7 +146,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n", ATTRIBUTE, i); + printf("%s[%" PRIuHSIZE "]:\n", ATTRIBUTE, i); printf("Serial number : %d\n", rdata[i].serial_no); printf("Location : %s\n", rdata[i].location); printf("Temperature (F) : %f\n", rdata[i].temperature); diff --git a/HDF5Examples/C/H5T/h5ex_t_cpxcmpd.c b/HDF5Examples/C/H5T/h5ex_t_cpxcmpd.c index 8506c08cbeb..370f7819bb4 100644 --- a/HDF5Examples/C/H5T/h5ex_t_cpxcmpd.c +++ b/HDF5Examples/C/H5T/h5ex_t_cpxcmpd.c @@ -293,7 +293,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n", DATASET, i); + printf("%s[%" PRIuHSIZE "]:\n", DATASET, i); printf(" Vehicle name :\n %s\n", rdata[i].name); printf(" Sensor locations :\n"); for (j = 0; j < rdata[i].sensors.len; j++) diff --git a/HDF5Examples/C/H5T/h5ex_t_cpxcmpdatt.c b/HDF5Examples/C/H5T/h5ex_t_cpxcmpdatt.c index c7efbce72aa..a55fb76f765 100644 --- a/HDF5Examples/C/H5T/h5ex_t_cpxcmpdatt.c +++ b/HDF5Examples/C/H5T/h5ex_t_cpxcmpdatt.c @@ -304,7 +304,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n", ATTRIBUTE, i); + printf("%s[%" PRIuHSIZE "]:\n", ATTRIBUTE, i); printf(" Vehicle name :\n %s\n", rdata[i].name); printf(" Sensor locations :\n"); for (j = 0; j < rdata[i].sensors.len; j++) diff --git a/HDF5Examples/C/H5T/h5ex_t_objref.c b/HDF5Examples/C/H5T/h5ex_t_objref.c index 660cc110a25..e6c2de95d66 100644 --- a/HDF5Examples/C/H5T/h5ex_t_objref.c +++ b/HDF5Examples/C/H5T/h5ex_t_objref.c @@ -135,7 +135,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n ->", DATASET, i); + printf("%s[%" PRIuHSIZE "]:\n ->", DATASET, i); /* * Open the referenced object, get its name and type. @@ -174,6 +174,9 @@ main(void) case H5O_TYPE_NAMED_DATATYPE: printf("Named Datatype"); break; + case H5O_TYPE_MAP: + printf("Map Object"); + break; case H5O_TYPE_UNKNOWN: case H5O_TYPE_NTYPES: printf("Unknown"); diff --git a/HDF5Examples/C/H5T/h5ex_t_objrefatt.c b/HDF5Examples/C/H5T/h5ex_t_objrefatt.c index 1d9d1feb91a..562364a203e 100644 --- a/HDF5Examples/C/H5T/h5ex_t_objrefatt.c +++ b/HDF5Examples/C/H5T/h5ex_t_objrefatt.c @@ -147,7 +147,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n ->", ATTRIBUTE, i); + printf("%s[%" PRIuHSIZE "]:\n ->", ATTRIBUTE, i); /* * Open the referenced object, get its name and type. @@ -186,6 +186,9 @@ main(void) case H5O_TYPE_NAMED_DATATYPE: printf("Named Datatype"); break; + case H5O_TYPE_MAP: + printf("Map Object"); + break; case H5O_TYPE_UNKNOWN: case H5O_TYPE_NTYPES: printf("Unknown"); diff --git a/HDF5Examples/C/H5T/h5ex_t_opaqueatt.c b/HDF5Examples/C/H5T/h5ex_t_opaqueatt.c index e88031ac9ab..67294921ae3 100644 --- a/HDF5Examples/C/H5T/h5ex_t_opaqueatt.c +++ b/HDF5Examples/C/H5T/h5ex_t_opaqueatt.c @@ -121,7 +121,7 @@ main(void) */ printf("Datatype tag for %s is: \"%s\"\n", ATTRIBUTE, tag); for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]: ", ATTRIBUTE, i); + printf("%s[%" PRIuHSIZE "]: ", ATTRIBUTE, i); for (j = 0; j < len; j++) printf("%c", rdata[j + i * len]); printf("\n"); diff --git a/HDF5Examples/C/H5T/h5ex_t_regref.c b/HDF5Examples/C/H5T/h5ex_t_regref.c index 39227259507..e6d4cef2c32 100644 --- a/HDF5Examples/C/H5T/h5ex_t_regref.c +++ b/HDF5Examples/C/H5T/h5ex_t_regref.c @@ -168,7 +168,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n ->", DATASET, i); + printf("%s[%" PRIuHSIZE "]:\n ->", DATASET, i); /* * Open the referenced object, retrieve its region as a diff --git a/HDF5Examples/C/H5T/h5ex_t_regrefatt.c b/HDF5Examples/C/H5T/h5ex_t_regrefatt.c index 5ed745d75b0..bb31b707c51 100644 --- a/HDF5Examples/C/H5T/h5ex_t_regrefatt.c +++ b/HDF5Examples/C/H5T/h5ex_t_regrefatt.c @@ -183,7 +183,7 @@ main(void) * Output the data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n ->", ATTRIBUTE, i); + printf("%s[%" PRIuHSIZE "]:\n ->", ATTRIBUTE, i); /* * Open the referenced object, retrieve its region as a diff --git a/HDF5Examples/C/H5T/h5ex_t_vlen.c b/HDF5Examples/C/H5T/h5ex_t_vlen.c index 7111a343e1c..b5649729c9b 100644 --- a/HDF5Examples/C/H5T/h5ex_t_vlen.c +++ b/HDF5Examples/C/H5T/h5ex_t_vlen.c @@ -118,7 +118,7 @@ main(void) * Output the variable-length data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n {", DATASET, i); + printf("%s[%" PRIuHSIZE "]:\n {", DATASET, i); ptr = rdata[i].p; for (j = 0; j < rdata[i].len; j++) { printf(" %d", ptr[j]); diff --git a/HDF5Examples/C/H5T/h5ex_t_vlenatt.c b/HDF5Examples/C/H5T/h5ex_t_vlenatt.c index db69aea4631..e173a20f3bc 100644 --- a/HDF5Examples/C/H5T/h5ex_t_vlenatt.c +++ b/HDF5Examples/C/H5T/h5ex_t_vlenatt.c @@ -128,7 +128,7 @@ main(void) * Output the variable-length data to the screen. */ for (i = 0; i < dims[0]; i++) { - printf("%s[%llu]:\n {", ATTRIBUTE, i); + printf("%s[%" PRIuHSIZE "]:\n {", ATTRIBUTE, i); ptr = rdata[i].p; for (j = 0; j < rdata[i].len; j++) { printf(" %d", ptr[j]); diff --git a/java/src/jni/h5pImp.c b/java/src/jni/h5pImp.c index ce9989d5043..e38701ff3ba 100644 --- a/java/src/jni/h5pImp.c +++ b/java/src/jni/h5pImp.c @@ -563,8 +563,8 @@ Java_hdf_hdf5lib_H5_H5Pisa_1class(JNIEnv *env, jclass clss, jlong plid, jlong pc JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Pget(JNIEnv *env, jclass clss, jlong plid, jstring name) { - const char *cstr = NULL; - jint val; + const char *cstr = NULL; + jint val = -1; herr_t status = FAIL; UNUSED(clss); diff --git a/src/H5FDsubfiling/H5FDsubfiling.c b/src/H5FDsubfiling/H5FDsubfiling.c index 71dd4bacd12..4c39f0f8729 100644 --- a/src/H5FDsubfiling/H5FDsubfiling.c +++ b/src/H5FDsubfiling/H5FDsubfiling.c @@ -2940,7 +2940,6 @@ translate_io_req_to_iovec(subfiling_context_t *sf_context, size_t iovec_idx, siz int64_t row_offset = 0; int64_t row_stripe_idx_start = 0; int64_t row_stripe_idx_final = 0; - int64_t cur_stripe_idx = 0; int64_t max_iovec_depth = 0; int64_t mem_offset = 0; size_t total_bytes = 0; @@ -3100,7 +3099,6 @@ translate_io_req_to_iovec(subfiling_context_t *sf_context, size_t iovec_idx, siz * vector components for each. Subfiles whose data size is * zero will not have I/O requests passed to them. */ - cur_stripe_idx = stripe_idx; for (int i = 0, subfile_idx = (int)first_subfile_idx; i < num_subfiles; i++) { H5_flexible_const_ptr_t *_io_bufs_ptr; H5FD_mem_t *_io_types_ptr; @@ -3295,7 +3293,6 @@ translate_io_req_to_iovec(subfiling_context_t *sf_context, size_t iovec_idx, siz offset_in_block += (int64_t)*_io_sizes_ptr; subfile_idx++; - cur_stripe_idx++; if (subfile_idx == num_subfiles) { subfile_idx = 0; diff --git a/src/H5Fint.c b/src/H5Fint.c index 1aa289dd874..9cdb9a00042 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -1300,7 +1300,8 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F if (NULL == (f->shared->mdc_log_location = (char *)H5MM_calloc((len + 1) * sizeof(char)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't allocate memory for mdc log file name"); - strncpy(f->shared->mdc_log_location, mdc_log_location, len); + strncpy(f->shared->mdc_log_location, mdc_log_location, len + 1); + f->shared->mdc_log_location[len] = '\0'; } else f->shared->mdc_log_location = NULL; diff --git a/test/atomic_writer.c b/test/atomic_writer.c index 89edfe75e57..54aa2f45349 100644 --- a/test/atomic_writer.c +++ b/test/atomic_writer.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #if !defined(WIN32) && !defined(__MINGW32__) @@ -184,13 +185,28 @@ main(int argc, char *argv[]) /* Write the series of integers to the file */ for (n = 0; n < num; n++) { + size_t seek_pos; /* Set up data to be written */ for (u = 0; u < num; u++) buf[u] = n; + seek_pos = n * sizeof(unsigned int); + if (sizeof(off_t) < 8) { + if (seek_pos > INT32_MAX) { + printf("WRITER: seek past range for lseek\n"); + goto error; + } + } + else { + if (seek_pos > INT64_MAX) { + printf("WRITER: seek past range for lseek\n"); + goto error; + } + } + /* Position the file to the proper location */ - if (lseek(fd, (n * sizeof(unsigned int)), SEEK_SET) < 0) { + if (lseek(fd, (off_t)seek_pos, SEEK_SET) < 0) { printf("WRITER: error from lseek\n"); goto error; } /* end if */ diff --git a/test/dsets.c b/test/dsets.c index 3203ed0b887..8f2d2c4d528 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -5891,8 +5891,10 @@ test_floattypes(hid_t file) (long double)1.0711093225222612990711093225222612, (long double)-9.8971679387636870998971679387636870e-1}}; long double new_data[2][5]; - size_t ld_spos, ld_epos, ld_esize, ld_mpos, ld_msize; - size_t tgt_precision = 128; +#if LDBL_MANT_DIG != 106 + size_t ld_spos, ld_epos, ld_esize, ld_mpos, ld_msize; + size_t tgt_precision = 128; +#endif TESTING(" long double (setup)"); From 2908dd1d12f82b3ad559e98ae639fac2c238bcdd Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Sun, 17 Mar 2024 11:34:21 -0700 Subject: [PATCH 13/46] Bump the size of the mirror VFD IP field (#4167) The IP address string isn't big enought to hold an IPv4-mapped IPv6 address. --- src/H5FDmirror.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5FDmirror.h b/src/H5FDmirror.h index 304f1e5a3f8..95ce936594d 100644 --- a/src/H5FDmirror.h +++ b/src/H5FDmirror.h @@ -35,7 +35,7 @@ #define H5FD_MIRROR_CURR_FAPL_T_VERSION 1 /** Max size of the remote_ip array in H5FD_mirror_fapl_t */ -#define H5FD_MIRROR_MAX_IP_LEN 32 +#define H5FD_MIRROR_MAX_IP_LEN 45 /* Max size of an IPv4-mapped IPv6 address */ /** *\struct H5FD_mirror_fapl_t From fa44de4cdbc125f644369834983c65811be12335 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Sun, 17 Mar 2024 18:47:59 -0700 Subject: [PATCH 14/46] Fix mirror VFD script (#4170) This had directory problems when running locally. --- test/test_mirror.sh.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_mirror.sh.in b/test/test_mirror.sh.in index ec77730624f..ab092e57719 100644 --- a/test/test_mirror.sh.in +++ b/test/test_mirror.sh.in @@ -41,14 +41,14 @@ while [ $# -gt 0 ]; do esac done - - RUN_DIR=mirror_vfd_test MIRROR_UTILS=../utils/mirror_vfd # TODO: presupposes from test/ -if [[ ! -d $RUN_DIR ]] ; then - mkdir $RUN_DIR +# Start clean +if test -d $RUN_DIR ; then + rm -rf $RUN_DIR fi +mkdir $RUN_DIR # Copy program files into dedicated test directory for FILE in $MIRROR_UTILS/mirror_* ; do @@ -65,7 +65,7 @@ cp mirror_vfd $RUN_DIR if [ -f $MIRROR_UTILS/.libs/mirror_server ] ; then RUN_LIBS=$RUN_DIR/.libs # Delete previous .libs directory, to remove any generated libtool files - if [[ -d $RUN_LIBS ]] ; then + if test -d $RUN_LIBS ; then rm -rf $RUN_LIBS fi mkdir $RUN_LIBS From 840476ead85229ac4d7be1b6c9dd87ad5f8e3a07 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Sun, 17 Mar 2024 20:48:16 -0500 Subject: [PATCH 15/46] Fix an issue where the Subfiling VFD's context cache grows too large (#4159) --- release_docs/RELEASE.txt | 15 ++++++++++++ src/H5FDsubfiling/H5FDsubfiling.c | 3 +++ src/H5FDsubfiling/H5subfiling_common.c | 33 ++++++++++++++++---------- src/H5FDsubfiling/H5subfiling_common.h | 1 + 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 2d8eb326643..d82167e1516 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -591,6 +591,21 @@ Bug Fixes since HDF5-1.14.0 release Library ------- + - Fixed an issue where the Subfiling VFD's context object cache could + grow too large + + The Subfiling VFD keeps a cache of its internal context objects to + speed up access to a context object for a particular file, as well + as access to that object across multiple opens of the same file. + However, opening a large amount of files with the Subfiling VFD over + the course of an application's lifetime could cause this cache to grow + too large and result in the application running out of available MPI + communicator objects. On file close, the Subfiling VFD now simply + evicts context objects out of its cache and frees them. It is assumed + that multiple opens of a file will be a less common use case for the + Subfiling VFD, but this can be revisited if it proves to be an issue + for performance. + - Fixed error when overwriting certain nested variable length types Previously, when using a datatype that included a variable length type diff --git a/src/H5FDsubfiling/H5FDsubfiling.c b/src/H5FDsubfiling/H5FDsubfiling.c index 4c39f0f8729..9594f676a88 100644 --- a/src/H5FDsubfiling/H5FDsubfiling.c +++ b/src/H5FDsubfiling/H5FDsubfiling.c @@ -1358,6 +1358,9 @@ H5FD__subfiling_close_int(H5FD_subfiling_t *file_ptr) H5MM_free(file_ptr->file_dir); file_ptr->file_dir = NULL; + if (file_ptr->context_id >= 0 && H5_free_subfiling_object(file_ptr->context_id) < 0) + H5_SUBFILING_DONE_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free subfiling context object"); + /* Release the file info */ file_ptr = H5FL_FREE(H5FD_subfiling_t, file_ptr); diff --git a/src/H5FDsubfiling/H5subfiling_common.c b/src/H5FDsubfiling/H5subfiling_common.c index 1127ae0a386..47d0624ef16 100644 --- a/src/H5FDsubfiling/H5subfiling_common.c +++ b/src/H5FDsubfiling/H5subfiling_common.c @@ -48,7 +48,6 @@ static int sf_file_map_size = 0; #define DEFAULT_TOPOLOGY_CACHE_SIZE 4 #define DEFAULT_FILE_MAP_ENTRIES 8 -static herr_t H5_free_subfiling_object(int64_t object_id); static herr_t H5_free_subfiling_object_int(subfiling_context_t *sf_context); static herr_t H5_free_subfiling_topology(sf_topology_t *topology); @@ -280,18 +279,25 @@ H5_get_subfiling_object(int64_t object_id) * Purpose: Frees the underlying subfiling object for a given subfiling * object ID. * - * NOTE: Currently we assume that all created subfiling - * objects are cached in the (very simple) context/topology - * cache until application exit, so the only time a subfiling - * object should be freed by this routine is if something - * fails right after creating one. Otherwise, the internal - * indexing for the relevant cache will be invalid. + * NOTE: Because we want to avoid the potentially large + * overhead of determining the application topology on every + * file open, we currently assume that all created subfiling + * topology objects are cached in the (very simple) topology + * cache until application exit. This allows us to quickly + * find and assign a cached topology object to a subfiling + * context object for a file when opened. Therefore, a + * subfiling topology object should (currently) only ever be + * freed by this routine if a function fails right after + * creating a topology object. Otherwise, the internal + * indexing for the topology cache will be invalid and we will + * either leak memory or assign invalid topology pointers to + * subfiling context objects after that point. * * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5_free_subfiling_object(int64_t object_id) { int64_t obj_type = (object_id >> 32) & 0x0FFFF; @@ -305,30 +311,31 @@ H5_free_subfiling_object(int64_t object_id) "couldn't get subfiling context for subfiling object ID"); if (H5_free_subfiling_object_int(sf_context) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "couldn't free subfiling object"); + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "couldn't free subfiling context object"); assert(sf_context_cache_num_entries > 0); assert(sf_context == sf_context_cache[sf_context_cache_num_entries - 1]); sf_context_cache[sf_context_cache_num_entries - 1] = NULL; sf_context_cache_num_entries--; } - else { + else if (obj_type == SF_TOPOLOGY) { sf_topology_t *sf_topology; - assert(obj_type == SF_TOPOLOGY); - if (NULL == (sf_topology = H5_get_subfiling_object(object_id))) H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "couldn't get subfiling context for subfiling object ID"); if (H5_free_subfiling_topology(sf_topology) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "couldn't free subfiling topology"); + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "couldn't free subfiling topology object"); assert(sf_topology_cache_num_entries > 0); assert(sf_topology == sf_topology_cache[sf_topology_cache_num_entries - 1]); sf_topology_cache[sf_topology_cache_num_entries - 1] = NULL; sf_topology_cache_num_entries--; } + else + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "couldn't free subfiling object - invalid object type"); done: H5_SUBFILING_FUNC_LEAVE; diff --git a/src/H5FDsubfiling/H5subfiling_common.h b/src/H5FDsubfiling/H5subfiling_common.h index 156902a5c5c..6b5cfa45c27 100644 --- a/src/H5FDsubfiling/H5subfiling_common.h +++ b/src/H5FDsubfiling/H5subfiling_common.h @@ -269,6 +269,7 @@ H5_DLL herr_t H5_close_subfiles(int64_t subfiling_context_id, MPI_Comm file_comm H5_DLL int64_t H5_new_subfiling_object_id(sf_obj_type_t obj_type); H5_DLL void *H5_get_subfiling_object(int64_t object_id); +H5_DLL herr_t H5_free_subfiling_object(int64_t object_id); H5_DLL herr_t H5_get_subfiling_config_from_file(FILE *config_file, int64_t *stripe_size, int64_t *num_subfiles); H5_DLL herr_t H5_resolve_pathname(const char *filepath, MPI_Comm comm, char **resolved_filepath); From eb0351efffe987a9c1882ccc9b03b5b0aec7f2dd Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Mon, 18 Mar 2024 06:05:45 -0700 Subject: [PATCH 16/46] Address code page issues w/ Windows file paths (#4172) On Windows, HDF5 attempted to convert file paths passed to open() and remove() to UTF-16 in order to handle Unicode file paths. This scheme does not work when the system uses code pages to handle non-ASCII file names. As suggested in the forum post below, we now also try to see if we can open the file with open(), which should handle systems where non-ASCII code pages are in use. https://forum.hdfgroup.org/t/open-create-hdf5-files-with-non-utf8-chars-such-as-shift-jis/11785 --- src/H5system.c | 78 ++++++++++++++++++++++++++++++++--------------- src/H5win32defs.h | 12 ++++---- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/src/H5system.c b/src/H5system.c index be886ae52f3..6057c6ff639 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -514,28 +514,22 @@ H5_get_utf16_str(const char *s) } /* end H5_get_utf16_str() */ /*------------------------------------------------------------------------- - * Function: Wopen_utf8 + * Function: Wopen * - * Purpose: UTF-8 equivalent of open(2) for use on Windows. - * Converts a UTF-8 input path to UTF-16 and then opens the - * file via _wopen() under the hood + * Purpose: Equivalent of open(2) for use on Windows. Necessary to + * handle code pages and Unicode on that platform. * * Return: Success: A POSIX file descriptor * Failure: -1 - * *------------------------------------------------------------------------- */ int -Wopen_utf8(const char *path, int oflag, ...) +Wopen(const char *path, int oflag, ...) { int fd = -1; /* POSIX file descriptor to be returned */ wchar_t *wpath = NULL; /* UTF-16 version of the path */ int pmode = 0; /* mode (optionally set via variable args) */ - /* Convert the input UTF-8 path to UTF-16 */ - if (NULL == (wpath = H5_get_utf16_str(path))) - goto done; - /* _O_BINARY must be set in Windows to avoid CR-LF <-> LF EOL * transformations when performing I/O. Note that this will * produce Unix-style text files, though. @@ -551,47 +545,83 @@ Wopen_utf8(const char *path, int oflag, ...) va_end(vl); } - /* Open the file */ + /* First try opening the file with the normal POSIX open() call. + * This will handle ASCII without additional processing as well as + * systems where code pages are being used instead of true Unicode. + */ + if ((fd = open(path, oflag, pmode)) >= 0) { + /* If this succeeds, we're done */ + goto done; + } + + if (errno == ENOENT) { + /* Not found, reset errno and try with UTF-16 */ + errno = 0; + } + else { + /* Some other error (like permissions), so just exit */ + goto done; + } + + /* Convert the input UTF-8 path to UTF-16 */ + if (NULL == (wpath = H5_get_utf16_str(path))) + goto done; + + /* Open the file using a UTF-16 path */ fd = _wopen(wpath, oflag, pmode); done: - if (wpath) - H5MM_xfree((void *)wpath); + H5MM_xfree(wpath); return fd; -} /* end Wopen_utf8() */ +} /* end Wopen() */ /*------------------------------------------------------------------------- - * Function: Wremove_utf8 + * Function: Wremove * - * Purpose: UTF-8 equivalent of remove(3) for use on Windows. - * Converts a UTF-8 input path to UTF-16 and then opens the - * file via _wremove() under the hood + * Purpose: Equivalent of remove(3) for use on Windows. Necessary to + * handle code pages and Unicode on that platform. * * Return: Success: 0 * Failure: -1 - * *------------------------------------------------------------------------- */ int -Wremove_utf8(const char *path) +Wremove(const char *path) { wchar_t *wpath = NULL; /* UTF-16 version of the path */ int ret = -1; + /* First try removing the file with the normal POSIX remove() call. + * This will handle ASCII without additional processing as well as + * systems where code pages are being used instead of true Unicode. + */ + if ((ret = remove(path)) >= 0) { + /* If this succeeds, we're done */ + goto done; + } + + if (errno == ENOENT) { + /* Not found, reset errno and try with UTF-16 */ + errno = 0; + } + else { + /* Some other error (like permissions), so just exit */ + goto done; + } + /* Convert the input UTF-8 path to UTF-16 */ if (NULL == (wpath = H5_get_utf16_str(path))) goto done; - /* Open the file */ + /* Remove the file using a UTF-16 path */ ret = _wremove(wpath); done: - if (wpath) - H5MM_xfree((void *)wpath); + H5MM_xfree(wpath); return ret; -} /* end Wremove_utf8() */ +} /* end Wremove() */ #endif /* H5_HAVE_WIN32_API */ diff --git a/src/H5win32defs.h b/src/H5win32defs.h index 9630c5e2d42..05d291ec03b 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -39,7 +39,7 @@ struct timezone { }; #endif -#define HDcreat(S, M) Wopen_utf8(S, O_CREAT | O_TRUNC | O_RDWR, M) +#define HDcreat(S, M) Wopen(S, O_CREAT | O_TRUNC | O_RDWR, M) #define HDflock(F, L) Wflock(F, L) #define HDfstat(F, B) _fstati64(F, B) #define HDftell(F) _ftelli64(F) @@ -59,13 +59,13 @@ struct timezone { */ #if (defined(_MSC_VER) && !defined(_MSVC_TRADITIONAL)) || _MSVC_TRADITIONAL /* Using the MSVC traditional preprocessor */ -#define HDopen(S, F, ...) Wopen_utf8(S, F, __VA_ARGS__) +#define HDopen(S, F, ...) Wopen(S, F, __VA_ARGS__) #else /* Using a standards conformant preprocessor */ -#define HDopen(S, F, ...) Wopen_utf8(S, F, ##__VA_ARGS__) +#define HDopen(S, F, ...) Wopen(S, F, ##__VA_ARGS__) #endif -#define HDremove(S) Wremove_utf8(S) +#define HDremove(S) Wremove(S) #define HDsetenv(N, V, O) Wsetenv(N, V, O) #define HDsetvbuf(F, S, M, Z) setvbuf(F, S, M, (Z > 1 ? Z : 2)) #define HDsleep(S) Sleep(S * 1000) @@ -89,8 +89,8 @@ H5_DLL int Wsetenv(const char *name, const char *value, int overwrite); H5_DLL int Wflock(int fd, int operation); H5_DLL herr_t H5_expand_windows_env_vars(char **env_var); H5_DLL wchar_t *H5_get_utf16_str(const char *s); -H5_DLL int Wopen_utf8(const char *path, int oflag, ...); -H5_DLL int Wremove_utf8(const char *path); +H5_DLL int Wopen(const char *path, int oflag, ...); +H5_DLL int Wremove(const char *path); H5_DLL int H5_get_win32_times(H5_timevals_t *tvs); H5_DLL char *H5_strndup(const char *s, size_t n); H5_DLL char *Wstrcasestr_wrap(const char *haystack, const char *needle); From 51fab96f4e025e5e4b87b6e434106756b882756d Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Mon, 18 Mar 2024 06:06:07 -0700 Subject: [PATCH 17/46] Add Doxygen to API calls in H5VLnative.h (#4173) --- src/H5VLnative.h | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/H5VLnative.h b/src/H5VLnative.h index 6eaefb63b96..6ca484ae329 100644 --- a/src/H5VLnative.h +++ b/src/H5VLnative.h @@ -515,16 +515,46 @@ extern "C" { #endif /* Token <--> address converters */ + /** * \ingroup H5VLNAT + * + * \brief Convert a haddr_t address to a native VOL connector token + * + * \fgdta_loc_obj_id{loc_id} + * \param[in] addr Object address + * \param[out] token Object token + * + * \return \herr_t + * + * \details This API call maps pre-VOL haddr_t native file format addresses + * to the more generic H5O_token_t tokens used by the VOL. + * + * \since 1.12.0 */ H5_DLL herr_t H5VLnative_addr_to_token(hid_t loc_id, haddr_t addr, H5O_token_t *token); /** * \ingroup H5VLNAT + * + * \brief Convert a native VOL connector token to a haddr_t address + * + * \fgdta_loc_obj_id{loc_id} + * \param[in] token Object token + * \param[out] addr Object address + * + * \return \herr_t + * + * \details This API call maps generic H5O_token_t tokens used by the VOL to + * pre-VOL haddr_t native file format addresses. + * + * \since 1.12.0 */ H5_DLL herr_t H5VLnative_token_to_addr(hid_t loc_id, H5O_token_t token, haddr_t *addr); -/* Not really public but must be included here */ +/** @private + * + * \brief Register the native VOL connector and retrieve an ID for it + */ H5_DLL hid_t H5VL_native_register(void); #ifdef __cplusplus From 344ba97feced8f630f876fb1df061a3398a9be00 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Mon, 18 Mar 2024 06:57:52 -0700 Subject: [PATCH 18/46] Allow H5Soffset_simple to accept NULL offsets (#4152) The reference manual states that the offset parameter of H5Soffset_simple() can be set to NULL to reset the offset of a simple dataspace to 0. This has never been true, and passing NULL was regarded as an error. The library will now accept NULL for the offset parameter and will correctly set the offset to zero. Fixes HDFFV-9299 --- release_docs/RELEASE.txt | 11 +++++++++++ src/H5Spkg.h | 1 + src/H5Spublic.h | 6 +++++- src/H5Sselect.c | 23 +++++++++++----------- src/H5Stest.c | 42 +++++++++++++++++++++++++++++++++++----- test/tselect.c | 26 +++++++++++++++++++++++++ 6 files changed, 91 insertions(+), 18 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index d82167e1516..dd568f3eb70 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -591,6 +591,17 @@ Bug Fixes since HDF5-1.14.0 release Library ------- + - Corrected H5Soffset_simple() when offset is NULL + + The reference manual states that the offset parameter of H5Soffset_simple() + can be set to NULL to reset the offset of a simple dataspace to 0. This + has never been true, and passing NULL was regarded as an error. + + The library will now accept NULL for the offset parameter and will + correctly set the offset to zero. + + Fixes HDFFV-9299 + - Fixed an issue where the Subfiling VFD's context object cache could grow too large diff --git a/src/H5Spkg.h b/src/H5Spkg.h index 3ae63a50964..fddd5e3c5fa 100644 --- a/src/H5Spkg.h +++ b/src/H5Spkg.h @@ -413,6 +413,7 @@ H5_DLL herr_t H5S__get_rebuild_status_test(hid_t space_id, H5S_diminfo_valid_t * H5S_diminfo_valid_t *status2); H5_DLL herr_t H5S__get_diminfo_status_test(hid_t space_id, H5S_diminfo_valid_t *status); H5_DLL htri_t H5S__internal_consistency_test(hid_t space_id); +H5_DLL herr_t H5S__verify_offsets(hid_t space_id, const hssize_t *offset); #endif /* H5S_TESTING */ #endif /*H5Spkg_H*/ diff --git a/src/H5Spublic.h b/src/H5Spublic.h index 5422d96bcbf..bf9e9118f64 100644 --- a/src/H5Spublic.h +++ b/src/H5Spublic.h @@ -832,12 +832,16 @@ H5_DLL herr_t H5Smodify_select(hid_t space1_id, H5S_seloper_t op, hid_t space2_i * \p space_id. The offset array must be the same number of * elements as the number of dimensions for the dataspace. If the * \p offset array is set to NULL, the offset for the dataspace is - * reset to 0. + * reset to 0 in all dimensions. * * This function allows the same shaped selection to be moved to * different locations within a dataspace without requiring it to * be redefined. * + * \note Until 1.14.4, setting the offset parameter to NULL was considered + * an error, despite the reference manual stating that it had the + * behavior described above. + * * \version 1.4.0 Fortran subroutine was introduced. * \since 1.0.0 * diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 477e0f840c1..7a032b26e85 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -81,11 +81,6 @@ H5FL_SEQ_EXTERN(hsize_t); Non-negative on success/Negative on failure DESCRIPTION Sets the selection offset for the dataspace - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - Only works for simple dataspaces currently - EXAMPLES - REVISION LOG --------------------------------------------------------------------------*/ herr_t H5S_select_offset(H5S_t *space, const hssize_t *offset) @@ -95,10 +90,14 @@ H5S_select_offset(H5S_t *space, const hssize_t *offset) /* Check args */ assert(space); assert(0 < space->extent.rank && space->extent.rank <= H5S_MAX_RANK); - assert(offset); - /* Copy the offset over */ - H5MM_memcpy(space->select.offset, offset, sizeof(hssize_t) * space->extent.rank); + /* Copy the offset over. As a special case, when offset is NULL, we + * reset all dimensions to zero. + */ + if (offset) + H5MM_memcpy(space->select.offset, offset, sizeof(hssize_t) * space->extent.rank); + else + memset(space->select.offset, 0, sizeof(hssize_t) * space->extent.rank); /* Indicate that the offset was changed */ space->select.offset_changed = true; @@ -133,12 +132,12 @@ H5Soffset_simple(hid_t space_id, const hssize_t *offset) /* Check args */ if (NULL == (space = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "not a dataspace"); + HGOTO_ERROR(H5E_DATASPACE, H5E_BADID, FAIL, "not a dataspace"); if (space->extent.rank == 0 || (H5S_GET_EXTENT_TYPE(space) == H5S_SCALAR || H5S_GET_EXTENT_TYPE(space) == H5S_NULL)) - HGOTO_ERROR(H5E_ID, H5E_UNSUPPORTED, FAIL, "can't set offset on scalar or null dataspace"); - if (offset == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no offset specified"); + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "can't set offset on scalar or null dataspace"); + + /* offset can be NULL (resets all dims to zero) */ /* Set the selection offset */ if (H5S_select_offset(space, offset) < 0) diff --git a/src/H5Stest.c b/src/H5Stest.c index 4978de0558e..422f7c1d20f 100644 --- a/src/H5Stest.c +++ b/src/H5Stest.c @@ -341,11 +341,6 @@ H5S__check_internal_consistency(const H5S_t *space) DESCRIPTION Check the states of internal data structures of the hyperslab, and see whether they are consistent or not - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - DO NOT USE THIS FUNCTION FOR ANYTHING EXCEPT TESTING - EXAMPLES - REVISION LOG --------------------------------------------------------------------------*/ htri_t H5S__internal_consistency_test(hid_t space_id) @@ -367,3 +362,40 @@ H5S__internal_consistency_test(hid_t space_id) done: FUNC_LEAVE_NOAPI(ret_value) } /* H5S__internal_consistency_test() */ + +/*-------------------------------------------------------------------------- + NAME + H5S__verify_offsets + PURPOSE + Verify that internal offsets match an array of offsets + USAGE + herr_t H5S__verify_offsets(hid_t space_id) + hid_t space_id; IN: dataspace id + const hssize_t *offset; IN: Offset to position the selection at + RETURNS + Non-negative true/false on success, negative on failure + DESCRIPTION + This function is necessary because there is no public API call + that lets you get the offsets +--------------------------------------------------------------------------*/ +herr_t +H5S__verify_offsets(hid_t space_id, const hssize_t *offset) +{ + H5S_t *space; /* Dataspace to modify */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + if (NULL == (space = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADID, FAIL, "not a dataspace"); + if (space->extent.rank == 0 || + (H5S_GET_EXTENT_TYPE(space) == H5S_SCALAR || H5S_GET_EXTENT_TYPE(space) == H5S_NULL)) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "can't set offset on scalar or null dataspace"); + + /* Check that the internal and passed-in offset data are the same */ + if (0 != memcmp(space->select.offset, offset, sizeof(hssize_t) * space->extent.rank)) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "internal offsets don't match parameters"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S__verify_offsets() */ diff --git a/test/tselect.c b/test/tselect.c index 55430f24ad5..20b85916739 100644 --- a/test/tselect.c +++ b/test/tselect.c @@ -4118,6 +4118,8 @@ test_select_hyper_offset(void) CHECK(ret, FAIL, "H5Soffset_simple"); valid = H5Sselect_valid(sid1); VERIFY(valid, true, "H5Sselect_valid"); + ret = H5S__verify_offsets(sid1, offset); + CHECK(ret, FAIL, "H5S__verify_offsets"); /* Check an invalid offset */ offset[0] = 10; @@ -4127,6 +4129,8 @@ test_select_hyper_offset(void) CHECK(ret, FAIL, "H5Soffset_simple"); valid = H5Sselect_valid(sid1); VERIFY(valid, false, "H5Sselect_valid"); + ret = H5S__verify_offsets(sid1, offset); + CHECK(ret, FAIL, "H5S__verify_offsets"); /* Reset offset */ offset[0] = 0; @@ -4136,6 +4140,28 @@ test_select_hyper_offset(void) CHECK(ret, FAIL, "H5Soffset_simple"); valid = H5Sselect_valid(sid1); VERIFY(valid, true, "H5Sselect_valid"); + ret = H5S__verify_offsets(sid1, offset); + CHECK(ret, FAIL, "H5S__verify_offsets"); + + /* Check behavior of NULL offset parameter */ + + /* Set a valid offset */ + offset[0] = -1; + offset[1] = 0; + offset[2] = 0; + ret = H5Soffset_simple(sid1, offset); + CHECK(ret, FAIL, "H5Soffset_simple"); + valid = H5Sselect_valid(sid1); + VERIFY(valid, true, "H5Sselect_valid"); + /* Reset using NULL */ + ret = H5Soffset_simple(sid1, NULL); + CHECK(ret, FAIL, "H5Soffset_simple"); + valid = H5Sselect_valid(sid1); + VERIFY(valid, true, "H5Sselect_valid"); + /* Validate offset */ + offset[0] = 0; + ret = H5S__verify_offsets(sid1, offset); + CHECK(ret, FAIL, "H5S__verify_offsets"); /* Select 15x26 hyperslab for memory dataset */ start[0] = 15; From 4e222bf5bb42a84378a89461fe28abd1874487c1 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:36:55 -0500 Subject: [PATCH 19/46] Add filter plugin user guide text. Fix registered URL in docs (#4169) --- doxygen/aliases | 1 - doxygen/dox/UsersGuide.dox | 2 + doxygen/examples/H5.format.1.1.html | 4 +- doxygen/examples/H5.format.2.0.html | 10 +- doxygen/examples/H5.format.html | 8 +- src/H5Dmodule.h | 83 +++++++-- src/H5PLmodule.h | 272 +++++++++++++++++++++++++--- src/H5Zmodule.h | 2 +- 8 files changed, 335 insertions(+), 47 deletions(-) diff --git a/doxygen/aliases b/doxygen/aliases index 4eb19621e74..ad868432bee 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -235,7 +235,6 @@ ALIASES += sa_metadata_ops="\sa \li H5Pget_all_coll_metadata_ops() \li H5Pget_co ################################################################################ ALIASES += ref_cons_semantics="Enabling a Strict Consistency Semantics Model in Parallel HDF5" -ALIASES += ref_dld_filters="HDF5 Dynamically Loaded Filters" ALIASES += ref_file_image_ops="HDF5 File Image Operations" ALIASES += ref_filter_pipe="Data Flow Pipeline for H5Dread()" ALIASES += ref_group_impls="Group implementations in HDF5" diff --git a/doxygen/dox/UsersGuide.dox b/doxygen/dox/UsersGuide.dox index 4f955e6074d..b6113ad15bd 100644 --- a/doxygen/dox/UsersGuide.dox +++ b/doxygen/dox/UsersGuide.dox @@ -134,6 +134,7 @@ HDF5 Release 1.14
  • \ref subsubsec_dataset_transfer_props
  • \ref subsubsec_dataset_transfer_store
  • \ref subsubsec_dataset_transfer_partial +
  • \ref subsubsec_dataset_transfer_dyn_filter \li \ref subsec_dataset_allocation
      @@ -147,6 +148,7 @@ HDF5 Release 1.14
    • \ref subsubsec_dataset_filters_nbit
    • \ref subsubsec_dataset_filters_scale
    • \ref subsubsec_dataset_filters_szip +
    • \ref subsubsec_dataset_filters_dyn
    \ref sec_datatype diff --git a/doxygen/examples/H5.format.1.1.html b/doxygen/examples/H5.format.1.1.html index f5e4c4e0fe5..f437d9be970 100644 --- a/doxygen/examples/H5.format.1.1.html +++ b/doxygen/examples/H5.format.1.1.html @@ -5436,8 +5436,8 @@

    Name: Data Storage - Filter Pipeline

    filters requested and supported by third parties. Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found at - - https://support.hdfgroup.org/services/contributions.html#filters. + + https://portal.hdfgroup.org/documentation/hdf5-docs/registered_filter_plugins.html. 1

    To request a filter identifier, please contact diff --git a/doxygen/examples/H5.format.2.0.html b/doxygen/examples/H5.format.2.0.html index bde030f3853..37cb7282eb1 100644 --- a/doxygen/examples/H5.format.2.0.html +++ b/doxygen/examples/H5.format.2.0.html @@ -12598,9 +12598,8 @@

    HDF5 Library and for filters requested and supported by third parties. Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found at - The HDF Group’s - Contributions page. + The HDF Group’s + Registered Filters page.

    @@ -12854,9 +12853,8 @@

    HDF5 Library and for filters requested and supported by third parties. Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found at - The HDF Group’s - Contributions page. + The HDF Group’s + Registered Filters page.

    diff --git a/doxygen/examples/H5.format.html b/doxygen/examples/H5.format.html index 7aba5fed440..30bd37cf10f 100644 --- a/doxygen/examples/H5.format.html +++ b/doxygen/examples/H5.format.html @@ -14226,8 +14226,8 @@

    IV.A.2.l. The Data Storage - Filter Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found at The HDF Group’s - - Contributions page.

    + + Registered Filters page.

    To request a filter identifier, please contact @@ -14488,8 +14488,8 @@

    IV.A.2.l. The Data Storage - Filter Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found at The HDF Group’s - - Contributions page.

    + + Registered Filters page.

    To request a filter identifier, please contact diff --git a/src/H5Dmodule.h b/src/H5Dmodule.h index 81f197d7147..27e5799968e 100644 --- a/src/H5Dmodule.h +++ b/src/H5Dmodule.h @@ -815,19 +815,10 @@ * * * - * + * * * * - * - * - * - * - * - * - * - * * * * @@ -844,6 +835,19 @@ * * * + * + * + * + * + * + * + * + * + * + * + * + * * *
    Data pipeline filters
    FilterBuilt-in FilterDescription
    gzip compressionData compression using zlib.
    Szip compressionData compression using the Szip library. See The HDF Group website for more information - * regarding the Szip filter.
    N-bit compressionData compression using an algorithm specialized for n-bit datatypes.
    Fletcher32Fletcher32 checksum for error-detection.
    Optional Built-in FilterDescription
    gzip compressionData compression using zlib.
    szip compressionData compression using the szip library. The HDF Group now uses the libaec library for the szip +filter.
    * @@ -861,6 +865,44 @@ * \li @see @ref subsubsec_dataset_filters_nbit * \li @see @ref subsubsec_dataset_filters_scale * + * \subsubsection subsubsec_dataset_transfer_dyn_filter Data Pipeline Dynamically Loaded Filters + * While the HDF5 “internal” compression methods work reasonably well on users’ + * datasets, there are certain drawbacks to this implementation. First, the “internal” compression + * methods may not provide the optimal compression ratio, as do some newly developed or specialized + * compression methods. Secondly, if a data provider wants to use a “non-internal” compression for + * storing the data in HDF5, they have to write a filter function that uses the new compression method + * and then register it with the library. Data consumers of such HDF5 files will need to have the new filter + * function and use it with their applications to read the data, or they will need a modified version of the + * HDF5 Library that has the new filter as a part of the library. + * + * If a user of such data does not have a modified HDF5 Library installed on his system, command-line tools + * such as h5dump or h5ls will not be able to display the compressed data. Furthermore, it would be + * practically impossible to determine the compression method used, making the data stored in HDF5 + * useless. + * + * It is clear that the internal HDF5 filter mechanism, while extensible, does not work well with third-party + * filters. It would be a maintenance nightmare to keep adding and supporting new compression methods + * in HDF5. For any set of HDF5 “internal” filters, there always will be data with which the “internal” +filters + * will not achieve the optimal performance needed to address data I/O and storage problems. Thus the + * internal HDF5 filter mechanism is enhanced to address the issues discussed above. + * + * We have a feature of HDF5 called “dynamically loaded filters in HDF5.” This feature + * makes the HDF5 third-party filters available to an application at runtime. The third-party HDF5 filter + * function has to be a part of the HDF5 filter plugin installed on the system as a shared library or DLL. + * + * To use a third-party filter an HDF5 application should call the #H5Pset_filter function when setting the + * filter pipeline for a dataset creation property. The HDF5 Library will register the filter with the library + * and the filter will be applied when data is written to the file. + * + * When an application reads data compressed with a third-party HDF5 filter, the HDF5 Library will search + * for the required filter plugin, register the filter with the library (if the filter function is not +registered) and + * apply it to the data on the read operation. + * + * For more information, + * \li @see @ref sec_filter_plugins + * * \subsubsection subsubsec_dataset_transfer_drive File Drivers * I/O is performed by the HDF5 virtual file layer. The file driver interface writes and reads blocks * of data; each driver module implements the interface using different I/O mechanisms. The table @@ -2685,8 +2727,25 @@ allocated if necessary. * and minimum values, and they will get a much larger minimum-bits (poor compression) *

    • * - * \subsubsection subsubsec_dataset_filters_szip Using the Szip Filter - * See The HDF Group website for further information regarding the Szip filter. + * \subsubsection subsubsec_dataset_filters_szip Using the SZip Filter + * See The HDF Group website for further information regarding the SZip filter. + * + * \subsubsection subsubsec_dataset_filters_dyn Using Dynamically-Loadable Filters + * \see \ref sec_filter_plugins for further information regarding the dynamically-loadable filters. + * + * HDF has a filter plugin repository of useful third-party plugins that can used + * + * + * + * + * + * + * + * + * + * + * + *
    FilterSetFilter Params
    BLOSCUD=32001,0,0
    BSHUFUD=32004,0,0
    BZIP2UD=307,0,1,9
    JPEGUD=32019,0,4,q,c,r,t
    LZ4 UD=32004,0,1,3
    LZFUD=32000,1,3,0,0,0
    SZUD=32017,1,5,2,7,20,40,0
    ZFPUD=32013,1,0,0
    ZSTDUD=32015,0,0
    * * Previous Chapter \ref sec_group - Next Chapter \ref sec_datatype * diff --git a/src/H5PLmodule.h b/src/H5PLmodule.h index 37654869746..436e8bdfa0a 100644 --- a/src/H5PLmodule.h +++ b/src/H5PLmodule.h @@ -27,7 +27,257 @@ #define H5_MY_PKG_ERR H5E_PLUGIN /** \page H5PL_UG The HDF5 Plugins - * @todo Under Construction + * + * \section sec_filter_plugins HDF5 Filter Plugins + * + * \subsection subsec_filter_plugins_intro Introduction + * HDF5 supports compression of data using a stackable pipeline of filters which can be + * implemented for reading and writing datasets, both at runtime and post‐process. + * These filters are supported as dynamically loadable plugins, and users can even + * implement custom filters of their own design. + * + * \subsection subsec_filter_plugins_model Programming Model for Applications + * This section describes the programming model for an application that uses a third-party HDF5 filter + * plugin to write or read data. For simplicity of presentation, it is assumed that the HDF5 filter plugin is + * available on the system in a default location. The HDF5 filter plugin is discussed in detail in the + * \ref subsec_filter_plugins_prog section. + * + * \subsubsection subsubsec_filter_plugins_model_apply Applying a Third-party Filter When Creating and Writing + * a Dataset A third-party filter can be added to the HDF5 filter pipeline by using the H5Pset_filter + * function, as a user would do in the past. The identification number and the filter parameters should be + * available to the application. For example, if the application intends to apply the HDF5 bzip2 compression + * filter that was registered with The HDF Group and has an identification number 307 + * (Registered + * Filters) then the application would follow the steps as outlined below: \code dcpl = H5Pcreate + * (H5P_DATASET_CREATE); status = H5Pset_filter (dcpl, (H5Z_filter_t)307, H5Z_FLAG_MANDATORY, (size_t)6, + * cd_values); dset = H5Dcreate (file, DATASET, H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, status = H5Dwrite + * (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]); \endcode + * + * \subsubsection subsubsec_filter_plugins_model_read Reading Data with an Applied Third-party Filter + * An application does not need to do anything special to read the data with a third-party filter applied. For + * example, if one wants to read data written in the previous example, the following regular steps should be + * taken: \code file = H5Fopen (FILE, H5F_ACC_RDONLY, H5P_DEFAULT); dset = H5Dopen (file, DATASET, + * H5P_DEFAULT); H5Dread (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata[0]); \endcode + * + * The command-line utility h5dump, for example, will read and display the data as shown: + * \code + * HDF5 "h5ex_d_bzip2.h5" { + * GROUP "/" { + * DATASET "DS1" { + * DATATYPE H5T_STD_I32LE + * DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + * STORAGE_LAYOUT { + * CHUNKED ( 4, 8 ) + * SIZE 6410 (1.278:1 COMPRESSION) + * } + * FILTERS { + * USER_DEFINED_FILTER { + * FILTER_ID 307 + * COMMENT HDF5 bzip2 filter; see http://www.hdfgroup.org/services/contributions.html + * PARAMS { 2 } + * } + * } + * FILLVALUE { + * FILL_TIME H5D_FILL_TIME_IFSET + * VALUE H5D_FILL_VALUE_DEFAULT + * } + * ALLOCATION_TIME { + * H5D_ALLOC_TIME_INCR + * } + * DATA { + * ... + * } + * } + * } + * } + * \endcode + * + * If the filter can not be loaded then h5dump will show the following: + * \code + * ... + * } + * DATA {h5dump error: unable to print data + * } + * ... + * \endcode + * + * \subsubsection subsubsec_filter_plugins_model_custom A Word of Caution When Using Custom Filters + * Data goes through the HDF5 filter pipeline only when it is written to the file or read into application + * memory space from the file. For example, the I/O operation is triggered with a call to #H5Fflush, or when + * a data item (HDF5 metadata or a raw data chunk) is evicted from the cache or brought into the cache. + * Please notice that #H5Dread/#H5Dwrite calls on the chunked datasets do not necessarily trigger I/O since + * the HDF5 Library uses a separate chunk cache. + * + * A data item may remain in the cache until the HDF5 Library is closed. If the HDF5 plugin that has to be + * applied to the data item becomes unavailable before the file and all objects in the file are closed, an + * error will occur. The following example demonstrates the issue. Please notice the position of the + * #H5Zunregister call: + * + * \code + * // Create a new group using compression. + * gcpl = H5Pcreate (H5P_GROUP_CREATE); + * status = H5Pset_filter(gcpl,H5Z_FILTER_BZIP2,H5Z_FLAG_MANDATORY,(size_t)1, cd_values); + * group = H5Gcreate (file, GNAME, H5P_DEFAULT, gcpl, H5P_DEFAULT); + * for (i=0; i < NGROUPS; i++) { + * sprintf(name, "group_%d", i); + * tmp_id = H5Gcreate (group, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + * status = H5Gclose(tmp_id); + * } + * + * status = H5Pclose (gcpl); + * status = H5Gclose (group); + * + * // Unregister the filter. Call to H5Fclose will fail because the library tries + * // to apply the filter that is not available anymore. This has a cascade effect + * // on H5Fclose. + * H5Zunregister(H5Z_FILTER_BZIP2); + * status = H5Fclose (file); + * \endcode + * + * Here is an error stack produced by the program: + * \code + * HDF5-DIAG: Error detected in HDF5 (xx.yy.zz) thread 0: + * #000: H5F.c line **** in H5Fclose(): decrementing file ID failed + * major: Object atom + * minor: Unable to close file + * #001: H5I.c line **** in H5I_dec_app_ref(): can't decrement ID ref count + * major: Object atom + * minor: Unable to decrement reference count + * #002: H5F.c line **** in H5F_close(): can't close file + * major: File accessibility + * minor: Unable to close file + * ... + * #026: H5Z.c line **** in H5Z_find(): required filter is not registered + * major: Data filters + * minor: Object not found + * \endcode + * + * To avoid the problem make sure to close all objects to which the filter is applied and flush them using + * the H5Fflush call before unregistering the filter. + * + * \subsection subsec_filter_plugins_prog Programming Model for HDF5 Filter Plugins + * This section describes how to create an HDF5 filter, an HDF5 filter plugin, and how to install the HDF5 + * plugin on the system. + * + * \subsubsection subsubsec_filter_plugins_prog_write Writing a Filter Function + * The HDF5 filter function for the dynamically loaded filter feature should be written as any custom filter + * described in Custom Filters. See the + * “Example” section, section 5, of that document to get an idea of the simple filter function, and see the + * example of the more sophisticated HDF5 bzip2 filter function in the “Building an HDF5 bzip2 Plugin Example” + * section. The HDF5 bzip2 filter function is also available for download from Filter Plugin Repository. + * + * The user has to remember a few things when writing an HDF5 filter function. + *
    • 1. An HDF5 filter is bidirectional. + * The filter handles both input and output to the file; a flag is passed to the filter to indicate the + * direction.
      • + *
    • 2. An HDF5 filter operates on a buffer. + * The filter reads data from a buffer, performs some sort of transformation on the data, places + * the result in the same or new buffer, and returns the buffer pointer and size to the caller.
      • + *
    • 3. An HDF5 filter should return zero in the case of failure.
    + * + * The signature of the HDF5 filter function and the accompanying filter structure (see the section below) + * are described in the HDF5 Reference Manual #H5Z_filter_t. + * + * \subsubsection subsubsec_filter_plugins_prog_reg Registering a Filter with The HDF Group + * If you are writing a filter that will be used by others, it would be a good idea to request a filter + * identification number and register it with The HDF Group. Please follow the procedure described at + * Registered + * Filters. + * + * The HDF Group anticipates that developers of HDF5 filter plugins will not only register new filters, but + * will also provide links to the source code and/or binaries for the corresponding HDF5 filter plugins. + * + * It is very important for the users of the filter that developers provide filter information in the “name” + * field of the filter structure, for example: + * \code + * const H5Z_class2_t H5Z_BZIP2[1] = {{ + * H5Z_CLASS_T_VERS, // H5Z_class_t version + * (H5Z_filter_t)H5Z_FILTER_BZIP2, // Filter id number + * 1, // encoder_present flag (set to true) + * 1, // decoder_present flag (set to true) + * "HDF5 bzip2 filter; see http://www.hdfgroup.org/services/contributions.html", + * // Filter name for debugging + * NULL, // The "can apply" callback + * NULL, // The "set local" callback + * (H5Z_func_t)H5Z_filter_bzip2, // The actual filter function + * }}; + * \endcode + * + * The HDF5 Library and command-line tools have access to the “name” field. An application can + * use the H5Pget_filter<*> functions to retrieve information about the filters. + * + * Using the example of the structure above, the h5dump tool will print the string “HDF5 bzip2 + * filter found at …” pointing users to the applied filter (see the example in the \ref + * subsubsec_filter_plugins_model_read section) thus solving the problem of the filter’s origin. + * + * \subsubsection subsubsec_filter_plugins_prog_create Creating an HDF5 Filter Plugin + * The HDF5 filter plugin source should include: + *
    • 1. The H5PLextern.h header file from the HDF5 distribution.
      • + *
    • 2. The definition of the filter structure (see the example shown in the section above).
      • + *
    • 3. The filter function (for example, H5Z_filter_bzip2).
      • + *
    • 4. The two functions necessary for the HDF5 Library to find the correct type of the plugin library + * while loading it at runtime and to get information about the filter function: + * + *
      H5PL_type_t H5PLget_plugin_type(void);
      const void* H5PLget_plugin_info(void);
      + * Here is an example of the functions above for the HDF5 bzip2 filter: + * + *
      H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;}
      const void* H5PLget_plugin_info(void) {return H5Z_BZIP2;}
      + *
      • + *
    • 5. Other functions such as the source of the compression library may also be included.
      • + *
    + * Build the HDF5 filter plugin as a shared library. The following steps should be taken: + *
    • 1. When compiling, point to the HDF5 header files.
      • + *
    • 2. Use the appropriate linking flags.
      • + *
    • 3. Link with any required external libraries.
      • + *
    • 4. For example, if libbz2.so is installed on a Linux system, the HDF5 bzip2 plugin library + * libH5Zbzip2.so may be linked with libbz2.so instead of including bzip2 source into the + * plugin library. + * The complete example of the HDF5 bzip2 plugin library is provided at + * BZIP2 Filter Plugin + * and can be adopted for other plugins.
    + * + * \subsubsection subsubsec_filter_plugins_prog_install Installing an HDF5 Filter Plugin + * The default directory for an HDF5 filter plugin library is defined on UNIX-like systems as + * \code + * “/usr/local/hdf5/lib/plugin” + * \endcode + * and on Windows systems as + * \code + * "%ALLUSERSPROFILE%/hdf5/lib/plugin". + * \endcode + * + * The default path can be overwritten by a user with the #HDF5_PLUGIN_PATH environment variable. + * Several directories can be specified for the search path using “:” as a path separator for UNIX-like + * systems and “;” for Windows. + * + * Readers are encouraged to try the example in the “Building an HDF5 bzip2 Plugin Example” section. + * + * \subsection subsec_filter_plugins_design Design + * Dynamic loading of the HDF5 filter plugin (or filter library) is triggered only by two events: when an + * application calls the #H5Pset_filter function to set the filter for the first time, or when the data to + * which the filter is applied is read for the first time. + * + * \subsection subsec_filter_plugins_build Building an HDF5 bzip2 Plugin Example + * The HDF Group provides an repository of the HDF5 filter plugin that can be checked out from + * BZIP2 Filter Plugin. + * + * It contains the source code for the bzip2 + * plugin library and an example that uses the plugin. It requires the HDF5 Library with the dynamically + * loaded feature and the bzip2 library being available on the system. + * The plugin and the example can be built using configure or CMake commands. For instructions on how + * to build with CMake, see the README.txt file in the source code distribution. The bzip2 library that can + * be built with CMake is available from: + * \code + * GIT_URL: "https://github.com/libarchive/bzip2.git" + * GIT_BRANCH: "master" + * \endcode + * + * See the documentation at + * hdf5_plugins/docs folder In + * particular: + * INSTALL_With_CMake + * USING_HDF5_AND_CMake */ /** @@ -36,26 +286,6 @@ * Use the functions in this module to manage the loading behavior of HDF5 * plugins. * - * - * - * - * - * - * - * - * - * - * - *
    CreateRead
    - * \snippet H5PL_examples.c create - * - * \snippet H5PL_examples.c read - *
    UpdateDelete
    - * \snippet H5PL_examples.c update - * - * \snippet H5PL_examples.c delete - *
    - * * \attention The loading behavior of HDF5 plugins can be controlled via the * functions described below and certain environment variables, such * as \c HDF5_PLUGIN_PRELOAD and \c HDF5_PLUGIN_PATH. diff --git a/src/H5Zmodule.h b/src/H5Zmodule.h index 8b1a0dedc4e..c7f8300fb7c 100644 --- a/src/H5Zmodule.h +++ b/src/H5Zmodule.h @@ -102,7 +102,7 @@ * Custom filters that have been registered with the library will have * additional unique identifiers. * - * See \ref_dld_filters for more information on how an HDF5 application can + * See \ref sec_filter_plugins for more information on how an HDF5 application can * apply a filter that is not registered with the HDF5 library. * * \defgroup H5ZPRE Predefined Filters From 330b80a2665bee74d987491622c4023ef1ab8343 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Mon, 18 Mar 2024 23:36:46 -0500 Subject: [PATCH 20/46] Add support for _Float16 16-bit floating point type (#4065) Fixed some conversion issues with Clang due to problematic undefined behavior when casting a negative floating-point value to an integer Fixed a bug in the library's software integer to floating-point conversion function where a user's conversion exception function returning H5T_CONV_UNHANDLED in the case of overflows would result in incorrect data after conversion Added configure checks for functions and macros related to _Float16 usage since some compilers expose the datatype but not the functions or macros Fixed a dt_arith test failure when H5_WANT_DCONV_EXCEPTION isn't defined Fixed a few warnings from not explicitly casting some _Float16 variables upwards --- config/cmake/ConfigureChecks.cmake | 83 +- config/cmake/ConversionTests.c | 122 ++ config/cmake/H5pubconf.h.in | 12 + configure.ac | 103 ++ doxygen/dox/DDLBNF112.dox | 2 +- doxygen/dox/DDLBNF114.dox | 654 ++++++++++ doxygen/dox/IntroHDF5.dox | 2 +- doxygen/dox/LearnBasics1.dox | 4 +- doxygen/dox/LearnBasics2.dox | 5 + doxygen/dox/Specifications.dox | 1 + .../examples/tables/predefinedDatatypes.dox | 12 + hl/src/H5LT.c | 13 +- hl/src/H5LTanalyze.c | 456 +++---- hl/src/H5LTanalyze.l | 3 + hl/src/H5LTparse.c | 789 ++++++------ hl/src/H5LTparse.h | 63 +- hl/src/H5LTparse.y | 8 +- java/src/hdf/hdf5lib/HDF5Constants.java | 12 + java/src/jni/h5Constants.c | 15 + java/src/jni/h5util.c | 8 +- release_docs/RELEASE.txt | 119 ++ src/H5Fmodule.h | 2 +- src/H5Gmodule.h | 2 +- src/H5T.c | 570 ++++++--- src/H5Tconv.c | 435 ++++++- src/H5Tinit_float.c | 48 +- src/H5Tmodule.h | 19 +- src/H5Tnative.c | 28 +- src/H5Tpkg.h | 96 ++ src/H5Tpublic.h | 19 + src/H5private.h | 38 +- src/H5trace.c | 8 + test/API/H5_api_dataset_test.c | 10 +- test/API/H5_api_test_util.c | 14 +- test/dt_arith.c | 1062 +++++++++++++++-- test/dtypes.c | 425 ++++++- test/ntypes.c | 121 ++ tools/lib/h5diff_array.c | 265 +++- tools/lib/h5diff_util.c | 10 +- tools/lib/h5tools_dump.c | 10 +- tools/lib/h5tools_str.c | 12 +- tools/lib/h5tools_type.c | 8 +- tools/src/h5import/h5import.c | 108 ++ tools/src/h5ls/h5ls.c | 13 +- tools/test/h5dump/CMakeTests.cmake | 8 + tools/test/h5dump/CMakeTestsXML.cmake | 8 + tools/test/h5dump/expected/tfloat16.ddl | 46 + tools/test/h5dump/expected/tfloat16_be.ddl | 46 + .../test/h5dump/expected/xml/tfloat16.h5.xml | 302 +++++ .../h5dump/expected/xml/tfloat16_be.h5.xml | 302 +++++ tools/test/h5dump/h5dumpgentest.c | 163 +++ tools/test/h5dump/testfiles/tfloat16.h5 | Bin 0 -> 2304 bytes tools/test/h5dump/testfiles/tfloat16_be.h5 | Bin 0 -> 2304 bytes tools/test/h5dump/testh5dump.sh.in | 8 + tools/test/h5dump/testh5dumpxml.sh.in | 8 + tools/test/h5ls/CMakeTests.cmake | 33 + tools/test/h5ls/expected/tfloat16.ls | 8 + tools/test/h5ls/expected/tfloat16_be.ls | 8 + .../h5ls/expected/tfloat16_be_nosupport.ls | 8 + .../test/h5ls/expected/tfloat16_nosupport.ls | 8 + tools/test/h5ls/testh5ls.sh.in | 22 +- 61 files changed, 5765 insertions(+), 1022 deletions(-) create mode 100644 doxygen/dox/DDLBNF114.dox create mode 100644 tools/test/h5dump/expected/tfloat16.ddl create mode 100644 tools/test/h5dump/expected/tfloat16_be.ddl create mode 100644 tools/test/h5dump/expected/xml/tfloat16.h5.xml create mode 100644 tools/test/h5dump/expected/xml/tfloat16_be.h5.xml create mode 100644 tools/test/h5dump/testfiles/tfloat16.h5 create mode 100644 tools/test/h5dump/testfiles/tfloat16_be.h5 create mode 100644 tools/test/h5ls/expected/tfloat16.ls create mode 100644 tools/test/h5ls/expected/tfloat16_be.ls create mode 100644 tools/test/h5ls/expected/tfloat16_be_nosupport.ls create mode 100644 tools/test/h5ls/expected/tfloat16_nosupport.ls diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 2905fe9ad08..a3eb80e6beb 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -808,7 +808,8 @@ macro (H5ConversionTests TEST def msg) ${CMAKE_BINARY_DIR} ${HDF_RESOURCES_DIR}/ConversionTests.c CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=-D${TEST}_TEST - OUTPUT_VARIABLE OUTPUT + COMPILE_OUTPUT_VARIABLE ${TEST}_COMPILE_OUTPUT + RUN_OUTPUT_VARIABLE ${TEST}_RUN_OUTPUT ) if (${TEST}_COMPILE) if (${TEST}_RUN EQUAL "0") @@ -818,14 +819,17 @@ macro (H5ConversionTests TEST def msg) set (${TEST} "" CACHE INTERNAL ${msg}) message (VERBOSE "${msg}... no") file (APPEND ${CMAKE_BINARY_DIR}/CMakeFiles/CMakeError.log - "Test ${TEST} Run failed with the following output and exit code:\n ${OUTPUT}\n" + "Test ${TEST} Compile succeeded with the following output:\n ${${TEST}_COMPILE_OUTPUT}\n" + ) + file (APPEND ${CMAKE_BINARY_DIR}/CMakeFiles/CMakeError.log + "Test ${TEST} Run failed with exit code ${${TEST}_RUN} and with the following output:\n ${${TEST}_RUN_OUTPUT}\n" ) endif () else () set (${TEST} "" CACHE INTERNAL ${msg}) message (VERBOSE "${msg}... no") file (APPEND ${CMAKE_BINARY_DIR}/CMakeFiles/CMakeError.log - "Test ${TEST} Compile failed with the following output:\n ${OUTPUT}\n" + "Test ${TEST} Compile failed with the following output:\n ${${TEST}_COMPILE_OUTPUT}\n" ) endif () else () @@ -887,3 +891,76 @@ H5ConversionTests (${HDF_PREFIX}_LLONG_TO_LDOUBLE_CORRECT TRUE "Checking IF corr # some long double values #----------------------------------------------------------------------------- H5ConversionTests (${HDF_PREFIX}_DISABLE_SOME_LDOUBLE_CONV FALSE "Checking IF the cpu is power9 and cannot correctly converting long double values") + +#----------------------------------------------------------------------------- +# Check if _Float16 type is available +#----------------------------------------------------------------------------- +message (STATUS "Checking if _Float16 support is available") +set (${HDF_PREFIX}_HAVE__FLOAT16 0) +HDF_CHECK_TYPE_SIZE (_Float16 ${HDF_PREFIX}_SIZEOF__FLOAT16) +if (${HDF_PREFIX}_SIZEOF__FLOAT16) + # Request _Float16 support + set (CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} "-D__STDC_WANT_IEC_60559_TYPES_EXT__") + + # Some compilers expose the _Float16 datatype, but not the macros and + # functions used with the datatype. We need the macros for proper + # datatype conversion support. Check for these here. + CHECK_SYMBOL_EXISTS (FLT16_EPSILON "float.h" h5_have_flt16_epsilon) + CHECK_SYMBOL_EXISTS (FLT16_MIN "float.h" h5_have_flt16_min) + CHECK_SYMBOL_EXISTS (FLT16_MAX "float.h" h5_have_flt16_max) + CHECK_SYMBOL_EXISTS (FLT16_MIN_10_EXP "float.h" h5_have_flt16_min_10_exp) + CHECK_SYMBOL_EXISTS (FLT16_MAX_10_EXP "float.h" h5_have_flt16_max_10_exp) + CHECK_SYMBOL_EXISTS (FLT16_MANT_DIG "float.h" h5_have_flt16_mant_dig) + + if (h5_have_flt16_epsilon AND h5_have_flt16_min AND + h5_have_flt16_max AND h5_have_flt16_min_10_exp AND + h5_have_flt16_max_10_exp AND h5_have_flt16_mant_dig) + # Some compilers like OneAPI on Windows appear to detect _Float16 support + # properly up to this point, and, in the absence of any architecture-specific + # tuning compiler flags, will generate code for H5Tconv.c that performs + # software conversions on _Float16 variables with compiler-internal functions + # such as __extendhfsf2, __truncsfhf2, or __truncdfhf2. However, these + # compilers will fail to link these functions into the build for currently + # unknown reasons and cause the build to fail. Since these are compiler-internal + # functions that we don't appear to have much control over, let's try to + # compile a program that will generate these functions to check for _Float16 + # support. If we fail to compile this program, we will simply disable + # _Float16 support for the time being. + H5ConversionTests ( + ${HDF_PREFIX}_FLOAT16_CONVERSION_FUNCS_LINK + FALSE + "Checking if compiler can convert _Float16 type with casts" + ) + + if (${${HDF_PREFIX}_FLOAT16_CONVERSION_FUNCS_LINK}) + # Finally, MacOS 13 appears to have a bug specifically when converting + # long double values to _Float16. Release builds of the dt_arith test + # would cause any assignments to a _Float16 variable to be elided, + # whereas Debug builds would perform incorrect hardware conversions by + # simply chopping off all the bytes of the value except for the first 2. + # These tests pass on MacOS 14, so let's perform a quick test to check + # if the hardware conversion is done correctly. + H5ConversionTests ( + ${HDF_PREFIX}_LDOUBLE_TO_FLOAT16_CORRECT + TRUE + "Checking if correctly converting long double to _Float16 values" + ) + + if (NOT ${${HDF_PREFIX}_LDOUBLE_TO_FLOAT16_CORRECT}) + message (VERBOSE "Conversions from long double to _Float16 appear to be incorrect. These will be emulated through a soft conversion function.") + endif () + + set (${HDF_PREFIX}_HAVE__FLOAT16 1) + + # Check if we can use fabsf16 + CHECK_FUNCTION_EXISTS (fabsf16 ${HDF_PREFIX}_HAVE_FABSF16) + else () + message (STATUS "_Float16 support has been disabled because the compiler couldn't compile and run a test program for _Float16 conversions") + message (STATUS "Check ${CMAKE_BINARY_DIR}/CMakeFiles/CMakeError.log for information on why the test program couldn't be compiled/run") + endif () + else () + message (STATUS "_Float16 support has been disabled since the required macros (FLT16_MAX, FLT16_EPSILON, etc. were not found)") + endif () +else () + message (STATUS "_Float16 support has been disabled since the _Float16 type was not found") +endif () diff --git a/config/cmake/ConversionTests.c b/config/cmake/ConversionTests.c index 725f0496f01..8e103bd3e97 100644 --- a/config/cmake/ConversionTests.c +++ b/config/cmake/ConversionTests.c @@ -285,3 +285,125 @@ int HDF_NO_UBSAN main(void) } #endif + +#ifdef H5_FLOAT16_CONVERSION_FUNCS_LINK_TEST + +#define __STDC_WANT_IEC_60559_TYPES_EXT__ + +#include +#include + +int HDF_NO_UBSAN main(void) +{ + _Float16 fl16_var; + signed char sc; + unsigned char usc; + short s; + unsigned short us; + int i; + unsigned int ui; + long l; + unsigned long ul; + long long ll; + unsigned long long ull; + float f; + double d; + long double ld; + int ret = 0; + + /* + * Cast the _Float16 type between all the different C datatypes + * we support conversions for in H5Tconv.c to check if the compiler + * properly links any software conversion functions it may generate + * for the casts, such as __extendhfsf2 or __truncdfhf2. + */ + + fl16_var = 3.0f16; + + sc = (signed char)fl16_var; + usc = (unsigned char)fl16_var; + s = (short)fl16_var; + us = (unsigned short)fl16_var; + i = (int)fl16_var; + ui = (unsigned int)fl16_var; + l = (long)fl16_var; + ul = (unsigned long)fl16_var; + ll = (long long)fl16_var; + ull = (unsigned long long)fl16_var; + f = (float)fl16_var; + d = (double)fl16_var; + ld = (long double)fl16_var; + + sc = (signed char)3; + fl16_var = (_Float16)sc; + + usc = (unsigned char)3; + fl16_var = (_Float16)usc; + + s = (short)3; + fl16_var = (_Float16)s; + + us = (unsigned short)3; + fl16_var = (_Float16)us; + + i = (int)3; + fl16_var = (_Float16)i; + + ui = (unsigned int)3; + fl16_var = (_Float16)ui; + + l = (long)3; + fl16_var = (_Float16)l; + + ul = (unsigned long)3; + fl16_var = (_Float16)ul; + + ll = (long long)3; + fl16_var = (_Float16)ll; + + ull = (unsigned long long)3; + fl16_var = (_Float16)ull; + + f = (float)3.0f; + fl16_var = (_Float16)f; + + d = (double)3.0; + fl16_var = (_Float16)d; + + ld = (long double)3.0l; + fl16_var = (_Float16)ld; + +done: + exit(ret); +} + +#endif + +#ifdef H5_LDOUBLE_TO_FLOAT16_CORRECT_TEST + +#define __STDC_WANT_IEC_60559_TYPES_EXT__ + +#include +#include +#include +#include + +int HDF_NO_UBSAN main(void) +{ + long double ld; + _Float16 half; + int ret = 1; + + ld = 32.0L; + half = 64.0f16; + + half = (_Float16)ld; + + if (fabsl(ld - (long double)half) < LDBL_EPSILON) + ret = 0; + +done: + exit(ret); +} + +#endif diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 8d866ec89a5..af8948d9f6e 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -131,6 +131,9 @@ /* Define if library information should be embedded in the executables */ #cmakedefine H5_HAVE_EMBEDDED_LIBINFO @H5_HAVE_EMBEDDED_LIBINFO@ +/* Define to 1 if you have the `fabsf16' function. */ +#cmakedefine H5_HAVE_FABSF16 @H5_HAVE_FABSF16@ + /* Define to 1 if you have the `fcntl' function. */ #cmakedefine H5_HAVE_FCNTL @H5_HAVE_FCNTL@ @@ -143,6 +146,9 @@ /* Define if support for szip filter is enabled */ #cmakedefine H5_HAVE_FILTER_SZIP @H5_HAVE_FILTER_SZIP@ +/* Determine if _Float16 is available */ +#cmakedefine H5_HAVE__FLOAT16 @H5_HAVE__FLOAT16@ + /* Determine if __float128 is available */ #cmakedefine H5_HAVE_FLOAT128 @H5_HAVE_FLOAT128@ @@ -384,6 +390,9 @@ /* Define if new-style references should be used with dimension scales */ #cmakedefine H5_DIMENSION_SCALES_WITH_NEW_REF @H5_DIMENSION_SCALES_WITH_NEW_REF@ +/* Define if your system can convert long double to _Float16 values correctly. */ +#cmakedefine H5_LDOUBLE_TO_FLOAT16_CORRECT @H5_LDOUBLE_TO_FLOAT16_CORRECT@ + /* Define if your system can convert long double to (unsigned) long long values correctly. */ #cmakedefine H5_LDOUBLE_TO_LLONG_ACCURATE @H5_LDOUBLE_TO_LLONG_ACCURATE@ @@ -586,6 +595,9 @@ /* The size of `_Quad', as computed by sizeof. */ #define H5_SIZEOF__QUAD @H5_SIZEOF__QUAD@ +/* The size of `_Float16', as computed by sizeof. */ +#define H5_SIZEOF__FLOAT16 @H5_SIZEOF__FLOAT16@ + /* The size of `__float128', as computed by sizeof. */ #define H5_SIZEOF___FLOAT128 @H5_SIZEOF___FLOAT128@ diff --git a/configure.ac b/configure.ac index b76f18ff29f..f33fb8b87b9 100644 --- a/configure.ac +++ b/configure.ac @@ -559,6 +559,109 @@ AC_CHECK_SIZEOF([float]) AC_CHECK_SIZEOF([double]) AC_CHECK_SIZEOF([long double]) +## ---------------------------------------------------------------------- +## Check if _Float16 support is available +## +AC_MSG_NOTICE([checking if _Float16 support is available]) +HAVE__FLOAT16="no" +AC_CHECK_SIZEOF([_Float16]) +if test "$ac_cv_sizeof__Float16" != 0; then + # Some compilers expose the _Float16 datatype, but not the macros and + # functions used with the datatype. We need the macros for proper + # datatype conversion support. Check for these here. + AC_CHECK_DECL([FLT16_EPSILON], [], [], [[ + #define __STDC_WANT_IEC_60559_TYPES_EXT__ + #include ]]) + AC_CHECK_DECL([FLT16_MIN], [], [], [[ + #define __STDC_WANT_IEC_60559_TYPES_EXT__ + #include ]]) + AC_CHECK_DECL([FLT16_MAX], [], [], [[ + #define __STDC_WANT_IEC_60559_TYPES_EXT__ + #include ]]) + AC_CHECK_DECL([FLT16_MIN_10_EXP], [], [], [[ + #define __STDC_WANT_IEC_60559_TYPES_EXT__ + #include ]]) + AC_CHECK_DECL([FLT16_MAX_10_EXP], [], [], [[ + #define __STDC_WANT_IEC_60559_TYPES_EXT__ + #include ]]) + AC_CHECK_DECL([FLT16_MANT_DIG], [], [], [[ + #define __STDC_WANT_IEC_60559_TYPES_EXT__ + #include ]]) + + if test "X$ac_cv_have_decl_FLT16_EPSILON" = "Xyes" && + test "X$ac_cv_have_decl_FLT16_MIN" = "Xyes" && + test "X$ac_cv_have_decl_FLT16_MAX" = "Xyes" && + test "X$ac_cv_have_decl_FLT16_MIN_10_EXP" = "Xyes" && + test "X$ac_cv_have_decl_FLT16_MAX_10_EXP" = "Xyes" && + test "X$ac_cv_have_decl_FLT16_MANT_DIG" = "Xyes" ; then + # Some compilers like OneAPI on Windows appear to detect _Float16 support + # properly up to this point, and, in the absence of any architecture-specific + # tuning compiler flags, will generate code for H5Tconv.c that performs + # software conversions on _Float16 variables with compiler-internal functions + # such as __extendhfsf2, __truncsfhf2, or __truncdfhf2. However, these + # compilers will fail to link these functions into the build for currently + # unknown reasons and cause the build to fail. Since these are compiler-internal + # functions that we don't appear to have much control over, let's try to + # compile a program that will generate these functions to check for _Float16 + # support. If we fail to compile this program, we will simply disable + # _Float16 support for the time being. + AC_MSG_CHECKING([if compiler can correctly compile and run a test program which converts _Float16 to other types with casts]) + TEST_SRC="`(echo \"#define H5_FLOAT16_CONVERSION_FUNCS_LINK_TEST 1\"; cat $srcdir/config/cmake/ConversionTests.c)`" + AC_CACHE_VAL([hdf5_cv_float16_conversion_funcs_link], + [AC_RUN_IFELSE( + [AC_LANG_SOURCE([$TEST_SRC])], + [hdf5_cv_float16_conversion_funcs_link=yes], [hdf5_cv_float16_conversion_funcs_link=no], [hdf5_cv_float16_conversion_funcs_link=no])]) + + if test ${hdf5_cv_float16_conversion_funcs_link} = "yes"; then + AC_MSG_RESULT([yes]) + + # Finally, MacOS 13 appears to have a bug specifically when converting + # long double values to _Float16. Release builds of the dt_arith test + # would cause any assignments to a _Float16 variable to be elided, + # whereas Debug builds would perform incorrect hardware conversions by + # simply chopping off all the bytes of the value except for the first 2. + # These tests pass on MacOS 14, so let's perform a quick test to check + # if the hardware conversion is done correctly. + AC_MSG_CHECKING([if compiler can correctly convert long double values to _Float16]) + TEST_SRC="`(echo \"#define H5_LDOUBLE_TO_FLOAT16_CORRECT_TEST 1\"; cat $srcdir/config/cmake/ConversionTests.c)`" + if test ${ac_cv_sizeof_long_double} = 0; then + hdf5_cv_ldouble_to_float16_correct=${hdf5_cv_ldouble_to_float16_correct=no} + else + AC_CACHE_VAL([hdf5_cv_ldouble_to_float16_correct], + [AC_RUN_IFELSE( + [AC_LANG_SOURCE([$TEST_SRC])], + [hdf5_cv_ldouble_to_float16_correct=yes], [hdf5_cv_ldouble_to_float16_correct=no], [hdf5_cv_ldouble_to_float16_correct=yes])]) + fi + + if test ${hdf5_cv_ldouble_to_float16_correct} = "yes"; then + AC_DEFINE([LDOUBLE_TO_FLOAT16_CORRECT], [1], + [Define if your system can convert long double to _Float16 values correctly.]) + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + AC_MSG_NOTICE([Conversions from long double to _Float16 appear to be incorrect. These will be emulated through a soft conversion function.]) + fi + + HAVE__FLOAT16="yes" + + # Check if we can use fabsf16 + AC_CHECK_FUNC([fabsf16], [AC_DEFINE([HAVE_FABSF16], [1], + [Define if has fabsf16 function])], []) + + # Define HAVE__FLOAT16 macro for H5pubconf.h if _Float16 is available. + AC_DEFINE([HAVE__FLOAT16], [1], [Determine if _Float16 is available]) + else + AC_MSG_RESULT([no]) + fi + fi + + AC_MSG_CHECKING([if _Float16 support is enabled]) + AC_MSG_RESULT([$HAVE__FLOAT16]) +fi + +# Define HAVE__FLOAT16 value to substitute into other files for conditional testing +AC_SUBST([HAVE__FLOAT16]) + ## ---------------------------------------------------------------------- ## Check if the Fortran interface should be enabled ## diff --git a/doxygen/dox/DDLBNF112.dox b/doxygen/dox/DDLBNF112.dox index 6809a0632ff..cfe34c321f9 100644 --- a/doxygen/dox/DDLBNF112.dox +++ b/doxygen/dox/DDLBNF112.dox @@ -1,4 +1,4 @@ -/** \page DDLBNF112 DDL in BNF for HDF5 1.12 and above +/** \page DDLBNF112 DDL in BNF for HDF5 1.12 through HDF5 1.14.3 \todo Revise this & break it up! diff --git a/doxygen/dox/DDLBNF114.dox b/doxygen/dox/DDLBNF114.dox new file mode 100644 index 00000000000..61e9157e560 --- /dev/null +++ b/doxygen/dox/DDLBNF114.dox @@ -0,0 +1,654 @@ +/** \page DDLBNF114 DDL in BNF for HDF5 1.14.4 and above + +\todo Revise this & break it up! + +\section intro114 Introduction + +This document contains the data description language (DDL) for an HDF5 file. The +description is in Backus-Naur Form (BNF). + +\section expo114 Explanation of Symbols + +This section contains a brief explanation of the symbols used in the DDL. + +\code{.unparsed} +::= defined as + a token with the name tname + | one of or + opt zero or one occurrence of + * zero or more occurrence of + + one or more occurrence of + [0-9] an element in the range between 0 and 9 + '[' the token within the quotes (used for special characters) + TBD To Be Decided +\endcode + +\section ddl114 The DDL + +\code{.unparsed} + ::= HDF5 { opt } + + ::= + + ::= SUPER_BLOCK { + SUPERBLOCK_VERSION + FREELIST_VERSION + SYMBOLTABLE_VERSION + OBJECTHEADER_VERSION + OFFSET_SIZE + LENGTH_SIZE + BTREE_RANK + BTREE_LEAF + ISTORE_K + + USER_BLOCK { + USERBLOCK_SIZE + } + } + + ::= FILE_SPACE_STRATEGY + FREE_SPACE_PERSIST + FREE_SPACE_SECTION_THRESHOLD + FILE_SPACE_PAGE_SIZE + + ::= H5F_FSPACE_STRATEGY_FSM_AGGR | H5F_FSPACE_STRATEGY_PAGE | + H5F_FSPACE_STRATEGY_AGGR | H5F_FSPACE_STRATEGY_NONE | + Unknown strategy + + ::= GROUP "/" { + * + opt + opt + * + * + } + + ::= | | | + + ::= DATATYPE { + + } + + ::= the assigned name for anonymous named type is + in the form of #oid, where oid is the object id + of the type + + ::= | | * Libraries and Tools Reference. - * The HDF5 DDL grammar is described in the document \ref DDLBNF110. + * The HDF5 DDL grammar is described in the document \ref DDLBNF114. * * \subsection subsec_file_summary File Function Summaries * General library (\ref H5 functions and macros), (\ref H5F functions), file related diff --git a/src/H5Gmodule.h b/src/H5Gmodule.h index c330fcdb400..fb9cf732c09 100644 --- a/src/H5Gmodule.h +++ b/src/H5Gmodule.h @@ -343,7 +343,7 @@ * * h5dump is described on the “HDF5 Tools” page of the \ref RM. * - * The HDF5 DDL grammar is described in the @ref DDLBNF110. + * The HDF5 DDL grammar is described in the @ref DDLBNF114. * * \subsection subsec_group_function Group Function Summaries * Functions that can be used with groups (\ref H5G functions) and property list functions that can used diff --git a/src/H5T.c b/src/H5T.c index 7d3db702a84..1fb4b214833 100644 --- a/src/H5T.c +++ b/src/H5T.c @@ -100,6 +100,30 @@ dt->shared->u.atomic.msb_pad = H5T_PAD_ZERO; \ } +/* Define the code templates for standard 16-bit floats for the "GUTS" in the H5T_INIT_TYPE macro */ +#define H5T_INIT_TYPE_FLOAT16_COMMON(ENDIANNESS) \ + { \ + H5T_INIT_TYPE_NUM_COMMON(ENDIANNESS) \ + dt->shared->u.atomic.u.f.sign = 15; \ + dt->shared->u.atomic.u.f.epos = 10; \ + dt->shared->u.atomic.u.f.esize = 5; \ + dt->shared->u.atomic.u.f.ebias = 0xf; \ + dt->shared->u.atomic.u.f.mpos = 0; \ + dt->shared->u.atomic.u.f.msize = 10; \ + dt->shared->u.atomic.u.f.norm = H5T_NORM_IMPLIED; \ + dt->shared->u.atomic.u.f.pad = H5T_PAD_ZERO; \ + } + +#define H5T_INIT_TYPE_FLOAT16LE_CORE \ + { \ + H5T_INIT_TYPE_FLOAT16_COMMON(H5T_ORDER_LE) \ + } + +#define H5T_INIT_TYPE_FLOAT16BE_CORE \ + { \ + H5T_INIT_TYPE_FLOAT16_COMMON(H5T_ORDER_BE) \ + } + /* Define the code templates for standard floats for the "GUTS" in the H5T_INIT_TYPE macro */ #define H5T_INIT_TYPE_FLOAT_COMMON(ENDIANNESS) \ { \ @@ -375,102 +399,105 @@ H5T_order_t H5T_native_order_g = H5T_ORDER_ERROR; * If more of these are added, the new ones must be added to the list of * types to reset in H5T_term_package(). */ -hid_t H5T_IEEE_F32BE_g = FAIL; -hid_t H5T_IEEE_F32LE_g = FAIL; -hid_t H5T_IEEE_F64BE_g = FAIL; -hid_t H5T_IEEE_F64LE_g = FAIL; - -hid_t H5T_VAX_F32_g = FAIL; -hid_t H5T_VAX_F64_g = FAIL; - -hid_t H5T_STD_I8BE_g = FAIL; -hid_t H5T_STD_I8LE_g = FAIL; -hid_t H5T_STD_I16BE_g = FAIL; -hid_t H5T_STD_I16LE_g = FAIL; -hid_t H5T_STD_I32BE_g = FAIL; -hid_t H5T_STD_I32LE_g = FAIL; -hid_t H5T_STD_I64BE_g = FAIL; -hid_t H5T_STD_I64LE_g = FAIL; -hid_t H5T_STD_U8BE_g = FAIL; -hid_t H5T_STD_U8LE_g = FAIL; -hid_t H5T_STD_U16BE_g = FAIL; -hid_t H5T_STD_U16LE_g = FAIL; -hid_t H5T_STD_U32BE_g = FAIL; -hid_t H5T_STD_U32LE_g = FAIL; -hid_t H5T_STD_U64BE_g = FAIL; -hid_t H5T_STD_U64LE_g = FAIL; -hid_t H5T_STD_B8BE_g = FAIL; -hid_t H5T_STD_B8LE_g = FAIL; -hid_t H5T_STD_B16BE_g = FAIL; -hid_t H5T_STD_B16LE_g = FAIL; -hid_t H5T_STD_B32BE_g = FAIL; -hid_t H5T_STD_B32LE_g = FAIL; -hid_t H5T_STD_B64BE_g = FAIL; -hid_t H5T_STD_B64LE_g = FAIL; -hid_t H5T_STD_REF_OBJ_g = FAIL; -hid_t H5T_STD_REF_DSETREG_g = FAIL; -hid_t H5T_STD_REF_g = FAIL; - -hid_t H5T_UNIX_D32BE_g = FAIL; -hid_t H5T_UNIX_D32LE_g = FAIL; -hid_t H5T_UNIX_D64BE_g = FAIL; -hid_t H5T_UNIX_D64LE_g = FAIL; - -hid_t H5T_C_S1_g = FAIL; - -hid_t H5T_FORTRAN_S1_g = FAIL; - -hid_t H5T_NATIVE_SCHAR_g = FAIL; -hid_t H5T_NATIVE_UCHAR_g = FAIL; -hid_t H5T_NATIVE_SHORT_g = FAIL; -hid_t H5T_NATIVE_USHORT_g = FAIL; -hid_t H5T_NATIVE_INT_g = FAIL; -hid_t H5T_NATIVE_UINT_g = FAIL; -hid_t H5T_NATIVE_LONG_g = FAIL; -hid_t H5T_NATIVE_ULONG_g = FAIL; -hid_t H5T_NATIVE_LLONG_g = FAIL; -hid_t H5T_NATIVE_ULLONG_g = FAIL; -hid_t H5T_NATIVE_FLOAT_g = FAIL; -hid_t H5T_NATIVE_DOUBLE_g = FAIL; -hid_t H5T_NATIVE_LDOUBLE_g = FAIL; -hid_t H5T_NATIVE_B8_g = FAIL; -hid_t H5T_NATIVE_B16_g = FAIL; -hid_t H5T_NATIVE_B32_g = FAIL; -hid_t H5T_NATIVE_B64_g = FAIL; -hid_t H5T_NATIVE_OPAQUE_g = FAIL; -hid_t H5T_NATIVE_HADDR_g = FAIL; -hid_t H5T_NATIVE_HSIZE_g = FAIL; -hid_t H5T_NATIVE_HSSIZE_g = FAIL; -hid_t H5T_NATIVE_HERR_g = FAIL; -hid_t H5T_NATIVE_HBOOL_g = FAIL; - -hid_t H5T_NATIVE_INT8_g = FAIL; -hid_t H5T_NATIVE_UINT8_g = FAIL; -hid_t H5T_NATIVE_INT_LEAST8_g = FAIL; -hid_t H5T_NATIVE_UINT_LEAST8_g = FAIL; -hid_t H5T_NATIVE_INT_FAST8_g = FAIL; -hid_t H5T_NATIVE_UINT_FAST8_g = FAIL; - -hid_t H5T_NATIVE_INT16_g = FAIL; -hid_t H5T_NATIVE_UINT16_g = FAIL; -hid_t H5T_NATIVE_INT_LEAST16_g = FAIL; -hid_t H5T_NATIVE_UINT_LEAST16_g = FAIL; -hid_t H5T_NATIVE_INT_FAST16_g = FAIL; -hid_t H5T_NATIVE_UINT_FAST16_g = FAIL; - -hid_t H5T_NATIVE_INT32_g = FAIL; -hid_t H5T_NATIVE_UINT32_g = FAIL; -hid_t H5T_NATIVE_INT_LEAST32_g = FAIL; -hid_t H5T_NATIVE_UINT_LEAST32_g = FAIL; -hid_t H5T_NATIVE_INT_FAST32_g = FAIL; -hid_t H5T_NATIVE_UINT_FAST32_g = FAIL; - -hid_t H5T_NATIVE_INT64_g = FAIL; -hid_t H5T_NATIVE_UINT64_g = FAIL; -hid_t H5T_NATIVE_INT_LEAST64_g = FAIL; -hid_t H5T_NATIVE_UINT_LEAST64_g = FAIL; -hid_t H5T_NATIVE_INT_FAST64_g = FAIL; -hid_t H5T_NATIVE_UINT_FAST64_g = FAIL; +hid_t H5T_IEEE_F16BE_g = H5I_INVALID_HID; +hid_t H5T_IEEE_F16LE_g = H5I_INVALID_HID; +hid_t H5T_IEEE_F32BE_g = H5I_INVALID_HID; +hid_t H5T_IEEE_F32LE_g = H5I_INVALID_HID; +hid_t H5T_IEEE_F64BE_g = H5I_INVALID_HID; +hid_t H5T_IEEE_F64LE_g = H5I_INVALID_HID; + +hid_t H5T_VAX_F32_g = H5I_INVALID_HID; +hid_t H5T_VAX_F64_g = H5I_INVALID_HID; + +hid_t H5T_STD_I8BE_g = H5I_INVALID_HID; +hid_t H5T_STD_I8LE_g = H5I_INVALID_HID; +hid_t H5T_STD_I16BE_g = H5I_INVALID_HID; +hid_t H5T_STD_I16LE_g = H5I_INVALID_HID; +hid_t H5T_STD_I32BE_g = H5I_INVALID_HID; +hid_t H5T_STD_I32LE_g = H5I_INVALID_HID; +hid_t H5T_STD_I64BE_g = H5I_INVALID_HID; +hid_t H5T_STD_I64LE_g = H5I_INVALID_HID; +hid_t H5T_STD_U8BE_g = H5I_INVALID_HID; +hid_t H5T_STD_U8LE_g = H5I_INVALID_HID; +hid_t H5T_STD_U16BE_g = H5I_INVALID_HID; +hid_t H5T_STD_U16LE_g = H5I_INVALID_HID; +hid_t H5T_STD_U32BE_g = H5I_INVALID_HID; +hid_t H5T_STD_U32LE_g = H5I_INVALID_HID; +hid_t H5T_STD_U64BE_g = H5I_INVALID_HID; +hid_t H5T_STD_U64LE_g = H5I_INVALID_HID; +hid_t H5T_STD_B8BE_g = H5I_INVALID_HID; +hid_t H5T_STD_B8LE_g = H5I_INVALID_HID; +hid_t H5T_STD_B16BE_g = H5I_INVALID_HID; +hid_t H5T_STD_B16LE_g = H5I_INVALID_HID; +hid_t H5T_STD_B32BE_g = H5I_INVALID_HID; +hid_t H5T_STD_B32LE_g = H5I_INVALID_HID; +hid_t H5T_STD_B64BE_g = H5I_INVALID_HID; +hid_t H5T_STD_B64LE_g = H5I_INVALID_HID; +hid_t H5T_STD_REF_OBJ_g = H5I_INVALID_HID; +hid_t H5T_STD_REF_DSETREG_g = H5I_INVALID_HID; +hid_t H5T_STD_REF_g = H5I_INVALID_HID; + +hid_t H5T_UNIX_D32BE_g = H5I_INVALID_HID; +hid_t H5T_UNIX_D32LE_g = H5I_INVALID_HID; +hid_t H5T_UNIX_D64BE_g = H5I_INVALID_HID; +hid_t H5T_UNIX_D64LE_g = H5I_INVALID_HID; + +hid_t H5T_C_S1_g = H5I_INVALID_HID; + +hid_t H5T_FORTRAN_S1_g = H5I_INVALID_HID; + +hid_t H5T_NATIVE_SCHAR_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UCHAR_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_SHORT_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_USHORT_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_LONG_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_ULONG_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_LLONG_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_ULLONG_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_FLOAT16_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_FLOAT_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_DOUBLE_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_LDOUBLE_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_B8_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_B16_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_B32_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_B64_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_OPAQUE_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_HADDR_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_HSIZE_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_HSSIZE_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_HERR_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_HBOOL_g = H5I_INVALID_HID; + +hid_t H5T_NATIVE_INT8_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT8_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_LEAST8_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_LEAST8_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_FAST8_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_FAST8_g = H5I_INVALID_HID; + +hid_t H5T_NATIVE_INT16_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT16_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_LEAST16_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_LEAST16_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_FAST16_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_FAST16_g = H5I_INVALID_HID; + +hid_t H5T_NATIVE_INT32_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT32_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_LEAST32_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_LEAST32_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_FAST32_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_FAST32_g = H5I_INVALID_HID; + +hid_t H5T_NATIVE_INT64_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT64_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_LEAST64_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_LEAST64_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_INT_FAST64_g = H5I_INVALID_HID; +hid_t H5T_NATIVE_UINT_FAST64_g = H5I_INVALID_HID; /* * Alignment constraints for HDF5 types. Accessing objects of these @@ -498,6 +525,7 @@ size_t H5T_NATIVE_LONG_ALIGN_g = 0; size_t H5T_NATIVE_ULONG_ALIGN_g = 0; size_t H5T_NATIVE_LLONG_ALIGN_g = 0; size_t H5T_NATIVE_ULLONG_ALIGN_g = 0; +size_t H5T_NATIVE_FLOAT16_ALIGN_g = 0; size_t H5T_NATIVE_FLOAT_ALIGN_g = 0; size_t H5T_NATIVE_DOUBLE_ALIGN_g = 0; size_t H5T_NATIVE_LDOUBLE_ALIGN_g = 0; @@ -533,6 +561,15 @@ size_t H5T_NATIVE_UINT_FAST64_ALIGN_g = 0; /* Useful floating-point values for conversion routines */ /* (+/- Inf for all floating-point types) */ +#ifdef H5_HAVE__FLOAT16 +/* Initialize these with a float literal since the f16 suffix + * is non-standard C and gives warnings when compiling the + * library with the -pedantic flag. These values will be + * overwritten anyway. + */ +H5__Float16 H5T_NATIVE_FLOAT16_POS_INF_g = 0.0f; +H5__Float16 H5T_NATIVE_FLOAT16_NEG_INF_g = 0.0f; +#endif float H5T_NATIVE_FLOAT_POS_INF_g = 0.0F; float H5T_NATIVE_FLOAT_NEG_INF_g = 0.0F; double H5T_NATIVE_DOUBLE_POS_INF_g = 0.0; @@ -685,6 +722,49 @@ H5T__init_inf(void) } /* end for */ } /* end if */ +#ifdef H5_HAVE__FLOAT16 + /* Get the _Float16 datatype */ + if (NULL == (dst_p = (H5T_t *)H5I_object(H5T_NATIVE_FLOAT16_g))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype"); + dst = &dst_p->shared->u.atomic; + + /* Check that we can re-order the bytes correctly */ + if (H5T_ORDER_LE != H5T_native_order_g && H5T_ORDER_BE != H5T_native_order_g) + HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unsupported byte order"); + + /* +Inf */ + d = (uint8_t *)&H5T_NATIVE_FLOAT16_POS_INF_g; + H5T__bit_set(d, dst->u.f.sign, (size_t)1, false); + H5T__bit_set(d, dst->u.f.epos, dst->u.f.esize, true); + H5T__bit_set(d, dst->u.f.mpos, dst->u.f.msize, false); + + /* Swap the bytes if the machine architecture is big-endian */ + if (H5T_ORDER_BE == H5T_native_order_g) { + half_size = dst_p->shared->size / 2; + for (u = 0; u < half_size; u++) { + uint8_t tmp = d[dst_p->shared->size - (u + 1)]; + d[dst_p->shared->size - (u + 1)] = d[u]; + d[u] = tmp; + } /* end for */ + } /* end if */ + + /* -Inf */ + d = (uint8_t *)&H5T_NATIVE_FLOAT16_NEG_INF_g; + H5T__bit_set(d, dst->u.f.sign, (size_t)1, true); + H5T__bit_set(d, dst->u.f.epos, dst->u.f.esize, true); + H5T__bit_set(d, dst->u.f.mpos, dst->u.f.msize, false); + + /* Swap the bytes if the machine architecture is big-endian */ + if (H5T_ORDER_BE == H5T_native_order_g) { + half_size = dst_p->shared->size / 2; + for (u = 0; u < half_size; u++) { + uint8_t tmp = d[dst_p->shared->size - (u + 1)]; + d[dst_p->shared->size - (u + 1)] = d[u]; + d[u] = tmp; + } /* end for */ + } /* end if */ +#endif + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5T__init_inf() */ @@ -738,6 +818,9 @@ H5T_init(void) herr_t status; bool copied_dtype = true; /* Flag to indicate whether datatype was copied or allocated (for error cleanup) */ +#ifdef H5_HAVE__FLOAT16 + H5T_t *native_float16 = NULL; /* Datatype structure for native _Float16 type */ +#endif herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -779,6 +862,10 @@ H5T_init(void) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype object"); if (NULL == (native_ullong = (H5T_t *)H5I_object(H5T_NATIVE_ULLONG_g))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype object"); +#ifdef H5_HAVE__FLOAT16 + if (NULL == (native_float16 = (H5T_t *)H5I_object(H5T_NATIVE_FLOAT16_g))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype object"); +#endif if (NULL == (native_float = (H5T_t *)H5I_object(H5T_NATIVE_FLOAT_g))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype object"); if (NULL == (native_double = (H5T_t *)H5I_object(H5T_NATIVE_DOUBLE_g))) @@ -823,6 +910,12 @@ H5T_init(void) *------------------------------------------------------------ */ + /* IEEE 2-byte little-endian float */ + H5T_INIT_TYPE(FLOAT16LE, H5T_IEEE_F16LE_g, COPY, native_double, SET, 2) + + /* IEEE 2-byte big-endian float */ + H5T_INIT_TYPE(FLOAT16BE, H5T_IEEE_F16BE_g, COPY, native_double, SET, 2) + /* IEEE 4-byte little-endian float */ H5T_INIT_TYPE(FLOATLE, H5T_IEEE_F32LE_g, COPY, native_double, SET, 4) @@ -1062,6 +1155,22 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "ldbl_flt", native_ldouble, native_float, H5T__conv_ldouble_float); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_dbl", native_ldouble, native_double, H5T__conv_ldouble_double); +#ifdef H5_HAVE__FLOAT16 + status |= + H5T__register_int(H5T_PERS_HARD, "flt16_flt", native_float16, native_float, H5T__conv__Float16_float); + status |= H5T__register_int(H5T_PERS_HARD, "flt16_dbl", native_float16, native_double, + H5T__conv__Float16_double); + status |= H5T__register_int(H5T_PERS_HARD, "flt16_ldbl", native_float16, native_ldouble, + H5T__conv__Float16_ldouble); + status |= + H5T__register_int(H5T_PERS_HARD, "flt_flt16", native_float, native_float16, H5T__conv_float__Float16); + status |= H5T__register_int(H5T_PERS_HARD, "dbl_flt16", native_double, native_float16, + H5T__conv_double__Float16); +#ifdef H5T_CONV_INTERNAL_LDOUBLE_FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "ldbl_flt16", native_ldouble, native_float16, + H5T__conv_ldouble__Float16); +#endif +#endif /* from long long */ status |= @@ -1220,6 +1329,10 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "schar_dbl", native_schar, native_double, H5T__conv_schar_double); status |= H5T__register_int(H5T_PERS_HARD, "schar_ldbl", native_schar, native_ldouble, H5T__conv_schar_ldouble); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "schar_flt16", native_schar, native_float16, + H5T__conv_schar__Float16); +#endif /* From unsigned char to floats */ status |= @@ -1228,6 +1341,10 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "uchar_dbl", native_uchar, native_double, H5T__conv_uchar_double); status |= H5T__register_int(H5T_PERS_HARD, "uchar_ldbl", native_uchar, native_ldouble, H5T__conv_uchar_ldouble); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "uchar_flt16", native_uchar, native_float16, + H5T__conv_uchar__Float16); +#endif /* From short to floats */ status |= @@ -1236,6 +1353,10 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "short_dbl", native_short, native_double, H5T__conv_short_double); status |= H5T__register_int(H5T_PERS_HARD, "short_ldbl", native_short, native_ldouble, H5T__conv_short_ldouble); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "short_flt16", native_short, native_float16, + H5T__conv_short__Float16); +#endif /* From unsigned short to floats */ status |= @@ -1244,23 +1365,39 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "ushort_dbl", native_ushort, native_double, H5T__conv_ushort_double); status |= H5T__register_int(H5T_PERS_HARD, "ushort_ldbl", native_ushort, native_ldouble, H5T__conv_ushort_ldouble); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "ushort_flt16", native_ushort, native_float16, + H5T__conv_ushort__Float16); +#endif /* From int to floats */ status |= H5T__register_int(H5T_PERS_HARD, "int_flt", native_int, native_float, H5T__conv_int_float); status |= H5T__register_int(H5T_PERS_HARD, "int_dbl", native_int, native_double, H5T__conv_int_double); status |= H5T__register_int(H5T_PERS_HARD, "int_ldbl", native_int, native_ldouble, H5T__conv_int_ldouble); +#ifdef H5_HAVE__FLOAT16 + status |= + H5T__register_int(H5T_PERS_HARD, "int_flt16", native_int, native_float16, H5T__conv_int__Float16); +#endif /* From unsigned int to floats */ status |= H5T__register_int(H5T_PERS_HARD, "uint_flt", native_uint, native_float, H5T__conv_uint_float); status |= H5T__register_int(H5T_PERS_HARD, "uint_dbl", native_uint, native_double, H5T__conv_uint_double); status |= H5T__register_int(H5T_PERS_HARD, "uint_ldbl", native_uint, native_ldouble, H5T__conv_uint_ldouble); +#ifdef H5_HAVE__FLOAT16 + status |= + H5T__register_int(H5T_PERS_HARD, "uint_flt16", native_uint, native_float16, H5T__conv_uint__Float16); +#endif /* From long to floats */ status |= H5T__register_int(H5T_PERS_HARD, "long_flt", native_long, native_float, H5T__conv_long_float); status |= H5T__register_int(H5T_PERS_HARD, "long_dbl", native_long, native_double, H5T__conv_long_double); status |= H5T__register_int(H5T_PERS_HARD, "long_ldbl", native_long, native_ldouble, H5T__conv_long_ldouble); +#ifdef H5_HAVE__FLOAT16 + status |= + H5T__register_int(H5T_PERS_HARD, "long_flt16", native_long, native_float16, H5T__conv_long__Float16); +#endif /* From unsigned long to floats */ status |= @@ -1269,6 +1406,10 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "ulong_dbl", native_ulong, native_double, H5T__conv_ulong_double); status |= H5T__register_int(H5T_PERS_HARD, "ulong_ldbl", native_ulong, native_ldouble, H5T__conv_ulong_ldouble); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "ulong_flt16", native_ulong, native_float16, + H5T__conv_ulong__Float16); +#endif /* From long long to floats */ status |= @@ -1279,6 +1420,10 @@ H5T_init(void) status |= H5T__register_int(H5T_PERS_HARD, "llong_ldbl", native_llong, native_ldouble, H5T__conv_llong_ldouble); #endif /* H5T_CONV_INTERNAL_LLONG_LDOUBLE */ +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "llong_flt16", native_llong, native_float16, + H5T__conv_llong__Float16); +#endif /* From unsigned long long to floats */ status |= @@ -1289,6 +1434,10 @@ H5T_init(void) status |= H5T__register_int(H5T_PERS_HARD, "ullong_ldbl", native_ullong, native_ldouble, H5T__conv_ullong_ldouble); #endif /* H5T_CONV_INTERNAL_ULLONG_LDOUBLE */ +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "ullong_flt16", native_ullong, native_float16, + H5T__conv_ullong__Float16); +#endif /* From floats to char */ status |= @@ -1297,6 +1446,10 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "dbl_schar", native_double, native_schar, H5T__conv_double_schar); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_schar", native_ldouble, native_schar, H5T__conv_ldouble_schar); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "flt16_schar", native_float16, native_schar, + H5T__conv__Float16_schar); +#endif /* From floats to unsigned char */ status |= @@ -1305,6 +1458,10 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "dbl_uchar", native_double, native_uchar, H5T__conv_double_uchar); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_uchar", native_ldouble, native_uchar, H5T__conv_ldouble_uchar); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "flt16_uchar", native_float16, native_uchar, + H5T__conv__Float16_uchar); +#endif /* From floats to short */ status |= @@ -1313,6 +1470,10 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "dbl_short", native_double, native_short, H5T__conv_double_short); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_short", native_ldouble, native_short, H5T__conv_ldouble_short); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "flt16_short", native_float16, native_short, + H5T__conv__Float16_short); +#endif /* From floats to unsigned short */ status |= @@ -1321,23 +1482,39 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "dbl_ushort", native_double, native_ushort, H5T__conv_double_ushort); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_ushort", native_ldouble, native_ushort, H5T__conv_ldouble_ushort); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "flt16_ushort", native_float16, native_ushort, + H5T__conv__Float16_ushort); +#endif /* From floats to int */ status |= H5T__register_int(H5T_PERS_HARD, "flt_int", native_float, native_int, H5T__conv_float_int); status |= H5T__register_int(H5T_PERS_HARD, "dbl_int", native_double, native_int, H5T__conv_double_int); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_int", native_ldouble, native_int, H5T__conv_ldouble_int); +#ifdef H5_HAVE__FLOAT16 + status |= + H5T__register_int(H5T_PERS_HARD, "flt16_int", native_float16, native_int, H5T__conv__Float16_int); +#endif /* From floats to unsigned int */ status |= H5T__register_int(H5T_PERS_HARD, "flt_uint", native_float, native_uint, H5T__conv_float_uint); status |= H5T__register_int(H5T_PERS_HARD, "dbl_uint", native_double, native_uint, H5T__conv_double_uint); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_uint", native_ldouble, native_uint, H5T__conv_ldouble_uint); +#ifdef H5_HAVE__FLOAT16 + status |= + H5T__register_int(H5T_PERS_HARD, "flt16_uint", native_float16, native_uint, H5T__conv__Float16_uint); +#endif /* From floats to long */ status |= H5T__register_int(H5T_PERS_HARD, "flt_long", native_float, native_long, H5T__conv_float_long); status |= H5T__register_int(H5T_PERS_HARD, "dbl_long", native_double, native_long, H5T__conv_double_long); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_long", native_ldouble, native_long, H5T__conv_ldouble_long); +#ifdef H5_HAVE__FLOAT16 + status |= + H5T__register_int(H5T_PERS_HARD, "flt16_long", native_float16, native_long, H5T__conv__Float16_long); +#endif /* From floats to unsigned long */ status |= @@ -1346,6 +1523,10 @@ H5T_init(void) H5T__register_int(H5T_PERS_HARD, "dbl_ulong", native_double, native_ulong, H5T__conv_double_ulong); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_ulong", native_ldouble, native_ulong, H5T__conv_ldouble_ulong); +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "flt16_ulong", native_float16, native_ulong, + H5T__conv__Float16_ulong); +#endif /* From floats to long long */ status |= @@ -1356,6 +1537,10 @@ H5T_init(void) status |= H5T__register_int(H5T_PERS_HARD, "ldbl_llong", native_ldouble, native_llong, H5T__conv_ldouble_llong); #endif /* H5T_CONV_INTERNAL_LDOUBLE_LLONG */ +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "flt16_llong", native_float16, native_llong, + H5T__conv__Float16_llong); +#endif /* From floats to unsigned long long */ status |= @@ -1366,6 +1551,10 @@ H5T_init(void) status |= H5T__register_int(H5T_PERS_HARD, "ldbl_ullong", native_ldouble, native_ullong, H5T__conv_ldouble_ullong); #endif /* H5T_CONV_INTERNAL_LDOUBLE_ULLONG */ +#ifdef H5_HAVE__FLOAT16 + status |= H5T__register_int(H5T_PERS_HARD, "flt16_ullong", native_float16, native_ullong, + H5T__conv__Float16_ullong); +#endif /* * The special no-op conversion is the fastest, so we list it last. The @@ -1546,99 +1735,102 @@ H5T_top_term_package(void) /* Reset all the datatype IDs */ if (H5T_IEEE_F32BE_g > 0) { - H5T_IEEE_F32BE_g = FAIL; - H5T_IEEE_F32LE_g = FAIL; - H5T_IEEE_F64BE_g = FAIL; - H5T_IEEE_F64LE_g = FAIL; - - H5T_STD_I8BE_g = FAIL; - H5T_STD_I8LE_g = FAIL; - H5T_STD_I16BE_g = FAIL; - H5T_STD_I16LE_g = FAIL; - H5T_STD_I32BE_g = FAIL; - H5T_STD_I32LE_g = FAIL; - H5T_STD_I64BE_g = FAIL; - H5T_STD_I64LE_g = FAIL; - H5T_STD_U8BE_g = FAIL; - H5T_STD_U8LE_g = FAIL; - H5T_STD_U16BE_g = FAIL; - H5T_STD_U16LE_g = FAIL; - H5T_STD_U32BE_g = FAIL; - H5T_STD_U32LE_g = FAIL; - H5T_STD_U64BE_g = FAIL; - H5T_STD_U64LE_g = FAIL; - H5T_STD_B8BE_g = FAIL; - H5T_STD_B8LE_g = FAIL; - H5T_STD_B16BE_g = FAIL; - H5T_STD_B16LE_g = FAIL; - H5T_STD_B32BE_g = FAIL; - H5T_STD_B32LE_g = FAIL; - H5T_STD_B64BE_g = FAIL; - H5T_STD_B64LE_g = FAIL; - H5T_STD_REF_OBJ_g = FAIL; - H5T_STD_REF_DSETREG_g = FAIL; - H5T_STD_REF_g = FAIL; - - H5T_UNIX_D32BE_g = FAIL; - H5T_UNIX_D32LE_g = FAIL; - H5T_UNIX_D64BE_g = FAIL; - H5T_UNIX_D64LE_g = FAIL; - - H5T_C_S1_g = FAIL; - - H5T_FORTRAN_S1_g = FAIL; - - H5T_NATIVE_SCHAR_g = FAIL; - H5T_NATIVE_UCHAR_g = FAIL; - H5T_NATIVE_SHORT_g = FAIL; - H5T_NATIVE_USHORT_g = FAIL; - H5T_NATIVE_INT_g = FAIL; - H5T_NATIVE_UINT_g = FAIL; - H5T_NATIVE_LONG_g = FAIL; - H5T_NATIVE_ULONG_g = FAIL; - H5T_NATIVE_LLONG_g = FAIL; - H5T_NATIVE_ULLONG_g = FAIL; - H5T_NATIVE_FLOAT_g = FAIL; - H5T_NATIVE_DOUBLE_g = FAIL; - H5T_NATIVE_LDOUBLE_g = FAIL; - H5T_NATIVE_B8_g = FAIL; - H5T_NATIVE_B16_g = FAIL; - H5T_NATIVE_B32_g = FAIL; - H5T_NATIVE_B64_g = FAIL; - H5T_NATIVE_OPAQUE_g = FAIL; - H5T_NATIVE_HADDR_g = FAIL; - H5T_NATIVE_HSIZE_g = FAIL; - H5T_NATIVE_HSSIZE_g = FAIL; - H5T_NATIVE_HERR_g = FAIL; - H5T_NATIVE_HBOOL_g = FAIL; - - H5T_NATIVE_INT8_g = FAIL; - H5T_NATIVE_UINT8_g = FAIL; - H5T_NATIVE_INT_LEAST8_g = FAIL; - H5T_NATIVE_UINT_LEAST8_g = FAIL; - H5T_NATIVE_INT_FAST8_g = FAIL; - H5T_NATIVE_UINT_FAST8_g = FAIL; - - H5T_NATIVE_INT16_g = FAIL; - H5T_NATIVE_UINT16_g = FAIL; - H5T_NATIVE_INT_LEAST16_g = FAIL; - H5T_NATIVE_UINT_LEAST16_g = FAIL; - H5T_NATIVE_INT_FAST16_g = FAIL; - H5T_NATIVE_UINT_FAST16_g = FAIL; - - H5T_NATIVE_INT32_g = FAIL; - H5T_NATIVE_UINT32_g = FAIL; - H5T_NATIVE_INT_LEAST32_g = FAIL; - H5T_NATIVE_UINT_LEAST32_g = FAIL; - H5T_NATIVE_INT_FAST32_g = FAIL; - H5T_NATIVE_UINT_FAST32_g = FAIL; - - H5T_NATIVE_INT64_g = FAIL; - H5T_NATIVE_UINT64_g = FAIL; - H5T_NATIVE_INT_LEAST64_g = FAIL; - H5T_NATIVE_UINT_LEAST64_g = FAIL; - H5T_NATIVE_INT_FAST64_g = FAIL; - H5T_NATIVE_UINT_FAST64_g = FAIL; + H5T_IEEE_F16BE_g = H5I_INVALID_HID; + H5T_IEEE_F16LE_g = H5I_INVALID_HID; + H5T_IEEE_F32BE_g = H5I_INVALID_HID; + H5T_IEEE_F32LE_g = H5I_INVALID_HID; + H5T_IEEE_F64BE_g = H5I_INVALID_HID; + H5T_IEEE_F64LE_g = H5I_INVALID_HID; + + H5T_STD_I8BE_g = H5I_INVALID_HID; + H5T_STD_I8LE_g = H5I_INVALID_HID; + H5T_STD_I16BE_g = H5I_INVALID_HID; + H5T_STD_I16LE_g = H5I_INVALID_HID; + H5T_STD_I32BE_g = H5I_INVALID_HID; + H5T_STD_I32LE_g = H5I_INVALID_HID; + H5T_STD_I64BE_g = H5I_INVALID_HID; + H5T_STD_I64LE_g = H5I_INVALID_HID; + H5T_STD_U8BE_g = H5I_INVALID_HID; + H5T_STD_U8LE_g = H5I_INVALID_HID; + H5T_STD_U16BE_g = H5I_INVALID_HID; + H5T_STD_U16LE_g = H5I_INVALID_HID; + H5T_STD_U32BE_g = H5I_INVALID_HID; + H5T_STD_U32LE_g = H5I_INVALID_HID; + H5T_STD_U64BE_g = H5I_INVALID_HID; + H5T_STD_U64LE_g = H5I_INVALID_HID; + H5T_STD_B8BE_g = H5I_INVALID_HID; + H5T_STD_B8LE_g = H5I_INVALID_HID; + H5T_STD_B16BE_g = H5I_INVALID_HID; + H5T_STD_B16LE_g = H5I_INVALID_HID; + H5T_STD_B32BE_g = H5I_INVALID_HID; + H5T_STD_B32LE_g = H5I_INVALID_HID; + H5T_STD_B64BE_g = H5I_INVALID_HID; + H5T_STD_B64LE_g = H5I_INVALID_HID; + H5T_STD_REF_OBJ_g = H5I_INVALID_HID; + H5T_STD_REF_DSETREG_g = H5I_INVALID_HID; + H5T_STD_REF_g = H5I_INVALID_HID; + + H5T_UNIX_D32BE_g = H5I_INVALID_HID; + H5T_UNIX_D32LE_g = H5I_INVALID_HID; + H5T_UNIX_D64BE_g = H5I_INVALID_HID; + H5T_UNIX_D64LE_g = H5I_INVALID_HID; + + H5T_C_S1_g = H5I_INVALID_HID; + + H5T_FORTRAN_S1_g = H5I_INVALID_HID; + + H5T_NATIVE_SCHAR_g = H5I_INVALID_HID; + H5T_NATIVE_UCHAR_g = H5I_INVALID_HID; + H5T_NATIVE_SHORT_g = H5I_INVALID_HID; + H5T_NATIVE_USHORT_g = H5I_INVALID_HID; + H5T_NATIVE_INT_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_g = H5I_INVALID_HID; + H5T_NATIVE_LONG_g = H5I_INVALID_HID; + H5T_NATIVE_ULONG_g = H5I_INVALID_HID; + H5T_NATIVE_LLONG_g = H5I_INVALID_HID; + H5T_NATIVE_ULLONG_g = H5I_INVALID_HID; + H5T_NATIVE_FLOAT16_g = H5I_INVALID_HID; + H5T_NATIVE_FLOAT_g = H5I_INVALID_HID; + H5T_NATIVE_DOUBLE_g = H5I_INVALID_HID; + H5T_NATIVE_LDOUBLE_g = H5I_INVALID_HID; + H5T_NATIVE_B8_g = H5I_INVALID_HID; + H5T_NATIVE_B16_g = H5I_INVALID_HID; + H5T_NATIVE_B32_g = H5I_INVALID_HID; + H5T_NATIVE_B64_g = H5I_INVALID_HID; + H5T_NATIVE_OPAQUE_g = H5I_INVALID_HID; + H5T_NATIVE_HADDR_g = H5I_INVALID_HID; + H5T_NATIVE_HSIZE_g = H5I_INVALID_HID; + H5T_NATIVE_HSSIZE_g = H5I_INVALID_HID; + H5T_NATIVE_HERR_g = H5I_INVALID_HID; + H5T_NATIVE_HBOOL_g = H5I_INVALID_HID; + + H5T_NATIVE_INT8_g = H5I_INVALID_HID; + H5T_NATIVE_UINT8_g = H5I_INVALID_HID; + H5T_NATIVE_INT_LEAST8_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_LEAST8_g = H5I_INVALID_HID; + H5T_NATIVE_INT_FAST8_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_FAST8_g = H5I_INVALID_HID; + + H5T_NATIVE_INT16_g = H5I_INVALID_HID; + H5T_NATIVE_UINT16_g = H5I_INVALID_HID; + H5T_NATIVE_INT_LEAST16_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_LEAST16_g = H5I_INVALID_HID; + H5T_NATIVE_INT_FAST16_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_FAST16_g = H5I_INVALID_HID; + + H5T_NATIVE_INT32_g = H5I_INVALID_HID; + H5T_NATIVE_UINT32_g = H5I_INVALID_HID; + H5T_NATIVE_INT_LEAST32_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_LEAST32_g = H5I_INVALID_HID; + H5T_NATIVE_INT_FAST32_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_FAST32_g = H5I_INVALID_HID; + + H5T_NATIVE_INT64_g = H5I_INVALID_HID; + H5T_NATIVE_UINT64_g = H5I_INVALID_HID; + H5T_NATIVE_INT_LEAST64_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_LEAST64_g = H5I_INVALID_HID; + H5T_NATIVE_INT_FAST64_g = H5I_INVALID_HID; + H5T_NATIVE_UINT_FAST64_g = H5I_INVALID_HID; n++; } /* end if */ diff --git a/src/H5Tconv.c b/src/H5Tconv.c index a37800bfed3..0e35fb5461b 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -112,6 +112,14 @@ * at least as wide as the destination. Overflow can occur * when the source magnitude is too large for the destination. * + * fX: Floating-point values to integers where the destination is at least + * as wide as the source. This case cannot generate overflows. + * + * Xf: Integers to floating-point values where the source is at least as + * wide as the destination. Overflows can occur when the destination is + * narrower than the source. + * + * * The macros take a subset of these arguments in the order listed here: * * CDATA: A pointer to the H5T_cdata_t structure that was passed to the @@ -704,6 +712,99 @@ H5T_CONV(H5T_CONV_Fx, STYPE, DTYPE, ST, DT, D_MIN, D_MAX, Y) \ } while (0) +#define H5T_CONV_fX(STYPE, DTYPE, ST, DT, D_MIN, D_MAX) \ + do { \ + HDcompile_assert(sizeof(ST) <= sizeof(DT)); \ + H5T_CONV(H5T_CONV_xX, STYPE, DTYPE, ST, DT, D_MIN, D_MAX, N) \ + } while (0) + +#define H5T_CONV_Xf_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX) \ + { \ + if (*(S) > (ST)(D_MAX) || (sprec < dprec && *(S) == (ST)(D_MAX))) { \ + H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)( \ + H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \ + conv_ctx->u.conv.cb_struct.user_data); \ + if (except_ret == H5T_CONV_UNHANDLED) \ + /* Let compiler convert if case is ignored by user handler*/ \ + *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _POS_INF_g); \ + else if (except_ret == H5T_CONV_ABORT) \ + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception"); \ + /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */ \ + } \ + else if (*(S) < (ST)(D_MIN)) { \ + H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)( \ + H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \ + conv_ctx->u.conv.cb_struct.user_data); \ + if (except_ret == H5T_CONV_UNHANDLED) \ + /* Let compiler convert if case is ignored by user handler*/ \ + *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _NEG_INF_g); \ + else if (except_ret == H5T_CONV_ABORT) \ + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception"); \ + /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */ \ + } \ + else if (sprec > dprec) { \ + unsigned low_bit_pos, high_bit_pos; \ + \ + /* Detect high & low bits set in source */ \ + H5T_HI_LO_BIT_SET(ST, *(S), low_bit_pos, high_bit_pos) \ + \ + /* Check for more bits of precision in src than available in dst */ \ + if ((high_bit_pos - low_bit_pos) >= dprec) { \ + H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)( \ + H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, \ + S, D, conv_ctx->u.conv.cb_struct.user_data); \ + if (except_ret == H5T_CONV_UNHANDLED) \ + /* Let compiler convert if case is ignored by user handler*/ \ + *(D) = (DT)(*(S)); \ + else if (except_ret == H5T_CONV_ABORT) \ + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception"); \ + /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */ \ + } \ + else \ + *(D) = (DT)(*(S)); \ + } \ + else \ + *(D) = (DT)(*(S)); \ + } +#define H5T_CONV_Xf_NOEX_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX) \ + { \ + if (*(S) > (ST)(D_MAX)) \ + *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _POS_INF_g); \ + else { \ + intmax_t s_cast = (intmax_t)(*(S)); \ + intmax_t d_cast = (intmax_t)(D_MAX); \ + \ + /* Check if source value would underflow destination. Do NOT do this \ + * by comparing against D_MIN casted to type ST here, as this will \ + * generally be undefined behavior (casting negative float value <= 1.0 \ + * to integer) for all floating point types and some compilers optimize \ + * this in a way that causes unexpected behavior. Instead, grab the \ + * absolute value of the source value first, then compare it to D_MAX. \ + */ \ + if (s_cast != INTMAX_MIN) \ + s_cast = imaxabs(s_cast); \ + else { \ + /* Handle two's complement integer representations where abs(INTMAX_MIN) \ + * can't be represented. Other representations will fall here as well, \ + * but this should be fine. \ + */ \ + s_cast = INTMAX_MAX; \ + d_cast -= 1; \ + } \ + \ + if (s_cast > d_cast) \ + *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _NEG_INF_g); \ + else \ + *(D) = (DT)(*(S)); \ + } \ + } + +#define H5T_CONV_Xf(STYPE, DTYPE, ST, DT, D_MIN, D_MAX) \ + do { \ + HDcompile_assert(sizeof(ST) >= sizeof(DT)); \ + H5T_CONV(H5T_CONV_Xf, STYPE, DTYPE, ST, DT, D_MIN, D_MAX, Y) \ + } while (0) + /* Since all "no exception" cores do the same thing (assign the value in the * source location to the destination location, using casting), use one "core" * to do them all. @@ -8215,6 +8316,255 @@ H5T__conv_ldouble_ullong(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_con } #endif /*H5T_CONV_INTERNAL_LDOUBLE_ULLONG*/ +/* Conversions for _Float16 type */ +#ifdef H5_HAVE__FLOAT16 +herr_t +H5T__conv_schar__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_xF(SCHAR, FLOAT16, signed char, H5__Float16, -, -); +} + +herr_t +H5T__conv_uchar__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_xF(UCHAR, FLOAT16, unsigned char, H5__Float16, -, -); +} + +herr_t +H5T__conv_short__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_xF(SHORT, FLOAT16, short, H5__Float16, -, -); +} + +herr_t +H5T__conv_ushort__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Xf(USHORT, FLOAT16, unsigned short, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +herr_t +H5T__conv_int__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Xf(INT, FLOAT16, int, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +herr_t +H5T__conv_uint__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Xf(UINT, FLOAT16, unsigned int, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +herr_t +H5T__conv_long__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Xf(LONG, FLOAT16, long, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +herr_t +H5T__conv_ulong__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Xf(ULONG, FLOAT16, unsigned long, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +herr_t +H5T__conv_llong__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Xf(LLONG, FLOAT16, long long, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +herr_t +H5T__conv_ullong__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Xf(ULLONG, FLOAT16, unsigned long long, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +herr_t +H5T__conv_float__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Ff(FLOAT, FLOAT16, float, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +herr_t +H5T__conv_double__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Ff(DOUBLE, FLOAT16, double, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} + +#ifdef H5T_CONV_INTERNAL_LDOUBLE_FLOAT16 +herr_t +H5T__conv_ldouble__Float16(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + /* Suppress warning about non-standard floating-point literal suffix */ + H5_GCC_CLANG_DIAG_OFF("pedantic") + H5T_CONV_Ff(LDOUBLE, FLOAT16, long double, H5__Float16, -FLT16_MAX, FLT16_MAX); + H5_GCC_CLANG_DIAG_ON("pedantic") +} +#endif + +herr_t +H5T__conv__Float16_schar(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5_GCC_CLANG_DIAG_OFF("float-equal") + H5T_CONV_Fx(FLOAT16, SCHAR, H5__Float16, signed char, SCHAR_MIN, SCHAR_MAX); + H5_GCC_CLANG_DIAG_ON("float-equal") +} + +herr_t +H5T__conv__Float16_uchar(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5_GCC_CLANG_DIAG_OFF("float-equal") + H5T_CONV_Fx(FLOAT16, UCHAR, H5__Float16, unsigned char, 0, UCHAR_MAX); + H5_GCC_CLANG_DIAG_ON("float-equal") +} + +herr_t +H5T__conv__Float16_short(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5_GCC_CLANG_DIAG_OFF("float-equal") + H5T_CONV_Fx(FLOAT16, SHORT, H5__Float16, short, SHRT_MIN, SHRT_MAX); + H5_GCC_CLANG_DIAG_ON("float-equal") +} + +herr_t +H5T__conv__Float16_ushort(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fX(FLOAT16, USHORT, H5__Float16, unsigned short, 0, USHRT_MAX); +} + +herr_t +H5T__conv__Float16_int(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fX(FLOAT16, INT, H5__Float16, int, INT_MIN, INT_MAX); +} + +herr_t +H5T__conv__Float16_uint(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fX(FLOAT16, UINT, H5__Float16, unsigned int, 0, UINT_MAX); +} + +herr_t +H5T__conv__Float16_long(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fX(FLOAT16, LONG, H5__Float16, long, LONG_MIN, LONG_MAX); +} + +herr_t +H5T__conv__Float16_ulong(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fX(FLOAT16, ULONG, H5__Float16, unsigned long, 0, ULONG_MAX); +} + +herr_t +H5T__conv__Float16_llong(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fX(FLOAT16, LLONG, H5__Float16, long long, LLONG_MIN, LLONG_MAX); +} + +herr_t +H5T__conv__Float16_ullong(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fX(FLOAT16, ULLONG, H5__Float16, unsigned long long, 0, ULLONG_MAX); +} + +herr_t +H5T__conv__Float16_float(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fF(FLOAT16, FLOAT, H5__Float16, float, -, -); +} + +herr_t +H5T__conv__Float16_double(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fF(FLOAT16, DOUBLE, H5__Float16, double, -, -); +} + +herr_t +H5T__conv__Float16_ldouble(H5T_t *st, H5T_t *dt, H5T_cdata_t *cdata, const H5T_conv_ctx_t *conv_ctx, + size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, + void H5_ATTR_UNUSED *bkg) +{ + H5T_CONV_fF(FLOAT16, LDOUBLE, H5__Float16, long double, -, -); +} +#endif + /*------------------------------------------------------------------------- * Function: H5T__conv_f_i * @@ -8249,8 +8599,9 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx uint8_t *int_buf = NULL; /*buffer for temporary value */ size_t buf_size; /*buffer size for temporary value */ size_t i; /*miscellaneous counters */ - size_t first; /*first bit(MSB) in an integer */ - ssize_t sfirst; /*a signed version of `first' */ + ssize_t msb_pos_s; /*first bit(MSB) in an integer */ + ssize_t new_msb_pos; /*MSB position after shifting mantissa by exponent */ + hssize_t shift_val; /*shift value when shifting mantissa by exponent */ bool truncated; /*if fraction value is dropped */ bool reverse; /*if reverse order of destination at the end */ H5T_conv_ret_t except_ret; /*return of callback function */ @@ -8314,8 +8665,11 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx /* Allocate enough space for the buffer holding temporary * converted value */ - buf_size = (size_t)(pow(2.0, (double)src.u.f.esize) / 8 + 1); - int_buf = (uint8_t *)H5MM_calloc(buf_size); + if (dst.prec / 8 > src_p->shared->size) + buf_size = (dst.prec + 7) / 8; + else + buf_size = src_p->shared->size; + int_buf = (uint8_t *)H5MM_calloc(buf_size); /* Allocate space for order-reversed source buffer */ src_rev = (uint8_t *)H5MM_calloc(src_p->shared->size); @@ -8580,36 +8934,47 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx if (H5T_NORM_IMPLIED == src.u.f.norm) H5T__bit_inc(int_buf, src.u.f.msize, 8 * buf_size - src.u.f.msize); + /* + * What is the bit position for the most significant bit(MSB) of S + * which is set? This is checked before shifting and before possibly + * converting to a negative integer. Note that later use of this value + * assumes that H5T__bit_shift will always shift in 0 during a right + * shift. + */ + msb_pos_s = H5T__bit_find(int_buf, (size_t)0, src.prec, H5T_BIT_MSB, true); + + /* + * The temporary buffer has no bits set and must therefore be + * zero; nothing to do. + */ + if (msb_pos_s < 0) + goto padding; + /* * Shift mantissa part by exponent minus mantissa size(right shift), * or by mantissa size minus exponent(left shift). Example: Sequence * 10...010111, expo=20, expo-msize=-3. Right-shift the sequence, we get * 00010...10. The last three bits were dropped. */ - H5T__bit_shift(int_buf, expo - (ssize_t)src.u.f.msize, (size_t)0, buf_size * 8); + shift_val = expo - (ssize_t)src.u.f.msize; + H5T__bit_shift(int_buf, shift_val, (size_t)0, buf_size * 8); + + /* Calculate the new position of the MSB after shifting and + * skip to the padding section if we shifted exactly to 0 + * (MSB position is -1) + */ + new_msb_pos = msb_pos_s + shift_val; + if (new_msb_pos == -1) + goto padding; /* - * If expo is less than mantissa size, the frantional value is dropped off + * If expo is less than mantissa size, the fractional value is dropped off * during conversion. Set exception type to be "truncate" */ if ((size_t)expo < src.u.f.msize && conv_ctx->u.conv.cb_struct.func) truncated = true; - /* - * What is the bit position for the most significant bit(MSB) of S - * which is set? This is checked before converted to negative - * integer. - */ - sfirst = H5T__bit_find(int_buf, (size_t)0, 8 * buf_size, H5T_BIT_MSB, true); - first = (size_t)sfirst; - - if (sfirst < 0) { - /* - * The source has no bits set and must therefore be zero. - * Set the destination to zero - nothing to do. - */ - } - else if (H5T_SGN_NONE == dst.u.i.sign) { /*destination is unsigned*/ + if (H5T_SGN_NONE == dst.u.i.sign) { /*destination is unsigned*/ /* * Destination is unsigned. Library's default way: If the source value * is greater than the maximal destination value then it overflows, the @@ -8639,7 +9004,7 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx } } else { /*source is positive*/ - if (first >= dst.prec) { + if (new_msb_pos >= (ssize_t)dst.prec) { /*overflow*/ if (conv_ctx->u.conv.cb_struct .func) { /*If user's exception handler is present, use it*/ @@ -8663,7 +9028,7 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception"); } - else if (first < dst.prec) { + else { if (truncated && conv_ctx->u.conv.cb_struct .func) { /*If user's exception handler is present, use it*/ /*reverse order first*/ @@ -8675,9 +9040,11 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx conv_ctx->u.conv.cb_struct.user_data); } - if (except_ret == H5T_CONV_UNHANDLED) + if (except_ret == H5T_CONV_UNHANDLED) { /*copy source value into it if case is ignored by user handler*/ - H5T__bit_copy(d, dst.offset, int_buf, (size_t)0, first + 1); + if (new_msb_pos >= 0) + H5T__bit_copy(d, dst.offset, int_buf, (size_t)0, (size_t)new_msb_pos + 1); + } else if (except_ret == H5T_CONV_HANDLED) { /*No need to reverse the order of destination because user handles it*/ reverse = false; @@ -8691,7 +9058,7 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx } else if (H5T_SGN_2 == dst.u.i.sign) { /*Destination is signed*/ if (sign) { /*source is negative*/ - if (first < dst.prec - 1) { + if ((new_msb_pos >= 0) && ((size_t)new_msb_pos < dst.prec - 1)) { if (truncated && conv_ctx->u.conv.cb_struct .func) { /*If user's exception handler is present, use it*/ /*reverse order first*/ @@ -8705,8 +9072,8 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx if (except_ret == H5T_CONV_UNHANDLED) { /*If this case ignored by user handler*/ /*Convert to integer representation. Equivalent to ~(value - 1).*/ - H5T__bit_dec(int_buf, (size_t)0, 8 * buf_size); - H5T__bit_neg(int_buf, (size_t)0, 8 * buf_size); + H5T__bit_dec(int_buf, (size_t)0, dst.prec); + H5T__bit_neg(int_buf, (size_t)0, dst.prec); /*copy source value into destination*/ H5T__bit_copy(d, dst.offset, int_buf, (size_t)0, dst.prec - 1); @@ -8749,7 +9116,7 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx } } else { /*source is positive*/ - if (first >= dst.prec - 1) { + if (new_msb_pos >= (ssize_t)dst.prec - 1) { /*overflow*/ if (conv_ctx->u.conv.cb_struct .func) { /*If user's exception handler is present, use it*/ @@ -8773,7 +9140,7 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx goto next; } } - else if (first < dst.prec - 1) { + else if (new_msb_pos < (ssize_t)dst.prec - 1) { if (truncated && conv_ctx->u.conv.cb_struct .func) { /*If user's exception handler is present, use it*/ /*reverse order first*/ @@ -8787,7 +9154,8 @@ H5T__conv_f_i(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx if (except_ret == H5T_CONV_UNHANDLED) { /*copy source value into it if case is ignored by user handler*/ - H5T__bit_copy(d, dst.offset, int_buf, (size_t)0, first + 1); + if (new_msb_pos >= 0) + H5T__bit_copy(d, dst.offset, int_buf, (size_t)0, (size_t)new_msb_pos + 1); } else if (except_ret == H5T_CONV_ABORT) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, @@ -8963,7 +9331,7 @@ H5T__conv_i_f(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx /* Allocate enough space for the buffer holding temporary * converted value */ - buf_size = (src.prec > dst.u.f.msize ? src.prec : dst.u.f.msize) / 8 + 1; + buf_size = ((src.prec > dst.u.f.msize ? src.prec : dst.u.f.msize) + 7) / 8; int_buf = (uint8_t *)H5MM_calloc(buf_size); /* Allocate space for order-reversed source buffer */ @@ -9189,7 +9557,8 @@ H5T__conv_i_f(H5T_t *src_p, H5T_t *dst_p, H5T_cdata_t *cdata, const H5T_conv_ctx goto padding; } } - else { + + if (!conv_ctx->u.conv.cb_struct.func || (except_ret == H5T_CONV_UNHANDLED)) { /*make destination infinity by setting exponent to maximal number and *mantissa to zero.*/ expo = expo_max; diff --git a/src/H5Tinit_float.c b/src/H5Tinit_float.c index 8384c3161df..de959b49735 100644 --- a/src/H5Tinit_float.c +++ b/src/H5Tinit_float.c @@ -54,8 +54,8 @@ * Purpose: This macro takes a floating point type like `double' and * and detects byte order, mantissa location, exponent location, * sign bit location, presence or absence of implicit mantissa - * bit, and exponent bias and initializes a detected_t structure - * with those properties. + * bit, and exponent bias and initializes a H5T_fpoint_det_t + * structure with those properties. * * Note that these operations can raise floating-point * exceptions and building with some compiler options @@ -307,14 +307,17 @@ H5T__fix_order(int n, int last, int *perm, H5T_order_t *order) if (last <= 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "failed to detect byte order"); - /* We have at least three points to consider */ - if (perm[last] < perm[last - 1] && perm[last - 1] < perm[last - 2]) { + if (perm[last] < perm[last - 1] && + /* Only check perm[last - 2] if we have more than 2 points to consider */ + ((last < 2) || (perm[last - 1] < perm[last - 2]))) { /* Little endian */ *order = H5T_ORDER_LE; for (int i = 0; i < n; i++) perm[i] = i; } - else if (perm[last] > perm[last - 1] && perm[last - 1] > perm[last - 2]) { + else if (perm[last] > perm[last - 1] && + /* Only check perm[last - 2] if we have more than 2 points to consider */ + ((last < 2) || (perm[last - 1] > perm[last - 2]))) { /* Big endian */ *order = H5T_ORDER_BE; for (int i = 0; i < n; i++) @@ -359,7 +362,7 @@ H5T__fix_order(int n, int last, int *perm, H5T_order_t *order) * Return: imp_bit will be set to 1 if the most significant bit * of the mantissa is discarded (ie, the mantissa has an * implicit `one' as the most significant bit). Otherwise, - * imp_bit will be set to zero zero. + * imp_bit will be set to zero. * * SUCCEED/FAIL *------------------------------------------------------------------------- @@ -571,6 +574,39 @@ H5T__init_native_float_types(void) */ H5T_native_order_g = det.order; +#ifdef H5_HAVE__FLOAT16 + /* H5T_NATIVE_FLOAT16 */ + + /* Get the type's characteristics */ + memset(&det, 0, sizeof(H5T_fpoint_det_t)); + DETECT_F(H5__Float16, det); + + /* Allocate and fill type structure */ + if (NULL == (dt = H5T__alloc())) + HGOTO_ERROR(H5E_DATATYPE, H5E_NOSPACE, FAIL, "datatype allocation failed"); + dt->shared->state = H5T_STATE_IMMUTABLE; + dt->shared->type = H5T_FLOAT; + dt->shared->size = det.size; + dt->shared->u.atomic.order = det.order; + dt->shared->u.atomic.offset = det.offset; + dt->shared->u.atomic.prec = det.prec; + dt->shared->u.atomic.lsb_pad = H5T_PAD_ZERO; + dt->shared->u.atomic.msb_pad = H5T_PAD_ZERO; + dt->shared->u.atomic.u.f.sign = det.sign; + dt->shared->u.atomic.u.f.epos = det.epos; + dt->shared->u.atomic.u.f.esize = det.esize; + dt->shared->u.atomic.u.f.ebias = det.ebias; + dt->shared->u.atomic.u.f.mpos = det.mpos; + dt->shared->u.atomic.u.f.msize = det.msize; + dt->shared->u.atomic.u.f.norm = det.norm; + dt->shared->u.atomic.u.f.pad = H5T_PAD_ZERO; + + /* Register the type and set global variables */ + if ((H5T_NATIVE_FLOAT16_g = H5I_register(H5I_DATATYPE, dt, false)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "can't register ID for built-in datatype"); + H5T_NATIVE_FLOAT16_ALIGN_g = det.comp_align; +#endif + done: /* Clear any FE_INVALID exceptions from NaN handling */ if (feclearexcept(FE_INVALID) != 0) diff --git a/src/H5Tmodule.h b/src/H5Tmodule.h index f1b7b175f2e..35e748bac3f 100644 --- a/src/H5Tmodule.h +++ b/src/H5Tmodule.h @@ -712,6 +712,14 @@ * * * + * #H5T_NATIVE_FLOAT16 + * + * + * _Float16 + * + * + * + * * #H5T_NATIVE_FLOAT * * @@ -3732,8 +3740,8 @@ filled according to the value of this property. The padding can be: * datatypes. * * The currently supported text format used by #H5LTtext_to_dtype and #H5LTdtype_to_text is the - * data description language (DDL) and conforms to the \ref DDLBNF110. The portion of the - * \ref DDLBNF110 that defines HDF5 datatypes appears below. + * data description language (DDL) and conforms to the \ref DDLBNF114. The portion of the + * \ref DDLBNF114 that defines HDF5 datatypes appears below. * The definition of HDF5 datatypes from the HDF5 DDL * \code * ::= | | | @@ -3753,10 +3761,11 @@ filled according to the value of this property. The padding can be: * H5T_NATIVE_INT | H5T_NATIVE_UINT | * H5T_NATIVE_LONG | H5T_NATIVE_ULONG | * H5T_NATIVE_LLONG | H5T_NATIVE_ULLONG - * ::= H5T_IEEE_F32BE | H5T_IEEE_F32LE | + * ::= H5T_IEEE_F16BE | H5T_IEEE_F16LE | + * H5T_IEEE_F32BE | H5T_IEEE_F32LE | * H5T_IEEE_F64BE | H5T_IEEE_F64LE | - * H5T_NATIVE_FLOAT | H5T_NATIVE_DOUBLE | - * H5T_NATIVE_LDOUBLE + * H5T_NATIVE_FLOAT16 | H5T_NATIVE_FLOAT | + * H5T_NATIVE_DOUBLE | H5T_NATIVE_LDOUBLE *