diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3278dcc88..9c3247496 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: Document workspace env: RUSTDOCFLAGS: "--cfg docsrs" - run: cargo doc --features static,zlib,blosc,lzf + run: cargo doc --features static,zlib,blosc,lzf,f16,complex brew: name: brew @@ -147,8 +147,8 @@ jobs: with: {toolchain: '${{matrix.rust}}'} - name: Build and test all crates run: cargo test --workspace -v --features hdf5-sys/static,hdf5-sys/zlib --exclude hdf5-derive - - name: Build and test with filters - run: cargo test --workspace -v --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc --exclude hdf5-derive + - name: Build and test with filters and other features + run: cargo test --workspace -v --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc,f16,complex --exclude hdf5-derive if: matrix.rust != 'stable-gnu' - name: Run examples run: | @@ -276,7 +276,7 @@ jobs: with: {toolchain: 1.64} - name: Build and test all crates run: - cargo test --workspace -vv --features=hdf5-sys/static --exclude=hdf5-derive + cargo test --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-derive wine: name: wine diff --git a/CHANGELOG.md b/CHANGELOG.md index 659450f05..e760225a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ - Add a `ByteReader` which implements `std::io::{Read, Seek}` for 1D `u8` datasets. Usage via `Dataset::as_byte_reader()`. - Add `chunk_visit` to visit all chunks in a dataset. -- Implement `H5Type` for `num_complex::Complex`. +- Add support for float16 values (`half::f16`), enabled via "f16" feature. +- Add support for complex numbers (`num_complex::Complex`), enabled via "complex". - Adding feature `static` for the `hdf5` crate which downloads and builds a bundled HDF5. ### Changed diff --git a/Cargo.toml b/Cargo.toml index e307a50a3..484d1325e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,9 +15,11 @@ edition = "2021" [workspace.dependencies] # external cfg-if = "1.0" +half = { version = "2.2", default-features = false } libc = "0.2" libz-sys = { version = "1.1", default-features = false } mpi-sys = "0.2" +num-complex = { version = "0.4", default-features = false } regex = "1.8" # internal hdf5-derive = { version = "0.8.1", path = "hdf5-derive" } # !V diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index f1331ce9e..6c040ec9e 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -175,8 +175,8 @@ pub struct Header { pub have_direct: bool, pub have_parallel: bool, pub have_threadsafe: bool, - pub have_zlib: bool, pub have_no_deprecated: bool, + pub have_filter_deflate: bool, pub version: Version, } @@ -203,7 +203,7 @@ impl Header { } else if name == "H5_HAVE_THREADSAFE" { hdr.have_threadsafe = value > 0; } else if name == "H5_HAVE_FILTER_DEFLATE" { - hdr.have_zlib = value > 0; + hdr.have_filter_deflate = value > 0; } else if name == "H5_NO_DEPRECATED_SYMBOLS" { hdr.have_no_deprecated = value > 0; } @@ -680,6 +680,10 @@ impl Config { println!("cargo:rustc-cfg=feature=\"have-threadsafe\""); println!("cargo:have_threadsafe=1"); } + if self.header.have_filter_deflate { + println!("cargo:rustc-cfg=feature=\"have-filter-deflate\""); + println!("cargo:have_filter_deflate=1"); + } } fn check_against_features_required(&self) { @@ -687,7 +691,7 @@ impl Config { for (flag, feature, native) in [ (!h.have_no_deprecated, "deprecated", "HDF5_ENABLE_DEPRECATED_SYMBOLS"), (h.have_threadsafe, "threadsafe", "HDF5_ENABLE_THREADSAFE"), - (h.have_zlib, "zlib", "HDF5_ENABLE_Z_LIB_SUPPORT"), + (h.have_filter_deflate, "zlib", "HDF5_ENABLE_Z_LIB_SUPPORT"), ] { if feature_enabled(&feature.to_ascii_uppercase()) { assert!( diff --git a/hdf5-types/Cargo.toml b/hdf5-types/Cargo.toml index cac06aa68..0ecf295e3 100644 --- a/hdf5-types/Cargo.toml +++ b/hdf5-types/Cargo.toml @@ -15,15 +15,20 @@ edition.workspace = true [features] h5-alloc = [] -complex = ["num-complex"] +complex = ["dep:num-complex"] +f16 = ["dep:half"] [dependencies] ascii = "1.1" cfg-if = { workspace = true } hdf5-sys = { workspace = true } libc = { workspace = true } -num-complex = { version = "0.4", optional = true, default-features = false } +num-complex = { workspace = true, optional = true } +half = { workspace = true, optional = true } [dev-dependencies] quickcheck = { version = "1.0", default-features = false } unindent = "0.2" + +[package.metadata.docs.rs] +features = ["f16", "complex"] diff --git a/hdf5-types/src/dyn_value.rs b/hdf5-types/src/dyn_value.rs index cc72cac20..1ef50ef33 100644 --- a/hdf5-types/src/dyn_value.rs +++ b/hdf5-types/src/dyn_value.rs @@ -115,10 +115,68 @@ impl From for DynValue<'_> { } #[derive(Copy, Clone, PartialEq)] -pub enum DynScalar { - Integer(DynInteger), +pub enum DynFloat { + #[cfg(feature = "f16")] + Float16(::half::f16), Float32(f32), Float64(f64), +} + +impl DynFloat { + pub(self) fn read(buf: &[u8], size: FloatSize) -> Self { + match size { + #[cfg(feature = "f16")] + FloatSize::U2 => Self::Float16(read_raw(buf)), + FloatSize::U4 => Self::Float32(read_raw(buf)), + FloatSize::U8 => Self::Float64(read_raw(buf)), + } + } +} + +unsafe impl DynClone for DynFloat { + fn dyn_clone(&mut self, out: &mut [u8]) { + match self { + #[cfg(feature = "f16")] + Self::Float16(x) => write_raw(out, *x), + Self::Float32(x) => write_raw(out, *x), + Self::Float64(x) => write_raw(out, *x), + } + } +} + +impl Debug for DynFloat { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + #[cfg(feature = "f16")] + Self::Float16(x) => Debug::fmt(&x, f), + Self::Float32(x) => Debug::fmt(&x, f), + Self::Float64(x) => Debug::fmt(&x, f), + } + } +} + +impl Display for DynFloat { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl From for DynScalar { + fn from(value: DynFloat) -> Self { + Self::Float(value) + } +} + +impl From for DynValue<'_> { + fn from(value: DynFloat) -> Self { + DynScalar::Float(value).into() + } +} + +#[derive(Copy, Clone, PartialEq)] +pub enum DynScalar { + Integer(DynInteger), + Float(DynFloat), Boolean(bool), } @@ -126,8 +184,7 @@ unsafe impl DynClone for DynScalar { fn dyn_clone(&mut self, out: &mut [u8]) { match self { Self::Integer(x) => x.dyn_clone(out), - Self::Float32(x) => write_raw(out, *x), - Self::Float64(x) => write_raw(out, *x), + Self::Float(x) => x.dyn_clone(out), Self::Boolean(x) => write_raw(out, *x), } } @@ -137,8 +194,7 @@ impl Debug for DynScalar { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Self::Integer(x) => Debug::fmt(&x, f), - Self::Float32(x) => Debug::fmt(&x, f), - Self::Float64(x) => Debug::fmt(&x, f), + Self::Float(x) => Debug::fmt(&x, f), Self::Boolean(x) => Debug::fmt(&x, f), } } @@ -637,8 +693,7 @@ impl<'a> DynValue<'a> { match tp { Integer(size) | Unsigned(size) => DynInteger::read(buf, true, *size).into(), - Float(FloatSize::U4) => DynScalar::Float32(read_raw(buf)).into(), - Float(FloatSize::U8) => DynScalar::Float64(read_raw(buf)).into(), + Float(size) => DynFloat::read(buf, *size).into(), Boolean => DynScalar::Boolean(read_raw(buf)).into(), Enum(ref tp) => DynEnum::new(tp, DynInteger::read(buf, tp.signed, tp.size)).into(), Compound(ref tp) => DynCompound::new(tp, buf).into(), diff --git a/hdf5-types/src/h5type.rs b/hdf5-types/src/h5type.rs index 7f50dc12a..f8dbf6dd6 100644 --- a/hdf5-types/src/h5type.rs +++ b/hdf5-types/src/h5type.rs @@ -40,18 +40,24 @@ impl IntSize { #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum FloatSize { + #[cfg(feature = "f16")] + U2 = 2, U4 = 4, U8 = 8, } impl FloatSize { pub const fn from_int(size: usize) -> Option { - if size == 4 { - Some(Self::U4) - } else if size == 8 { - Some(Self::U8) - } else { - None + #[cfg(feature = "f16")] + { + if size == 2 { + return Some(Self::U2); + } + } + match size { + 4 => Some(Self::U4), + 8 => Some(Self::U8), + _ => None, } } } @@ -167,6 +173,8 @@ impl Display for TypeDescriptor { TypeDescriptor::Unsigned(IntSize::U2) => write!(f, "uint16"), TypeDescriptor::Unsigned(IntSize::U4) => write!(f, "uint32"), TypeDescriptor::Unsigned(IntSize::U8) => write!(f, "uint64"), + #[cfg(feature = "f16")] + TypeDescriptor::Float(FloatSize::U2) => write!(f, "float16"), TypeDescriptor::Float(FloatSize::U4) => write!(f, "float32"), TypeDescriptor::Float(FloatSize::U8) => write!(f, "float64"), TypeDescriptor::Boolean => write!(f, "bool"), @@ -251,6 +259,8 @@ impl_h5type!(u8, Unsigned, IntSize::U1); impl_h5type!(u16, Unsigned, IntSize::U2); impl_h5type!(u32, Unsigned, IntSize::U4); impl_h5type!(u64, Unsigned, IntSize::U8); +#[cfg(feature = "f16")] +impl_h5type!(::half::f16, Float, FloatSize::U2); impl_h5type!(f32, Float, FloatSize::U4); impl_h5type!(f64, Float, FloatSize::U8); diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index 714ee0567..9300f0e3a 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -15,11 +15,21 @@ edition.workspace = true [features] default = [] -mpio = ["dep:mpi-sys", "hdf5-sys/mpio"] -lzf = ["dep:lzf-sys", "dep:errno"] -blosc = ["dep:blosc-sys"] +# Compile and statically link bundled HDF5 library. static = ["hdf5-sys/static"] +# Enable zlib compression filter. zlib = ["hdf5-sys/zlib"] +# Enable LZF compression filter. +lzf = ["dep:lzf-sys", "dep:errno"] +# Enable blosc compression filters. +blosc = ["dep:blosc-sys"] +# Enable MPI support. +mpio = ["dep:mpi-sys", "hdf5-sys/mpio"] +# Enable complex number type support. +complex = ["hdf5-types/complex"] +# Enable float16 type support. +f16 = ["hdf5-types/f16"] + # The features with version numbers such as 1.10.3, 1.12.0 are metafeatures # and is only available when the HDF5 library is at least this version. # Features have_direct and have_parallel are also metafeatures and dependent @@ -44,6 +54,8 @@ hdf5-sys = { workspace = true } hdf5-types = { workspace = true } [dev-dependencies] +half = { workspace = true } +num-complex = { workspace = true } paste = "1.0" pretty_assertions = "1.3" rand = { version = "0.8", features = ["small_rng"] } @@ -52,5 +64,5 @@ scopeguard = "1.1" tempfile = "3.6" [package.metadata.docs.rs] -features = ["hdf5-sys/static", "hdf5-sys/zlib", "blosc", "lzf"] +features = ["static", "zlib", "blosc", "lzf", "f16", "complex"] rustdoc-args = ["--cfg", "docsrs"] diff --git a/hdf5/build.rs b/hdf5/build.rs index caccb2813..7a74dbbf8 100644 --- a/hdf5/build.rs +++ b/hdf5/build.rs @@ -9,6 +9,7 @@ fn main() { "DEP_HDF5_HAVE_DIRECT" => print_feature("have-direct"), "DEP_HDF5_HAVE_PARALLEL" => print_feature("have-parallel"), "DEP_HDF5_HAVE_THREADSAFE" => print_feature("have-threadsafe"), + "DEP_HDF5_HAVE_FILTER_DEFLATE" => print_feature("have-filter-deflate"), // internal config flags "DEP_HDF5_MSVC_DLL_INDIRECTION" => print_cfg("msvc_dll_indirection"), // public version features diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs index d07021622..a84ac68ae 100644 --- a/hdf5/src/hl/chunks.rs +++ b/hdf5/src/hl/chunks.rs @@ -72,7 +72,7 @@ mod v1_14_0 { use super::*; use hdf5_sys::h5d::H5Dchunk_iter; - /// Borrowed version of [ChunkInfo](crate::dataset::ChunkInfo) + /// Borrowed version of [`ChunkInfo`](crate::dataset::ChunkInfo) #[derive(Clone, Debug, PartialEq, Eq)] pub struct ChunkInfoRef<'a> { pub offset: &'a [hsize_t], diff --git a/hdf5/src/hl/datatype.rs b/hdf5/src/hl/datatype.rs index 2e90febe5..9c9ebe58a 100644 --- a/hdf5/src/hl/datatype.rs +++ b/hdf5/src/hl/datatype.rs @@ -327,6 +327,16 @@ impl Datatype { Ok(string_id) } + #[cfg(feature = "f16")] + unsafe fn f16_type() -> Result { + use hdf5_sys::h5t::{H5Tset_ebias, H5Tset_fields}; + let f16_id = be_le!(H5T_IEEE_F32BE, H5T_IEEE_F32LE); + h5try!(H5Tset_fields(f16_id, 15, 10, 5, 0, 10)); // cf. h5py/h5py#339 + h5try!(H5Tset_size(f16_id, 2)); + h5try!(H5Tset_ebias(f16_id, 15)); + Ok(f16_id) + } + let datatype_id: Result<_> = h5lock!({ match *desc { TD::Integer(size) => Ok(match size { @@ -342,6 +352,8 @@ impl Datatype { IntSize::U8 => be_le!(H5T_STD_U64BE, H5T_STD_U64LE), }), TD::Float(size) => Ok(match size { + #[cfg(feature = "f16")] + FloatSize::U2 => f16_type()?, FloatSize::U4 => be_le!(H5T_IEEE_F32BE, H5T_IEEE_F32LE), FloatSize::U8 => be_le!(H5T_IEEE_I16BE, H5T_IEEE_F64LE), }), diff --git a/tests/common/gen.rs b/hdf5/tests/common/gen.rs similarity index 94% rename from tests/common/gen.rs rename to hdf5/tests/common/gen.rs index 5f69a458f..4a08a5734 100644 --- a/tests/common/gen.rs +++ b/hdf5/tests/common/gen.rs @@ -1,12 +1,16 @@ use std::convert::TryFrom; -use std::fmt; +use std::fmt::{self, Debug}; use std::iter; use hdf5::types::{FixedAscii, FixedUnicode, VarLenArray, VarLenAscii, VarLenUnicode}; use hdf5::H5Type; +use half::f16; use ndarray::{ArrayD, SliceInfo, SliceInfoElem}; +use num_complex::Complex; +use rand::distributions::Standard; use rand::distributions::{Alphanumeric, Uniform}; +use rand::prelude::Distribution; use rand::prelude::{Rng, SliceRandom}; pub fn gen_shape(rng: &mut R, ndim: usize) -> Vec { @@ -93,6 +97,21 @@ macro_rules! impl_gen_tuple { impl_gen_tuple! { A, B, C, D, E, F, G, H, I, J, K, L } +impl Gen for f16 { + fn gen(rng: &mut R) -> Self { + Self::from_f32(rng.gen()) + } +} + +impl Gen for Complex +where + Standard: Distribution, +{ + fn gen(rng: &mut R) -> Self { + Self::new(rng.gen(), rng.gen()) + } +} + pub fn gen_vec(rng: &mut R, size: usize) -> Vec { iter::repeat(()).map(|_| T::gen(rng)).take(size).collect() } diff --git a/tests/common/macros.rs b/hdf5/tests/common/macros.rs similarity index 100% rename from tests/common/macros.rs rename to hdf5/tests/common/macros.rs diff --git a/tests/common/mod.rs b/hdf5/tests/common/mod.rs similarity index 100% rename from tests/common/mod.rs rename to hdf5/tests/common/mod.rs diff --git a/tests/common/util.rs b/hdf5/tests/common/util.rs similarity index 100% rename from tests/common/util.rs rename to hdf5/tests/common/util.rs diff --git a/tests/test_dataset.rs b/hdf5/tests/test_dataset.rs similarity index 97% rename from tests/test_dataset.rs rename to hdf5/tests/test_dataset.rs index ffacbdac1..caefc5c4e 100644 --- a/tests/test_dataset.rs +++ b/hdf5/tests/test_dataset.rs @@ -313,6 +313,21 @@ fn test_read_write_primitive() -> hdf5::Result<()> { Ok(()) } +#[cfg(feature = "f16")] +#[test] +fn test_read_write_f16() -> hdf5::Result<()> { + test_read_write::<::half::f16>()?; + Ok(()) +} + +#[cfg(feature = "complex")] +#[test] +fn test_read_write_complex() -> hdf5::Result<()> { + test_read_write::<::num_complex::Complex32>()?; + test_read_write::<::num_complex::Complex64>()?; + Ok(()) +} + #[test] fn test_read_write_enum() -> hdf5::Result<()> { test_read_write::() @@ -352,6 +367,7 @@ fn test_create_on_databuilder() { } #[test] +#[cfg(feature = "have-filter-deflate")] fn test_issue_223() { let file = new_in_memory_file().unwrap(); diff --git a/tests/test_datatypes.rs b/hdf5/tests/test_datatypes.rs similarity index 97% rename from tests/test_datatypes.rs rename to hdf5/tests/test_datatypes.rs index 8289ae1b4..1b83638f9 100644 --- a/tests/test_datatypes.rs +++ b/hdf5/tests/test_datatypes.rs @@ -26,6 +26,8 @@ pub fn test_datatype_roundtrip() { check_roundtrip!(u16, TD::Unsigned(IntSize::U2)); check_roundtrip!(u32, TD::Unsigned(IntSize::U4)); check_roundtrip!(u64, TD::Unsigned(IntSize::U8)); + #[cfg(feature = "f16")] + check_roundtrip!(::half::f16, TD::Float(FloatSize::U2)); check_roundtrip!(f32, TD::Float(FloatSize::U4)); check_roundtrip!(f64, TD::Float(FloatSize::U8)); check_roundtrip!(bool, TD::Boolean); diff --git a/tests/test_plist.rs b/hdf5/tests/test_plist.rs similarity index 100% rename from tests/test_plist.rs rename to hdf5/tests/test_plist.rs diff --git a/tests/tests.rs b/hdf5/tests/tests.rs similarity index 100% rename from tests/tests.rs rename to hdf5/tests/tests.rs