diff --git a/.travis.yml b/.travis.yml index aa45f1f2..4d3cc245 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ addons: packages: - doxygen - graphviz + - libhdf5-dev before_install: - wget -O cmake.sh https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-3.14.6-Linux-x86_64.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 6156084f..bc961069 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.13 FATAL_ERROR) -project(libpressio VERSION "0.17.2" LANGUAGES CXX C) +project(libpressio VERSION "0.18.0" LANGUAGES CXX C) #correct was to set a default build type # https://blog.kitware.com/cmake-and-the-default-build-type/ @@ -45,6 +45,7 @@ check_cpp_standard(exclusive_scan) set(LIBPRESSIO_FEATURES "") +set(LIBPRESSIO_COMPRESSORS "") add_library(libpressio #core implementation @@ -99,7 +100,7 @@ target_compile_features(libpressio PUBLIC cxx_std_17) option(LIBPRESSIO_HAS_MGARD "build the MGARD plugin" OFF) if(LIBPRESSIO_HAS_MGARD) - set(LIBPRESSIO_FEATURES "${LIBPRESSIO_FEATURES} mgard") + set(LIBPRESSIO_COMPRESSORS "${LIBPRESSIO_COMPRESSORS} mgard") find_package(mgard REQUIRED) target_sources(libpressio PRIVATE @@ -111,7 +112,7 @@ endif() option(LIBPRESSIO_HAS_ZFP "build the ZFP plugin" ON) if(LIBPRESSIO_HAS_ZFP) - set(LIBPRESSIO_FEATURES "${LIBPRESSIO_FEATURES} zfp") + set(LIBPRESSIO_COMPRESSORS "${LIBPRESSIO_COMPRESSORS} zfp") find_package(ZFP REQUIRED) target_sources(libpressio PRIVATE @@ -123,7 +124,7 @@ endif() option(LIBPRESSIO_HAS_SZ "build the SZ plugin" ON) if(LIBPRESSIO_HAS_SZ) - set(LIBPRESSIO_FEATURES "${LIBPRESSIO_FEATURES} sz") + set(LIBPRESSIO_COMPRESSORS "${LIBPRESSIO_COMPRESSORS} sz") find_package(SZ REQUIRED) find_package(ZLIB REQUIRED) find_package(PkgConfig REQUIRED) @@ -137,6 +138,20 @@ if(LIBPRESSIO_HAS_SZ) target_link_libraries(libpressio PUBLIC SZ) endif() +option(LIBPRESSIO_HAS_HDF "build the hdf5 io plugin" ON) +if(LIBPRESSIO_HAS_HDF) + set(LIBPRESSIO_FEATURES "${LIBPRESSIO_FEATURES} hdf5") + find_package(HDF5 REQUIRED COMPONENTS C) + target_link_libraries(libpressio PUBLIC ${HDF5_C_LIBRARIES}) + target_include_directories(libpressio PUBLIC ${HDF5_C_INCLUDE_DIRS}) + target_compile_definitions(libpressio PUBLIC ${HDF5_C_DEFINITIONS}) + target_sources(libpressio + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/plugins/io/hdf5.cc + ${CMAKE_CURRENT_SOURCE_DIR}/include/libpressio_ext/io/hdf5.h + ) +endif() + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/include/pressio_version.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/pressio_version.h diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index f4b3f260..80d9de70 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -1,6 +1,6 @@ Copyright © 2019 , UChicago Argonne, LLC All Rights Reserved -[libpressio, Version 0.17.2] +[libpressio, Version 0.18.0] Robert Underwood Argonne National Laboratory diff --git a/README.md b/README.md index bbf7f1ca..a728f2c0 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Libpressio additionally optionally requires: + `sz` commit `7b7463411f02be4700d13aac6737a6a9662806b4` or later and its dependencies to provide the ZFP plugin + `numpy` version `1.14.5` or later and its dependencies to provide the python bindings + `Doxygen` version 1.8.15 or later to generate documentation ++ `HDF5` version 1.10.0 or later for HDF5 data support ## Configuring LibPressio diff --git a/include/libpressio_ext/cpp/options.h b/include/libpressio_ext/cpp/options.h index 6c06d933..ae9fc546 100644 --- a/include/libpressio_ext/cpp/options.h +++ b/include/libpressio_ext/cpp/options.h @@ -28,18 +28,25 @@ using option_type = std::variant enum pressio_option_type pressio_type_to_enum; +/** maps to int to pressio_option_int32_type */ template <> inline constexpr enum pressio_option_type pressio_type_to_enum = pressio_option_int32_type; +/** maps to unsigned int to pressio_option_uint32_type */ template <> inline constexpr enum pressio_option_type pressio_type_to_enum = pressio_option_uint32_type; +/** maps to float to pressio_option_float_type */ template <> inline constexpr enum pressio_option_type pressio_type_to_enum = pressio_option_float_type; +/** maps to double to pressio_option_double_type */ template <> inline constexpr enum pressio_option_type pressio_type_to_enum = pressio_option_double_type; +/** maps to std::string to pressio_option_charptr_type */ template <> inline constexpr enum pressio_option_type pressio_type_to_enum = pressio_option_charptr_type; +/** maps to const char* to pressio_option_charptr_type */ template <> inline constexpr enum pressio_option_type pressio_type_to_enum = pressio_option_charptr_type; +/** maps to void* to pressio_option_userptr_type */ template <> inline constexpr enum pressio_option_type pressio_type_to_enum = pressio_option_userptr_type; diff --git a/include/libpressio_ext/io/hdf5.h b/include/libpressio_ext/io/hdf5.h new file mode 100644 index 00000000..9d69626e --- /dev/null +++ b/include/libpressio_ext/io/hdf5.h @@ -0,0 +1,49 @@ +/** + * \file + * \brief IO functions for simple HDF5 files + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef PRESSIO_HDF5_IO +#define PRESSIO_HDF5_IO + +struct pressio_data; + +/** + * reads files that use simple datatypes and dataspaces into a pressio_data + * + * The following are not supported: + * + complex datatypes or dataspaces + * + unlimited dimension datasets. + * + * \param[in] file_name name of the file to read + * \param[in] dataset_name name of the dataset to read from the file + * + * \returns a pressio_data structure or nullptr if there was an error + */ +struct pressio_data* +pressio_io_data_path_h5read(const char* file_name, const char* dataset_name); + +/** + * writes files that use simple datatypes and dataspaces from a pressio_data + * + * if the file already exists, it will be opened. if the file does not exist, it will be created. + * if the dataset already exists, it will be overwritten. if the dataset does not exist, it will be created. + * + * \param[in] data the data to be written + * \param[in] file_name name of the file to written + * \param[in] dataset_name name of the dataset to written to the file + * + * \returns a 0 if successful, nonzero on error + */ +int +pressio_io_data_path_h5write(struct pressio_data const* data, const char* file_name, const char* dataset_name); + +#endif + +#ifdef __cplusplus +} +#endif diff --git a/include/pressio.h b/include/pressio.h index 021ecda6..4b0c7ce9 100644 --- a/include/pressio.h +++ b/include/pressio.h @@ -57,15 +57,20 @@ int pressio_error_code(struct pressio* library); const char* pressio_error_msg(struct pressio* library); /** + * it will not return more information than the tailored functions below * \returns a string with version and feature information */ const char* pressio_version(); /** - * \returns a string containing all the compressor_ids supported by this version separated by a space - * it will not return more information than the tailored functions below + * \returns a string containing all the features supported by this version separated by a space. Some features are compressors, but not all are. * \see pressio_get_compressor the compressor_ids may be passed to pressio_get_compressor */ const char* pressio_features(); +/** + * \returns a string containing all the compressors supported by this version separated by a space + * \see pressio_get_supported_compressors the compressor_ids may be passed to pressio_get_compressor + */ +const char* pressio_supported_compressors(); /** * \returns the major version of the library */ diff --git a/include/pressio_version.h.in b/include/pressio_version.h.in index 5e269d82..e50bf181 100644 --- a/include/pressio_version.h.in +++ b/include/pressio_version.h.in @@ -1,5 +1,6 @@ -#define LIBPRESSIO_VERSION "@PROJECT_VERSION@ @LIBPRESSIO_FEATURES@" -#define LIBPRESSIO_FEATURES "@LIBPRESSIO_FEATURES@" +#define LIBPRESSIO_VERSION "@PROJECT_VERSION@ @LIBPRESSIO_FEATURES@ @LIBPRESSIO_COMPRESSORS@" +#define LIBPRESSIO_COMPRESSORS "@LIBPRESSIO_COMPRESSORS@" +#define LIBPRESSIO_FEATURES "@LIBPRESSIO_FEATURES@@LIBPRESSIO_COMPRESSORS@" #define LIBPRESSIO_MAJOR_VERSION @PROJECT_VERSION_MAJOR@ #define LIBPRESSIO_MINOR_VERSION @PROJECT_VERSION_MINOR@ #define LIBPRESSIO_PATCH_VERSION @PROJECT_VERSION_PATCH@ diff --git a/src/plugins/io/hdf5.cc b/src/plugins/io/hdf5.cc new file mode 100644 index 00000000..b00478b1 --- /dev/null +++ b/src/plugins/io/hdf5.cc @@ -0,0 +1,198 @@ +#include +#include +#include +#include +#include +#include + +namespace { + std::optional h5t_to_pressio(hid_t h5type) { + if(H5Tequal(h5type, H5T_NATIVE_INT8) > 0) return pressio_int8_dtype; + if(H5Tequal(h5type, H5T_NATIVE_INT16) > 0) return pressio_int16_dtype; + if(H5Tequal(h5type, H5T_NATIVE_INT32) > 0) return pressio_int32_dtype; + if(H5Tequal(h5type, H5T_NATIVE_INT64) > 0) return pressio_int64_dtype; + if(H5Tequal(h5type, H5T_NATIVE_UINT8) > 0) return pressio_uint8_dtype; + if(H5Tequal(h5type, H5T_NATIVE_UINT16) > 0) return pressio_uint16_dtype; + if(H5Tequal(h5type, H5T_NATIVE_UINT32) > 0) return pressio_uint32_dtype; + if(H5Tequal(h5type, H5T_NATIVE_UINT64) > 0) return pressio_uint64_dtype; + if(H5Tequal(h5type, H5T_NATIVE_FLOAT) > 0) return pressio_float_dtype; + if(H5Tequal(h5type, H5T_NATIVE_DOUBLE) > 0) return pressio_double_dtype; + return std::nullopt; + } + hid_t pressio_to_h5t(pressio_dtype dtype) { + switch(dtype) { + case pressio_double_dtype: + return H5T_NATIVE_DOUBLE; + case pressio_float_dtype: + return H5T_NATIVE_FLOAT; + case pressio_uint8_dtype: + return H5T_NATIVE_UINT8; + case pressio_uint16_dtype: + return H5T_NATIVE_UINT16; + case pressio_uint32_dtype: + return H5T_NATIVE_UINT32; + case pressio_uint64_dtype: + return H5T_NATIVE_UINT64; + case pressio_int8_dtype: + return H5T_NATIVE_INT8; + case pressio_int16_dtype: + return H5T_NATIVE_INT16; + case pressio_int32_dtype: + return H5T_NATIVE_INT32; + case pressio_int64_dtype: + return H5T_NATIVE_INT64; + case pressio_byte_dtype: + return H5T_NATIVE_UCHAR; + default: + assert(false && "unexpected type"); + //shutup gcc + return H5T_NATIVE_UCHAR; + } + } + + bool hdf_path_exists(hid_t file, std::string const& path) { + if(path == "" or path == "/") return true; + else { + //check for parent path + auto last_slash_pos = path.find_last_of('/'); + if(last_slash_pos != std::string::npos) + { + //recurse to check for parent + auto parent = path.substr(0, last_slash_pos - 1); + if (not hdf_path_exists(file, parent)) return false; + } + + //check the path passed in + return H5Lexists(file, path.c_str(), H5P_DEFAULT); + } + } + + /** + * this class is a standard c++ idiom for closing resources + * it calls the function passed in during the destructor. + */ + template + class cleanup { + public: + cleanup(Function f) noexcept: cleanup_fn(std::move(f)), do_cleanup(true) {} + cleanup(cleanup&& rhs) noexcept: cleanup_fn(std::move(rhs.cleanup_fn)), do_cleanup(true) { + do_cleanup = false; + } + cleanup(cleanup const&)=delete; + cleanup& operator=(cleanup const&)=delete; + cleanup& operator=(cleanup && rhs) noexcept { + do_cleanup = std::exchange(rhs.do_cleanup, false); + cleanup_fn = std::move(rhs.cleanup_fn); + } + ~cleanup() { if(do_cleanup) cleanup_fn(); } + + private: + Function cleanup_fn; + bool do_cleanup; + }; +} + +extern "C" { + +struct pressio_data* +pressio_io_data_path_h5read(const char* file_name, const char* dataset_name) +{ + hid_t file = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT); + if(file < 0) return nullptr; + cleanup cleanup_file([&]{ H5Fclose(file); }); + + hid_t dataset = H5Dopen2(file, dataset_name, H5P_DEFAULT); + if(dataset < 0) return nullptr; + cleanup cleanup_dataset([&]{ H5Dclose(dataset); }); + + hid_t dataspace = H5Dget_space(dataset); + if(dataspace < 0) return nullptr; + cleanup cleanup_dataspace([&]{ H5Sclose(dataspace); }); + + int ndims = H5Sget_simple_extent_ndims(dataspace); + std::vector dims(ndims); + H5Sget_simple_extent_dims(dataspace, dims.data(), nullptr); + + //convert to size_t from hsize_t + std::vector pressio_dims(std::begin(dims), std::end(dims)); + + hid_t type = H5Dget_type(dataset); + if(type < 0) return nullptr; + cleanup cleanup_type([&]{ H5Tclose(type);}) ; + + if(auto dtype = h5t_to_pressio(type); dtype) { + auto ret = pressio_data_new_owning(*dtype, pressio_dims.size(), pressio_dims.data()); + auto ptr = pressio_data_ptr(ret, nullptr); + + H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, ptr); + + return ret; + } else { + return nullptr; + } +} + +int +pressio_io_data_path_h5write(struct pressio_data const* data, const char* file_name, const char* dataset_name) +{ + //check if the file exists + hid_t file; + int perms_ok = access(file_name, W_OK); + if(perms_ok == 0) + { + file = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT); + } else { + if(errno == ENOENT) { + file = H5Fcreate(file_name, H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT); + } else { + return 1; + } + } + if(file < 0) return 1; + cleanup cleanup_file([&]{ H5Fclose(file); }); + + + //prepare the dataset for writing + std::vector dims; + for (size_t i = 0; i < pressio_data_num_dimensions(data); ++i) { + dims.push_back(pressio_data_get_dimension(data, i)); + } + + std::vector h5_dims(std::begin(dims), std::end(dims)); + hid_t dataspace = H5Screate_simple( + h5_dims.size(), + h5_dims.data(), + nullptr + ); + if(dataspace < 0) return 1; + cleanup cleanup_space([&]{ H5Sclose(dataspace);}); + + hid_t dataset; + if (hdf_path_exists(file, dataset_name)) + { + dataset = H5Dopen(file, dataset_name, H5P_DEFAULT); + } else { + dataset = H5Dcreate2(file, + dataset_name, + pressio_to_h5t(pressio_data_dtype(data)), + dataspace, + H5P_DEFAULT, + H5P_DEFAULT, + H5P_DEFAULT + ); + } + if(dataset < 0) return 1; + cleanup cleanup_dataset([&]{ H5Dclose(dataset);}); + + //write the dataset + return (H5Dwrite( + dataset, + pressio_to_h5t(pressio_data_dtype(data)), + H5S_ALL, + H5S_ALL, + H5P_DEFAULT, + pressio_data_ptr(data,nullptr) + ) < 0); +} + +} diff --git a/src/pressio.cc b/src/pressio.cc index a3a2f82f..d9aa54d5 100644 --- a/src/pressio.cc +++ b/src/pressio.cc @@ -129,6 +129,9 @@ const char* pressio_version() { const char* pressio_features() { return LIBPRESSIO_FEATURES; } +const char* pressio_supported_compressors() { + return LIBPRESSIO_COMPRESSORS; +} unsigned int pressio_major_version() { return LIBPRESSIO_MAJOR_VERSION; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 328474fd..4241ffc7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,6 +39,10 @@ target_include_directories(test_pressio_data PRIVATE ${CMAKE_SOURCE_DIR}/src) add_gtest(test_pressio_options.cc) add_gtest(test_io.cc) +if(LIBPRESSIO_HAS_HDF) + add_gtest(test_hdf5.cc) +endif() + if(LIBPRESSIO_HAS_SZ) add_gtest(test_sz_plugin.cc) target_include_directories(test_sz_plugin PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../include) diff --git a/test/test_hdf5.cc b/test/test_hdf5.cc new file mode 100644 index 00000000..9f2286a6 --- /dev/null +++ b/test/test_hdf5.cc @@ -0,0 +1,107 @@ +#include +#include +#include "libpressio_ext/io/hdf5.h" +#include "pressio_data.h" +#include "gtest/gtest.h" + +class PressioIOHDFTests: public ::testing::Test { + + void SetUp() override { + tmp_fd = mkstemps(test_file, 3); + } + void TearDown() override{ + close(tmp_fd); + } + + protected: + char test_file[14] = "testXXXXXX.h5"; + int tmp_fd; +}; + +TEST_F(PressioIOHDFTests, read_and_write) { + std::vector floats(3*3*3); + std::vector dims{3,3,3}; + std::iota(std::begin(floats), std::end(floats), 0.0); + + pressio_data* data = pressio_data_new_nonowning(pressio_float_dtype, floats.data(), dims.size(), dims.data()); + + + { + auto ret = pressio_io_data_path_h5write(data, test_file, "testing"); + EXPECT_EQ(ret, 0); + } + + { + pressio_data* result = pressio_io_data_path_h5read(test_file, "testing"); + ASSERT_NE(result, nullptr); + ASSERT_EQ(pressio_float_dtype, pressio_data_dtype(result)); + ASSERT_EQ(3, pressio_data_num_dimensions(result)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 0)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 1)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 2)); + std::vector result_data(3*3*3); + size_t size_in_bytes; + void* result_ptr = pressio_data_ptr(result, &size_in_bytes); + memcpy(result_data.data(), result_ptr, size_in_bytes); + EXPECT_EQ(floats, result_data); + } + + pressio_data_free(data); +} + +TEST_F(PressioIOHDFTests, write_twice) { + std::vector floats(3*3*3); + std::vector dims{3,3,3}; + std::iota(std::begin(floats), std::end(floats), 0.0); + + pressio_data* data = pressio_data_new_nonowning(pressio_float_dtype, floats.data(), dims.size(), dims.data()); + + + { + auto ret = pressio_io_data_path_h5write(data, test_file, "testing"); + EXPECT_EQ(ret, 0); + } + + { + pressio_data* result = pressio_io_data_path_h5read(test_file, "testing"); + ASSERT_NE(result, nullptr); + ASSERT_EQ(pressio_float_dtype, pressio_data_dtype(result)); + ASSERT_EQ(3, pressio_data_num_dimensions(result)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 0)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 1)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 2)); + std::vector result_data(3*3*3); + size_t size_in_bytes; + void* result_ptr = pressio_data_ptr(result, &size_in_bytes); + memcpy(result_data.data(), result_ptr, size_in_bytes); + ASSERT_EQ(result_data.front(), 0.0f); + EXPECT_EQ(floats, result_data); + } + + std::iota(std::rbegin(floats), std::rend(floats), 0); + + + { + auto ret = pressio_io_data_path_h5write(data, test_file, "testing"); + EXPECT_EQ(ret, 0); + } + + { + pressio_data* result = pressio_io_data_path_h5read(test_file, "testing"); + ASSERT_NE(result, nullptr); + ASSERT_EQ(pressio_float_dtype, pressio_data_dtype(result)); + ASSERT_EQ(3, pressio_data_num_dimensions(result)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 0)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 1)); + ASSERT_EQ(3, pressio_data_get_dimension(result, 2)); + std::vector result_data(3*3*3); + size_t size_in_bytes; + void* result_ptr = pressio_data_ptr(result, &size_in_bytes); + memcpy(result_data.data(), result_ptr, size_in_bytes); + ASSERT_EQ(result_data.back(), 0.0f); + EXPECT_EQ(floats, result_data); + } + + + pressio_data_free(data); +}