From a45e1e95a996cc02644eacf24388bc0b13d0b12a Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 15:30:48 -0500 Subject: [PATCH 01/15] Checks HDF5 build features (cmake) --- CMakeLists.txt | 14 +++++++++++++- README.md | 2 +- docs/source/gettingstarted.rst | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 52e7358..590b87c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,10 +32,22 @@ if(NOT HDF5_VOL_ASYNC_INSTALL_TEST_DIR) set(HDF5_VOL_ASYNC_INSTALL_TEST_DIR ${CMAKE_INSTALL_PREFIX}/test) endif() +include(CheckSymbolExists) +list(APPEND CMAKE_REQUIRED_INCLUDES ${HDF5_INCLUDE_DIRS}) +check_symbol_exists(H5_HAVE_PARALLEL "H5pubconf.h" HDF5_HAVE_PARALLEL) +if(NOT HDF5_HAVE_PARALLEL) + message(FATAL_ERROR "HDF5 library needs to enable parallel support.") +endif() + +check_symbol_exists(H5_HAVE_THREADSAFE "H5pubconf.h" HDF5_HAVE_THREADSAFE) +if(NOT HDF5_HAVE_THREADSAFE) + message(FATAL_ERROR "HDF5 library needs to enable threadsafe support.") +endif() + set(HDF5_VOL_ASYNC_PACKAGE "hdf5_vol_async") set(HDF5_VOL_ASYNC_PACKAGE_NAME "HDF5_VOL_ASYNC") set(HDF5_VOL_ASYNC_PACKAGE_DESCRIPTION "HDF5 Asynchronous I/O VOL connector") -set(HDF5_VOL_ASYNC_PACKAGE_URL "https://github.com/hpc-io/vol-async") +set(HDF5_VOL_ASYNC_PACKAGE_URL "https://github.com/HDFGroup/vol-async") set(HDF5_VOL_ASYNC_PACKAGE_VENDOR "HPC IO") #----------------------------------------------------------------------------- diff --git a/README.md b/README.md index dd16868..8cf2b31 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![linux](https://github.com/hpc-io/vol-async/actions/workflows/linux.yml/badge.svg?branch=develop)](https://github.com/hpc-io/vol-async/actions/workflows/linux.yml) +[![linux](https://github.com/HDFGroup/vol-async/actions/workflows/linux.yml/badge.svg?branch=develop)](https://github.com/HDFGroup/vol-async/actions/workflows/linux.yml) # HDF5 Asynchronous I/O VOL Connector Asynchronous I/O is becoming increasingly popular with the large amount of data access required by scientific applications. They can take advantage of an asynchronous interface by scheduling I/O as early as possible and overlap computation or communication with I/O operations, which hides the cost associated with I/O and improves the overall performance. This work is part of the [ECP-ExaIO](https://www.exascaleproject.org/research-project/exaio) project. diff --git a/docs/source/gettingstarted.rst b/docs/source/gettingstarted.rst index 457d516..6c53fc6 100644 --- a/docs/source/gettingstarted.rst +++ b/docs/source/gettingstarted.rst @@ -32,7 +32,7 @@ Define the following configuration parameters, which may be used in the instruct .. code-block:: - git clone --recursive https://github.com/hpc-io/vol-async.git + git clone --recursive https://github.com/HDFGroup/vol-async.git 2. Download the HDF5 source code From be1cb93944b6f9b713cb89fc260d36b899186d06 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 16:03:08 -0500 Subject: [PATCH 02/15] require HDF5 version 1.14 and greater --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 590b87c..c07be81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ set (CMAKE_POSITION_INDEPENDENT_CODE ON) find_package(MPI REQUIRED) find_package(ABT REQUIRED) -find_package(HDF5 REQUIRED COMPONENTS C) +find_package(HDF5 1.14 REQUIRED COMPONENTS C) include_directories(${MPI_INCLUDE_PATH}) include_directories(${HDF5_INCLUDE_DIRS}) From 6df8afb124236528b3b908cdea1857332f155a48 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 16:12:20 -0500 Subject: [PATCH 03/15] removed invalid hdf5 build option --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 588fa84..f221215 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -49,7 +49,7 @@ jobs: export HDF5_LIBTOOL=/usr/bin/libtoolize ./autogen.sh ./configure --prefix=$HDF5_DIR --enable-parallel --enable-threadsafe --enable-unsupported \ - --disable-tests --disable-hl --disable-fortran --disable-tools --disable-hltools + --disable-tests --disable-hl --disable-fortran --disable-tools make && make install # Compile Argobots @@ -138,7 +138,7 @@ jobs: export HDF5_LIBTOOL=/usr/bin/libtoolize ./autogen.sh ./configure --prefix=$HDF5_DIR --enable-parallel --enable-threadsafe --enable-unsupported \ - --disable-tests --disable-hl --disable-fortran --disable-tools --disable-hltools + --disable-tests --disable-hl --disable-fortran --disable-tools make && make install # Compile Argobots From ca8e269534b4ada2e28056f4caa2816b3155887c Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:10:45 -0500 Subject: [PATCH 04/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f221215..160a664 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -14,7 +14,7 @@ on: # Allows to run this workflow manually from the Actions tab workflow_dispatch: jobs: - vol-async-hdf5-1_14_3-Makefile: + vol-async-hdf5-latest-Makefile: runs-on: ubuntu-latest timeout-minutes: 60 @@ -24,12 +24,12 @@ jobs: with: submodules: 'true' - - name: Checkout HDF5 1.14.3 - uses: actions/checkout@v3 + - name: Checkout HDF5 latest + uses: actions/download-artifact@v4 with: repository: HDFGroup/hdf5 - ref: hdf5-1_14_3 - path: hdf5 + name: hdf5.tar.gz + path: releases/latest/download - name: Dependencies run: | @@ -45,6 +45,8 @@ jobs: mkdir $HDF5_DIR # Compile HDF5 + tar xzf hdf5.tar.gz + ln -sf hdf5* hdf5 cd hdf5 export HDF5_LIBTOOL=/usr/bin/libtoolize ./autogen.sh From 41908324f7dc12a6f01fb9daabcc953f48f8410e Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:18:12 -0500 Subject: [PATCH 05/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 160a664..7948647 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -23,13 +23,6 @@ jobs: uses: actions/checkout@v3 with: submodules: 'true' - - - name: Checkout HDF5 latest - uses: actions/download-artifact@v4 - with: - repository: HDFGroup/hdf5 - name: hdf5.tar.gz - path: releases/latest/download - name: Dependencies run: | @@ -45,6 +38,7 @@ jobs: mkdir $HDF5_DIR # Compile HDF5 + wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz tar xzf hdf5.tar.gz ln -sf hdf5* hdf5 cd hdf5 From 80e390e4539844e3903b9f71bbc2c4aeebe89f9b Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:21:48 -0500 Subject: [PATCH 06/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7948647..77d421c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -36,11 +36,11 @@ jobs: export ABT_DIR=$mydir/argobots export VOL_DIR=$mydir - mkdir $HDF5_DIR # Compile HDF5 wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz tar xzf hdf5.tar.gz ln -sf hdf5* hdf5 + mkdir $HDF5_DIR cd hdf5 export HDF5_LIBTOOL=/usr/bin/libtoolize ./autogen.sh From 29a52119d526f09e798e2aa1d4408756d5fe3b99 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:23:55 -0500 Subject: [PATCH 07/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 77d421c..0e58890 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -36,10 +36,10 @@ jobs: export ABT_DIR=$mydir/argobots export VOL_DIR=$mydir - # Compile HDF5 + # Download and Compile latest release of HDF5 wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz tar xzf hdf5.tar.gz - ln -sf hdf5* hdf5 + ln -sf hdf5-* hdf5 mkdir $HDF5_DIR cd hdf5 export HDF5_LIBTOOL=/usr/bin/libtoolize From 889dc7caa37fdf057b73e894090a4197ef943316 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:25:43 -0500 Subject: [PATCH 08/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0e58890..8af0b50 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -39,6 +39,7 @@ jobs: # Download and Compile latest release of HDF5 wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz tar xzf hdf5.tar.gz + ls -aolF ln -sf hdf5-* hdf5 mkdir $HDF5_DIR cd hdf5 From 9a1d725a7c01d11f4b176894dfd0410c2cebb830 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:33:07 -0500 Subject: [PATCH 09/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 8af0b50..92d7042 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -28,6 +28,12 @@ jobs: run: | sudo apt-get update sudo apt-get install libtool libopenmpi-dev zlib1g-dev python3 + + - name: Checkout latest HDF5 release + run: | + wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz + tar xzf hdf5.tar.gz + ln -sf hdf5-* hdf5 - name: Installation run: | @@ -36,11 +42,6 @@ jobs: export ABT_DIR=$mydir/argobots export VOL_DIR=$mydir - # Download and Compile latest release of HDF5 - wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz - tar xzf hdf5.tar.gz - ls -aolF - ln -sf hdf5-* hdf5 mkdir $HDF5_DIR cd hdf5 export HDF5_LIBTOOL=/usr/bin/libtoolize From 6e4fffd1263d088e7fc4216b22123d6bdd2bba99 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:37:02 -0500 Subject: [PATCH 10/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 92d7042..e89af29 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -43,9 +43,8 @@ jobs: export VOL_DIR=$mydir mkdir $HDF5_DIR + # Compile HDF5 cd hdf5 - export HDF5_LIBTOOL=/usr/bin/libtoolize - ./autogen.sh ./configure --prefix=$HDF5_DIR --enable-parallel --enable-threadsafe --enable-unsupported \ --disable-tests --disable-hl --disable-fortran --disable-tools make && make install @@ -191,7 +190,7 @@ jobs: make check date - vol-async-hdf5-1_14_3-cmake: + vol-async-hdf5-latest-cmake: runs-on: ubuntu-latest timeout-minutes: 30 @@ -201,12 +200,11 @@ jobs: with: submodules: 'true' - - name: Checkout HDF5 1.14.3 - uses: actions/checkout@v3 - with: - repository: HDFGroup/hdf5 - ref: hdf5-1_14_3 - path: hdf5 + - name: Checkout latest HDF5 release + run: | + wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz + tar xzf hdf5.tar.gz + ln -sf hdf5-* hdf5 - name: Dependencies run: | From 8a28d59f6571478264a7a68e4187eaccff4d57f8 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:40:34 -0500 Subject: [PATCH 11/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index e89af29..7fb2809 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -199,18 +199,18 @@ jobs: uses: actions/checkout@v3 with: submodules: 'true' + + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install libtool libopenmpi-dev zlib1g-dev python3 cmake - name: Checkout latest HDF5 release run: | wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz tar xzf hdf5.tar.gz ln -sf hdf5-* hdf5 - - - name: Dependencies - run: | - sudo apt-get update - sudo apt-get install libtool libopenmpi-dev zlib1g-dev python3 cmake - + - name: Installation run: | mydir="$PWD" From a30a4a5190af6985a9493b395c2b4775cbe535e6 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 17:42:41 -0500 Subject: [PATCH 12/15] change to testing with latest HDF5 --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7fb2809..6f307ab 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -224,7 +224,7 @@ jobs: mkdir build cd build cmake -DCMAKE_INSTALL_PREFIX=$HDF5_DIR -DHDF5_ENABLE_PARALLEL=ON -DHDF5_ENABLE_THREADSAFE=ON \ - -DALLOW_UNSUPPORTED=ON -DBUILD_TESTING=OFF -DHDF5_BUILD_HL_LIB=OFF -DHDF5_BUILD_HL_TOOLS=OFF \ + -DALLOW_UNSUPPORTED=ON -DBUILD_TESTING=OFF -DHDF5_BUILD_HL_LIB=OFF \ -DHDF5_BUILD_EXAMPLES=OFF -DHDF5_BUILD_FORTRAN=OFF -DCMAKE_C_COMPILER=mpicc .. make -j && make install @@ -305,7 +305,7 @@ jobs: mkdir build cd build cmake -DCMAKE_INSTALL_PREFIX=$HDF5_DIR -DHDF5_ENABLE_PARALLEL=ON -DHDF5_ENABLE_THREADSAFE=ON \ - -DALLOW_UNSUPPORTED=ON -DBUILD_TESTING=OFF -DHDF5_BUILD_HL_LIB=OFF -DHDF5_BUILD_HL_TOOLS=OFF \ + -DALLOW_UNSUPPORTED=ON -DBUILD_TESTING=OFF -DHDF5_BUILD_HL_LIB=OFF \ -DHDF5_BUILD_EXAMPLES=OFF -DHDF5_BUILD_FORTRAN=OFF -DCMAKE_C_COMPILER=mpicc .. make -j && make install From 9a7f0dc00da82972b460403535b5cd857e48d94d Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 19:36:31 -0500 Subject: [PATCH 13/15] change to testing with latest HDF5 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c07be81..590b87c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ set (CMAKE_POSITION_INDEPENDENT_CODE ON) find_package(MPI REQUIRED) find_package(ABT REQUIRED) -find_package(HDF5 1.14 REQUIRED COMPONENTS C) +find_package(HDF5 REQUIRED COMPONENTS C) include_directories(${MPI_INCLUDE_PATH}) include_directories(${HDF5_INCLUDE_DIRS}) From b34c027e1dbfbacd1215fe1958953584ff96db4c Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 30 May 2024 22:59:36 -0500 Subject: [PATCH 14/15] change to testing with latest HDF5 --- CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 590b87c..d11a670 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,13 @@ if(NOT HDF5_HAVE_THREADSAFE) message(FATAL_ERROR "HDF5 library needs to enable threadsafe support.") endif() +#Check if HDF5 version is 1.14 or greater +if(HDF5_FOUND) + if (HDF5_VERSION VERSION_LESS "1.14.0") + message (FATAL_ERROR "HDF5 ${HDF5_VERSION}: required HDF5 is at least 1.14 ") + endif () +endif () + set(HDF5_VOL_ASYNC_PACKAGE "hdf5_vol_async") set(HDF5_VOL_ASYNC_PACKAGE_NAME "HDF5_VOL_ASYNC") set(HDF5_VOL_ASYNC_PACKAGE_DESCRIPTION "HDF5 Asynchronous I/O VOL connector") From f9a9170448977d143f0288a5558330bb2a9b9543 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Fri, 31 May 2024 09:08:24 -0500 Subject: [PATCH 15/15] change to testing with latest HDF5 --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d11a670..e35a4bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,10 +44,10 @@ if(NOT HDF5_HAVE_THREADSAFE) message(FATAL_ERROR "HDF5 library needs to enable threadsafe support.") endif() -#Check if HDF5 version is 1.14 or greater +# Check if HDF5 version is 1.14 or greater if(HDF5_FOUND) if (HDF5_VERSION VERSION_LESS "1.14.0") - message (FATAL_ERROR "HDF5 ${HDF5_VERSION}: required HDF5 is at least 1.14 ") + message (FATAL_ERROR "HDF5 ${HDF5_VERSION}: required HDF5 version is at least 1.14 ") endif () endif () @@ -55,7 +55,7 @@ set(HDF5_VOL_ASYNC_PACKAGE "hdf5_vol_async") set(HDF5_VOL_ASYNC_PACKAGE_NAME "HDF5_VOL_ASYNC") set(HDF5_VOL_ASYNC_PACKAGE_DESCRIPTION "HDF5 Asynchronous I/O VOL connector") set(HDF5_VOL_ASYNC_PACKAGE_URL "https://github.com/HDFGroup/vol-async") -set(HDF5_VOL_ASYNC_PACKAGE_VENDOR "HPC IO") +set(HDF5_VOL_ASYNC_PACKAGE_VENDOR "HDF Group") #----------------------------------------------------------------------------- # Options