Test h5migrate in CI

.github/workflows/ci.yml

@@ -130,7 +130,7 @@ jobs:
           sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev
           if test "${{ matrix.os.dist }}" = 'ubuntu-22.04'
           then
-            sudo apt --yes --no-install-recommends install meson
+            sudo apt --yes --no-install-recommends install meson python3 python3-pip python3-setuptools python3-wheel
           else
             sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel
             sudo pip3 install meson
@@ -342,6 +342,22 @@ jobs:
           sleep 10
           ./scripts/test.sh -r /hdf5
           ./scripts/setup.sh stop
+      - name: Test julea-h5migrate
+        if: false
+        env:
+          LSAN_OPTIONS: exitcode=0
+        run: |
+          . scripts/environment.sh
+          pip3 -qqq install h5py numpy
+          python3 scripts/gen_hdf5_test_file.py
+          ./scripts/setup.sh start
+          HDF5_VOL_CONNECTOR=julea-db julea-h5migrate testfile.h5 julea-db://imported.h5
+          HDF5_VOL_CONNECTOR=julea-db julea-h5migrate julea-db://imported.h5 exported.h5
+          # tail cuts the filename
+          h5dump testfile.h5 | tail -n+2 > original
+          h5dump exported.h5 | tail -n+2 > exported 
+          diff original exported 
+          ./scripts/setup.sh stop
   doxygen:
     name: Doxygen
     runs-on: ubuntu-22.04

scripts/gen_hdf5_test_file.py

@@ -0,0 +1,51 @@
+#!/bin/env python3
+
+# JULEA - Flexible storage framework
+# Copyright (C) 2023 Timm Leon Erxleben
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# This is a simple script to generate HDF5 files for tests.
+
+import h5py
+import numpy as np
+
+rng = np.random.default_rng()
+
+f = h5py.File('testfile.h5', 'w')
+
+# create some groups
+g1 = f.create_group("g1")
+g1_1 = g1.create_group("g1_1")
+g2 = f.create_group("g2")
+
+# create some datasets with different data types and sizes
+f["2D-f64"] = rng.random(size=(10, 8), dtype=np.float64)
+g1["3D-f64"] = rng.random(size=(5, 10, 3), dtype=np.float64)
+g1_1["1D-f32"] = rng.random(size=(10), dtype=np.float32)
+g2["0D-int"] = rng.integers(0, 500, dtype=np.uint32)
+g2["4D-int64"] = rng.integers(0, 500, size=(2,2,2,2), dtype=np.int64)
+g1["4D-int32"] = rng.integers(0, 500, size=(2,2,2,2), dtype=np.int32)
+g2["string-set"] = np.bytes_("I am a fixed size string!")
+#2["var-string-set"] = "I am a variable sized string!"
+
+# create some attributes
+f.attrs["file-attr"] = 42
+#g1.attrs["group1-attr"] = "I am a group attribute and a variable sized string"
+g1.attrs["group1-attr-fixed-str"] = np.bytes_("I am a group attribute and a fixed size string")
+g2.attrs["group2-attr"] = rng.random(size=(2,2))
+g2["string-set"].attrs["dset-attr"] = 3.14159
+
+f.close()
+

scripts/spack

@@ -156,7 +156,9 @@ spack_get_dependencies ()
 
 	if test -n "${CI}"
 	then
+		dependencies="${dependencies} python"
 		dependencies="${dependencies} py-gcovr"
+		dependencies="${dependencies} py-pip"
 	fi
 
 	#dependencies="${dependencies} enzo@main"