Don't require bit reproducable for bump on tail and landau restart tests

Set an absolute and relative tolerance of 1e-14 on parallel tests (OpenMP and Cuda). The tolerance is set to 0.0 for serial tests. This is a stop-gap solution until a physically relevant tolerance can be found. See merge request gysela-developpers/gyselalibxx!454 -------------------------------------------- Co-authored-by: Emily Bourne <[email protected]>
gyselax · Apr 16, 2024 · e2a284b · e2a284b
1 parent c9095c0
commit e2a284b
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 35 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -6,6 +6,9 @@ cmake_tests:
   extends: .cmake_tests
   image: ghcr.io/gyselax/gyselalibxx_env
   timeout: 2 hours
+  variables:
+    RELATIVE_RESTART_TOLERANCE: '0.0'
+    ABSOLUTE_RESTART_TOLERANCE: '0.0'
   script: |
     set -x
     git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
@@ -24,6 +27,9 @@ cmake_tests:
 cmake_tests_Release:
   extends: .cmake_tests
   image: ghcr.io/gyselax/gyselalibxx_env
+  variables:
+    RELATIVE_RESTART_TOLERANCE: '0.0'
+    ABSOLUTE_RESTART_TOLERANCE: '0.0'
   script: |
     set -x
     git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
@@ -47,6 +53,9 @@ cmake_tests_Release:
 cmake_tests_Release_OMP:
   extends: .cmake_tests
   image: ghcr.io/gyselax/gyselalibxx_env
+  variables:
+    RELATIVE_RESTART_TOLERANCE: '1e-14'
+    ABSOLUTE_RESTART_TOLERANCE: '1e-14'
   script: |
     set -x
     git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
@@ -71,6 +80,9 @@ cmake_tests_Release_GPU:
   extends: .cmake_tests
   tags:
     - persee
+  variables:
+    RELATIVE_RESTART_TOLERANCE: '1e-14'
+    ABSOLUTE_RESTART_TOLERANCE: '1e-14'
   before_script: |
     . /data/gyselarunner/spack-0.20.0/share/spack/setup-env.sh
     spack load gcc@11

diff --git a/post-process/PythonScripts/compare_hdf5_results.py b/post-process/PythonScripts/compare_hdf5_results.py
@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: MIT
+""" Compare HDF5 results between two files.
+"""
+from argparse import ArgumentParser
+import h5py as h5
+
+import numpy as np
+
+if __name__ == '__main__':
+    parser = ArgumentParser(
+        description='Compare HDF5 results between two files')
+    parser.add_argument('file1',
+                        type=h5.File,
+                        help='File name of the first HDF5 file')
+    parser.add_argument('file2',
+                        type=h5.File,
+                        help='File name of the second HDF5 file')
+    parser.add_argument('obj',
+                        type=str,
+                        help='Name of an HDF5 object, in absolute path')
+    parser.add_argument('-R', '--relative',
+            nargs='?',
+            type=float,
+            metavar='TOL',
+            help='The tolerance for the relative difference')
+    parser.add_argument('-A', '--absolute',
+            nargs='?',
+            type=float,
+            metavar='TOL',
+            help='The tolerance for the absolute difference')
+
+    args = parser.parse_args()
+
+    vals1 = np.array(args.file1[args.obj])
+    vals2 = np.array(args.file2[args.obj])
+
+    abs_error = np.abs(vals1 - vals2)
+    rel_error = abs_error / np.abs(vals2)
+
+    print("Maximum absolute error found : ", abs_error.max())
+    print("Maximum relative error found : ", rel_error.max())
+
+    args.file1.close()
+    args.file2.close()
+
+    if args.relative:
+        assert args.absolute
+        assert np.allclose(vals1, vals2, rtol=args.relative, atol=args.absolute)
+    else:
+        assert np.array_equal(vals1, vals2)
diff --git a/tests/geometryXVx/bump_on_tail/CMakeLists.txt b/tests/geometryXVx/bump_on_tail/CMakeLists.txt
@@ -1,7 +1,5 @@
 # SPDX-License-Identifier: MIT
 
-cmake_minimum_required(VERSION 3.15)
-
 find_package(Python3 REQUIRED COMPONENTS Interpreter)
 
 add_test(NAME TestSimulationBumpontailFFT
@@ -17,6 +15,7 @@ add_test(NAME TestSimulationBumpontailRestartFFT_XVx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_bumpontail_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:bumpontail_fft>"
+        "$<TARGET_FILE:Python3::Interpreter>"
         "restart")
 set_property(TEST TestSimulationBumpontailRestartFFT_XVx PROPERTY TIMEOUT 200)
 set_property(TEST TestSimulationBumpontailRestartFFT_XVx PROPERTY COST 100)
@@ -35,6 +34,7 @@ add_test(NAME TestSimulationBumpontailRestartFemUniform_xperiod_vx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_bumpontail_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:bumpontail_fem_uniform_xperiod_vx>"
+        "$<TARGET_FILE:Python3::Interpreter>"
         "restart")
 set_property(TEST TestSimulationBumpontailRestartFemUniform_xperiod_vx PROPERTY TIMEOUT 200)
 set_property(TEST TestSimulationBumpontailRestartFemUniform_xperiod_vx PROPERTY COST 100)

diff --git a/tests/geometryXVx/bump_on_tail/test_bumpontail_restart.sh b/tests/geometryXVx/bump_on_tail/test_bumpontail_restart.sh
@@ -1,14 +1,15 @@
 #!/bin/bash
 set -xe
 
-if [ $# -ne 3 ]
+if [ $# -ne 4 ]
 then
-    echo "Usage: $0 <VOICEXX_SRCDIR> <VOICEXX_EXEC> <SIMULATION_NAME>"
+    echo "Usage: $0 <VOICEXX_SRCDIR> <VOICEXX_EXEC> <PYTHON3_EXE> <SIMULATION_NAME>"
     exit 1
 fi
 VOICEXX_SRCDIR="$1"
 VOICEXX_EXEC="$2"
-SIMULATION_NAME="$3"
+PYTHON3_EXE="$3"
+SIMULATION_NAME="$4"
 
 OUTDIR="${PWD}/${SIMULATION_NAME}"
 
@@ -45,21 +46,18 @@ cp "${RSTDIR}/bumpontail.yaml" bumpontail_restart.yaml
 sed -i 's/^  nbiter: .*/  nbiter: 2/' bumpontail_restart.yaml
 sed -i 's/^  time_diag: .*/  time_diag: 0.5/' bumpontail_restart.yaml
 
-h5ls -d ${PWD}/VOICEXX_00006.h5/time_saved ${RSTDIR}/VOICEXX_00005.h5/time_saved
-command="h5diff ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 time_saved"
-eval $command
+h5ls -d ${PWD}/VOICEXX_00005.h5/time_saved ${RSTDIR}/VOICEXX_00005.h5/time_saved
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 time_saved -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
 fi
 
-command="h5diff ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 electrostatic_potential"
-eval $command
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 electrostatic_potential -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
 fi
 
-command="h5diff ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 fdistribu"
-eval $command
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 fdistribu -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
-fi
+fi
diff --git a/tests/geometryXVx/landau/CMakeLists.txt b/tests/geometryXVx/landau/CMakeLists.txt
@@ -1,7 +1,5 @@
 # SPDX-License-Identifier: MIT
 
-cmake_minimum_required(VERSION 3.15)
-
 find_package(Python3 REQUIRED COMPONENTS Interpreter)
 
 add_test(NAME TestSimulationLandauFFT_XVx
@@ -17,6 +15,7 @@ add_test(NAME TestSimulationLandauRestartFFT_XVx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_landau_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:landau_fft>"
+        "$<TARGET_FILE:Python3::Interpreter>"
         "restart")
 set_property(TEST TestSimulationLandauRestartFFT_XVx PROPERTY TIMEOUT 200)
 set_property(TEST TestSimulationLandauRestartFFT_XVx PROPERTY COST 100)
@@ -35,6 +34,7 @@ add_test(NAME TestSimulationLandauRestartFemUniform_xperiod_vx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_landau_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:landau_fem_uniform_xperiod_vx>"
+        "$<TARGET_FILE:Python3::Interpreter>"
         "restart")
 set_property(TEST TestSimulationLandauRestartFemUniform_xperiod_vx PROPERTY TIMEOUT 200)
 set_property(TEST TestSimulationLandauRestartFemUniform_xperiod_vx PROPERTY COST 100)

diff --git a/tests/geometryXVx/landau/test_landau_restart.sh b/tests/geometryXVx/landau/test_landau_restart.sh
@@ -1,14 +1,15 @@
 #!/bin/bash
 set -xe
 
-if [ $# -ne 3 ]
+if [ $# -ne 4 ]
 then
-    echo "Usage: $0 <VOICEXX_SRCDIR> <VOICEXX_EXEC> <SIMULATION_NAME>"
+    echo "Usage: $0 <VOICEXX_SRCDIR> <VOICEXX_EXEC> <PYTHON3_EXE> <SIMULATION_NAME>"
     exit 1
 fi
 VOICEXX_SRCDIR="$1"
 VOICEXX_EXEC="$2"
-SIMULATION_NAME="$3"
+PYTHON3_EXE="$3"
+SIMULATION_NAME="$4"
 
 OUTDIR="${PWD}/${SIMULATION_NAME}"
 
@@ -45,20 +46,17 @@ cp "${RSTDIR}/landau.yaml" landau_restart.yaml
 sed -i 's/^  nbiter: .*/  nbiter: 2/' landau_restart.yaml
 sed -i 's/^  time_diag: .*/  time_diag: 0.5/' landau_restart.yaml
 
-command="h5diff ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 time_saved"
-eval $command
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 time_saved -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
 fi
 
-command="h5diff ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 electrostatic_potential"
-eval $command
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 electrostatic_potential -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
 fi
 
-command="h5diff ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 fdistribu"
-eval $command
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 fdistribu -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
-fi
+fi
diff --git a/tests/geometryXVx/sheath/CMakeLists.txt b/tests/geometryXVx/sheath/CMakeLists.txt
@@ -1,11 +1,12 @@
 # SPDX-License-Identifier: MIT
 
-cmake_minimum_required(VERSION 3.15)
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
 
 add_test(NAME TestSimulationSheathRestart_xperiod_vx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_sheath_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:sheath_xperiod_vx>"
+        "$<TARGET_FILE:Python3::Interpreter>"
         "restart")
 set_property(TEST TestSimulationSheathRestart_xperiod_vx PROPERTY TIMEOUT 200)
 set_property(TEST TestSimulationSheathRestart_xperiod_vx PROPERTY COST 100)

diff --git a/tests/geometryXVx/sheath/test_sheath_restart.sh b/tests/geometryXVx/sheath/test_sheath_restart.sh
@@ -1,14 +1,15 @@
 #!/bin/bash
 set -xe
 
-if [ $# -ne 3 ]
+if [ $# -ne 4 ]
 then
-    echo "Usage: $0 <VOICEXX_SRCDIR> <VOICEXX_EXEC> <SIMULATION_NAME>"
+    echo "Usage: $0 <VOICEXX_SRCDIR> <VOICEXX_EXEC> <PYTHON3_EXE> <SIMULATION_NAME>"
     exit 1
 fi
 VOICEXX_SRCDIR="$1"
 VOICEXX_EXEC="$2"
-SIMULATION_NAME="$3"
+PYTHON3_EXE="$3"
+SIMULATION_NAME="$4"
 
 OUTDIR="${PWD}/${SIMULATION_NAME}"
 
@@ -46,20 +47,17 @@ sed -i 's/^  nbiter: .*/  nbiter: 2/' sheath_restart.yaml
 sed -i 's/^  time_diag: .*/  time_diag: 0.5/' sheath_restart.yaml
 
 h5ls -d ${PWD}/VOICEXX_00005.h5/time_saved ${RSTDIR}/VOICEXX_00005.h5/time_saved
-command="h5diff --use-system-epsilon -r ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 time_saved"
-eval $command
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 time_saved -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
 fi
 
-command="h5diff --use-system-epsilon -r ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 electrostatic_potential"
-eval $command
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 electrostatic_potential -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
 fi
 
-command="h5diff --use-system-epsilon -r ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 fdistribu"
-eval $command
+${PYTHON3_EXE} ${VOICEXX_SRCDIR}/post-process/PythonScripts/compare_hdf5_results.py ${PWD}/VOICEXX_00005.h5 ${RSTDIR}/VOICEXX_00005.h5 fdistribu -R ${RELATIVE_RESTART_TOLERANCE} -A ${ABSOLUTE_RESTART_TOLERANCE}
 if [ $? -ne 0 ]; then
     exit 1
 fi