Run examples as part of the test suite

.gitlab-ci.yml

@@ -127,7 +127,6 @@ build/cuda101/openmpi/gcc/all/debug/shared:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
     BUILD_PAPI_SDE: "ON"
-    RUN_EXAMPLES: "ON"
     CUDA_ARCH: 35
 
 build/cuda101/nompi/clang/all/release/static:
@@ -404,7 +403,6 @@ build/amd/nompi/gcc/rocm45/release/shared:
     BUILD_OMP: "ON"
     BUILD_HIP: "ON"
     NONDEFAULT_STREAM: "ON"
-    RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Release"
     BUILD_HWLOC: "OFF"
 
@@ -418,7 +416,6 @@ build/amd/nompi/clang/rocm45/release/static:
     CXX_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_HIP: "ON"
-    RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
 
@@ -432,7 +429,6 @@ build/amd/nompi/clang/rocm45/debug/shared:
     CXX_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_HIP: "ON"
-    RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Debug"
 
 # ROCm 5.0.2 and friends
@@ -445,7 +441,6 @@ build/amd/nompi/gcc/rocm502/debug/static:
   variables:
     BUILD_OMP: "ON"
     BUILD_HIP: "ON"
-    RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Debug"
     BUILD_SHARED_LIBS: "OFF"
 
@@ -459,7 +454,6 @@ build/amd/nompi/clang/rocm502/release/shared:
     CXX_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_HIP: "ON"
-    RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Release"
 
 # without omp
@@ -473,7 +467,6 @@ build/amd/nompi/gcc/rocm502_wo_omp/release/shared:
     BUILD_OMP: "OFF"
     BUILD_MPI: "OFF"
     BUILD_HIP: "ON"
-    RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Release"
 
 # no cuda but latest gcc and clang

.gitlab/scripts.yml

@@ -50,7 +50,6 @@
         -DGINKGO_FAST_TESTS=${FAST_TESTS}
         -DGINKGO_TEST_NONDEFAULT_STREAM=${NONDEFAULT_STREAM}
         -DGINKGO_MIXED_PRECISION=${MIXED_PRECISION}
-        -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_DPCPP_SINGLE_MODE=${DPCPP_SINGLE_MODE}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
@@ -95,7 +94,6 @@
         -DGINKGO_MIXED_PRECISION=${MIXED_PRECISION}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_DPCPP_SINGLE_MODE=${DPCPP_SINGLE_MODE}
-        -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
     - awk '!/^#/ { print ($2 - $1)/1000 " " $4 }' .ninja_log | sort -nr
     - |
@@ -106,27 +104,6 @@
     - pushd test/test_install
     - ninja install
     - popd
-    - |
-      if [ "${RUN_EXAMPLES}" == "ON" ]; then
-         export EX_ARG="reference"
-         ninja run_all_examples
-         ninja validate_all_examples
-         if [ "{BUILD_OMP}" == "ON" ]; then
-            export EX_ARG="omp"
-            ninja run_all_examples
-            ninja validate_all_examples
-        fi
-        if [ "{BUILD_CUDA}" == "ON" ]; then
-            export EX_ARG="cuda"
-            ninja run_all_examples
-            ninja validate_all_examples
-        fi
-        if [ "{BUILD_HIP}" == "ON" ]; then
-            export EX_ARG="hip"
-            ninja run_all_examples
-            ninja validate_all_examples
-        fi
-      fi
     - if [ -n "${SYCL_DEVICE_TYPE}" ]; then unset SYCL_DEVICE_TYPE; fi
     - if [ -n "${SYCL_DEVICE_FILTER}" ]; then unset SYCL_DEVICE_FILTER; fi
     - PKG_CONFIG_PATH=${INSTALL_PREFIX}/lib/pkgconfig:$PKG_CONFIG_PATH LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:$LD_LIBRARY_PATH ninja test_pkgconfig

.gitlab/variables.yml

@@ -19,7 +19,6 @@
     NONDEFAULT_STREAM: "OFF"
     DPCPP_SINGLE_MODE: "OFF"
     MIXED_PRECISION: "ON"
-    RUN_EXAMPLES: "OFF"
     CONFIG_LOG: "ON"
     CXX_FLAGS: "-Wpedantic"
     EXTRA_CMAKE_FLAGS: ""

examples/CMakeLists.txt

@@ -1,5 +1,4 @@
 option(GINKGO_BUILD_EXTLIB_EXAMPLE "Build the external-lib-interfacing with deal.II, you need to link the deal.II library." OFF)
-option(GINKGO_RUN_EXAMPLES " Compile run and validation targets for the examples." ON)
 
 set(EXAMPLES_EXEC_LIST
     adaptiveprecision-blockjacobi
@@ -73,24 +72,39 @@ foreach(example ${EXAMPLES_LIST})
     add_subdirectory(${example})
 endforeach()
 
-if(GINKGO_RUN_EXAMPLES)
-    foreach(example ${EXAMPLES_LIST})
-        set(example_path "${CMAKE_CURRENT_BINARY_DIR}/${example}")
-        file(WRITE ${example_path}/target-wrapper.sh "${example_path}/${example} \$\{EX_ARG\}")
-        add_custom_target("run-${example}"
-            COMMAND chmod +x ${example_path}/target-wrapper.sh && ${example_path}/target-wrapper.sh > ${example_path}/${example}.out
-            WORKING_DIRECTORY ${example_path})
-        file(WRITE ${example_path}/diff-command "#!/bin/bash
-diff <(sed -n '8,$p' ${example_path}/${example}.out | sed -E 's/([^a-z,\":\\s\\)\\(\\{\\}_]+)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([^a-z,\":\\s\\)\\(\\{\\}_]+)//g')")
-        add_custom_target("validate-${example}"
-            COMMAND chmod +x ${example_path}/diff-command && ${example_path}/diff-command
-            WORKING_DIRECTORY ${example_path})
-    endforeach()
-
-    add_custom_target(run_all_examples)
-    add_custom_target(validate_all_examples)
-    foreach(run_ex ${EXAMPLES_EXEC_LIST})
-        add_dependencies(run_all_examples "run-${run_ex}")
-        add_dependencies(validate_all_examples "validate-${run_ex}")
+if(GINKGO_BUILD_TESTS)
+    find_package(Python3 COMPONENTS Interpreter REQUIRED)
+    set(executors)
+    if(GINKGO_BUILD_REFERENCE)
+        list(APPEND executors reference)
+    endif()
+    # OpenMP support is needed for some host kernels
+    if(GINKGO_BUILD_CUDA AND GINKGO_BUILD_OMP)
+        list(APPEND executors cuda)
+    endif()
+    if(GINKGO_BUILD_HIP AND GINKGO_BUILD_OMP)
+        list(APPEND executors hip)
+    endif()
+    # DPC++ support is incomplete
+    #if(GINKGO_BUILD_SYCL)
+    #    list(APPEND executors dpcpp)
+    #endif()
+    if(GINKGO_BUILD_OMP)
+        list(APPEND executors omp)
+    endif()
+    foreach(example IN LISTS EXAMPLES_EXEC_LIST)
+        foreach(executor IN LISTS executors)
+            add_test(NAME example_${example}_${executor}
+                     COMMAND
+                     ${Python3_EXECUTABLE}
+                     "${CMAKE_CURRENT_SOURCE_DIR}/compare-output.py"
+                     "${CMAKE_CURRENT_BINARY_DIR}/${example}"
+                     "$<TARGET_FILE_DIR:ginkgo>"
+                     "${CMAKE_CURRENT_SOURCE_DIR}/${example}/doc/results.dox"
+                     "$<TARGET_FILE:${example}>"
+                     "${executor}"
+                     WORKING_DIRECTORY
+                     "${CMAKE_CURRENT_SOURCE_DIR}/${example}")
+        endforeach()
     endforeach()
 endif()

examples/compare-output.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+import re
+import os
+import difflib
+
+if len(sys.argv) < 5:
+    print(
+        "Usage: compare-output.py <working-dir> <path-to-ginkgo-lib> <reference-file> <full-path-to-executable> <args>..."
+    )
+    sys.exit(1)
+
+cwd = sys.argv[1]
+ginkgo_path = sys.argv[2]
+reference = open(sys.argv[3]).readlines()
+args = sys.argv[4:]
+env = os.environ.copy()
+if os.name == "nt":
+    # PATH is hopefully never empty
+    env["PATH"] += ";" + ginkgo_path
+else:
+    if "LD_LIBRARY_PATH" in env:
+        env["LD_LIBRARY_PATH"] += ":" + ginkgo_path
+    else:
+        env["LD_LIBRARY_PATH"] = ginkgo_path
+
+result = subprocess.run(
+    args=args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env
+)
+
+stdout = result.stdout.decode().splitlines()
+stderr = result.stderr.decode().splitlines()
+
+if len(stderr) > 0:
+    print("FAIL: stderr not empty")
+    print("".join(stderr))
+    sys.exit(1)
+
+# skip header/footer
+stdout = stdout[7:]
+reference = reference[5:-4]
+# remove everything that looks like a number
+remove_pattern = "[-+0-9.]+(e[-+0-9]+)?"
+stdout = [re.sub(remove_pattern, "", line.strip()) for line in stdout]
+reference = [re.sub(remove_pattern, "", line.strip()) for line in reference]
+# compare result
+if stdout != reference:
+    print("FAIL: stdout differs")
+    print("\n".join(difflib.unified_diff(reference, stdout)))
+    sys.exit(1)
+print("PASS")