From 78cb4e1f1a5df1730bb10765f84971a603a1cdeb Mon Sep 17 00:00:00 2001
From: Mikael Simberg <mikael.simberg@iki.fi>
Date: Tue, 20 Feb 2024 09:53:39 +0100
Subject: [PATCH] Add thread sanitizer CI pipeline

---
 ci/.gitlab-ci.yml     | 37 +++++++++++++++++++------------------
 ci/cpu/tsan.yml       | 35 +++++++++++++++++++++++++++++++++++
 ci/ctest_to_gitlab.sh |  6 +++---
 ci/docker/tsan.yaml   | 33 +++++++++++++++++++++++++++++++++
 4 files changed, 90 insertions(+), 21 deletions(-)
 create mode 100644 ci/cpu/tsan.yml
 create mode 100644 ci/docker/tsan.yaml

diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml
index 0204a001ee..9030be2a84 100644
--- a/ci/.gitlab-ci.yml
+++ b/ci/.gitlab-ci.yml
@@ -1,19 +1,20 @@
 include:
-  - local: 'ci/cpu/asan_ubsan_lsan.yml'
-  - local: 'ci/cpu/clang15_release_cxx20.yml'
-  - local: 'ci/cpu/clang15_release_stdexec.yml'
-  - local: 'ci/cpu/clang15_release.yml'
-  - local: 'ci/cpu/clang16_release.yml'
-  - local: 'ci/cpu/clang18_release.yml'
-  - local: 'ci/cpu/gcc11_release_stdexec.yml'
-  - local: 'ci/cpu/gcc11_debug_stdexec.yml'
-  - local: 'ci/cpu/gcc12_release_cxx20.yml'
-  - local: 'ci/cpu/gcc13_codecov.yml'
-  - local: 'ci/cpu/gcc13_release.yml'
-  - local: 'ci/cuda/gcc11_release.yml'
-  - local: 'ci/cuda/gcc11_release_scalapack.yml'
-  - local: 'ci/cuda/gcc11_codecov.yml'
-  - local: 'ci/cuda/gcc11_debug_scalapack.yml'
-  - local: 'ci/cuda/gcc13_release_stdexec.yml'
-  - local: 'ci/rocm/clang14_release.yml'
-  - local: 'ci/rocm/clang14_release_stdexec.yml'
+  # - local: 'ci/cpu/asan_ubsan_lsan.yml'
+  - local: 'ci/cpu/tsan.yml'
+  # - local: 'ci/cpu/clang15_release_cxx20.yml'
+  # - local: 'ci/cpu/clang15_release_stdexec.yml'
+  # - local: 'ci/cpu/clang15_release.yml'
+  # - local: 'ci/cpu/clang16_release.yml'
+  # - local: 'ci/cpu/clang18_release.yml'
+  # - local: 'ci/cpu/gcc11_release_stdexec.yml'
+  # - local: 'ci/cpu/gcc11_debug_stdexec.yml'
+  # - local: 'ci/cpu/gcc12_release_cxx20.yml'
+  # - local: 'ci/cpu/gcc13_codecov.yml'
+  # - local: 'ci/cpu/gcc13_release.yml'
+  # - local: 'ci/cuda/gcc11_release.yml'
+  # - local: 'ci/cuda/gcc11_release_scalapack.yml'
+  # - local: 'ci/cuda/gcc11_codecov.yml'
+  # - local: 'ci/cuda/gcc11_debug_scalapack.yml'
+  # - local: 'ci/cuda/gcc13_release_stdexec.yml'
+  # - local: 'ci/rocm/clang14_release.yml'
+  # - local: 'ci/rocm/clang14_release_stdexec.yml'
diff --git a/ci/cpu/tsan.yml b/ci/cpu/tsan.yml
new file mode 100644
index 0000000000..e4f592f69c
--- /dev/null
+++ b/ci/cpu/tsan.yml
@@ -0,0 +1,35 @@
+include:
+  - local: 'ci/common-ci.yml'
+
+cpu tsan deps:
+  extends: .build_deps_common
+  variables:
+    EXTRA_APTGET: "clang-18 libclang-rt-18-dev libomp-18-dev"
+    COMPILER: clang@18
+    SPACK_ENVIRONMENT: ci/docker/tsan.yaml
+    USE_MKL: "ON"
+    BUILD_IMAGE: $CSCS_REGISTRY_PATH/cpu-tsan/deps
+
+cpu tsan build:
+  extends:
+    - .build_common
+    - .build_for_daint-mc
+  needs:
+    - cpu tsan deps
+  variables:
+    DLAF_IMAGE: $CSCS_REGISTRY_PATH/cpu-tsan/dlaf:$CI_COMMIT_SHA
+
+cpu tsan test:
+  extends: .run_common
+  needs:
+    - cpu tsan build
+  variables:
+    TSAN_OPTIONS: "halt_on_error=1:history_size=0"
+    PIKA_SMALL_STACK_SIZE: "0x100000"
+    PIKA_MEDIUM_STACK_SIZE: "0x100000"
+    # Override use of libSegFault, not necessary with sanitizers
+    LD_PRELOAD: ""
+  trigger:
+    include:
+      - artifact: pipeline.yml
+        job: cpu tsan build
diff --git a/ci/ctest_to_gitlab.sh b/ci/ctest_to_gitlab.sh
index cedf894f80..1398cf9ed8 100755
--- a/ci/ctest_to_gitlab.sh
+++ b/ci/ctest_to_gitlab.sh
@@ -61,7 +61,7 @@ JOB_TEMPLATE="
     USE_MPI: 'YES'
     DISABLE_AFTER_SCRIPT: 'YES'
     DLAF_HDF5_TEST_OUTPUT_PATH: \$CI_PROJECT_DIR
-  script: mpi-ctest -L {{LABEL}}
+  script: mpi-ctest --timeout 300 -E eigensolver -L {{LABEL}}
   artifacts:
     paths:
       - codecov-reports/"
@@ -93,14 +93,14 @@ JOB_TEMPLATE="
   variables:
     SLURM_CPUS_PER_TASK: {{CPUS_PER_TASK}}
     SLURM_NTASKS: {{NTASKS}}
-    SLURM_TIMELIMIT: '20:00'
+    SLURM_TIMELIMIT: '2:00:00'
     SLURM_UNBUFFEREDIO: 1
     SLURM_WAIT: 0
     PULL_IMAGE: 'YES'
     USE_MPI: 'YES'
     DISABLE_AFTER_SCRIPT: 'YES'
     DLAF_HDF5_TEST_OUTPUT_PATH: \$CI_PROJECT_DIR
-  script: mpi-ctest -L {{LABEL}}"
+  script: mpi-ctest --timeout 300 -E eigensolver -L {{LABEL}}"
 fi
 
 JOBS=""
diff --git a/ci/docker/tsan.yaml b/ci/docker/tsan.yaml
new file mode 100644
index 0000000000..6feff7b489
--- /dev/null
+++ b/ci/docker/tsan.yaml
@@ -0,0 +1,33 @@
+#
+# Distributed Linear Algebra with Future (DLAF)
+#
+# Copyright (c) 2018-2024, ETH Zurich
+# All rights reserved.
+#
+# Please, refer to the LICENSE file in the root directory.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+spack:
+  include:
+  - /spack_environment/common.yaml
+
+  view: false
+  concretizer:
+    unify:
+      true
+
+  specs:
+    - dla-future@master build_type=Debug cxxflags="-fsanitize=thread -fno-omit-frame-pointer" ldflags="-fsanitize=thread" +miniapps +ci-test
+
+  packages:
+    all:
+      variants:
+        - 'build_type=RelWithDebInfo'
+    pika:
+      require:
+        - 'build_type=Debug'
+        - 'malloc=system'
+        - '+sanitizers'
+        - 'cxxflags="-fsanitize=thread -fno-omit-frame-pointer"'
+        - 'ldflags="-fsanitize=thread"'