Add [email protected] % [email protected] to exp/cpu/dev

Note, however, the Spack package for metis (and parmetis) needs to be
updated to use the latest patched versions of [email protected] from the PETSc
project's fork for a number of reasons [1].

[1] awslabs/palace#170

etc/spack/sdsc/exp/cpu/dev/specs/gcc@13.3.0/metis@5.1.0.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+#SBATCH [email protected]
+#SBATCH --account=use300
+#SBATCH --clusters=expanse
+#SBATCH --partition=ind-shared
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=16
+#SBATCH --mem=32G
+#SBATCH --time=00:30:00
+#SBATCH --output=%x.o%j.%N
+
+declare -xir UNIX_TIME="$(date +'%s')"
+declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')"
+
+declare -xr JOB_SCRIPT="$(scontrol show job ${SLURM_JOB_ID} | awk -F= '/Command=/{print $2}')"
+declare -xr JOB_SCRIPT_MD5="$(md5sum ${JOB_SCRIPT} | awk '{print $1}')"
+declare -xr JOB_SCRIPT_SHA256="$(sha256sum ${JOB_SCRIPT} | awk '{print $1}')"
+declare -xr JOB_SCRIPT_NUMBER_OF_LINES="$(wc -l ${JOB_SCRIPT} | awk '{print $1}')"
+
+declare -xr SCHEDULER_NAME='slurm'
+declare -xr SCHEDULER_MAJOR='23'
+declare -xr SCHEDULER_MINOR='02'
+declare -xr SCHEDULER_REVISION='7'
+declare -xr SCHEDULER_VERSION="${SCHEDULER_MAJOR}.${SCHEDULER_MINOR}.${SCHEDULER_REVISION}"
+declare -xr SCHEDULER_MODULE="${SCHEDULER_NAME}/${SLURM_CLUSTER_NAME}/${SCHEDULER_VERSION}"
+
+declare -xr SPACK_MAJOR='0'
+declare -xr SPACK_MINOR='21'
+declare -xr SPACK_REVISION='2'
+declare -xr SPACK_VERSION="${SPACK_MAJOR}.${SPACK_MINOR}.${SPACK_REVISION}"
+declare -xr SPACK_INSTANCE_NAME='cpu'
+declare -xr SPACK_INSTANCE_VERSION='dev'
+declare -xr SPACK_INSTANCE_DIR='/home/mkandes/software/spack/repos/sdsc/spack'
+
+declare -xr TMPDIR="${SLURM_TMPDIR}/spack-stage"
+declare -xr TMP="${TMPDIR}"
+
+echo "${UNIX_TIME} ${LOCAL_TIME} ${SLURM_JOB_ID} ${JOB_SCRIPT_MD5} ${JOB_SCRIPT_SHA256} ${JOB_SCRIPT_NUMBER_OF_LINES} ${JOB_SCRIPT}"
+cat  "${JOB_SCRIPT}"
+
+module purge
+module load "${SCHEDULER_MODULE}"
+module list
+. "${SPACK_INSTANCE_DIR}/share/spack/setup-env.sh"
+
+declare -xr SPACK_PACKAGE='[email protected]'
+declare -xr SPACK_COMPILER='[email protected]'
+declare -xr SPACK_VARIANTS='~gdb ~int64 ~real64 +shared'
+declare -xr SPACK_DEPENDENCIES=''
+declare -xr SPACK_SPEC="${SPACK_PACKAGE} % ${SPACK_COMPILER} ${SPACK_VARIANTS} ${SPACK_DEPENDENCIES}"
+
+printenv
+
+spack config get compilers  
+spack config get config  
+spack config get mirrors
+spack config get modules
+spack config get packages
+spack config get repos
+spack config get upstreams
+
+time -p spack spec --long --namespaces --types --reuse "$(echo ${SPACK_SPEC})"
+if [[ "${?}" -ne 0 ]]; then
+  echo 'ERROR: spack concretization failed.'
+  exit 1
+fi
+
+mkdir -p "${TMPDIR}"
+
+time -p spack install --jobs "${SLURM_CPUS_PER_TASK}" --fail-fast --yes-to-all --reuse "$(echo ${SPACK_SPEC})"
+if [[ "${?}" -ne 0 ]]; then
+  echo 'ERROR: spack install failed.'
+  exit 1
+fi

var/spack/repos/sdsc/packages/metis/gklib_nomisleadingindentation_warning.patch

@@ -0,0 +1,12 @@
+diff -Naur metis-5.1.0/GKlib/GKlibSystem.cmake metis-5.1.0.patched/GKlib/GKlibSystem.cmake
+--- metis-5.1.0/GKlib/GKlibSystem.cmake	2013-03-30 12:24:45.000000000 -0400
++++ metis-5.1.0.patched/GKlib/GKlibSystem.cmake	2018-12-07 16:22:58.491427261 -0500
+@@ -33,7 +33,7 @@
+       set(GKlib_COPTIONS "${GKlib_COPTIONS} -fPIC")
+   endif(NOT MINGW)
+ # GCC warnings.
+-  set(GKlib_COPTIONS "${GKlib_COPTIONS} -Wall -pedantic -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unknown-pragmas")
++  set(GKlib_COPTIONS "${GKlib_COPTIONS} -Wall -pedantic -Wno-misleading-indentation -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unknown-pragmas")
+ elseif(${CMAKE_C_COMPILER_ID} MATCHES "Sun")
+ # Sun insists on -xc99.
+   set(GKlib_COPTIONS "${GKlib_COPTIONS} -xc99")

var/spack/repos/sdsc/packages/metis/gklib_path.patch

@@ -0,0 +1,11 @@
+--- a/CMakeLists.txt	2022-07-20 21:17:20.352231603 +0200
++++ b/CMakeLists.txt	2022-07-20 21:19:28.998269385 +0200
+@@ -1,7 +1,7 @@
+ cmake_minimum_required(VERSION 2.8)
+ project(METIS)
+
+-set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
++set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/GKlib" CACHE PATH "path to GKlib")
+ set(SHARED FALSE CACHE BOOL "build a shared library")
+
+ if(MSVC)

var/spack/repos/sdsc/packages/metis/install_gklib_defs_rename.patch

@@ -0,0 +1,22 @@
+# HG changeset patch
+# User Sean Farley <[email protected]>
+# Date 1332269671 18000
+#      Tue Mar 20 13:54:31 2012 -0500
+# Node ID b95c0c2e1d8bf8e3273f7d45e856f0c0127d998e
+# Parent  88049269953c67c3fdcc4309bf901508a875f0dc
+cmake: add gklib headers to install into include
+
+diff -r 88049269953c -r b95c0c2e1d8b libmetis/CMakeLists.txt
+Index: libmetis/CMakeLists.txt
+===================================================================
+--- a/libmetis/CMakeLists.txt Tue Mar 20 13:54:29 2012 -0500
++++ b/libmetis/CMakeLists.txt Tue Mar 20 13:54:31 2012 -0500
+@@ -12,6 +12,8 @@ endif()
+ if(METIS_INSTALL)
+   install(TARGETS metis
+     LIBRARY DESTINATION lib
+     RUNTIME DESTINATION lib
+     ARCHIVE DESTINATION lib)
++  install(FILES gklib_defs.h DESTINATION include)
++  install(FILES gklib_rename.h DESTINATION include)
+ endif()

var/spack/repos/sdsc/packages/metis/package.py

@@ -0,0 +1,237 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+import sys
+
+import spack.build_systems.cmake
+import spack.build_systems.makefile
+from spack.package import *
+
+
+class Metis(CMakePackage, MakefilePackage):
+    """METIS is a set of serial programs for partitioning graphs, partitioning
+    finite element meshes, and producing fill reducing orderings for sparse
+    matrices.
+
+    The algorithms implemented in METIS are based on the multilevel
+    recursive-bisection, multilevel k-way, and multi-constraint partitioning schemes.
+    """
+
+    homepage = "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
+    url = "https://bitbucket.org/petsc/pkg-metis/get/v5.1.0.tar.gz"
+
+    # not a metis developer, just package reviewer!
+    maintainers("mthcrts")
+
+    version("5.1.0", sha256="414bbda33b0f71194dbc15d3e93c269e9e0c92ba010f32546376fa91366541d5")
+
+    build_system(
+        conditional("cmake", when="@5:"), conditional("makefile", when="@:4"), default="cmake"
+    )
+    variant("shared", default=True, description="Build shared libraries")
+    with when("build_system=cmake"):
+        variant("gdb", default=False, description="Enable gdb support")
+        variant("int64", default=False, description="Use index type of 64 bit")
+        variant("real64", default=False, description="Use real type of 64 bit")
+
+        # Use the correct path to GKLIB when building out of source
+        patch("gklib_path.patch")
+        # Install both gklib_defs.h and gklib_rename.h
+        patch("install_gklib_defs_rename.patch")
+        # Disable the "misleading indentation" warning when compiling
+        patch("gklib_nomisleadingindentation_warning.patch", when="%gcc@6:")
+
+    with when("build_system=makefile"):
+        variant("debug", default=False, description="Compile in debug mode")
+
+    def patch(self):
+        if not self.spec.satisfies("build_system=cmake"):
+            return
+
+        source_path = self.stage.source_path
+        metis_header = FileFilter(join_path(source_path, "include", "metis.h"))
+
+        metis_header.filter(
+            r"(\b)(IDXTYPEWIDTH )(\d+)(\b)",
+            r"\1\2{0}\4".format("64" if "+int64" in self.spec else "32"),
+        )
+        metis_header.filter(
+            r"(\b)(REALTYPEWIDTH )(\d+)(\b)",
+            r"\1\2{0}\4".format("64" if "+real64" in self.spec else "32"),
+        )
+
+        # Make clang 7.3 happy.
+        # Prevents "ld: section __DATA/__thread_bss extends beyond end of file"
+        # See upstream LLVM issue https://llvm.org/bugs/show_bug.cgi?id=27059
+        # and https://github.com/Homebrew/homebrew-science/blob/master/metis.rb
+        if self.spec.satisfies("%[email protected]"):
+            filter_file(
+                "#define MAX_JBUFS 128",
+                "#define MAX_JBUFS 24",
+                join_path(source_path, "GKlib", "error.c"),
+            )
+
+
+class SetupEnvironment:
+    def setup_build_environment(self, env):
+        # Ignore warnings/errors re unrecognized omp pragmas on %intel
+        if "%intel@14:" in self.spec:
+            env.append_flags("CFLAGS", "-diag-disable 3180")
+        # Ignore some warnings to get it to compile with %nvhpc
+        #   111: statement is unreachable
+        #   177: variable "foo" was declared but never referenced
+        #   188: enumerated type mixed with another type
+        #   550: variable "foo" was set but never used
+        if "%nvhpc" in self.spec:
+            env.append_flags("CFLAGS", "--display_error_number")
+            env.append_flags("CFLAGS", "--diag_suppress 111")
+            env.append_flags("CFLAGS", "--diag_suppress 177")
+            env.append_flags("CFLAGS", "--diag_suppress 188")
+            env.append_flags("CFLAGS", "--diag_suppress 550")
+
+
+class MakefileBuilder(spack.build_systems.makefile.MakefileBuilder, SetupEnvironment):
+    @property
+    def build_targets(self):
+        options = []
+        if "+shared" in self.spec:
+            options.append("COPTIONS={0}".format(self.pkg.compiler.cc_pic_flag))
+        if "+debug" in self.spec:
+            options.append("OPTFLAGS=-g -O0")
+        return options
+
+    def install(self, pkg, spec, prefix):
+        # Compile and install library files
+        ccompile = Executable(pkg.compiler.cc)
+
+        mkdir(prefix.bin)
+        binfiles = (
+            "pmetis",
+            "kmetis",
+            "oemetis",
+            "onmetis",
+            "partnmesh",
+            "partdmesh",
+            "mesh2nodal",
+            "mesh2dual",
+            "graphchk",
+        )
+        for binfile in binfiles:
+            install(binfile, prefix.bin)
+
+        mkdir(prefix.lib)
+        install("libmetis.a", prefix.lib)
+
+        mkdir(prefix.include)
+        install(join_path("Lib", "*.h"), prefix.include)
+
+        mkdir(prefix.share)
+        sharefiles = (
+            ("Graphs", "4elt.graph"),
+            ("Graphs", "metis.mesh"),
+            ("Graphs", "test.mgraph"),
+        )
+        for sharefile in tuple(join_path(*sf) for sf in sharefiles):
+            install(sharefile, prefix.share)
+
+        if "+shared" in spec:
+            shared_flags = [pkg.compiler.cc_pic_flag, "-shared"]
+            if sys.platform == "darwin":
+                shared_suffix = "dylib"
+                shared_flags.extend(["-Wl,-all_load", "libmetis.a"])
+            else:
+                shared_suffix = "so"
+                shared_flags.extend(["-Wl,-whole-archive", "libmetis.a", "-Wl,-no-whole-archive"])
+
+            shared_out = "%s/libmetis.%s" % (prefix.lib, shared_suffix)
+            shared_flags.extend(["-o", shared_out])
+
+            ccompile(*shared_flags)
+
+        # Set up and run tests on installation
+        ccompile(
+            "-I%s" % prefix.include,
+            "-L%s" % prefix.lib,
+            (pkg.compiler.cc_rpath_arg + prefix.lib if "+shared" in spec else ""),
+            join_path("Programs", "io.o"),
+            join_path("Test", "mtest.c"),
+            "-o",
+            "%s/mtest" % prefix.bin,
+            "-lmetis",
+            "-lm",
+        )
+
+    def check(self):
+        test_bin = lambda testname: join_path(prefix.bin, testname)
+        test_graph = lambda graphname: join_path(prefix.share, graphname)
+
+        graph = test_graph("4elt.graph")
+        os.system("%s %s" % (test_bin("mtest"), graph))
+        os.system("%s %s 40" % (test_bin("kmetis"), graph))
+        os.system("%s %s" % (test_bin("onmetis"), graph))
+        graph = test_graph("test.mgraph")
+        os.system("%s %s 2" % (test_bin("pmetis"), graph))
+        os.system("%s %s 2" % (test_bin("kmetis"), graph))
+        os.system("%s %s 5" % (test_bin("kmetis"), graph))
+        graph = test_graph("metis.mesh")
+        os.system("%s %s 10" % (test_bin("partnmesh"), graph))
+        os.system("%s %s 10" % (test_bin("partdmesh"), graph))
+        os.system("%s %s" % (test_bin("mesh2dual"), graph))
+
+
+class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder, SetupEnvironment):
+    def cmake_args(self):
+        options = [
+            self.define_from_variant("SHARED", "shared"),
+            self.define_from_variant("GDB", "gdb"),
+        ]
+
+        if self.spec.satisfies("~shared"):
+            # Remove all RPATH options
+            # (RPATHxxx options somehow trigger cmake to link dynamically)
+            rpath_options = []
+            for o in options:
+                if o.find("RPATH") >= 0:
+                    rpath_options.append(o)
+            for o in rpath_options:
+                options.remove(o)
+
+        return options
+
+    @run_after("install")
+    def install_headers(self):
+        with working_dir(self.build_directory):
+            # install all headers, which will be needed for ParMETIS and other programs
+            directories = ["GKlib", "libmetis", "programs"]
+            for directory in directories:
+                inc_dist = join_path(self.prefix.include, directory)
+                mkdirp(inc_dist)
+                install(join_path(self.stage.source_path, directory, "*.h"), inc_dist)
+
+    def check(self):
+        # On some systems, the installed binaries for METIS cannot
+        # be executed without first being read.
+        ls = which("ls")
+        ls("-a", "-l", self.prefix.bin)
+
+        graphchk = Executable(join_path(self.prefix.bin, "graphchk"))
+        gpmetis = Executable(join_path(self.prefix.bin, "gpmetis"))
+        ndmetis = Executable(join_path(self.prefix.bin, "ndmetis"))
+        mpmetis = Executable(join_path(self.prefix.bin, "mpmetis"))
+        for f in ["4elt", "copter2", "mdual"]:
+            graph = join_path(self.stage.source_path, "graphs", "%s.graph" % f)
+            graphchk(graph)
+            gpmetis(graph, "2")
+            ndmetis(graph)
+
+        graph = join_path(self.stage.source_path, "graphs", "test.mgraph")
+        gpmetis(graph, "2")
+        graph = join_path(self.stage.source_path, "graphs", "metis.mesh")
+        mpmetis(graph, "2")
+
+    @run_after("install", when="+shared platform=darwin")
+    def darwin_fix(self):
+        # The shared library is not installed correctly on Darwin; fix this
+        fix_darwin_install_name(prefix.lib)