Improve ability to use system packages for GMP/RE2 (#25184)

Improves Chapels ability to use system packages for GMP and RE2.

This PR adds some `pkg-config`/`brew` logic to use CHPL_GMP=true when
GMP is not in the library path. It also adds some extra detection for
GMP. This means that if a user has GMP installed, `CHPL_GMP` will
default to `system`

This PR makes CHPL_RE2=system a config time error, because Chapel relies
on a special version of RE2 that is not available in any package
manager. This should improve errors if a user tries to use a system
install of RE2

This PR also puts some utils in place to check if GMP is installed on
the system, but it does not switch the default for CHPL_GMP

This PR also now requires `pkg-config` on systems with homebrew, and
makes the necessary adjustments to hwloc, jemalloc, gmp, and the
homebrew formula

Testing:
- ran `start_test test/library/standard/BigInteger/apiTest.chpl` with
CHPL_GMP=system
- tested that CHPL_RE2=system is a config time error
- tested various error conditions (e.g. "pkg-config" not installed,
packages missing)

[Reviewed by @jhh67]

runtime/make/Makefile.runtime.gmp-system

@@ -16,6 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-RUNTIME_DEFS += -DCHPL_HAS_GMP
-# assumes gmp.h is in system path.
-
+RUNTIME_DEFS += -DCHPL_HAS_GMP $(shell $(CHPL_MAKE_PYTHON) $(CHPL_MAKE_HOME)/util/chplenv/chpl_gmp.py --compile)

third-party/Makefile

@@ -112,13 +112,15 @@ $(GASNET_INSTALL_DIR): $(GASNET_DEPEND)
 
 try-gmp: FORCE
   ifneq ($(CHPL_MAKE_GMP_IS_OVERRIDDEN), True)
+  ifneq ($(CHPL_MAKE_GMP), system)
     ifeq ($(wildcard $(GMP_BUILD_DIR)),)
 	@echo "Speculatively attempting to build gmp"
 	-@$(MAKE) GMP_SPECULATIVE=yes gmp
     else ifeq ($(wildcard $(GMP_H_FILE)),)
 	$(info Speculative build of gmp squashed due to previous failures.)
     endif
   endif
+  endif
 
 gmp: $(GMP_H_FILE)
 $(GMP_H_FILE): $(GMP_DEPEND)

util/chplenv/chpl_compiler.py

@@ -5,6 +5,7 @@
 import sys
 
 import chpl_platform, chpl_locale_model, overrides
+import homebrew_utils
 from utils import which, error, memoize, warning
 
 
@@ -428,7 +429,7 @@ def get_system_compile_args(flag):
         paths.append('-I/usr/local/include')
 
     # Add Homebrew include directory if Homebrew is installed
-    homebrew_prefix = chpl_platform.get_homebrew_prefix()
+    homebrew_prefix = homebrew_utils.get_homebrew_prefix()
     if homebrew_prefix:
         paths.append('-I' + homebrew_prefix + '/include')
 
@@ -474,7 +475,7 @@ def get_system_link_args(flag):
         paths.append('-L/usr/local/lib')
 
     # Add Homebrew lib directory if Homebrew is installed
-    homebrew_prefix = chpl_platform.get_homebrew_prefix()
+    homebrew_prefix = homebrew_utils.get_homebrew_prefix()
     if homebrew_prefix:
         paths.append('-L' + homebrew_prefix + '/lib')
 

util/chplenv/chpl_gmp.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 import os
 import sys
+import optparse
 
 import chpl_compiler, chpl_platform, overrides, third_party_utils
 from chpl_home_utils import get_chpl_third_party
-from utils import memoize, warning
+from utils import memoize, warning, error
 
 # returns True if CHPL_GMP was set by the user
 # (i.e. not inferred to be the default)
@@ -37,7 +38,6 @@ def get():
 
     return gmp_val
 
-
 @memoize
 def get_uniq_cfg_path():
     return third_party_utils.default_uniq_cfg_path()
@@ -49,6 +49,13 @@ def get_compile_args():
     gmp_val = get()
     if gmp_val == 'bundled':
          return third_party_utils.get_bundled_compile_args('gmp')
+    elif gmp_val == 'system':
+        # try pkg-config
+        args = third_party_utils.pkgconfig_get_system_compile_args('gmp')
+        if args != (None, None):
+            return args
+        else:
+            third_party_utils.could_not_find_pkgconfig_pkg("gmp", "CHPL_GMP")
 
     return ([ ], [ ])
 
@@ -61,13 +68,35 @@ def get_link_args():
          return third_party_utils.pkgconfig_get_bundled_link_args('gmp')
 
     elif gmp_val == 'system':
-        return ([ ], ['-lgmp'])
+        # try pkg-config
+        args = third_party_utils.pkgconfig_get_system_link_args('gmp')
+        if args != (None, None):
+            return args
+        else:
+            third_party_utils.could_not_find_pkgconfig_pkg("gmp", "CHPL_GMP")
 
     return ([ ], [ ])
 
+
 def _main():
     gmp_val = get()
-    sys.stdout.write("{0}\n".format(gmp_val))
+
+    parser = optparse.OptionParser(usage='usage: %prog [--prefix] [--compile] [--link]')
+    parser.add_option('--compile', dest='action',
+                      action='store_const',
+                      const='compile', default='')
+    parser.add_option('--link', dest='action',
+                      action='store_const',
+                      const='link', default='')
+
+    (options, args) = parser.parse_args()
+
+    if options.action == 'compile':
+        sys.stdout.write("{0}\n".format(get_compile_args()))
+    elif options.action == 'link':
+        sys.stdout.write("{0}\n".format(get_link_args()))
+    else:
+        sys.stdout.write("{0}\n".format(gmp_val))
 
 
 if __name__ == '__main__':

util/chplenv/chpl_hwloc.py

@@ -39,12 +39,8 @@ def get_compile_args():
         args = third_party_utils.pkgconfig_get_system_compile_args('hwloc')
         if args != (None, None):
             return args
-        # try homebrew
-        hwloc_prefix = chpl_platform.get_homebrew_prefix('hwloc')
-        if hwloc_prefix:
-            return ([], ['-I{0}'.format(os.path.join(hwloc_prefix, 'include'))])
-
-        error("Could not find a suitable hwloc installation.  Please install hwloc or set CHPL_HWLOC=bundled or CHPL_HWLOC=none.")
+        else:
+            third_party_utils.could_not_find_pkgconfig_pkg("hwloc", "CHPL_HWLOC")
 
     return ([ ], [ ])
 
@@ -66,14 +62,8 @@ def get_link_args():
                 error("CHPL_HWLOC=system requires hwloc >= 2.1", ValueError)
 
             return third_party_utils.pkgconfig_get_system_link_args('hwloc')
-        # try homebrew
-        hwloc_prefix = chpl_platform.get_homebrew_prefix('hwloc')
-        if hwloc_prefix:
-            # TODO: this should also check the version
-            return ([], ['-L{0}'.format(os.path.join(hwloc_prefix, 'lib')),
-                         '-lhwloc'])
-
-        error("Could not find a suitable hwloc installation. Please install hwloc or set CHPL_HWLOC=bundled or CHPL_HWLOC=none.")
+        else:
+            third_party_utils.could_not_find_pkgconfig_pkg("hwloc", "CHPL_HWLOC")
 
     return ([ ], [ ])
 
@@ -89,12 +79,8 @@ def get_prefix():
             prefix = run_command(['pkg-config', '--variable', 'prefix', 'hwloc'])
             if prefix:
                 return prefix.strip()
-        # try homebrew
-        hwloc_prefix = chpl_platform.get_homebrew_prefix('hwloc')
-        if hwloc_prefix:
-            return hwloc_prefix.strip()
-
-        error("Could not find a suitable hwloc installation.  Please install hwloc or set CHPL_HWLOC=bundled or CHPL_HWLOC=none.")
+        else:
+            third_party_utils.could_not_find_pkgconfig_pkg("hwloc", "CHPL_HWLOC")
 
     return ''
 

util/chplenv/chpl_jemalloc.py

@@ -4,6 +4,7 @@
 import optparse
 
 import chpl_bin_subdir, chpl_compiler, chpl_mem, chpl_platform, overrides, third_party_utils
+import homebrew_utils
 from utils import error, memoize, run_command, warning
 
 
@@ -97,13 +98,9 @@ def get_compile_args(flag):
         args = third_party_utils.pkgconfig_get_system_compile_args('jemalloc')
         if args != (None, None):
             return args
-        # try homebrew
-        jemalloc_prefix = chpl_platform.get_homebrew_prefix('jemalloc')
-        if jemalloc_prefix:
-            return ([], ['-I{0}'.format(os.path.join(jemalloc_prefix, 'include'))])
-
-        envname = "CHPL_TARGET_JEMALLOC" if flag == "target" else "CHPL_HOST_JEMALLOC"
-        error("Could not find a suitable jemalloc installation. Please install jemalloc or set {}=bundled".format(envname, envname))
+        else:
+            envname = "CHPL_TARGET_JEMALLOC" if flag == "target" else "CHPL_HOST_JEMALLOC"
+            third_party_utils.could_not_find_pkgconfig_pkg("jemalloc", envname)
 
     return ([ ], [ ])
 
@@ -133,15 +130,9 @@ def get_link_args(flag):
         args = third_party_utils.pkgconfig_get_system_link_args('jemalloc')
         if args != (None, None):
             return args
-        # try homebrew
-        jemalloc_prefix = chpl_platform.get_homebrew_prefix('jemalloc')
-        if jemalloc_prefix:
-            return ([], ['-L{0}'.format(os.path.join(jemalloc_prefix, 'lib')),
-                         '-ljemalloc'])
-
-        envname = "CHPL_TARGET_JEMALLOC" if flag == "target" else "CHPL_HOST_JEMALLOC"
-        error("Could not find a suitable jemalloc installation. Please install jemalloc or set {}=bundled or {}=none.".format(envname, envname))
-
+        else:
+            envname = "CHPL_TARGET_JEMALLOC" if flag == "target" else "CHPL_HOST_JEMALLOC"
+            third_party_utils.could_not_find_pkgconfig_pkg("jemalloc", envname)
     return ([ ], [ ])
 
 def _main():

util/chplenv/chpl_llvm.py

@@ -8,6 +8,7 @@
 import chpl_bin_subdir, chpl_arch, chpl_compiler, chpl_platform, overrides
 from chpl_home_utils import get_chpl_third_party, get_chpl_home
 import chpl_gpu
+import homebrew_utils
 from utils import which, memoize, error, run_command, try_run_command, warning
 from collections import defaultdict
 
@@ -254,7 +255,7 @@ def find_system_llvm_config():
         return llvm_config
 
 
-    homebrew_prefix = chpl_platform.get_homebrew_prefix()
+    homebrew_prefix = homebrew_utils.get_homebrew_prefix()
 
     paths = [ ]
     for vers in llvm_versions():

util/chplenv/chpl_platform.py

@@ -92,22 +92,6 @@ def is_arch_linux():
     arch_file = "/etc/arch-release"
     return os.path.exists(arch_file)
 
-# if running on a system with homebrew, return the homebrew prefix
-# if not, return None
-@memoize
-def get_homebrew_prefix(pkg=None):
-    # Check to see if Homebrew is installed. If it is, return the prefix.
-    cmd = ['brew', '--prefix']
-    if pkg is not None:
-        cmd.append(str(pkg))
-    exists, retcode, my_out, my_err = try_run_command(cmd)
-    if exists and retcode == 0:
-        # Make sure to include homebrew search path
-        homebrew_prefix = my_out.strip()
-        return homebrew_prefix
-
-    return None
-
 def _main():
     parser = optparse.OptionParser(usage='usage: %prog [--host|target])')
     parser.add_option('--host', dest='flag', action='store_const',

util/chplenv/chpl_re2.py

@@ -4,7 +4,7 @@
 
 import chpl_compiler, chpl_platform, overrides, third_party_utils
 from chpl_home_utils import get_chpl_third_party
-from utils import memoize, warning
+from utils import memoize, warning, error
 
 
 # returns True if CHPL_RE2 was set by the user
@@ -21,7 +21,9 @@ def is_overridden():
 @memoize
 def get():
     re2 = overrides.get('CHPL_RE2')
-    regexp = overrides.get('CHPL_REGEXP')
+    if re2 == "system":
+        error("CHPL_RE2=system is not supported. Please use CHPL_RE2=bundled or CHL_RE2=none instead.")
+
     if not re2:
         re2_header = os.path.join(get_chpl_third_party(), 're2',
                                   'install', get_uniq_cfg_path(),

util/chplenv/homebrew_utils.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import sys
+
+from utils import error, memoize, try_run_command
+
+@memoize
+def get_homebrew_prefix(pkg=None):
+    """
+    If running on a system with Homebrew, return the Homebrew prefix.
+    If not, return None.
+
+    If pkg is provided, return the Homebrew prefix for that package.
+    """
+    cmd = ['brew', '--prefix']
+    if pkg is not None:
+        cmd.append(str(pkg))
+    exists, retcode, my_out, _ = try_run_command(cmd)
+    if exists and retcode == 0:
+        # Make sure to include homebrew search path
+        homebrew_prefix = my_out.strip()
+        return homebrew_prefix
+
+    return None
+
+@memoize
+def homebrew_exists():
+    """Check if Homebrew is installed on the system."""
+    return get_homebrew_prefix() is not None
+
+@memoize
+def homebrew_pkg_exists(pkg):
+    """Check if a Homebrew package is installed on the system."""
+    cmd = ['brew', 'list', pkg]
+    exists, retcode, _, _ = try_run_command(cmd)
+    return exists and retcode == 0
+
+def _main():
+    sys.stdout.write("{0}\n".format(get_homebrew_prefix()))
+
+
+if __name__ == '__main__':
+    _main()

util/chplenv/third_party_utils.py

@@ -5,6 +5,7 @@
 import chpl_lib_pic, chpl_locale_model, chpl_platform
 from chpl_home_utils import get_chpl_home, get_chpl_third_party, using_chapel_module
 from utils import error, memoize, run_command, warning, try_run_command
+import homebrew_utils
 
 #
 # This is the default unique configuration path which
@@ -109,7 +110,7 @@ def filter_libs(bundled_libs, system_libs):
 def pkgconfig_get_system_compile_args(pkg):
     # check that pkg-config knows about the package in question
     exists, returncode, my_stdout, my_stderr = try_run_command(['pkg-config', '--exists', pkg])
-    if returncode:
+    if not exists or returncode:
         return (None, None)
     # run pkg-config to get the cflags
     cflags_line = run_command(['pkg-config', '--cflags'] + [pkg]);
@@ -426,3 +427,12 @@ def read_bundled_pkg_config_file(pkg, ucp='', pcfile=''):
     replace_path = install_path
 
     return (read_pkg_config_file(pcpath, find_path, replace_path), pcpath)
+
+
+def could_not_find_pkgconfig_pkg(pkg, envname):
+    if homebrew_utils.homebrew_exists() and homebrew_utils.homebrew_pkg_exists(pkg):
+        # tell user to install pkg-config as well
+        error("{0} is installed via homebrew, but pkg-config is not installed. Please install pkg-config with `brew install pkg-config`.".format(pkg))
+    else:
+        install_str = " with `brew install {0}` ".format(pkg) if homebrew_utils.homebrew_exists() else ""
+        error("Could not find a suitable {0} installation. Please install {0}{1}or set {2}=bundled`.".format(pkg, install_str, envname))

util/packaging/homebrew/chapel-main.rb

@@ -22,6 +22,7 @@ class Chapel < Formula
   depends_on "jemalloc"
   depends_on "llvm@17"
   depends_on "[email protected]"
+  depends_on "pkg-config" => :build
 
   # LLVM is built with gcc11 and we will fail on linux with gcc version 5.xx
   fails_with gcc: "5"