[WIP] Use yb-gcc-installer

.github/workflows/ci.yml

@@ -52,9 +52,7 @@ jobs:
             runs_on: ubuntu-24.04
             docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2024-09-20T23_57_46
             build_thirdparty_args: >-
-              --compiler-prefix=/usr
-              --compiler-family=gcc
-              --compiler-suffix=-12
+              --toolchain=gcc12
             architecture: x86_64
 
           - name: ubuntu2204-x86_64-clang17
@@ -115,7 +113,7 @@ jobs:
             runs_on: ubuntu-24.04
             docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2024-09-20T20_33_55
             build_thirdparty_args: >-
-              --devtoolset=12
+              --toolchain=gcc12
             architecture: x86_64
 
           - name: almalinux8-x86_64-gcc13

python/yugabyte_db_thirdparty/builder.py

@@ -214,6 +214,8 @@ def determine_compiler_family_and_prefix(self) -> Tuple[str, Optional[str]]:
             self.toolchain.write_url_and_path_files()
             if self.args.toolchain.startswith('llvm'):
                 compiler_family = 'clang'
+            else:
+                compiler_family = 'gcc'
         elif self.args.devtoolset:
             compiler_family = 'gcc'
         elif self.args.compiler_prefix:
@@ -267,6 +269,8 @@ def parse_args(self) -> None:
         if self.args.expected_major_compiler_version is None:
             if self.args.toolchain and re.match('^llvm[0-9]+$', self.args.toolchain):
                 self.args.expected_major_compiler_version = int(self.args.toolchain[4:])
+            if self.args.toolchain and re.match('^gcc[0-9]+$', self.args.toolchain):
+                self.args.expected_major_compiler_version = int(self.args.toolchain[3:])
             elif self.args.devtoolset is not None:
                 self.args.expected_major_compiler_version = self.args.devtoolset
             elif re.match('^-[0-9]+$', self.args.compiler_suffix):
@@ -281,7 +285,8 @@ def parse_args(self) -> None:
             compiler_suffix=self.args.compiler_suffix,
             devtoolset=self.args.devtoolset,
             use_ccache=self.args.use_ccache,
-            expected_major_compiler_version=self.args.expected_major_compiler_version
+            expected_major_compiler_version=self.args.expected_major_compiler_version,
+            from_installer=self.args.toolchain is not None
         )
 
         llvm_major_version: Optional[int] = self.compiler_choice.get_llvm_major_version()
@@ -323,9 +328,9 @@ def populate_dependencies(self) -> None:
             if (self.compiler_choice.is_clang() and
                     llvm_major_version is not None and llvm_major_version >= 10):
                 if self.toolchain:
-                    llvm_version_str = self.toolchain.get_llvm_version_str()
+                    llvm_version_str = self.toolchain.get_installer_version_str()
                 else:
-                    llvm_version_str = self.compiler_choice.get_llvm_version_str()
+                    llvm_version_str = self.compiler_choice.get_installer_version_str()
 
                 self.dependencies.append(
                     get_build_def_module('llvm_libunwind').LlvmLibUnwindDependency(
@@ -488,6 +493,10 @@ def init_compiler_independent_flags(self, dep: Dependency) -> None:
             self.add_include_path(os.path.join(build_type_parent_dir, 'include'))
             self.add_lib_dir_and_rpath(os.path.join(build_type_parent_dir, 'lib'))
 
+        if self.toolchain:
+            self.add_lib_dir_and_rpath(os.path.join(self.toolchain.toolchain_root, 'lib'))
+            self.add_lib_dir_and_rpath(os.path.join(self.toolchain.toolchain_root, 'lib64'))
+
         self.compiler_flags += ['-fno-omit-frame-pointer', '-fPIC', '-O3', '-Wall', '-DNDEBUG']
         if is_linux():
             # On Linux, ensure we set a long enough rpath so we can change it later with chrpath,
@@ -983,7 +992,7 @@ def init_flags(self, dep: Dependency) -> None:
         """
         self.init_compiler_independent_flags(dep)
 
-        if not is_macos() and self.compiler_choice.using_clang():
+        if not is_macos() and self.compiler_choice.is_clang():
             # Special setup for Clang on Linux.
             compiler_choice = self.compiler_choice
             llvm_major_version: Optional[int] = compiler_choice.get_llvm_major_version()
@@ -992,7 +1001,7 @@ def init_flags(self, dep: Dependency) -> None:
             else:
                 raise ValueError(f"Unknown or unsupproted LLVM major version: {llvm_major_version}")
 
-        if self.compiler_choice.using_gcc():
+        if self.compiler_choice.is_gcc():
             self.cxx_flags.append('-fext-numeric-literals')
 
         if is_linux():

python/yugabyte_db_thirdparty/compiler_choice.py

@@ -50,6 +50,7 @@ class CompilerChoice:
     cxx_identification: Optional[CompilerIdentification]
     compiler_version_str: Optional[str]
     expected_major_compiler_version: Optional[int]
+    from_installer: bool
 
     def __init__(
             self,
@@ -58,7 +59,8 @@ def __init__(
             compiler_suffix: str,
             devtoolset: Optional[int],
             use_ccache: bool,
-            expected_major_compiler_version: Optional[int]) -> None:
+            expected_major_compiler_version: Optional[int],
+            from_installer: bool) -> None:
         assert compiler_family in ['gcc', 'clang']
         self.compiler_family = compiler_family
         self.compiler_prefix = compiler_prefix
@@ -78,6 +80,8 @@ def __init__(
 
         self.expected_major_compiler_version = expected_major_compiler_version
 
+        self.from_installer = from_installer
+
         self.find_compiler()
         self.identify_compiler_version()
 
@@ -169,11 +173,16 @@ def find_clang(self) -> Tuple[str, str]:
                 os.path.join(clang_bin_dir, 'clang++') + self.compiler_suffix)
 
     def is_clang(self) -> bool:
+        assert self.compiler_family is not None
         return self.compiler_family == 'clang'
 
     def is_gcc(self) -> bool:
+        assert self.compiler_family is not None
         return self.compiler_family == 'gcc'
 
+    def is_installer_gcc(self) -> bool:
+        return self.is_gcc() and self.from_installer
+
     def is_gcc_major_version_at_least(self, expected_major_version: int) -> bool:
         gcc_major_version = self.get_gcc_major_version()
         return gcc_major_version is not None and gcc_major_version >= expected_major_version
@@ -245,8 +254,8 @@ def identify_compiler_version(self) -> None:
                 self.cxx_identification.version_str)
         self.compiler_version_str = self.cc_identification.version_str
 
-    def get_llvm_version_str(self) -> str:
-        assert self.is_clang()
+    def get_installer_version_str(self) -> str:
+        assert self.is_clang() or self.is_installer_gcc()
         assert self.compiler_version_str is not None
         return self.compiler_version_str
 
@@ -257,7 +266,7 @@ def get_compiler_major_version(self) -> int:
     def get_llvm_major_version(self) -> Optional[int]:
         if not self.is_clang():
             return None
-        return extract_major_version(self.get_llvm_version_str())
+        return extract_major_version(self.get_installer_version_str())
 
     def is_llvm_major_version_at_least(self, lower_bound: int) -> bool:
         llvm_major_version = self.get_llvm_major_version()
@@ -266,17 +275,13 @@ def is_llvm_major_version_at_least(self, lower_bound: int) -> bool:
         return llvm_major_version >= lower_bound
 
     def get_gcc_major_version(self) -> Optional[int]:
-        if self.compiler_family != 'gcc':
+        if not self.is_gcc():
             return None
+        if self.is_installer_gcc():
+            return extract_major_version(self.get_installer_version_str())
         assert self.compiler_version_str is not None
         return extract_major_version(self.compiler_version_str)
 
-    def using_gcc_major_version_at_least(self, required_gcc_major_version: int) -> bool:
-        gcc_major_version = self.get_gcc_major_version()
-        if gcc_major_version is None:
-            return False
-        return gcc_major_version >= required_gcc_major_version
-
     def check_compiler_major_version(self) -> None:
         assert self.expected_major_compiler_version is not None
         actual_major_version = self.get_compiler_major_version()
@@ -293,14 +298,6 @@ def check_compiler_major_version(self) -> None:
                     self.cxx_identification
                 ))
 
-    def using_clang(self) -> bool:
-        assert self.compiler_family is not None
-        return self.compiler_family == 'clang'
-
-    def using_gcc(self) -> bool:
-        assert self.compiler_family is not None
-        return self.compiler_family == 'gcc'
-
     def get_compiler_family_and_version(self) -> str:
         return '%s%d' % (self.compiler_family, self.get_compiler_major_version())
 

python/yugabyte_db_thirdparty/library_checking.py

@@ -152,12 +152,12 @@ def __init__(self, fs_layout: FileSystemLayout) -> None:
         self.tp_installed_dir = fs_layout.tp_installed_dir
 
     def configure_for_compiler(self, compiler_choice: CompilerChoice) -> None:
-        if compiler_choice.using_gcc():
+        if compiler_choice.is_gcc():
             # The GCC toolchain links with the libstdc++ library in a system-wide location.
             self.allowed_system_libraries.add('libstdc++')
 
-        if (compiler_choice.using_gcc() or
-                compiler_choice.using_clang() and
+        if (compiler_choice.is_gcc() or
+                compiler_choice.is_clang() and
                 compiler_choice.is_llvm_major_version_at_least(13)):
             # For GCC and Clang 13+, there are some issues with removing the libgcc_s dependency
             # from libraries even if it is apparently not needed as shown by "ldd -u".
@@ -166,7 +166,7 @@ def configure_for_compiler(self, compiler_choice: CompilerChoice) -> None:
             # For Clang 12, it looks like we can safely remove the libgcc_s dependency.
             self.needed_libs_to_remove.add('libgcc_s')
 
-        if compiler_choice.using_gcc_major_version_at_least(11):
+        if compiler_choice.is_gcc_major_version_at_least(11):
             # When building DiskANN with GCC 11+, we end up using the system OpenMP library called
             # libgomp.so.1.
             self.allowed_system_libraries.add('libgomp')

python/yugabyte_db_thirdparty/toolchain.py

@@ -14,6 +14,7 @@
 import os
 import re
 from llvm_installer import LlvmInstaller
+from yb_gcc_installer import GCCInstaller
 
 from yugabyte_db_thirdparty.download_manager import DownloadManager
 from yugabyte_db_thirdparty.util import YB_THIRDPARTY_DIR, write_file
@@ -24,21 +25,22 @@
 from sys_detection import SHORT_OS_NAME_REGEX_STR, is_compatible_os_and_version
 
 
-LLVM_VERSION_FROM_ARCHIVE_NAME_RE = re.compile(
-        rf'^yb-llvm-v(.*)-[0-9]+-[0-9a-f]+-.*')
-
-
-def get_llvm_url(tag: str) -> str:
-    return 'https://github.com/yugabyte/build-clang/releases/download/%s/yb-llvm-%s.tar.gz' % (
-            tag, tag)
+INSTALLER_VERSION_FROM_ARCHIVE_NAME_RE = re.compile(
+        rf'^yb-(?:llvm|gcc)-v(.*)-[0-9]+-[0-9a-f]+-.*')
 
 
 MIN_LLVM_VERSION = 14
 MAX_LLVM_VERSION = 19
+MIN_GCC_VERSION = 12
+MAX_GCC_VERSION = 15
 
 LLVM_VERSIONS = list(range(MIN_LLVM_VERSION, MAX_LLVM_VERSION + 1))
+GCC_VERSIONS = list(range(MIN_GCC_VERSION, MAX_GCC_VERSION + 1))
 
-TOOLCHAIN_TYPES = ['llvm%d' % v for v in LLVM_VERSIONS]
+TOOLCHAIN_TYPES = (
+    ['llvm%d' % v for v in LLVM_VERSIONS] +
+    ['gcc%d' % v for v in GCC_VERSIONS]
+)
 
 
 class Toolchain:
@@ -76,14 +78,14 @@ def write_url_and_path_files(self) -> None:
         write_file(os.path.join(YB_THIRDPARTY_DIR, 'toolchain_url.txt'), self.toolchain_url)
         write_file(os.path.join(YB_THIRDPARTY_DIR, 'toolchain_path.txt'), self.toolchain_root)
 
-    def get_llvm_version_str(self) -> str:
-        if not self.toolchain_type.startswith('llvm'):
-            raise ValueError('Expected an LLVM toolchain type, found: %s' % self.toolchain_type)
+    def get_installer_version_str(self) -> str:
+        if not self.toolchain_type.startswith('llvm') and not self.toolchain_type.startswith('gcc'):
+            raise ValueError('Expected an LLVM/GCC toolchain type, found: %s' % self.toolchain_type)
         archive_name = os.path.basename(self.toolchain_url)
-        url_match = LLVM_VERSION_FROM_ARCHIVE_NAME_RE.match(archive_name)
+        url_match = INSTALLER_VERSION_FROM_ARCHIVE_NAME_RE.match(archive_name)
         if not url_match:
             raise ValueError(
-                    'Could not extract LLVM version from download URL: %s' % archive_name)
+                    'Could not extract LLVM/GCC version from download URL: %s' % archive_name)
         return url_match.group(1)
 
 
@@ -103,13 +105,19 @@ def ensure_toolchains_installed(
 
 
 def get_toolchain_url(toolchain_type: str) -> str:
-    assert toolchain_type.startswith('llvm')
     local_sys_conf = sys_detection.local_sys_conf()
-    major_llvm_version = int(toolchain_type[4:])
-    llvm_installer = LlvmInstaller(
-        short_os_name_and_version=local_sys_conf.short_os_name_and_version(),
-        architecture=local_sys_conf.architecture)
-    return llvm_installer.get_llvm_url(major_llvm_version=major_llvm_version)
+    if toolchain_type.startswith('llvm'):
+        major_version = int(toolchain_type[4:])
+        llvm_installer = LlvmInstaller(
+            short_os_name_and_version=local_sys_conf.short_os_name_and_version(),
+            architecture=local_sys_conf.architecture)
+        return llvm_installer.get_llvm_url(major_llvm_version=major_version)
+    else:
+        major_version = int(toolchain_type[3:])
+        gcc_installer = GCCInstaller(
+            short_os_name_and_version=local_sys_conf.short_os_name_and_version(),
+            architecture=local_sys_conf.architecture)
+        return gcc_installer.get_gcc_url(major_gcc_version=major_version)
 
 
 def ensure_toolchain_installed(
@@ -124,6 +132,8 @@ def ensure_toolchain_installed(
 
     if toolchain_type.startswith('llvm'):
         parent_dir = '/opt/yb-build/llvm'
+    elif toolchain_type.startswith('gcc'):
+        parent_dir = '/opt/yb-build/gcc'
     else:
         raise RuntimeError(
             f"We don't know where to install toolchain of type f{toolchain_type}")

requirements.txt

@@ -7,5 +7,6 @@ packaging
 pycodestyle
 ruamel.yaml
 sys-detection
+yb-gcc-installer @ git+https://github.com/es1024/yb-gcc-installer
 yugabyte_pycommon
 argparse_utils

requirements_frozen.txt

@@ -41,5 +41,6 @@ twine==4.0.2
 typing-extensions==4.8.0
 urllib3==2.0.6
 websocket-client==1.6.4
+yb-gcc-installer @ git+https://github.com/es1024/yb-gcc-installer@3b683e6c37a6073eb4af9582b3521ff7fa6dcfeb
 yugabyte-pycommon==1.9.15
 zipp==3.17.0