diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be07b034..d3b89094 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,9 +52,7 @@ jobs: runs_on: ubuntu-24.04 docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2024-09-20T23_57_46 build_thirdparty_args: >- - --compiler-prefix=/usr - --compiler-family=gcc - --compiler-suffix=-12 + --toolchain=gcc12 architecture: x86_64 - name: ubuntu2204-x86_64-clang17 @@ -115,7 +113,7 @@ jobs: runs_on: ubuntu-24.04 docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2024-09-20T20_33_55 build_thirdparty_args: >- - --devtoolset=12 + --toolchain=gcc12 architecture: x86_64 - name: almalinux8-x86_64-gcc13 diff --git a/python/yugabyte_db_thirdparty/builder.py b/python/yugabyte_db_thirdparty/builder.py index 8bd31ff5..4167fb5c 100644 --- a/python/yugabyte_db_thirdparty/builder.py +++ b/python/yugabyte_db_thirdparty/builder.py @@ -214,6 +214,8 @@ def determine_compiler_family_and_prefix(self) -> Tuple[str, Optional[str]]: self.toolchain.write_url_and_path_files() if self.args.toolchain.startswith('llvm'): compiler_family = 'clang' + else: + compiler_family = 'gcc' elif self.args.devtoolset: compiler_family = 'gcc' elif self.args.compiler_prefix: @@ -267,6 +269,8 @@ def parse_args(self) -> None: if self.args.expected_major_compiler_version is None: if self.args.toolchain and re.match('^llvm[0-9]+$', self.args.toolchain): self.args.expected_major_compiler_version = int(self.args.toolchain[4:]) + if self.args.toolchain and re.match('^gcc[0-9]+$', self.args.toolchain): + self.args.expected_major_compiler_version = int(self.args.toolchain[3:]) elif self.args.devtoolset is not None: self.args.expected_major_compiler_version = self.args.devtoolset elif re.match('^-[0-9]+$', self.args.compiler_suffix): @@ -281,7 +285,8 @@ def parse_args(self) -> None: compiler_suffix=self.args.compiler_suffix, devtoolset=self.args.devtoolset, use_ccache=self.args.use_ccache, - expected_major_compiler_version=self.args.expected_major_compiler_version + expected_major_compiler_version=self.args.expected_major_compiler_version, + from_installer=self.args.toolchain is not None ) llvm_major_version: Optional[int] = self.compiler_choice.get_llvm_major_version() @@ -323,9 +328,9 @@ def populate_dependencies(self) -> None: if (self.compiler_choice.is_clang() and llvm_major_version is not None and llvm_major_version >= 10): if self.toolchain: - llvm_version_str = self.toolchain.get_llvm_version_str() + llvm_version_str = self.toolchain.get_installer_version_str() else: - llvm_version_str = self.compiler_choice.get_llvm_version_str() + llvm_version_str = self.compiler_choice.get_installer_version_str() self.dependencies.append( get_build_def_module('llvm_libunwind').LlvmLibUnwindDependency( @@ -488,6 +493,10 @@ def init_compiler_independent_flags(self, dep: Dependency) -> None: self.add_include_path(os.path.join(build_type_parent_dir, 'include')) self.add_lib_dir_and_rpath(os.path.join(build_type_parent_dir, 'lib')) + if self.toolchain: + self.add_lib_dir_and_rpath(os.path.join(self.toolchain.toolchain_root, 'lib')) + self.add_lib_dir_and_rpath(os.path.join(self.toolchain.toolchain_root, 'lib64')) + self.compiler_flags += ['-fno-omit-frame-pointer', '-fPIC', '-O3', '-Wall', '-DNDEBUG'] if is_linux(): # On Linux, ensure we set a long enough rpath so we can change it later with chrpath, @@ -983,7 +992,7 @@ def init_flags(self, dep: Dependency) -> None: """ self.init_compiler_independent_flags(dep) - if not is_macos() and self.compiler_choice.using_clang(): + if not is_macos() and self.compiler_choice.is_clang(): # Special setup for Clang on Linux. compiler_choice = self.compiler_choice llvm_major_version: Optional[int] = compiler_choice.get_llvm_major_version() @@ -992,7 +1001,7 @@ def init_flags(self, dep: Dependency) -> None: else: raise ValueError(f"Unknown or unsupproted LLVM major version: {llvm_major_version}") - if self.compiler_choice.using_gcc(): + if self.compiler_choice.is_gcc(): self.cxx_flags.append('-fext-numeric-literals') if is_linux(): diff --git a/python/yugabyte_db_thirdparty/compiler_choice.py b/python/yugabyte_db_thirdparty/compiler_choice.py index 12f8659b..d6f6b729 100644 --- a/python/yugabyte_db_thirdparty/compiler_choice.py +++ b/python/yugabyte_db_thirdparty/compiler_choice.py @@ -50,6 +50,7 @@ class CompilerChoice: cxx_identification: Optional[CompilerIdentification] compiler_version_str: Optional[str] expected_major_compiler_version: Optional[int] + from_installer: bool def __init__( self, @@ -58,7 +59,8 @@ def __init__( compiler_suffix: str, devtoolset: Optional[int], use_ccache: bool, - expected_major_compiler_version: Optional[int]) -> None: + expected_major_compiler_version: Optional[int], + from_installer: bool) -> None: assert compiler_family in ['gcc', 'clang'] self.compiler_family = compiler_family self.compiler_prefix = compiler_prefix @@ -78,6 +80,8 @@ def __init__( self.expected_major_compiler_version = expected_major_compiler_version + self.from_installer = from_installer + self.find_compiler() self.identify_compiler_version() @@ -169,11 +173,16 @@ def find_clang(self) -> Tuple[str, str]: os.path.join(clang_bin_dir, 'clang++') + self.compiler_suffix) def is_clang(self) -> bool: + assert self.compiler_family is not None return self.compiler_family == 'clang' def is_gcc(self) -> bool: + assert self.compiler_family is not None return self.compiler_family == 'gcc' + def is_installer_gcc(self) -> bool: + return self.is_gcc() and self.from_installer + def is_gcc_major_version_at_least(self, expected_major_version: int) -> bool: gcc_major_version = self.get_gcc_major_version() return gcc_major_version is not None and gcc_major_version >= expected_major_version @@ -245,8 +254,8 @@ def identify_compiler_version(self) -> None: self.cxx_identification.version_str) self.compiler_version_str = self.cc_identification.version_str - def get_llvm_version_str(self) -> str: - assert self.is_clang() + def get_installer_version_str(self) -> str: + assert self.is_clang() or self.is_installer_gcc() assert self.compiler_version_str is not None return self.compiler_version_str @@ -257,7 +266,7 @@ def get_compiler_major_version(self) -> int: def get_llvm_major_version(self) -> Optional[int]: if not self.is_clang(): return None - return extract_major_version(self.get_llvm_version_str()) + return extract_major_version(self.get_installer_version_str()) def is_llvm_major_version_at_least(self, lower_bound: int) -> bool: llvm_major_version = self.get_llvm_major_version() @@ -266,17 +275,13 @@ def is_llvm_major_version_at_least(self, lower_bound: int) -> bool: return llvm_major_version >= lower_bound def get_gcc_major_version(self) -> Optional[int]: - if self.compiler_family != 'gcc': + if not self.is_gcc(): return None + if self.is_installer_gcc(): + return extract_major_version(self.get_installer_version_str()) assert self.compiler_version_str is not None return extract_major_version(self.compiler_version_str) - def using_gcc_major_version_at_least(self, required_gcc_major_version: int) -> bool: - gcc_major_version = self.get_gcc_major_version() - if gcc_major_version is None: - return False - return gcc_major_version >= required_gcc_major_version - def check_compiler_major_version(self) -> None: assert self.expected_major_compiler_version is not None actual_major_version = self.get_compiler_major_version() @@ -293,14 +298,6 @@ def check_compiler_major_version(self) -> None: self.cxx_identification )) - def using_clang(self) -> bool: - assert self.compiler_family is not None - return self.compiler_family == 'clang' - - def using_gcc(self) -> bool: - assert self.compiler_family is not None - return self.compiler_family == 'gcc' - def get_compiler_family_and_version(self) -> str: return '%s%d' % (self.compiler_family, self.get_compiler_major_version()) diff --git a/python/yugabyte_db_thirdparty/library_checking.py b/python/yugabyte_db_thirdparty/library_checking.py index 2a1865b4..52ef724d 100644 --- a/python/yugabyte_db_thirdparty/library_checking.py +++ b/python/yugabyte_db_thirdparty/library_checking.py @@ -152,12 +152,12 @@ def __init__(self, fs_layout: FileSystemLayout) -> None: self.tp_installed_dir = fs_layout.tp_installed_dir def configure_for_compiler(self, compiler_choice: CompilerChoice) -> None: - if compiler_choice.using_gcc(): + if compiler_choice.is_gcc(): # The GCC toolchain links with the libstdc++ library in a system-wide location. self.allowed_system_libraries.add('libstdc++') - if (compiler_choice.using_gcc() or - compiler_choice.using_clang() and + if (compiler_choice.is_gcc() or + compiler_choice.is_clang() and compiler_choice.is_llvm_major_version_at_least(13)): # For GCC and Clang 13+, there are some issues with removing the libgcc_s dependency # from libraries even if it is apparently not needed as shown by "ldd -u". @@ -166,7 +166,7 @@ def configure_for_compiler(self, compiler_choice: CompilerChoice) -> None: # For Clang 12, it looks like we can safely remove the libgcc_s dependency. self.needed_libs_to_remove.add('libgcc_s') - if compiler_choice.using_gcc_major_version_at_least(11): + if compiler_choice.is_gcc_major_version_at_least(11): # When building DiskANN with GCC 11+, we end up using the system OpenMP library called # libgomp.so.1. self.allowed_system_libraries.add('libgomp') diff --git a/python/yugabyte_db_thirdparty/toolchain.py b/python/yugabyte_db_thirdparty/toolchain.py index 28a72382..58621674 100644 --- a/python/yugabyte_db_thirdparty/toolchain.py +++ b/python/yugabyte_db_thirdparty/toolchain.py @@ -14,6 +14,7 @@ import os import re from llvm_installer import LlvmInstaller +from yb_gcc_installer import GCCInstaller from yugabyte_db_thirdparty.download_manager import DownloadManager from yugabyte_db_thirdparty.util import YB_THIRDPARTY_DIR, write_file @@ -24,21 +25,22 @@ from sys_detection import SHORT_OS_NAME_REGEX_STR, is_compatible_os_and_version -LLVM_VERSION_FROM_ARCHIVE_NAME_RE = re.compile( - rf'^yb-llvm-v(.*)-[0-9]+-[0-9a-f]+-.*') - - -def get_llvm_url(tag: str) -> str: - return 'https://github.com/yugabyte/build-clang/releases/download/%s/yb-llvm-%s.tar.gz' % ( - tag, tag) +INSTALLER_VERSION_FROM_ARCHIVE_NAME_RE = re.compile( + rf'^yb-(?:llvm|gcc)-v(.*)-[0-9]+-[0-9a-f]+-.*') MIN_LLVM_VERSION = 14 MAX_LLVM_VERSION = 19 +MIN_GCC_VERSION = 12 +MAX_GCC_VERSION = 15 LLVM_VERSIONS = list(range(MIN_LLVM_VERSION, MAX_LLVM_VERSION + 1)) +GCC_VERSIONS = list(range(MIN_GCC_VERSION, MAX_GCC_VERSION + 1)) -TOOLCHAIN_TYPES = ['llvm%d' % v for v in LLVM_VERSIONS] +TOOLCHAIN_TYPES = ( + ['llvm%d' % v for v in LLVM_VERSIONS] + + ['gcc%d' % v for v in GCC_VERSIONS] +) class Toolchain: @@ -76,14 +78,14 @@ def write_url_and_path_files(self) -> None: write_file(os.path.join(YB_THIRDPARTY_DIR, 'toolchain_url.txt'), self.toolchain_url) write_file(os.path.join(YB_THIRDPARTY_DIR, 'toolchain_path.txt'), self.toolchain_root) - def get_llvm_version_str(self) -> str: - if not self.toolchain_type.startswith('llvm'): - raise ValueError('Expected an LLVM toolchain type, found: %s' % self.toolchain_type) + def get_installer_version_str(self) -> str: + if not self.toolchain_type.startswith('llvm') and not self.toolchain_type.startswith('gcc'): + raise ValueError('Expected an LLVM/GCC toolchain type, found: %s' % self.toolchain_type) archive_name = os.path.basename(self.toolchain_url) - url_match = LLVM_VERSION_FROM_ARCHIVE_NAME_RE.match(archive_name) + url_match = INSTALLER_VERSION_FROM_ARCHIVE_NAME_RE.match(archive_name) if not url_match: raise ValueError( - 'Could not extract LLVM version from download URL: %s' % archive_name) + 'Could not extract LLVM/GCC version from download URL: %s' % archive_name) return url_match.group(1) @@ -103,13 +105,19 @@ def ensure_toolchains_installed( def get_toolchain_url(toolchain_type: str) -> str: - assert toolchain_type.startswith('llvm') local_sys_conf = sys_detection.local_sys_conf() - major_llvm_version = int(toolchain_type[4:]) - llvm_installer = LlvmInstaller( - short_os_name_and_version=local_sys_conf.short_os_name_and_version(), - architecture=local_sys_conf.architecture) - return llvm_installer.get_llvm_url(major_llvm_version=major_llvm_version) + if toolchain_type.startswith('llvm'): + major_version = int(toolchain_type[4:]) + llvm_installer = LlvmInstaller( + short_os_name_and_version=local_sys_conf.short_os_name_and_version(), + architecture=local_sys_conf.architecture) + return llvm_installer.get_llvm_url(major_llvm_version=major_version) + else: + major_version = int(toolchain_type[3:]) + gcc_installer = GCCInstaller( + short_os_name_and_version=local_sys_conf.short_os_name_and_version(), + architecture=local_sys_conf.architecture) + return gcc_installer.get_gcc_url(major_gcc_version=major_version) def ensure_toolchain_installed( @@ -124,6 +132,8 @@ def ensure_toolchain_installed( if toolchain_type.startswith('llvm'): parent_dir = '/opt/yb-build/llvm' + elif toolchain_type.startswith('gcc'): + parent_dir = '/opt/yb-build/gcc' else: raise RuntimeError( f"We don't know where to install toolchain of type f{toolchain_type}") diff --git a/requirements.txt b/requirements.txt index a1c8d9cb..521c3d02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,6 @@ packaging pycodestyle ruamel.yaml sys-detection +yb-gcc-installer @ git+https://github.com/es1024/yb-gcc-installer yugabyte_pycommon argparse_utils diff --git a/requirements_frozen.txt b/requirements_frozen.txt index de9bd840..5b4c11de 100644 --- a/requirements_frozen.txt +++ b/requirements_frozen.txt @@ -41,5 +41,6 @@ twine==4.0.2 typing-extensions==4.8.0 urllib3==2.0.6 websocket-client==1.6.4 +yb-gcc-installer @ git+https://github.com/es1024/yb-gcc-installer@3b683e6c37a6073eb4af9582b3521ff7fa6dcfeb yugabyte-pycommon==1.9.15 zipp==3.17.0