From 2f32d9de82a23064f87ab6fba39564dfa24e11ed Mon Sep 17 00:00:00 2001 From: Alex Lopez Date: Fri, 19 Jan 2024 11:31:06 +0100 Subject: [PATCH] Split builder setup and actual build and make the former optional --- .builders/build.py | 40 ++++++-- .builders/images/macos/build.sh | 130 ------------------------ .builders/images/macos/builder_setup.sh | 105 +++++++++++++++++++ .github/workflows/build-deps-macos.yml | 8 +- 4 files changed, 142 insertions(+), 141 deletions(-) delete mode 100644 .builders/images/macos/build.sh create mode 100644 .builders/images/macos/builder_setup.sh diff --git a/.builders/build.py b/.builders/build.py index 564d5e06e4e49..f2166f07a4da2 100644 --- a/.builders/build.py +++ b/.builders/build.py @@ -69,37 +69,59 @@ def build_macos(): parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False) parser.add_argument('output_dir') parser.add_argument('--python', default='3') - parser.add_argument('--cache-dir', - help='Path to directory to load or save a build cache.') + parser.add_argument('--builder-root', required=True, + help='Path to a folder where things will be installed during builder setup.') + parser.add_argument('--skip-setup', default=False, action='store_true', + help='Skip builder setup, assuming it has already been set up.') args = parser.parse_args() context_path = HERE / 'images' / 'macos' + builder_root = Path(args.builder_root).absolute() + builder_root.mkdir(exist_ok=True) with temporary_directory() as temp_dir: - build_context_dir = shutil.copytree(context_path, temp_dir, dirs_exist_ok=True) + mount_dir = temp_dir / 'mnt' + mount_dir.mkdir() + + build_context_dir = shutil.copytree(context_path, mount_dir / 'build_context', dirs_exist_ok=True) # Copy utilities shared by multiple images for entry in context_path.parent.iterdir(): if entry.is_file(): shutil.copy2(entry, build_context_dir) - mount_dir = temp_dir / 'mnt' - mount_dir.mkdir() - dependency_file = mount_dir / 'requirements.in' dependency_file.write_text('\n'.join(chain.from_iterable(read_dependencies().values()))) shutil.copy(HERE.parent / '.deps' / 'build_dependencies.txt', mount_dir) shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') + prefix_path = builder_root / 'prefix' env = { **os.environ, - 'DD_PREFIX_CACHE': args.cache_dir or '', 'DD_MOUNT_DIR': mount_dir, + # Paths to pythons + 'DD_PY3_BUILDENV_PATH': builder_root / 'py3' / 'bin' / 'python', + 'DD_PY2_BUILDENV_PATH': builder_root / 'py2' / 'bin' / 'python', + # Path where we'll install libraries that we build + 'DD_PREFIX_PATH': prefix_path, + # Common compilation flags + 'LDFLAGS': f'-Wl,-rpath,{prefix_path}/lib -L{prefix_path}/lib', + 'CFLAGS': f'-I{prefix_path}/include -O2', + # Build command for extra platform-specific build steps + 'DD_BUILD_COMMAND': f'bash {build_context_dir}/extra_build.sh' } + + if not args.skip_setup: + check_process( + ['bash', str(HERE / 'images' / 'macos' / 'builder_setup.sh')], + env=env, + cwd=builder_root, + ) + check_process( - ['bash', str(HERE / 'images' / 'macos' / 'build.sh'), '--python', args.python], + [os.environ['DD_PYTHON3'], str(mount_dir / 'scripts' / 'build_wheels.py'), '--python', args.python], env=env, - cwd=build_context_dir, + cwd=builder_root, ) output_dir = Path(args.output_dir) diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh deleted file mode 100644 index a02b694db653a..0000000000000 --- a/.builders/images/macos/build.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -export MACOSX_DEPLOYMENT_TARGET="10.12" - -"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip -"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv -"${DD_PYTHON3}" -m virtualenv py3 -export DD_PY3_BUILDENV_PATH="$(pwd)/py3/bin/python" - -"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip -"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv -"${DD_PYTHON2}" -m virtualenv py2 -export DD_PY2_BUILDENV_PATH="$(pwd)/py2/bin/python" - -# Path where we'll install libraries that we build -export DD_PREFIX_PATH="$(pwd)/prefix" - -export LDFLAGS="-Wl,-rpath,${DD_PREFIX_PATH}/lib -L${DD_PREFIX_PATH}/lib" -export CFLAGS="-I${DD_PREFIX_PATH}/include -O2" -export PATH="${DD_PREFIX_PATH}/bin:${PATH}" - -"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt" - -# Install always with our own prefix path -install-from-source() { - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" "$@" -} - -# mqi -IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit -curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg -sudo installer -pkg /tmp/mq_client.pkg -target / -rm -rf /tmp/mq_client.pkg - -# Restore cache if it exists -if [[ -n ${DD_PREFIX_CACHE:-} && -d ${DD_PREFIX_CACHE:-} ]]; then - echo "Using provided cache for built libraries." - cp -r "${DD_PREFIX_CACHE}" "${DD_PREFIX_PATH}" -else - # openssl - DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \ - VERSION="3.0.12" \ - SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \ - RELATIVE_PATH="openssl-{{version}}" \ - CONFIGURE_SCRIPT="./config" \ - install-from-source \ - -fPIC shared \ - no-module \ - no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost - - # libxml & libxslt for lxml - DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \ - VERSION="2.10.3" \ - SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \ - RELATIVE_PATH="libxml2-{{version}}" \ - install-from-source \ - --without-iconv \ - --without-python \ - --without-icu \ - --without-debug \ - --without-mem-debug \ - --without-run-debug \ - --without-legacy \ - --without-catalog \ - --without-docbook \ - --disable-static - - DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \ - VERSION="1.1.37" \ - SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \ - RELATIVE_PATH="libxslt-{{version}}" \ - install-from-source \ - --with-libxml-prefix="${DD_PREFIX_PATH}" \ - --without-python \ - --without-crypto \ - --without-profiler \ - --without-debugger \ - --disable-static - - # curl - DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \ - VERSION="8.4.0" \ - SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \ - RELATIVE_PATH="curl-{{version}}" \ - install-from-source \ - --disable-manual \ - --disable-debug \ - --enable-optimize \ - --disable-static \ - --disable-ldap \ - --disable-ldaps \ - --disable-rtsp \ - --enable-proxy \ - --disable-dependency-tracking \ - --enable-ipv6 \ - --without-libidn \ - --without-gnutls \ - --without-librtmp \ - --without-libssh2 \ - --with-ssl="${DD_PREFIX_PATH}" - # Remove the binary installed so that we consistenly use the same original `curl` binary - rm "${DD_PREFIX_PATH}/bin/curl" - - # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support - # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation - # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being - # compatible with that in the Agent, the rest shouldn't matter much - DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \ - VERSION="0.9.29" \ - SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \ - RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \ - CONFIGURE_SCRIPT="true" \ - install-from-source - DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \ - VERSION="2.1.28" \ - SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ - RELATIVE_PATH="cyrus-sasl-{{version}}" \ - install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework - - # Cache everything under prefix - if [[ -n ${DD_PREFIX_CACHE:-} ]]; then - cp -r "${DD_PREFIX_PATH}" "${DD_PREFIX_CACHE}" - fi -fi - -export DD_BUILD_COMMAND="bash $(pwd)/extra_build.sh" - -"${DD_PYTHON3}" "${DD_MOUNT_DIR}/scripts/build_wheels.py" "$@" diff --git a/.builders/images/macos/builder_setup.sh b/.builders/images/macos/builder_setup.sh new file mode 100644 index 0000000000000..c3b90b250a4a8 --- /dev/null +++ b/.builders/images/macos/builder_setup.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip +"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv +"${DD_PYTHON3}" -m virtualenv py3 + +"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip +"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv +"${DD_PYTHON2}" -m virtualenv py2 + +"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "${DD_MOUNT_DIR}/build_context/runner_dependencies.txt" + +# Install always with our own prefix path +cp "${DD_MOUNT_DIR}/build_context/install-from-source.sh" . +install-from-source() { + bash "install-from-source.sh" --prefix="${DD_PREFIX_PATH}" "$@" +} + +# mqi +IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit +curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg +sudo installer -pkg /tmp/mq_client.pkg -target / +rm -rf /tmp/mq_client.pkg + +# openssl +DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \ +VERSION="3.0.12" \ +SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \ +RELATIVE_PATH="openssl-{{version}}" \ +CONFIGURE_SCRIPT="./config" \ + install-from-source \ + -fPIC shared \ + no-module \ + no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost + +# libxml & libxslt for lxml +DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \ +VERSION="2.10.3" \ +SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \ +RELATIVE_PATH="libxml2-{{version}}" \ + install-from-source \ + --without-iconv \ + --without-python \ + --without-icu \ + --without-debug \ + --without-mem-debug \ + --without-run-debug \ + --without-legacy \ + --without-catalog \ + --without-docbook \ + --disable-static + +DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \ +VERSION="1.1.37" \ +SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \ +RELATIVE_PATH="libxslt-{{version}}" \ + install-from-source \ + --with-libxml-prefix="${DD_PREFIX_PATH}" \ + --without-python \ + --without-crypto \ + --without-profiler \ + --without-debugger \ + --disable-static + +# curl +DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \ +VERSION="8.4.0" \ +SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \ +RELATIVE_PATH="curl-{{version}}" \ + install-from-source \ + --disable-manual \ + --disable-debug \ + --enable-optimize \ + --disable-static \ + --disable-ldap \ + --disable-ldaps \ + --disable-rtsp \ + --enable-proxy \ + --disable-dependency-tracking \ + --enable-ipv6 \ + --without-libidn \ + --without-gnutls \ + --without-librtmp \ + --without-libssh2 \ + --with-ssl="${DD_PREFIX_PATH}" +# Remove the binary installed so that we consistenly use the same original `curl` binary +rm "${DD_PREFIX_PATH}/bin/curl" + +# Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support +# Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation +# with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being +# compatible with that in the Agent, the rest shouldn't matter much +DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \ +VERSION="0.9.29" \ +SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \ +RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \ +CONFIGURE_SCRIPT="true" \ + install-from-source +DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \ +VERSION="2.1.28" \ +SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ +RELATIVE_PATH="cyrus-sasl-{{version}}" \ + install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml index 729753ea68356..682c52e357cf5 100644 --- a/.github/workflows/build-deps-macos.yml +++ b/.github/workflows/build-deps-macos.yml @@ -34,7 +34,11 @@ jobs: env: DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3" DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python" + # This sets the minimum mac os version compatible for all built artifacts + MACOSX_DEPLOYMENT_TARGET: "10.12" run: | ${DD_PYTHON3} -m pip install packaging - ${DD_PYTHON3} .builders/build.py --python 3 out_py3 - ${DD_PYTHON3} .builders/build.py --python 2 out_py2 + + mkdir builder_root + ${DD_PYTHON3} .builders/build.py --builder-root builder_root --python 3 out_py3 + ${DD_PYTHON3} .builders/build.py --builder-root builder_root --skip-setup --python 2 out_py2