diff --git a/.github/workflows/build_python_runtime.yml b/.github/workflows/build_python_runtime.yml index d47fefcaa..8a149832b 100644 --- a/.github/workflows/build_python_runtime.yml +++ b/.github/workflows/build_python_runtime.yml @@ -26,7 +26,7 @@ env: # Unfortunately these jobs cannot be easily written as a matrix since `matrix.exclude` does not # support expression syntax, and the `inputs` context is not available inside the job `if` key. jobs: - build-and-upload-heroku-20: + heroku-20: runs-on: pub-hk-ubuntu-22.04-xlarge env: STACK_VERSION: "20" @@ -34,15 +34,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Build Docker image - run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/ + run: docker build --platform="linux/amd64" --pull --tag buildenv --build-arg=STACK_VERSION builds/ - name: Build and package Python runtime - run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}" + run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}" - name: Upload Python runtime archive to S3 if: (!inputs.dry_run) run: aws s3 sync ./upload "s3://${S3_BUCKET}" - build-and-upload-heroku-22: - # We only support Python 3.9+ on Heroku-22. + heroku-22: + # On Heroku-22 we only support Python 3.9+. if: (!startsWith(inputs.python_version,'3.8.')) runs-on: pub-hk-ubuntu-22.04-xlarge env: @@ -51,9 +51,43 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Build Docker image - run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/ + run: docker build --platform="linux/amd64" --pull --tag buildenv --build-arg=STACK_VERSION builds/ - name: Build and package Python runtime - run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}" + run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}" + - name: Upload Python runtime archive to S3 + if: (!inputs.dry_run) + run: aws s3 sync ./upload "s3://${S3_BUCKET}" + + # TODO: Arch + heroku-24: + # On Heroku-24 we only support Python 3.12+. + if: (startsWith(inputs.python_version,'3.12.')) + strategy: + fail-fast: false + matrix: + architecture: ["amd64", "arm64"] + runs-on: "${{ matrix.architecture == 'arm64' && 'pub-hk-ubuntu-22.04-arm-large' || 'pub-hk-ubuntu-22.04-xlarge' }}" + env: + STACK_VERSION: "24" + steps: + - name: Checkout + uses: actions/checkout@v4 + # The beta Arm64 runners don't yet ship with the normal installed tools. + - name: Install Docker and AWS CLI (ARM64 only) + if: matrix.architecture == 'arm64' + run: | + sudo apt-get update --error-on=any + sudo apt-get install -y --no-install-recommends acl docker.io docker-buildx unzip + sudo usermod -aG docker $USER + sudo setfacl --modify user:$USER:rw /var/run/docker.sock + curl https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install + rm -rf awscliv2.zip ./aws/ + - name: Build Docker image + run: docker build --platform="linux/${{ matrix.architecture }}" --pull --tag buildenv --build-arg=STACK_VERSION builds/ + - name: Build and package Python runtime + run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}" - name: Upload Python runtime archive to S3 if: (!inputs.dry_run) run: aws s3 sync ./upload "s3://${S3_BUCKET}" diff --git a/bin/steps/python b/bin/steps/python index b231e92bb..bab836dfb 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -8,8 +8,10 @@ runtime-fixer runtime.txt || true PYTHON_VERSION=$(cat runtime.txt) -# The location of the pre-compiled python binary. -PYTHON_URL="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz" +# The Python runtime archive filename is of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst' +ARCH=$(dpkg --print-architecture) +UBUNTU_VERSION=$(lsb_release --short --release 2>/dev/null) +PYTHON_URL="${S3_BASE_URL}/${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-${ARCH}.tar.zst" if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}"; then puts-warn "Requested runtime '${PYTHON_VERSION}' is not available for this stack (${STACK})." @@ -135,7 +137,7 @@ else # Prepare destination directory. mkdir -p .heroku/python - if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar -zxC .heroku/python; then + if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar --zstd --extract --directory .heroku/python; then # The Python version was confirmed to exist previously, so any failure here is due to # a networking issue or archive/buildpack bug rather than the runtime not existing. puts-warn "Failed to download/install ${PYTHON_VERSION}" diff --git a/builds/Dockerfile b/builds/Dockerfile index 37e25c26f..8dcd1f7cb 100644 --- a/builds/Dockerfile +++ b/builds/Dockerfile @@ -1,11 +1,14 @@ -ARG STACK_VERSION="22" -FROM --platform=linux/amd64 heroku/heroku:${STACK_VERSION}-build +ARG STACK_VERSION="24" +FROM heroku/heroku:${STACK_VERSION}-build ARG STACK_VERSION ENV STACK="heroku-${STACK_VERSION}" -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ +# For Heroku-24 and newer, the build image sets a non-root default `USER`. +USER root + +RUN apt-get update --error-on=any \ + && apt-get install -y --no-install-recommends \ libsqlite3-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/builds/build_python_runtime.sh b/builds/build_python_runtime.sh index 379505e63..a2739dcaf 100755 --- a/builds/build_python_runtime.sh +++ b/builds/build_python_runtime.sh @@ -5,9 +5,14 @@ set -euo pipefail PYTHON_VERSION="${1:?"Error: The Python version to build must be specified as the first argument."}" PYTHON_MAJOR_VERSION="${PYTHON_VERSION%.*}" -INSTALL_DIR="/app/.heroku/python" +ARCH=$(dpkg --print-architecture) + +# Python is relocated at build time to different locations for classic vs CNB (which works since +# we set LD_LIBRARY_PATH and PYTHONHOME appropriately), so for packaging purposes here we "install" +# Python here into an arbitrary location that intentionally matches neither location. +INSTALL_DIR="/python" SRC_DIR="/tmp/src" -ARCHIVES_DIR="/tmp/upload/${STACK}/runtimes" +UPLOAD_DIR="/tmp/upload" function error() { echo "Error: ${1}" >&2 @@ -15,6 +20,11 @@ function error() { } case "${STACK}" in + heroku-24) + SUPPORTED_PYTHON_VERSIONS=( + "3.12" + ) + ;; heroku-22) SUPPORTED_PYTHON_VERSIONS=( "3.9" @@ -60,14 +70,14 @@ case "${PYTHON_MAJOR_VERSION}" in ;; esac -echo "Building Python ${PYTHON_VERSION} for ${STACK}..." +echo "Building Python ${PYTHON_VERSION} for ${STACK} (${ARCH})..." SOURCE_URL="https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" SIGNATURE_URL="${SOURCE_URL}.asc" set -o xtrace -mkdir -p "${SRC_DIR}" "${INSTALL_DIR}" "${ARCHIVES_DIR}" +mkdir -p "${SRC_DIR}" "${INSTALL_DIR}" "${UPLOAD_DIR}" curl --fail --retry 3 --retry-connrefused --connect-timeout 10 --max-time 60 -o python.tgz "${SOURCE_URL}" curl --fail --retry 3 --retry-connrefused --connect-timeout 10 --max-time 60 -o python.tgz.asc "${SIGNATURE_URL}" @@ -78,10 +88,28 @@ gpg --batch --verify python.tgz.asc python.tgz tar --extract --file python.tgz --strip-components=1 --directory "${SRC_DIR}" cd "${SRC_DIR}" +# explicit architecture +# https://github.com/docker-library/python/pull/198 +# https://github.com/docker-library/memcached/pull/13 + +# https://github.com/docker-library/python/blob/master/3.12/bookworm/Dockerfile + +# hardening flags: https://github.com/docker-library/python/issues/810 + +# dpkg-buildflags --get CFLAGS +# -g -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -ffile-prefix-map=/=. -flto=auto -ffat-lto-objects -fstack-protector-strong -fstack-clash-protection -Wformat -Werror=format-security -mbranch-protection=standard + +# dpkg-buildflags --get LDFLAGS +# -Wl,-Bsymbolic-functions -flto=auto -ffat-lto-objects -Wl,-z,relro + # Aim to keep this roughly consistent with the options used in the Python Docker images, # for maximum compatibility / most battle-tested build configuration: # https://github.com/docker-library/python CONFIGURE_OPTS=( + # Explicitly set the target architecture rather than auto-detecting based on the host CPU. + # This only affects targets like i386 (for which we don't build), but we pass it anyway for + # completeness and parity with the Python Docker image builds. + "--build=$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" # Support loadable extensions in the `_sqlite` extension module. "--enable-loadable-sqlite-extensions" # Enable recommended release build performance optimisations such as PGO. @@ -133,11 +161,17 @@ fi ./configure "${CONFIGURE_OPTS[@]}" -# Using LDFLAGS we instruct the linker to omit all symbol information from the final binary -# and shared libraries, to reduce the size of the build. We have to use `--strip-all` and +# dpkg-buildflags returns the distro's default compiler/linker options, which enables +# various security/hardening best practices. See: +# - https://wiki.ubuntu.com/ToolChain/CompilerFlags +# - https://wiki.debian.org/Hardening +# We also use `--strip-all` to instruct the linker to omit all symbol information from the final +# binary and shared libraries, to reduce the size of the build. We have to use `--strip-all` and # not `--strip-unneeded` since `ld` only understands the former (unlike the `strip` command), # however it's safe to use since these options don't apply to static libraries. -make -j "$(nproc)" LDFLAGS='-Wl,--strip-all' +make -j "$(nproc)" \ + "EXTRA_CFLAGS=$(dpkg-buildflags --get CFLAGS)" \ + "LDFLAGS=$(dpkg-buildflags --get LDFLAGS),--strip-all" make install if [[ "${PYTHON_MAJOR_VERSION}" == 3.[8-9] ]]; then @@ -191,13 +225,11 @@ LD_LIBRARY_PATH="${SRC_DIR}" "${SRC_DIR}/python" -m compileall -f --invalidation # This symlink must be relative, to ensure that the Python install remains relocatable. ln -srvT "${INSTALL_DIR}/bin/python3" "${INSTALL_DIR}/bin/python" -cd "${ARCHIVES_DIR}" - -# The tar file is gzipped separately, so we can set a higher gzip compression level than -# the default. In the future we'll also want to create a second archive that used zstd. -TAR_FILENAME="python-${PYTHON_VERSION}.tar" -tar --create --format=pax --sort=name --verbose --file "${TAR_FILENAME}" --directory="${INSTALL_DIR}" . -gzip --best "${TAR_FILENAME}" +# Results in a compressed archive filename of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst' +UBUNTU_VERSION=$(lsb_release --short --release 2>/dev/null) +TAR_FILEPATH="${UPLOAD_DIR}/python-${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-${ARCH}.tar" +tar --create --format=pax --sort=name --file "${TAR_FILEPATH}" --directory="${INSTALL_DIR}" . +zstd -T0 -22 --ultra --long --no-progress --rm "${TAR_FILEPATH}" du --max-depth 1 --human-readable "${INSTALL_DIR}" -du --all --human-readable "${ARCHIVES_DIR}" +du --all --human-readable "${UPLOAD_DIR}"