WIP

.github/workflows/build_python_runtime.yml

@@ -26,23 +26,23 @@ env:
 # Unfortunately these jobs cannot be easily written as a matrix since `matrix.exclude` does not
 # support expression syntax, and the `inputs` context is not available inside the job `if` key.
 jobs:
-  build-and-upload-heroku-20:
+  heroku-20:
     runs-on: pub-hk-ubuntu-22.04-xlarge
     env:
       STACK_VERSION: "20"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Build Docker image
-        run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/
+        run: docker build --platform="linux/amd64" --pull --tag buildenv --build-arg=STACK_VERSION builds/
       - name: Build and package Python runtime
-        run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+        run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
       - name: Upload Python runtime archive to S3
         if: (!inputs.dry_run)
         run: aws s3 sync ./upload "s3://${S3_BUCKET}"
 
-  build-and-upload-heroku-22:
-    # We only support Python 3.9+ on Heroku-22.
+  heroku-22:
+    # On Heroku-22 we only support Python 3.9+.
     if: (!startsWith(inputs.python_version,'3.8.'))
     runs-on: pub-hk-ubuntu-22.04-xlarge
     env:
@@ -51,9 +51,43 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Build Docker image
-        run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/
+        run: docker build --platform="linux/amd64" --pull --tag buildenv --build-arg=STACK_VERSION builds/
       - name: Build and package Python runtime
-        run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+        run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+      - name: Upload Python runtime archive to S3
+        if: (!inputs.dry_run)
+        run: aws s3 sync ./upload "s3://${S3_BUCKET}"
+
+  # TODO: Arch
+  heroku-24:
+    # On Heroku-24 we only support Python 3.12+.
+    if: (startsWith(inputs.python_version,'3.12.'))
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture: ["amd64", "arm64"]
+    runs-on: "${{ matrix.architecture == 'arm64' && 'pub-hk-ubuntu-22.04-arm-large' || 'pub-hk-ubuntu-22.04-xlarge' }}"
+    env:
+      STACK_VERSION: "24"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # The beta Arm64 runners don't yet ship with the normal installed tools.
+      - name: Install Docker and AWS CLI (ARM64 only)
+        if: matrix.architecture == 'arm64'
+        run: |
+          sudo apt-get update --error-on=any
+          sudo apt-get install -y --no-install-recommends acl docker.io docker-buildx unzip
+          sudo usermod -aG docker $USER
+          sudo setfacl --modify user:$USER:rw /var/run/docker.sock
+          curl https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip -o awscliv2.zip
+          unzip awscliv2.zip
+          sudo ./aws/install
+          rm -rf awscliv2.zip ./aws/
+      - name: Build Docker image
+        run: docker build --platform="linux/${{ matrix.architecture }}" --pull --tag buildenv --build-arg=STACK_VERSION builds/
+      - name: Build and package Python runtime
+        run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
       - name: Upload Python runtime archive to S3
         if: (!inputs.dry_run)
         run: aws s3 sync ./upload "s3://${S3_BUCKET}"

bin/steps/python

@@ -8,8 +8,10 @@
 runtime-fixer runtime.txt || true
 PYTHON_VERSION=$(cat runtime.txt)
 
-# The location of the pre-compiled python binary.
-PYTHON_URL="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz"
+# The Python runtime archive filename is of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst'
+ARCH=$(dpkg --print-architecture)
+UBUNTU_VERSION=$(lsb_release --short --release 2>/dev/null)
+PYTHON_URL="${S3_BASE_URL}/${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-${ARCH}.tar.zst"
 
 if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}"; then
   puts-warn "Requested runtime '${PYTHON_VERSION}' is not available for this stack (${STACK})."
@@ -135,7 +137,7 @@ else
   # Prepare destination directory.
   mkdir -p .heroku/python
 
-  if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar -zxC .heroku/python; then
+  if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar --zstd --extract --directory .heroku/python; then
     # The Python version was confirmed to exist previously, so any failure here is due to
     # a networking issue or archive/buildpack bug rather than the runtime not existing.
     puts-warn "Failed to download/install ${PYTHON_VERSION}"

builds/Dockerfile

@@ -1,11 +1,14 @@
-ARG STACK_VERSION="22"
-FROM --platform=linux/amd64 heroku/heroku:${STACK_VERSION}-build
+ARG STACK_VERSION="24"
+FROM heroku/heroku:${STACK_VERSION}-build
 
 ARG STACK_VERSION
 ENV STACK="heroku-${STACK_VERSION}"
 
-RUN apt-get update \
-    && apt-get install --no-install-recommends -y \
+# For Heroku-24 and newer, the build image sets a non-root default `USER`.
+USER root
+
+RUN apt-get update --error-on=any \
+    && apt-get install -y --no-install-recommends \
       libsqlite3-dev \
     && rm -rf /var/lib/apt/lists/*
 

builds/build_python_runtime.sh

@@ -5,16 +5,26 @@ set -euo pipefail
 PYTHON_VERSION="${1:?"Error: The Python version to build must be specified as the first argument."}"
 PYTHON_MAJOR_VERSION="${PYTHON_VERSION%.*}"
 
-INSTALL_DIR="/app/.heroku/python"
+ARCH=$(dpkg --print-architecture)
+
+# Python is relocated at build time to different locations for classic vs CNB (which works since
+# we set LD_LIBRARY_PATH and PYTHONHOME appropriately), so for packaging purposes here we "install"
+# Python here into an arbitrary location that intentionally matches neither location.
+INSTALL_DIR="/python"
 SRC_DIR="/tmp/src"
-ARCHIVES_DIR="/tmp/upload/${STACK}/runtimes"
+UPLOAD_DIR="/tmp/upload"
 
 function error() {
   echo "Error: ${1}" >&2
   exit 1
 }
 
 case "${STACK}" in
+  heroku-24)
+    SUPPORTED_PYTHON_VERSIONS=(
+      "3.12"
+    )
+    ;;
   heroku-22)
     SUPPORTED_PYTHON_VERSIONS=(
       "3.9"
@@ -60,14 +70,14 @@ case "${PYTHON_MAJOR_VERSION}" in
     ;;
 esac
 
-echo "Building Python ${PYTHON_VERSION} for ${STACK}..."
+echo "Building Python ${PYTHON_VERSION} for ${STACK} (${ARCH})..."
 
 SOURCE_URL="https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz"
 SIGNATURE_URL="${SOURCE_URL}.asc"
 
 set -o xtrace
 
-mkdir -p "${SRC_DIR}" "${INSTALL_DIR}" "${ARCHIVES_DIR}"
+mkdir -p "${SRC_DIR}" "${INSTALL_DIR}" "${UPLOAD_DIR}"
 
 curl --fail --retry 3 --retry-connrefused --connect-timeout 10 --max-time 60 -o python.tgz "${SOURCE_URL}"
 curl --fail --retry 3 --retry-connrefused --connect-timeout 10 --max-time 60 -o python.tgz.asc "${SIGNATURE_URL}"
@@ -78,10 +88,28 @@ gpg --batch --verify python.tgz.asc python.tgz
 tar --extract --file python.tgz --strip-components=1 --directory "${SRC_DIR}"
 cd "${SRC_DIR}"
 
+# explicit architecture
+# https://github.com/docker-library/python/pull/198
+# https://github.com/docker-library/memcached/pull/13
+
+# https://github.com/docker-library/python/blob/master/3.12/bookworm/Dockerfile
+
+# hardening flags: https://github.com/docker-library/python/issues/810
+
+# dpkg-buildflags --get CFLAGS
+# -g -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -ffile-prefix-map=/=. -flto=auto -ffat-lto-objects -fstack-protector-strong -fstack-clash-protection -Wformat -Werror=format-security -mbranch-protection=standard
+
+# dpkg-buildflags --get LDFLAGS
+# -Wl,-Bsymbolic-functions -flto=auto -ffat-lto-objects -Wl,-z,relro
+
 # Aim to keep this roughly consistent with the options used in the Python Docker images,
 # for maximum compatibility / most battle-tested build configuration:
 # https://github.com/docker-library/python
 CONFIGURE_OPTS=(
+  # Explicitly set the target architecture rather than auto-detecting based on the host CPU.
+  # This only affects targets like i386 (for which we don't build), but we pass it anyway for
+  # completeness and parity with the Python Docker image builds.
+  "--build=$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"
   # Support loadable extensions in the `_sqlite` extension module.
   "--enable-loadable-sqlite-extensions"
   # Enable recommended release build performance optimisations such as PGO.
@@ -133,11 +161,17 @@ fi
 
 ./configure "${CONFIGURE_OPTS[@]}"
 
-# Using LDFLAGS we instruct the linker to omit all symbol information from the final binary
-# and shared libraries, to reduce the size of the build. We have to use `--strip-all` and
+# dpkg-buildflags returns the distro's default compiler/linker options, which enables
+# various security/hardening best practices. See:
+# - https://wiki.ubuntu.com/ToolChain/CompilerFlags
+# - https://wiki.debian.org/Hardening
+# We also use `--strip-all` to instruct the linker to omit all symbol information from the final
+# binary and shared libraries, to reduce the size of the build. We have to use `--strip-all` and
 # not `--strip-unneeded` since `ld` only understands the former (unlike the `strip` command),
 # however it's safe to use since these options don't apply to static libraries.
-make -j "$(nproc)" LDFLAGS='-Wl,--strip-all'
+make -j "$(nproc)" \
+  "EXTRA_CFLAGS=$(dpkg-buildflags --get CFLAGS)" \
+  "LDFLAGS=$(dpkg-buildflags --get LDFLAGS),--strip-all"
 make install
 
 if [[ "${PYTHON_MAJOR_VERSION}" == 3.[8-9] ]]; then
@@ -191,13 +225,11 @@ LD_LIBRARY_PATH="${SRC_DIR}" "${SRC_DIR}/python" -m compileall -f --invalidation
 # This symlink must be relative, to ensure that the Python install remains relocatable.
 ln -srvT "${INSTALL_DIR}/bin/python3" "${INSTALL_DIR}/bin/python"
 
-cd "${ARCHIVES_DIR}"
-
-# The tar file is gzipped separately, so we can set a higher gzip compression level than
-# the default. In the future we'll also want to create a second archive that used zstd.
-TAR_FILENAME="python-${PYTHON_VERSION}.tar"
-tar --create --format=pax --sort=name --verbose --file "${TAR_FILENAME}" --directory="${INSTALL_DIR}" .
-gzip --best "${TAR_FILENAME}"
+# Results in a compressed archive filename of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst'
+UBUNTU_VERSION=$(lsb_release --short --release 2>/dev/null)
+TAR_FILEPATH="${UPLOAD_DIR}/python-${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-${ARCH}.tar"
+tar --create --format=pax --sort=name --file "${TAR_FILEPATH}" --directory="${INSTALL_DIR}" .
+zstd -T0 -22 --ultra --long --no-progress --rm "${TAR_FILEPATH}"
 
 du --max-depth 1 --human-readable "${INSTALL_DIR}"
-du --all --human-readable "${ARCHIVES_DIR}"
+du --all --human-readable "${UPLOAD_DIR}"