feat: [NODE-1355] Upgrade build container to 24.04

.github/workflows-source/ci-main.yml

@@ -229,7 +229,7 @@ jobs:
         run: |
           set -xeuo pipefail
           export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies
-          pip3 install --ignore-installed -r requirements.txt
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
           cd ci/src
           pytest -m "not fails_on_merge_train" -v -o junit_family=xunit1 \
             --junitxml=../../test_report.xml --cov=. --cov-report=term \

.github/workflows-source/ci-pr-only.yml

@@ -148,7 +148,7 @@ jobs:
         id: setup-python-deps
         shell: bash
         run: |
-          pip3 install --ignore-installed -r requirements.txt
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
       - name: Dependency Scan for Pull Request
         id: dependencies-check
         shell: bash

.github/workflows-source/release-testing.yml

@@ -152,7 +152,7 @@ jobs:
         id: setup-python-deps
         shell: bash
         run: |
-          pip3 install --ignore-installed -r requirements.txt
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
       - name: Dependency Scan for Release
         id: dependency-scan-release-cut
         shell: bash

.github/workflows-source/schedule-daily.yml

@@ -198,7 +198,7 @@ jobs:
         id: setup-environment-deps
         shell: bash
         run: |
-          pip3 install --ignore-installed -r requirements.txt
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
           cargo install wasm-pack --version "${CARGO_WASMPACK_VERSION}"
           source "${NVM_DIR}/nvm.sh"
           nvm use ${DEFAULT_NODE_VERSION}

.github/workflows/ci-main.yml

@@ -298,7 +298,7 @@ jobs:
         run: |
           set -xeuo pipefail
           export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies
-          pip3 install --ignore-installed -r requirements.txt
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
           cd ci/src
           pytest -m "not fails_on_merge_train" -v -o junit_family=xunit1 \
             --junitxml=../../test_report.xml --cov=. --cov-report=term \

.github/workflows/ci-pr-only.yml

@@ -151,7 +151,7 @@ jobs:
         id: setup-python-deps
         shell: bash
         run: |
-          pip3 install --ignore-installed -r requirements.txt
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
       - name: Dependency Scan for Pull Request
         id: dependencies-check
         shell: bash

.github/workflows/release-testing.yml

@@ -207,7 +207,7 @@ jobs:
         id: setup-python-deps
         shell: bash
         run: |
-          pip3 install --ignore-installed -r requirements.txt
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
       - name: Dependency Scan for Release
         id: dependency-scan-release-cut
         shell: bash

.github/workflows/schedule-daily.yml

@@ -253,7 +253,7 @@ jobs:
         id: setup-environment-deps
         shell: bash
         run: |
-          pip3 install --ignore-installed -r requirements.txt
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
           cargo install wasm-pack --version "${CARGO_WASMPACK_VERSION}"
           source "${NVM_DIR}/nvm.sh"
           nvm use ${DEFAULT_NODE_VERSION}

.pre-commit-config.yaml

@@ -66,15 +66,10 @@ repos:
     stages: [manual]
     pass_filenames: false
 
-- repo: https://github.com/willthames/ansible-lint.git
-  rev: v4.2.0
+- repo: https://github.com/ansible/ansible-lint.git
+  rev: v24.5.0
   hooks:
     - id: ansible-lint
       always_run: false
-      files: (^|/)testnet/.+\.(yaml|yml)$
-      exclude: |
-          (?x)^(
-              testnet/tests/.*|.*/docker-compose.yml|testnet/env/shared-config\.yml
-          )$
-      # To be uncommented upon moving to new ansible-lint
-      # args: ['-i', 'testnet/ansible/.ansible-lint-ignore', 'testnet/ansible']
+      files: (^|/)testnet/ansible/.+\.(yaml|yml)$
+      args: ['-i', 'testnet/ansible/.ansible-lint-ignore', 'testnet/ansible']

WORKSPACE.bazel

@@ -50,20 +50,6 @@ distroless_register_toolchains()
 
 load("@rules_distroless//apt:index.bzl", "deb_index")
 
-# Packageset based on an Ubuntu focal snapshot, see manifest file
-# for details
-# To update, comment out the `lock` field below and run:
-#   bazel run @focal//:lock
-deb_index(
-    name = "focal",
-    lock = "//bazel:focal.lock.json",
-    manifest = "//bazel:focal.yaml",
-)
-
-load("@focal//:packages.bzl", "focal_packages")
-
-focal_packages()
-
 # Packageset based on an Ubuntu noble snapshot, see manifest file
 # for details
 # To update, comment out the `lock` field below and run:
@@ -143,20 +129,13 @@ oci_pull(
 # used by rosetta image
 oci_pull(
     name = "rust_base",
-    image = "gcr.io/distroless/cc-debian11@sha256:8e94f031353596c3fc9db6a2499bcc82dacc40cb71e0703476f9fad41677efdf",
+    image = "gcr.io/distroless/cc-debian12@sha256:3310655aac0d85eb9d579792387af1ff3eb7a1667823478be58020ab0e0d97a8",
     platforms = ["linux/amd64"],
 )
 
 # used in various places as base
 oci_pull(
     name = "ubuntu_base",
-    image = "docker.io/library/ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea",
-    platforms = ["linux/amd64"],
-)
-
-# used in various places as base
-oci_pull(
-    name = "ubuntu_noble_base",
     image = "docker.io/library/ubuntu@sha256:77d57fd89366f7d16615794a5b53e124d742404e20f035c22032233f1826bd6a",
     platforms = ["linux/amd64"],
 )

ci/container/Dockerfile

@@ -1,6 +1,6 @@
 # https://hub.docker.com/_/ubuntu
-# focal-20240216
-FROM ubuntu@sha256:48c35f3de33487442af224ed4aabac19fd9bfbd91ee90e9471d412706b20ba73
+# noble-20240605
+FROM ubuntu@sha256:2e863c44b718727c860746568e1d54afd13b2fa71b160f5cd9058fc436217b30
 ENV TZ=UTC
 
 COPY --chmod=0700 ./ci/container/files/known_hosts /etc/ssh/ssh_known_hosts
@@ -28,7 +28,7 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /
 
 # install afl & gsutils deps for bazel-fuzzers
 RUN curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | apt-key add - && \
-    echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main" | tee -a /etc/apt/sources.list.d/llvm.list && \
+    echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main" | tee -a /etc/apt/sources.list.d/llvm.list && \
     echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && \
     apt -yq update && \
@@ -55,6 +55,25 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     cd .. && \
     rm -rf e2fsdroid
 
+# Install an older libtinfo for rules_haskell
+RUN mkdir libtinfo && \
+    cd libtinfo && \
+    wget "http://security.ubuntu.com/ubuntu/pool/universe/n/ncurses/libtinfo5_6.4-2ubuntu0.1_amd64.deb" && \
+    apt -yqq install ./libtinfo5_6.4-2ubuntu0.1_amd64.deb && \
+    cd .. && \
+    rm -rf libtinfo
+
+# Build libunwind from source. The packaged version breaks linking
+RUN apt -yqq install libtool && \
+    git clone --branch v1.8.1 --depth 1 "https://github.com/libunwind/libunwind" && \
+    cd libunwind && \
+    autoreconf -i && \
+    ./configure --libdir=/lib/x86_64-linux-gnu --includedir=/usr/include/x86_64-linux-gnu --disable-tests && \
+    make && \
+    make install && \
+    cd .. && \
+    rm -rf libunwind
+
 ARG sdk_version=0.12.0
 ARG sdk_sha=40da56ad27774d5e1b2cbc35f94c17368be8c8da557aca19878940264bd82a0a
 RUN mkdir -p /tmp/sdk && curl -fsSL https://github.com/dfinity/sdk/releases/download/${sdk_version}/dfx-${sdk_version}-x86_64-linux.tar.gz -o /tmp/sdk/dfx.tar.gz && \
@@ -84,9 +103,8 @@ RUN . /opt/nvm/nvm.sh && \
     nvm install ${dependency_mgmt_nns_dapp_node_version} && \
     nvm install ${dependency_mgmt_default_node_version}
 
-RUN groupadd -g 1000 ubuntu && useradd -ms /bin/bash -u 1000 -g 1000 ubuntu && \
-    # needed for github actions runner \
-    groupadd -g 1001 buildifier && useradd -ms /bin/bash -u 1001 -g 1001 buildifier && \
+# needed for github actions runner
+RUN groupadd -g 1001 buildifier && useradd -ms /bin/bash -u 1001 -g 1001 -G ubuntu buildifier && \
     # CI before script requires sudo \
     echo "ALL ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 
@@ -125,9 +143,9 @@ RUN curl -sSL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/
 
 # Add mold linker
 ARG MOLD_BIN="/usr/local/bin/mold"
-ARG MOLD_VERSION=2.4.1
+ARG MOLD_VERSION=2.31.0
 RUN curl -sSL "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/mold-${MOLD_VERSION}-$(uname -m)-linux.tar.gz" | tar -C /usr/local --strip-components=1 -xzf - && \
-    echo "4d34b489a0810e71a937103f38e4b6c951abac36b5b60c58bc07b73efa7139cd  ${MOLD_BIN}" | shasum -a 256 -c - && \
+    echo "557c1f19b575e42714771727afa6c4f7fa9b98e41c65f281f7d6cd8f9863a817  ${MOLD_BIN}" | shasum -a 256 -c - && \
     ln -sf "${MOLD_BIN}" "$(realpath /usr/bin/ld)"
 
 # Add kubectl

ci/container/build-image.sh

@@ -38,8 +38,7 @@ fi
 
 DOCKER_BUILDKIT=1 docker "${ARGS[@]}" build "${BUILD_ARGS[@]}" \
     -t ic-build:"$DOCKER_IMG_TAG" \
-    -t docker.io/dfinity/ic-build:"$DOCKER_IMG_TAG" \
-    -t docker.io/dfinity/ic-build:latest \
+    -t ghcr.io/dfinity/ic-build:latest \
     -t ghcr.io/dfinity/ic-build:"$DOCKER_IMG_TAG" \
     --build-arg RUST_VERSION="$RUST_VERSION" \
     -f ci/container/Dockerfile .

ci/container/container-run.sh

@@ -65,9 +65,12 @@ IMAGE="$IMAGE:$IMAGE_TAG"
 if ! sudo podman "${PODMAN_ARGS[@]}" image exists $IMAGE; then
     if ! sudo podman "${PODMAN_ARGS[@]}" pull $IMAGE; then
         # fallback to building the image
-        docker() { sudo podman "${PODMAN_ARGS[@]}" "$@" --network=host; }
+        docker() {
+            PODMAN_ARGS=(${PODMAN_ARGS})
+            sudo podman "${PODMAN_ARGS[@]}" "$@" --network=host
+        }
         export -f docker
-        "$REPO_ROOT"/ci/container/build-image.sh "${BUILD_ARGS[@]}"
+        PODMAN_ARGS="${PODMAN_ARGS[@]}" "$REPO_ROOT"/ci/container/build-image.sh "${BUILD_ARGS[@]}"
         unset -f docker
     fi
 fi

ci/container/files/packages.common

@@ -36,23 +36,22 @@ gcc
 lld
 pkg-config
 libssl-dev
-libunwind-dev
 libusb-1.0-0-dev
 libsqlite3-dev
 zlib1g-dev
-libclang-10-dev
+libclang-18-dev
 protobuf-compiler
 llvm
 liblmdb-dev
 liblzma-dev
 
 # Haskell
-libtinfo5
 libtinfo-dev
 libffi-dev
 libgmp-dev
 
 # IC-OS
+fdisk
 cryptsetup-bin
 dosfstools
 fakeroot

ci/src/dependencies/scanner/manager/bazel_rust_dependency_manager.py

@@ -110,8 +110,11 @@ def __transitive_bazel_string_to_dependency(bazel_string: str) -> typing.Optiona
             # ['zstd', 'sys', '2.0.2', 'zstd.1.5.2']
             for split_string in parse_result:
                 # 2.0.2
-                if isinstance(version.parse(split_string), version.Version):
-                    version_str = split_string
+                try:
+                    if isinstance(version.parse(split_string), version.Version):
+                        version_str = split_string
+                except version.InvalidVersion:
+                    continue
             # split with -2.0.2
             # noinspection PyUnboundLocalVariable
             name = result[0].split(f"-{version_str}", 1)[0]