From e735ea871cf41fabdd154340acde04d90b2f56f2 Mon Sep 17 00:00:00 2001 From: Igor Borodin Date: Fri, 27 Oct 2023 11:24:25 +0200 Subject: [PATCH] ci: Migrate zk-environment to Debian 12 (#329) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What ❔ - Migrate core non-CUDA zk-environment Docker image to Debian 12 due to mystical ~50% CI load test performance degradation observed on Ubuntu 22.04 based image - Refactor and tidy-up CI and Dockerfile structure for Ubuntu 20.04 based CUDA images ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. --- .../workflows/zk-environment-old-publish.yml | 30 --- .github/workflows/zk-environment-publish.yml | 56 ++++-- docker-compose-backup-test.yml | 25 --- docker-compose-cpu-runner.yml | 2 +- .../20.04_amd64_cuda_11_8.Dockerfile | 183 ++++++++++++++++++ .../20.04_amd64_cuda_12_0.Dockerfile} | 16 +- .../zk-environment/20.04_amd64_old.Dockerfile | 101 ---------- docker/zk-environment/Dockerfile | 130 ++----------- 8 files changed, 248 insertions(+), 295 deletions(-) delete mode 100644 .github/workflows/zk-environment-old-publish.yml delete mode 100644 docker-compose-backup-test.yml create mode 100644 docker/zk-environment/20.04_amd64_cuda_11_8.Dockerfile rename docker/{zk-environment-cuda-12-0/Dockerfile => zk-environment/20.04_amd64_cuda_12_0.Dockerfile} (92%) delete mode 100644 docker/zk-environment/20.04_amd64_old.Dockerfile diff --git a/.github/workflows/zk-environment-old-publish.yml b/.github/workflows/zk-environment-old-publish.yml deleted file mode 100644 index aa3c6a02e0c..00000000000 --- a/.github/workflows/zk-environment-old-publish.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Temp workflow to build and push old Ubuntu 22.04 image for debugging purposes -# TODO: delete me -name: Publish zk-environment Docker images - -on: - push: - branches: - - main - -jobs: - zk_environment_old: - name: Build and push zk-environment "old" Docker images to Docker Hub - runs-on: matterlabs-ci-runner - steps: - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c - with: - submodules: "recursive" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2 - - name: Log in to Docker Hub - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and optionally push zk-environment lightweight - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 - with: - file: docker/zk-environment/20.04_amd64_old.Dockerfile - tags: matterlabs/zk-environment:old-lightweight - push: true diff --git a/.github/workflows/zk-environment-publish.yml b/.github/workflows/zk-environment-publish.yml index b018cce7b55..f220891977d 100644 --- a/.github/workflows/zk-environment-publish.yml +++ b/.github/workflows/zk-environment-publish.yml @@ -36,6 +36,7 @@ jobs: changed_files: outputs: zk_environment: ${{ steps.changed-files-yaml.outputs.zk_env_any_changed }} + zk_environment_cuda_11_8: ${{ steps.changed-files-yaml.outputs.zk_env_cuda_11_8_any_changed }} zk_environment_cuda_12: ${{ steps.changed-files-yaml.outputs.zk_env_cuda_12_any_changed }} runs-on: ubuntu-latest steps: @@ -51,8 +52,11 @@ jobs: zk_env: - docker/zk-environment/Dockerfile - .github/workflows/zk-environment.publish.yml + zk_env_cuda_11_8: + - docker/zk-environment/20.04_amd64_cuda_11_8.Dockerfile + - .github/workflows/zk-environment.publish.yml zk_env_cuda_12: - - docker/zk-environment-cuda-12-0/Dockerfile + - docker/zk-environment/20.04_amd64_cuda_12_0.Dockerfile - .github/workflows/zk-environment.publish.yml get_short_sha: @@ -110,14 +114,6 @@ jobs: tags: "matterlabs/zk-environment:${{ needs.get_short_sha.outputs.short_sha }}-lightweight-nightly-${{ matrix.arch }}" build-args: ARCH=${{ matrix.arch }} push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - - name: Build and optionally push zk-environment CUDA 11.8 - if: matrix.arch == 'amd64' - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 - with: - file: docker/zk-environment/Dockerfile - target: nvidia-tools - tags: "matterlabs/zk-environment:latest2.0" - push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} zk_environment_multiarch_manifest: if: needs.changed_files.outputs.zk_environment == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -148,34 +144,52 @@ jobs: docker buildx imagetools create --tag "${multiarch_tag}" "${individual_images[@]}" done - zk_environment_cuda_12: - if: needs.changed_files.outputs.zk_environment_cuda_12 == 'true' + zk_environment_cuda: needs: changed_files - name: Build and optionally push zk-environment CUDA 12 Docker image runs-on: [matterlabs-ci-runner] + strategy: + matrix: + cuda_version: ['11_8', '12_0'] steps: - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + - name: Evaluate condition + id: condition + run: | + key="zk_environment_cuda_${{ matrix.cuda_version }}" + changed_files_output=$(echo '${{ toJson(needs.changed_files.outputs) }}' | jq -r ".$key") + echo "should_run=$changed_files_output" >> "$GITHUB_OUTPUT" + + - name: Checkout code + if: steps.condition.outputs.should_run == 'true' + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c with: submodules: "recursive" - - name: Log in to us-central1 GAR - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + - name: Log in to US GAR + if: steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' run: | gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + - name: Log in to Docker Hub - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + if: steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU + if: steps.condition.outputs.should_run == 'true' uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2 + if: steps.condition.outputs.should_run == 'true' + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 + - name: Build and optionally push + if: steps.condition.outputs.should_run == 'true' uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 with: - file: docker/zk-environment-cuda-12-0/Dockerfile + file: docker/zk-environment/20.04_amd64_cuda_${{ matrix.cuda_version }}.Dockerfile push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} tags: | - us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zk-environment-cuda-12-0:latest - matterlabs/zk-environment:cuda-12-0-latest + us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zk-environment-cuda-${{ matrix.cuda_version }}:latest + matterlabs/zk-environment:cuda-${{ matrix.cuda_version }}-latest diff --git a/docker-compose-backup-test.yml b/docker-compose-backup-test.yml deleted file mode 100644 index dfadd7cad58..00000000000 --- a/docker-compose-backup-test.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3.2' -services: - - geth: - image: "matterlabs/geth:latest" - environment: - - PLUGIN_CONFIG - - zk: - image: "matterlabs/zk-environment:latest2.0" - depends_on: - - geth - command: cargo run --bin zksync_core - volumes: - - .:/usr/src/zksync - - /usr/src/cache:/usr/src/cache - - /usr/src/keys:/usr/src/keys - environment: - - IN_DOCKER=1 - - CACHE_DIR=/usr/src/cache - - SCCACHE_CACHE_SIZE=50g - - DB_PATH=/usr/src/db - - CI=1 - env_file: - - ./etc/env/dev.env diff --git a/docker-compose-cpu-runner.yml b/docker-compose-cpu-runner.yml index fa2cfb890f7..1f04481dbef 100644 --- a/docker-compose-cpu-runner.yml +++ b/docker-compose-cpu-runner.yml @@ -6,7 +6,7 @@ services: - PLUGIN_CONFIG zk: - image: "matterlabs/zk-environment:latest2.0" + image: "matterlabs/zk-environment:latest2.0-lightweight" depends_on: - geth - postgres diff --git a/docker/zk-environment/20.04_amd64_cuda_11_8.Dockerfile b/docker/zk-environment/20.04_amd64_cuda_11_8.Dockerfile new file mode 100644 index 00000000000..d55df7065be --- /dev/null +++ b/docker/zk-environment/20.04_amd64_cuda_11_8.Dockerfile @@ -0,0 +1,183 @@ +FROM ubuntu:20.04@sha256:3246518d9735254519e1b2ff35f95686e4a5011c90c85344c1f38df7bae9dd37 as base + +WORKDIR /usr/src/zksync +ENV DEBIAN_FRONTEND noninteractive + +# Install required dependencies +RUN apt-get update && apt-get install -y \ + cmake \ + make \ + bash \ + git \ + openssl \ + libssl-dev \ + gcc \ + g++ \ + curl \ + pkg-config \ + software-properties-common \ + jq \ + openssh-server \ + openssh-client \ + wget \ + vim \ + ca-certificates \ + gnupg2 \ + postgresql-client \ + wget \ + bzip2 + +# Install dependencies for RocksDB. `liburing` is not available for Ubuntu 20.04, +# so we use a PPA with the backport +RUN add-apt-repository ppa:savoury1/virtualisation && \ + apt-get update && \ + apt-get install -y \ + curl \ + gnutls-bin git \ + build-essential \ + clang-7 \ + lldb-7 \ + lld-7 \ + liburing-dev + +# Install docker engine +RUN wget -c -O - https://download.docker.com/linux/ubuntu/gpg | apt-key add - +RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +RUN apt update; apt install -y docker-ce-cli + +# Configurate git to fetch submodules correctly (https://stackoverflow.com/questions/38378914/how-to-fix-git-error-rpc-failed-curl-56-gnutls) +RUN git config --global http.postBuffer 1048576000 + +# Install node and yarn +RUN wget -c -O - https://deb.nodesource.com/setup_18.x | bash - +RUN apt-get install -y nodejs +RUN npm install -g yarn + +# Install Rust and required cargo packages +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH + +ENV GCLOUD_VERSION=403.0.0 +# Install gloud for gcr login and gcfuze for mounting buckets +RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ + wget -c -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update -y && apt-get install google-cloud-cli=${GCLOUD_VERSION}-0 --no-install-recommends -y && \ + gcloud config set core/disable_usage_reporting true && \ + gcloud config set component_manager/disable_update_check true && \ + gcloud config set metrics/environment github_docker_image + +RUN wget -c -O - https://sh.rustup.rs | bash -s -- -y +RUN rustup install nightly-2023-07-21 +RUN rustup default stable +RUN cargo install --version=0.5.13 sqlx-cli +RUN cargo install cargo-tarpaulin +RUN cargo install cargo-nextest + +# Copy compiler (both solc and zksolc) binaries +# Obtain `solc` 0.8.20. +RUN wget -c https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.20%2Bcommit.a1b79de6 \ + && mv solc-linux-amd64-v0.8.20+commit.a1b79de6 /usr/bin/solc \ + && chmod +x /usr/bin/solc +# Obtain `zksolc` 1.3.13. +RUN wget -c https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.13 \ + && mv zksolc-linux-amd64-musl-v1.3.13 /usr/bin/zksolc \ + && chmod +x /usr/bin/zksolc + +# Somehow it is installed with some other packages +RUN apt-get remove valgrind -y + +# We need valgrind 3.20, which is unavailable in repos or ppa, so we will build it from source +RUN wget -c https://sourceware.org/pub/valgrind/valgrind-3.20.0.tar.bz2 && \ + tar -xf valgrind-3.20.0.tar.bz2 && \ + cd valgrind-3.20.0 && ./configure && make && make install && \ + cd ../ && rm -rf valgrind-3.20.0.tar.bz2 && rm -rf valgrind-3.20.0 + +# Setup the environment +ENV ZKSYNC_HOME=/usr/src/zksync +ENV PATH="${ZKSYNC_HOME}/bin:${PATH}" +ENV CI=1 +RUN cargo install sccache +ENV RUSTC_WRAPPER=/usr/local/cargo/bin/sccache + +FROM base as nvidia-tools + +# Setup nvidia-cuda env +ENV NVARCH x86_64 + +ENV NVIDIA_REQUIRE_CUDA "cuda>=11.8 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511 brand=nvidiartx,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516" +ENV NV_CUDA_CUDART_VERSION 11.8.89-1 +ENV NV_CUDA_COMPAT_PACKAGE cuda-compat-11-8 + +RUN wget -c -O - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/${NVARCH}/3bf863cc.pub | apt-key add - && \ + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/${NVARCH} /" > /etc/apt/sources.list.d/cuda.list + +ENV CUDA_VERSION 11.8.0 + +# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a +RUN apt-get update && apt-get install -y --no-install-recommends \ + cuda-cudart-11-8=${NV_CUDA_CUDART_VERSION} \ + ${NV_CUDA_COMPAT_PACKAGE} \ + && ln -s cuda-11.8 /usr/local/cuda && \ + rm -rf /var/lib/apt/lists/* + +# Required for nvidia-docker v1 +RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \ + && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf + +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 + +# nvidia-container-runtime +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +ENV NV_CUDA_LIB_VERSION 11.8.0-1 + +ENV NV_NVTX_VERSION 11.8.86-1 +ENV NV_LIBNPP_VERSION 11.8.0.86-1 +ENV NV_LIBNPP_PACKAGE libnpp-11-8=${NV_LIBNPP_VERSION} +ENV NV_LIBCUSPARSE_VERSION 11.7.5.86-1 + +ENV NV_LIBCUBLAS_PACKAGE_NAME libcublas-11-8 +ENV NV_LIBCUBLAS_VERSION 11.11.3.6-1 +ENV NV_LIBCUBLAS_PACKAGE ${NV_LIBCUBLAS_PACKAGE_NAME}=${NV_LIBCUBLAS_VERSION} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + cuda-libraries-11-8=${NV_CUDA_LIB_VERSION} \ + ${NV_LIBNPP_PACKAGE} \ + cuda-nvtx-11-8=${NV_NVTX_VERSION} \ + libcusparse-11-8=${NV_LIBCUSPARSE_VERSION} \ + ${NV_LIBCUBLAS_PACKAGE} \ + && rm -rf /var/lib/apt/lists/* + +# Keep apt from auto upgrading the cublas and nccl packages. See https://gitlab.com/nvidia/container-images/cuda/-/issues/88 +RUN apt-mark hold ${NV_LIBCUBLAS_PACKAGE_NAME} + +ENV NV_CUDA_CUDART_DEV_VERSION 11.8.89-1 +ENV NV_NVML_DEV_VERSION 11.8.86-1 +ENV NV_LIBCUSPARSE_DEV_VERSION 11.7.5.86-1 +ENV NV_LIBNPP_DEV_VERSION 11.8.0.86-1 +ENV NV_LIBNPP_DEV_PACKAGE libnpp-dev-11-8=${NV_LIBNPP_DEV_VERSION} + +ENV NV_LIBCUBLAS_DEV_VERSION 11.11.3.6-1 +ENV NV_LIBCUBLAS_DEV_PACKAGE_NAME libcublas-dev-11-8 +ENV NV_LIBCUBLAS_DEV_PACKAGE ${NV_LIBCUBLAS_DEV_PACKAGE_NAME}=${NV_LIBCUBLAS_DEV_VERSION} + +ENV NV_NVPROF_VERSION 11.8.87-1 +ENV NV_NVPROF_DEV_PACKAGE cuda-nvprof-11-8=${NV_NVPROF_VERSION} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libtinfo5 libncursesw5 \ + cuda-cudart-dev-11-8=${NV_CUDA_CUDART_DEV_VERSION} \ + cuda-command-line-tools-11-8=${NV_CUDA_LIB_VERSION} \ + cuda-minimal-build-11-8=${NV_CUDA_LIB_VERSION} \ + cuda-libraries-dev-11-8=${NV_CUDA_LIB_VERSION} \ + cuda-nvml-dev-11-8=${NV_NVML_DEV_VERSION} \ + ${NV_NVPROF_DEV_PACKAGE} \ + ${NV_LIBNPP_DEV_PACKAGE} \ + libcusparse-dev-11-8=${NV_LIBCUSPARSE_DEV_VERSION} \ + ${NV_LIBCUBLAS_DEV_PACKAGE} \ + && rm -rf /var/lib/apt/lists/* + +ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs diff --git a/docker/zk-environment-cuda-12-0/Dockerfile b/docker/zk-environment/20.04_amd64_cuda_12_0.Dockerfile similarity index 92% rename from docker/zk-environment-cuda-12-0/Dockerfile rename to docker/zk-environment/20.04_amd64_cuda_12_0.Dockerfile index 5331ca77219..00d545a0201 100644 --- a/docker/zk-environment-cuda-12-0/Dockerfile +++ b/docker/zk-environment/20.04_amd64_cuda_12_0.Dockerfile @@ -39,7 +39,7 @@ RUN add-apt-repository ppa:savoury1/virtualisation && \ liburing-dev # Install docker engine -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +RUN wget -c -O - https://download.docker.com/linux/ubuntu/gpg | apt-key add - RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" RUN apt update; apt install -y docker-ce-cli @@ -47,7 +47,7 @@ RUN apt update; apt install -y docker-ce-cli RUN git config --global http.postBuffer 1048576000 # Install node and yarn -RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - +RUN wget -c -O - https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install -y nodejs RUN npm install -g yarn @@ -59,13 +59,13 @@ ENV RUSTUP_HOME=/usr/local/rustup \ ENV GCLOUD_VERSION=451.0.1 # Install gloud for gcr login and gcfuze for mounting buckets RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + wget -c -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ apt-get update -y && apt-get install google-cloud-cli=${GCLOUD_VERSION}-0 --no-install-recommends -y && \ gcloud config set core/disable_usage_reporting true && \ gcloud config set component_manager/disable_update_check true && \ gcloud config set metrics/environment github_docker_image -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +RUN wget -c -O - https://sh.rustup.rs | bash -s -- -y RUN rustup install nightly-2023-07-21 RUN rustup default stable RUN cargo install --version=0.5.13 sqlx-cli @@ -74,11 +74,11 @@ RUN cargo install cargo-nextest # Copy compiler (both solc and zksolc) binaries # Obtain `solc` 0.8.20. -RUN wget https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.20%2Bcommit.a1b79de6 \ +RUN wget -c https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.20%2Bcommit.a1b79de6 \ && mv solc-linux-amd64-v0.8.20+commit.a1b79de6 /usr/bin/solc \ && chmod +x /usr/bin/solc # Obtain `zksolc` 1.3.13. -RUN wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.13 \ +RUN wget -c https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.13 \ && mv zksolc-linux-amd64-musl-v1.3.13 /usr/bin/zksolc \ && chmod +x /usr/bin/zksolc @@ -113,7 +113,7 @@ ENV NV_CUDA_COMPAT_PACKAGE cuda-compat-12-0 # curl purging is removed, it's required in next steps RUN apt-get update && apt-get install -y --no-install-recommends \ gnupg2 curl ca-certificates && \ - curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/${NVARCH}/3bf863cc.pub | apt-key add - && \ + wget -c -O - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/${NVARCH}/3bf863cc.pub | apt-key add - && \ echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/${NVARCH} /" > /etc/apt/sources.list.d/cuda.list && \ rm -rf /var/lib/apt/lists/* @@ -241,6 +241,6 @@ RUN apt-mark hold ${NV_LIBCUBLAS_DEV_PACKAGE_NAME} ${NV_LIBNCCL_DEV_PACKAGE_NAME ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs # Install cmake 3.24, as we need it for boojum-cuda -RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.24.3/cmake-3.24.3-linux-x86_64.sh && \ +RUN wget -c https://github.com/Kitware/CMake/releases/download/v3.24.3/cmake-3.24.3-linux-x86_64.sh && \ chmod +x cmake-3.24.3-linux-x86_64.sh && \ ./cmake-3.24.3-linux-x86_64.sh --skip-license --prefix=/usr/local diff --git a/docker/zk-environment/20.04_amd64_old.Dockerfile b/docker/zk-environment/20.04_amd64_old.Dockerfile deleted file mode 100644 index bb7f1da60df..00000000000 --- a/docker/zk-environment/20.04_amd64_old.Dockerfile +++ /dev/null @@ -1,101 +0,0 @@ -FROM ubuntu:20.04@sha256:3246518d9735254519e1b2ff35f95686e4a5011c90c85344c1f38df7bae9dd37 as base - -WORKDIR /usr/src/zksync -ENV DEBIAN_FRONTEND noninteractive - -# Install required dependencies -RUN apt-get update && apt-get install -y \ - cmake \ - make \ - bash \ - git \ - openssl \ - libssl-dev \ - gcc \ - g++ \ - curl \ - pkg-config \ - software-properties-common \ - jq \ - openssh-server \ - openssh-client \ - wget \ - vim \ - ca-certificates \ - gnupg2 \ - postgresql-client \ - wget \ - bzip2 - -# Install dependencies for RocksDB. `liburing` is not available for Ubuntu 20.04, -# so we use a PPA with the backport -RUN add-apt-repository ppa:savoury1/virtualisation && \ - apt-get update && \ - apt-get install -y \ - curl \ - gnutls-bin git \ - build-essential \ - clang-7 \ - lldb-7 \ - lld-7 \ - liburing-dev - -# Install docker engine -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -RUN apt update; apt install -y docker-ce-cli - -# Configurate git to fetch submodules correctly (https://stackoverflow.com/questions/38378914/how-to-fix-git-error-rpc-failed-curl-56-gnutls) -RUN git config --global http.postBuffer 1048576000 - -# Install node and yarn -RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - -RUN apt-get install -y nodejs -RUN npm install -g yarn - -# Install Rust and required cargo packages -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH - -ENV GCLOUD_VERSION=403.0.0 -# Install gloud for gcr login and gcfuze for mounting buckets -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get update -y && apt-get install google-cloud-cli=${GCLOUD_VERSION}-0 --no-install-recommends -y && \ - gcloud config set core/disable_usage_reporting true && \ - gcloud config set component_manager/disable_update_check true && \ - gcloud config set metrics/environment github_docker_image - -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y -RUN rustup install nightly-2023-07-21 -RUN rustup default stable -RUN cargo install --version=0.5.13 sqlx-cli -RUN cargo install cargo-tarpaulin -RUN cargo install cargo-nextest - -# Copy compiler (both solc and zksolc) binaries -# Obtain `solc` 0.8.20. -RUN wget https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.20%2Bcommit.a1b79de6 \ - && mv solc-linux-amd64-v0.8.20+commit.a1b79de6 /usr/bin/solc \ - && chmod +x /usr/bin/solc -# Obtain `zksolc` 1.3.13. -RUN wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.13 \ - && mv zksolc-linux-amd64-musl-v1.3.13 /usr/bin/zksolc \ - && chmod +x /usr/bin/zksolc - -# Somehow it is installed with some other packages -RUN apt-get remove valgrind -y - -# We need valgrind 3.20, which is unavailable in repos or ppa, so we will build it from source -RUN wget https://sourceware.org/pub/valgrind/valgrind-3.20.0.tar.bz2 && \ - tar -xf valgrind-3.20.0.tar.bz2 && \ - cd valgrind-3.20.0 && ./configure && make && make install && \ - cd ../ && rm -rf valgrind-3.20.0.tar.bz2 && rm -rf valgrind-3.20.0 - -# Setup the environment -ENV ZKSYNC_HOME=/usr/src/zksync -ENV PATH="${ZKSYNC_HOME}/bin:${PATH}" -ENV CI=1 -RUN cargo install sccache -ENV RUSTC_WRAPPER=/usr/local/cargo/bin/sccache diff --git a/docker/zk-environment/Dockerfile b/docker/zk-environment/Dockerfile index 6005573f0a6..8bbb46ea636 100644 --- a/docker/zk-environment/Dockerfile +++ b/docker/zk-environment/Dockerfile @@ -1,5 +1,5 @@ # Building solidity from sources so binary is available for any architecture -FROM ubuntu:22.04 as solidity-builder +FROM debian:bookworm as solidity-builder # Install prerequisites RUN apt-get update && apt-get install -y \ @@ -23,7 +23,7 @@ WORKDIR /solidity/build RUN cmake .. RUN make -FROM ubuntu:22.04 as rust-lightweight +FROM debian:bookworm as rust-lightweight ARG ARCH=amd64 @@ -65,15 +65,22 @@ RUN apt-get update && \ lld-15 \ liburing-dev -# Install docker engine -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -RUN add-apt-repository "deb [arch=${ARCH}] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -RUN apt update; apt install -y docker-ce-cli - -# Install node and yarn +# Install Docker +RUN apt-get update && \ + apt-get install ca-certificates curl gnupg && \ + install -m 0755 -d /etc/apt/keyrings && \ + wget -c -O - https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ + chmod a+r /etc/apt/keyrings/docker.gpg && \ + echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Install Node and yarn ENV NODE_MAJOR=18 RUN mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + wget -c -O - https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ apt-get update && apt-get install nodejs -y && \ npm install -g yarn @@ -86,13 +93,13 @@ ENV RUSTUP_HOME=/usr/local/rustup \ # Install gloud for GCR/GAR login ENV GCLOUD_VERSION=451.0.1 RUN echo "deb [arch=${ARCH}] http://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + wget -c -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ apt-get update -y && apt-get install google-cloud-cli=${GCLOUD_VERSION}-0 --no-install-recommends -y && \ gcloud config set core/disable_usage_reporting true && \ gcloud config set component_manager/disable_update_check true && \ gcloud config set metrics/environment github_docker_image -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ +RUN wget -c -O - https://sh.rustup.rs | bash -s -- -y && \ rustup default stable RUN cargo install --version=0.5.13 sqlx-cli @@ -100,11 +107,11 @@ RUN cargo install cargo-tarpaulin RUN cargo install cargo-nextest # Copy compiler (both solc and zksolc) binaries -# Obtain `solc` 0.8.20 +# Obtain `solc` 0.8.20. COPY --from=solidity-builder /solidity/build/solc/solc /usr/bin/ RUN chmod +x /usr/bin/solc # Obtain `zksolc` 1.3.15. -RUN wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-${ARCH}/zksolc-linux-${ARCH}-musl-v1.3.15 \ +RUN wget -c https://github.com/matter-labs/zksolc-bin/raw/main/linux-${ARCH}/zksolc-linux-${ARCH}-musl-v1.3.15 \ && mv zksolc-linux-${ARCH}-musl-v1.3.15 /usr/bin/zksolc \ && chmod +x /usr/bin/zksolc @@ -112,7 +119,7 @@ RUN wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-${ARCH}/zksolc RUN apt-get remove valgrind -y # We need valgrind 3.20, which is unavailable in repos or ppa, so we will build it from source -RUN wget https://sourceware.org/pub/valgrind/valgrind-3.20.0.tar.bz2 && \ +RUN wget -c https://sourceware.org/pub/valgrind/valgrind-3.20.0.tar.bz2 && \ tar -xf valgrind-3.20.0.tar.bz2 && \ cd valgrind-3.20.0 && ./configure && make && make install && \ cd ../ && rm -rf valgrind-3.20.0.tar.bz2 && rm -rf valgrind-3.20.0 @@ -124,101 +131,6 @@ ENV CI=1 RUN cargo install sccache ENV RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -FROM rust-lightweight as nvidia-tools - -# Install Rust and required cargo packages -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH - -# Setup the environment -ENV ZKSYNC_HOME=/usr/src/zksync -ENV PATH="${ZKSYNC_HOME}/bin:${PATH}" -ENV CI=1 -ENV RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -ENV DEBIAN_FRONTEND noninteractive - -# Setup nvidia-cuda env -ENV NVARCH x86_64 - -ENV NVIDIA_REQUIRE_CUDA "cuda>=11.8 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511 brand=nvidiartx,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516" -ENV NV_CUDA_CUDART_VERSION 11.8.89-1 -ENV NV_CUDA_COMPAT_PACKAGE cuda-compat-11-8 - -RUN curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/${NVARCH}/3bf863cc.pub | apt-key add - && \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/${NVARCH} /" > /etc/apt/sources.list.d/cuda.list - -ENV CUDA_VERSION 11.8.0 - -# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-cudart-11-8=${NV_CUDA_CUDART_VERSION} \ - ${NV_CUDA_COMPAT_PACKAGE} \ - && ln -s cuda-11.8 /usr/local/cuda && \ - rm -rf /var/lib/apt/lists/* - -# Required for nvidia-docker v1 -RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \ - && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf - -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} -ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 - -# nvidia-container-runtime -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility - -ENV NV_CUDA_LIB_VERSION 11.8.0-1 - -ENV NV_NVTX_VERSION 11.8.86-1 -ENV NV_LIBNPP_VERSION 11.8.0.86-1 -ENV NV_LIBNPP_PACKAGE libnpp-11-8=${NV_LIBNPP_VERSION} -ENV NV_LIBCUSPARSE_VERSION 11.7.5.86-1 - -ENV NV_LIBCUBLAS_PACKAGE_NAME libcublas-11-8 -ENV NV_LIBCUBLAS_VERSION 11.11.3.6-1 -ENV NV_LIBCUBLAS_PACKAGE ${NV_LIBCUBLAS_PACKAGE_NAME}=${NV_LIBCUBLAS_VERSION} - -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-libraries-11-8=${NV_CUDA_LIB_VERSION} \ - ${NV_LIBNPP_PACKAGE} \ - cuda-nvtx-11-8=${NV_NVTX_VERSION} \ - libcusparse-11-8=${NV_LIBCUSPARSE_VERSION} \ - ${NV_LIBCUBLAS_PACKAGE} \ - && rm -rf /var/lib/apt/lists/* - -# Keep apt from auto upgrading the cublas and nccl packages. See https://gitlab.com/nvidia/container-images/cuda/-/issues/88 -RUN apt-mark hold ${NV_LIBCUBLAS_PACKAGE_NAME} - -ENV NV_CUDA_CUDART_DEV_VERSION 11.8.89-1 -ENV NV_NVML_DEV_VERSION 11.8.86-1 -ENV NV_LIBCUSPARSE_DEV_VERSION 11.7.5.86-1 -ENV NV_LIBNPP_DEV_VERSION 11.8.0.86-1 -ENV NV_LIBNPP_DEV_PACKAGE libnpp-dev-11-8=${NV_LIBNPP_DEV_VERSION} - -ENV NV_LIBCUBLAS_DEV_VERSION 11.11.3.6-1 -ENV NV_LIBCUBLAS_DEV_PACKAGE_NAME libcublas-dev-11-8 -ENV NV_LIBCUBLAS_DEV_PACKAGE ${NV_LIBCUBLAS_DEV_PACKAGE_NAME}=${NV_LIBCUBLAS_DEV_VERSION} - -ENV NV_NVPROF_VERSION 11.8.87-1 -ENV NV_NVPROF_DEV_PACKAGE cuda-nvprof-11-8=${NV_NVPROF_VERSION} - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libtinfo5 libncursesw5 \ - cuda-cudart-dev-11-8=${NV_CUDA_CUDART_DEV_VERSION} \ - cuda-command-line-tools-11-8=${NV_CUDA_LIB_VERSION} \ - cuda-minimal-build-11-8=${NV_CUDA_LIB_VERSION} \ - cuda-libraries-dev-11-8=${NV_CUDA_LIB_VERSION} \ - cuda-nvml-dev-11-8=${NV_NVML_DEV_VERSION} \ - ${NV_NVPROF_DEV_PACKAGE} \ - ${NV_LIBNPP_DEV_PACKAGE} \ - libcusparse-dev-11-8=${NV_LIBCUSPARSE_DEV_VERSION} \ - ${NV_LIBCUBLAS_DEV_PACKAGE} \ - && rm -rf /var/lib/apt/lists/* - -ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs - - FROM rust-lightweight as rust-lightweight-nightly RUN rustup install nightly-2023-07-21 && \