feat: build F3 sidecar FFI in Dockerfile

.github/workflows/docker.yml

@@ -17,28 +17,128 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
 
 env:
-  FOREST_TEST_IMAGE: forest-image
-  FOREST_TEST_ARCHIVE: forest-image.tar
+  CI: 1
+  CARGO_INCREMENTAL: 0
+  CACHE_TIMEOUT_MINUTES: 5
+  SCRIPT_TIMEOUT_MINUTES: 30
+  AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
+  AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
+  RUSTC_WRAPPER: sccache
+  CC: sccache clang
+  CXX: sccache clang++
 
 jobs:
-  build-and-push-docker-image:
+  build-ubuntu-2204-amd64:
+    name: Build forest binaries on Ubuntu-22.04-amd64
+    runs-on: ubuntu-22.04
     # Run the job only if the PR is not a draft.
     # This is done to limit the runner cost.
     if: github.event.pull_request.draft == false
+    steps:
+      - name: Configure SCCache variables
+        run: |
+          # External PRs do not have access to 'vars' or 'secrets'.
+          if [[ "${{secrets.AWS_ACCESS_KEY_ID}}" != "" ]]; then
+            echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" >> $GITHUB_ENV
+            echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" >> $GITHUB_ENV
+            echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" >> $GITHUB_ENV
+          fi
+      - run: lscpu
+      - name: Show IP
+        run: curl ifconfig.me
+        continue-on-error: true
+      - name: Checkout Sources
+        uses: actions/checkout@v4
+      - name: Setup sccache
+        uses: mozilla-actions/[email protected]
+        timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
+        continue-on-error: true
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.work"
+      - name: Cargo Install
+        env:
+          # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times
+          RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld"
+        run: make install
+      - uses: actions/upload-artifact@v4
+        with:
+          name: 'forest-linux-amd64'
+          path: |
+            ~/.cargo/bin/forest*
+          if-no-files-found: error
+
+  build-ubuntu-2204-arm64:
+    name: Build forest binaries on Ubuntu-22.04-arm64
+    runs-on: buildjet-4vcpu-ubuntu-2204-arm
+    # Run the job only if the PR is not a draft.
+    # This is done to limit the runner cost.
+    if: github.event.pull_request.draft == false
+    steps:
+      - name: Configure SCCache variables
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+        run: |
+          # External PRs do not have access to 'vars' or 'secrets'.
+          if [[ "${{secrets.AWS_ACCESS_KEY_ID}}" != "" ]]; then
+            echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" >> $GITHUB_ENV
+            echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" >> $GITHUB_ENV
+            echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" >> $GITHUB_ENV
+          fi
+      - run: lscpu
+      - name: Show IP
+        run: curl ifconfig.me
+        continue-on-error: true
+      - name: Checkout Sources
+        uses: actions/checkout@v4
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+      - name: Setup sccache
+        uses: mozilla-actions/[email protected]
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+        timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
+        continue-on-error: true
+      - uses: actions/setup-go@v5
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+        with:
+          go-version-file: "go.work"
+      - name: Cargo Install
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+        run: make install
+      - uses: actions/upload-artifact@v4
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+        with:
+          name: 'forest-linux-arm64'
+          path: |
+            ~/.cargo/bin/forest*
+          if-no-files-found: error
+
+  build-and-push-docker-image:
     name: Build images and push to GHCR
-    # Change to `buildjet-8vcpu-ubuntu-2204` if `fuzzy` is down.
-    # runs-on: fuzzy
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-22.04
+    # Run the job only if the PR is not a draft.
+    # This is done to limit the runner cost.
+    if: github.event.pull_request.draft == false
     timeout-minutes: 30
+    needs:
+      - build-ubuntu-2204-amd64
+      - build-ubuntu-2204-arm64
     steps:
       - name: List cached docker images
         run: docker image ls
 
       - name: Checkout code
         uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: 'forest-linux-amd64'
+          path: /tmp/forest/forest-linux-amd64
+
+      - uses: actions/download-artifact@v4
+        # Compile Docker image only for AMD64 for a regular PR to save some CI time.
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+        with:
+          name: 'forest-linux-arm64'
+          path: /tmp/forest/forest-linux-arm64
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -78,7 +178,8 @@ jobs:
       - name: Build fat image and push to GitHub Container Registry
         uses: docker/build-push-action@v6
         with:
-          context: .
+          context: /tmp/forest
+          file: ./Dockerfile-ci
           tags: ${{ steps.metafat.outputs.tags }}
           labels: ${{ steps.metafat.outputs.labels }}
           push: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}
@@ -104,7 +205,8 @@ jobs:
       - name: Build slim image and push to GitHub Container Registry
         uses: docker/build-push-action@v6
         with:
-          context: .
+          context: /tmp/forest
+          file: ./Dockerfile-ci
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           push: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}

Dockerfile

@@ -6,54 +6,27 @@
 # docker run --init -it forest
 # ```
 # 
-# Build and manually push to Github Container Registry (see https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)
-# ```
-# docker build -t ghcr.io/chainsafe/forest:latest .
-# docker push ghcr.io/chainsafe/forest:latest
-# ```
-
-##
-# Build stage
-# Use github action runner cached images to avoid being rate limited
-# https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md#cached-docker-images
-## 
 
-# Cross-compilation helpers
-# https://github.com/tonistiigi/xx
-FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx
-
-FROM --platform=$BUILDPLATFORM ubuntu:22.04 AS build-env
+FROM golang:1.22-bullseye AS build-env
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # install dependencies
 RUN apt-get update && \
     apt-get install --no-install-recommends -y build-essential clang curl git ca-certificates
 RUN update-ca-certificates
 
+# install Rust
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --profile minimal
 ENV PATH="/root/.cargo/bin:${PATH}"
 
-# Copy the cross-compilation scripts 
-COPY --from=xx / /
-
-# export TARGETPLATFORM
-ARG TARGETPLATFORM
-
-# Install those packages for the target architecture
-RUN xx-apt-get update && \
-    xx-apt-get install -y libc6-dev g++
-
 WORKDIR /forest
 COPY . .
 
-# TODO(forest): https://github.com/ChainSafe/forest/issues/4758
-ENV FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT=1
-
 # Install Forest. Move it out of the cache for the prod image.
 RUN --mount=type=cache,sharing=private,target=/root/.cargo/registry \
     --mount=type=cache,sharing=private,target=/root/.rustup \
     --mount=type=cache,sharing=private,target=/forest/target \
-    make install-xx && \
+    make install && \
     mkdir /forest_out && \
     cp /root/.cargo/bin/forest* /forest_out
 

Dockerfile-ci

@@ -0,0 +1,56 @@
+# This Dockerfile is for building official Forest docker multiplatform images on CI,
+# linux/amd64 and linux/arm64 are currently supported.
+#
+# This Dockerfile composes Forest binaries that are prebuilt in other CI steps, to take
+# better advantage of build cache and reduce build time and cost.
+#
+# Build and manually push to Github Container Registry (see https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)
+# ```
+# docker build -t ghcr.io/chainsafe/forest:latest .
+# docker push ghcr.io/chainsafe/forest:latest
+# ```
+
+##
+# Prod image for forest binary
+# Use github action runner cached images to avoid being rate limited
+# https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md#cached-docker-images
+##
+# A slim image contains only forest binaries
+FROM ubuntu:22.04 AS slim-image
+
+# export TARGETPLATFORM TARGETOS and TARGETARCH
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+
+ENV DEBIAN_FRONTEND="noninteractive"
+# Install binary dependencies
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+RUN update-ca-certificates
+
+# Assuming prebuilt Forest binaries are under `forest-linux-[amd64|arm64]`
+COPY --chmod=555 ./forest-${TARGETOS}-${TARGETARCH}/* /usr/local/bin
+
+# Basic verification of dynamically linked dependencies
+RUN forest -V && forest-cli -V && forest-tool -V && forest-wallet -V
+
+ENTRYPOINT ["forest"]
+
+# A fat image contains forest binaries and fil proof parameter files under $FIL_PROOFS_PARAMETER_CACHE
+FROM slim-image AS fat-image
+
+# Move FIL_PROOFS_PARAMETER_CACHE out of forest data dir since users always need to mount the data dir
+ENV FIL_PROOFS_PARAMETER_CACHE="/var/tmp/filecoin-proof-parameters"
+
+# Populate $FIL_PROOFS_PARAMETER_CACHE
+RUN forest-tool fetch-params --keys
+
+# Cache actor bundle in the image
+ENV FOREST_ACTOR_BUNDLE_PATH="/var/tmp/forest_actor_bundle.car.zst"
+
+# Populate $FOREST_ACTOR_BUNDLE_PATH
+RUN forest-tool state-migration actor-bundle $FOREST_ACTOR_BUNDLE_PATH
+
+ENTRYPOINT ["forest"]

Makefile

@@ -1,8 +1,3 @@
-# Using https://github.com/tonistiigi/xx
-# Use in Docker images when cross-compiling.
-install-xx:
-	xx-cargo install --locked --path . --force
-
 # Redundancy tracked by #2991
 install-cli:
 	cargo install --locked --path . --force

build.rs

@@ -12,12 +12,14 @@ fn main() {
         std::env::set_var("GOWORK", "off");
         rust2go::Builder::default()
             .with_go_src("./f3-sidecar")
-            .with_regen_arg(rust2go::RegenArgs {
-                src: "./src/f3/go_ffi.rs".into(),
-                dst: "./f3-sidecar/ffi_gen.go".into(),
-                without_main: true,
-                ..Default::default()
-            })
+            // the generated Go file has been commited to the git repository,
+            // uncomment to regenerate the code locally
+            // .with_regen_arg(rust2go::RegenArgs {
+            //     src: "./src/f3/go_ffi.rs".into(),
+            //     dst: "./f3-sidecar/ffi_gen.go".into(),
+            //     without_main: true,
+            //     ..Default::default()
+            // })
             .build();
     }
 }
@@ -28,15 +30,14 @@ fn is_docs_rs() -> bool {
 }
 
 fn is_sidecar_ffi_enabled() -> bool {
-    // Note: arm64 is disabled for now as cross-compilation is not yet supported in rust2go
-    // and it's reported rust2go build does not work on arm64 MacOS
-    if cfg!(target_arch = "x86_64") {
+    // Note: arm64 is disabled on MacOS for now as it's reported rust2go build does not work there
+    if cfg!(all(target_arch = "aarch64", target_os = "macos")) {
+        false
+    } else {
         // Opt-out building the F3 sidecar staticlib
         match std::env::var("FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT") {
             Ok(value) => !matches!(value.to_lowercase().as_str(), "1" | "true"),
             _ => true,
         }
-    } else {
-        false
     }
 }