feat: bootstrapping tests

.github/workflows/forest.yml

@@ -412,6 +412,25 @@ jobs:
       - name: Dump docker logs
         if: always()
         uses: jwalton/gh-docker-logs@v2
+  bootstrap-checks:
+    needs:
+      - build-ubuntu
+    name: Bootstrap checks
+    runs-on: ubuntu-latest
+    env:
+      # We use a custom Dockerfile for CI to speed up the build process.
+      FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: 'forest-${{ runner.os }}'
+      - name: Run bootstrap tests
+        run: ./scripts/tests/bootstrapper/setup.sh
+        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
+      - name: Dump docker logs
+        if: always()
+        uses: jwalton/gh-docker-logs@v2
   snapshot-parity-checks:
     needs:
       - build-ubuntu

scripts/tests/bootstrapper/.env

@@ -0,0 +1,11 @@
+# Note: this should be a `fat` image so that it contains the pre-downloaded filecoin proof parameters
+LOTUS_VERSION=v1.28.0-rc5
+FIL_PROOFS_PARAMETER_CACHE=/var/tmp/filecoin-proof-parameters
+LOTUS_RPC_PORT=1234
+FOREST_RPC_PORT=2345
+FOREST_P2P_PORT=12345
+# Pre-generated keypair for the forest node. This is required to easily connect to the forest node from the lotus node.
+FOREST_PEER_KEYPAIR=7PCBrDPUebd7Pj+DqhbzNuKBWmldP9r2K5eEnbYelUoK4xd+ng8c6C9gDa/q31/U5b6FIlNnHDQLQ4WSop1y6w==
+# The PeerID is derived from the `FOREST_PEER_KEYPAIR`.
+FOREST_BOOTSTRAPPER_ADDRESS=/dns/forest-bootstrapper/tcp/12345/p2p/12D3KooWAYs5zbzniHaL9RnnH2RKdNvibuj3BCS4b3bHtYvC81yL
+CHAIN=calibnet

scripts/tests/bootstrapper/README.md

@@ -0,0 +1,32 @@
+# Forest as a bootstrapper test
+
+The setup here creates a single Forest bootstrap peer with a well-known peer id
+and p2p listening port. Then, a secondary Forest and Lotus are created and
+connected to that peer. The assertion succeeds if both secondary peers are able
+to sync the chain from the bootstrap peer and have multiple peers in their
+peerstores.
+
+This is illustrated in the following flowchart:
+
+```mermaid
+flowchart TD
+    A[Init] -->|Download proofs and snapshot| B(Start the Forest bootstrapper)
+    B --> C(Start Forest-Peer)
+    B --> D(Start Lotus)
+    C -->|Wait for sync| E(Assert peer store populated)
+    D -->|Wait for sync| F(Assert peer store populated)
+    F --> G
+    E --> G(Finish)
+```
+
+## Usage
+
+```bash
+./setup.sh
+```
+
+## Teardown
+
+```bash
+docker compose down -v --rmi all
+```

scripts/tests/bootstrapper/docker-compose.yml

@@ -0,0 +1,201 @@
+# Docker compose file to run Forest bootstrap tests.
+
+services:
+  init: 
+    build:
+      context: ../../../.
+      dockerfile: ${FOREST_DOCKERFILE_OVERRIDE:-Dockerfile}
+    volumes:
+      - node-data:/data
+      - filecoin-proofs:${FIL_PROOFS_PARAMETER_CACHE}
+    networks:
+      - bootstrap-tests
+    environment:
+      - FIL_PROOFS_PARAMETER_CACHE=${FIL_PROOFS_PARAMETER_CACHE}
+    entrypoint: [ "/bin/bash", "-c" ]
+    user: 0:0
+    command:
+      - |
+        set -euxo pipefail
+        # fetch parameter files
+        forest-tool fetch-params --keys
+        # if there are some files in the data directory, then we don't need to fetch the snapshot
+        if [ "$$(ls -A /data/*.car.zst)" ]; then
+          echo "Snapshot already fetched"
+        else
+          forest-tool snapshot fetch --chain ${CHAIN} -d /data
+        fi
+  forest-bootstrapper:
+    depends_on:
+      init:
+        condition: service_completed_successfully
+    build:
+      context: ../../../.
+      dockerfile: ${FOREST_DOCKERFILE_OVERRIDE:-Dockerfile}
+    volumes:
+      - node-data:/data
+      - filecoin-proofs:${FIL_PROOFS_PARAMETER_CACHE}
+    networks:
+      - bootstrap-tests
+    environment:
+      - FIL_PROOFS_PARAMETER_CACHE=${FIL_PROOFS_PARAMETER_CACHE}
+      - FULLNODE_API_INFO=/dns/forest/tcp/${FOREST_RPC_PORT}/http
+    entrypoint: [ "/bin/bash", "-c" ]
+    user: 0:0
+    command:
+      - |
+        set -euxo pipefail
+        # Import the P2P keypair
+        mkdir -p /root/.local/share/forest/libp2p
+        echo ${FOREST_PEER_KEYPAIR} | base64 --decode  > /root/.local/share/forest/libp2p/keypair
+
+        forest --chain ${CHAIN} --encrypt-keystore false --no-gc \
+          --p2p-listen-address /ip4/0.0.0.0/tcp/$FOREST_P2P_PORT \
+          --import-snapshot $(ls /data/*.car.zst | tail -n 1)
+  forest-peer:
+    depends_on:
+      init:
+        condition: service_completed_successfully
+    build:
+      context: ../../../.
+      dockerfile: ${FOREST_DOCKERFILE_OVERRIDE:-Dockerfile}
+    volumes:
+      - node-data:/data
+      - filecoin-proofs:${FIL_PROOFS_PARAMETER_CACHE}
+    networks:
+      - bootstrap-tests
+    environment:
+      - FIL_PROOFS_PARAMETER_CACHE=${FIL_PROOFS_PARAMETER_CACHE}
+    entrypoint: [ "/bin/bash", "-c" ]
+    user: 0:0
+    command:
+      - |
+        set -euxo pipefail
+        base64 --decode <<< ${FOREST_PEER_KEYPAIR} > keypair
+        PEER_ID=$(forest-tool shed peer-id-from-key-pair keypair)
+
+        # Make sure to use the Forest bootstrapper as the only bootstrap peer
+        cat > config.toml <<EOF
+        [network]
+        bootstrap_peers = ["/dns/forest-bootstrapper/tcp/$FOREST_P2P_PORT/p2p/$${PEER_ID}"]
+        EOF
+
+        forest --chain ${CHAIN} --encrypt-keystore false --no-gc \
+          --config config.toml \
+          --rpc-address 0.0.0.0:${FOREST_RPC_PORT} \
+          --import-snapshot $(ls /data/*.car.zst | tail -n 1)
+    healthcheck:
+      test: [ "CMD", "forest-cli", "sync", "wait" ]
+      interval: 15s
+      timeout: 10m
+      retries: 3
+      start_period: 10m
+  forest-peers-wait:
+    depends_on:
+      init:
+        condition: service_completed_successfully
+      forest-peer:
+        condition: service_healthy
+    build:
+      context: ../../../.
+      dockerfile: ${FOREST_DOCKERFILE_OVERRIDE:-Dockerfile}
+    volumes:
+      - node-data:/data
+      - filecoin-proofs:${FIL_PROOFS_PARAMETER_CACHE}
+    networks:
+      - bootstrap-tests
+    environment:
+      - FIL_PROOFS_PARAMETER_CACHE=${FIL_PROOFS_PARAMETER_CACHE}
+      - FULLNODE_API_INFO=/dns/forest-peer/tcp/${FOREST_RPC_PORT}/http
+    entrypoint: [ "/bin/bash", "-c" ]
+    user: 0:0
+    command:
+      - |
+        set -euxo pipefail
+        forest-cli sync wait
+
+        # at this point, forest node is synced and it should have multiple peers. Assert the latter.
+        if [ $$(forest-cli net peers | wc -l) -lt 2 ]; then
+          echo "Forest node should have at least 2 peers"
+          exit 1
+        fi
+        echo "Forest properly bootstrapped"
+  lotus:
+    depends_on:
+      init:
+        condition: service_completed_successfully
+    build:
+      dockerfile: ./lotus.dockerfile
+      context: .
+      args:
+        - LOTUS_VERSION=${LOTUS_VERSION}
+        - BOOTSTRAPPER=${FOREST_BOOTSTRAPPER_ADDRESS}
+    volumes:
+      - node-data:/data
+      - lotus-data:/var/lib/lotus
+      - filecoin-proofs:${FIL_PROOFS_PARAMETER_CACHE}
+    networks:
+      - bootstrap-tests
+    environment:
+      - FIL_PROOFS_PARAMETER_CACHE=${FIL_PROOFS_PARAMETER_CACHE}
+      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/${LOTUS_RPC_PORT}/http
+      - LOTUS_SYNC_BOOTSTRAP_PEERS=1
+      - FULLNODE_API_INFO=/dns/lotus/tcp/${LOTUS_RPC_PORT}/http
+    entrypoint: [ "/bin/bash", "-c" ]
+    command:
+      - |
+        set -euxo pipefail
+        lotus daemon --remove-existing-chain --import-snapshot $(ls /data/*.car.zst | tail -n 1)
+  lotus-peers-wait:
+    depends_on:
+      lotus:
+        condition: service_started
+    build:
+      dockerfile: ./lotus.dockerfile
+      context: .
+      args:
+        - LOTUS_VERSION=${LOTUS_VERSION}
+        - BOOTSTRAPPER=${FOREST_BOOTSTRAPPER_ADDRESS}
+    volumes:
+      - node-data:/data
+      - lotus-data:/var/lib/lotus
+      - filecoin-proofs:${FIL_PROOFS_PARAMETER_CACHE}
+    networks:
+      - bootstrap-tests
+    environment:
+      - FULLNODE_API_INFO=/dns/lotus/tcp/${LOTUS_RPC_PORT}/http
+    entrypoint: [ "/bin/bash", "-c" ]
+    command:
+      - |
+        set -euxo pipefail
+        lotus wait-api --timeout 10m
+        lotus sync wait
+        # At this point, lotus node is synced and it should have multiple peers. Assert the latter.
+        if [ $$(lotus net peers | wc -l) -lt 2 ]; then
+          echo "Lotus node should have at least 2 peers"
+          exit 1
+        fi
+        echo "Lotus properly bootstrapped"
+  post-setup:
+    depends_on:
+      lotus-peers-wait:
+        condition: service_completed_successfully
+      forest-peers-wait:
+        condition: service_completed_successfully
+    image: busybox
+    networks:
+      - bootstrap-tests
+    entrypoint: [ "/bin/sh", "-c" ]
+    command:
+      - |
+        set -euxo pipefail
+        echo "Success"
+
+volumes:
+  filecoin-proofs:
+  node-data:
+  # mount this to /var/lib/lotus to avoid creating random volumes
+  lotus-data:
+
+networks:
+  bootstrap-tests:

scripts/tests/bootstrapper/lotus.dockerfile

@@ -0,0 +1,53 @@
+# Lotus binaries image, to be used in the local devnet with Forest.
+FROM golang:1.21-bullseye AS lotus-builder
+
+RUN apt-get update && apt-get install -y curl ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev 
+
+WORKDIR /lotus
+
+# Install rust toolchain for rebuilding `filecoin-ffi`
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --profile minimal
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+ARG LOTUS_VERSION
+RUN git clone --depth 1 --branch ${LOTUS_VERSION} https://github.com/filecoin-project/lotus.git .
+
+# Replace the default bootstrap peers with a pre-defined one. This is needed on this level because the bootstrap peers are compiled into the binary and not configurable at runtime.
+ARG BOOTSTRAPPER
+RUN echo ${BOOTSTRAPPER} > build/bootstrap/calibnet.pi
+
+# Apply a patch to Lotus so that local IPs are not discarded
+COPY patch-lotus.diff .
+RUN git apply patch-lotus.diff
+
+# https://github.com/Filecoin-project/filecoin-ffi?tab=readme-ov-file#building-from-source
+RUN CGO_CFLAGS_ALLOW="-D__BLST_PORTABLE__" \
+    CGO_CFLAGS="-D__BLST_PORTABLE__" \
+    FFI_USE_BLST_PORTABLE="1" \
+    FFI_USE_GPU="0" \
+    make calibnet && strip lotus*
+
+FROM ubuntu:22.04
+
+# Need to copy the relevant shared libraries from the builder image.
+# See https://github.com/filecoin-project/lotus/blob/master/Dockerfile
+COPY --from=lotus-builder /etc/ssl/certs            /etc/ssl/certs
+COPY --from=lotus-builder /lib/*/libdl.so.2         /lib/
+COPY --from=lotus-builder /lib/*/librt.so.1         /lib/
+COPY --from=lotus-builder /lib/*/libgcc_s.so.1      /lib/
+COPY --from=lotus-builder /lib/*/libutil.so.1       /lib/
+COPY --from=lotus-builder /usr/lib/*/libltdl.so.7   /lib/
+COPY --from=lotus-builder /usr/lib/*/libnuma.so.1   /lib/
+COPY --from=lotus-builder /usr/lib/*/libhwloc.so.*  /lib/
+COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
+
+# Copy only the binaries relevant for the bootstrap test
+COPY --from=lotus-builder /lotus/lotus /usr/local/bin/
+
+WORKDIR /lotus
+
+# Basic verification of dynamically linked dependencies
+RUN lotus -v
+
+CMD ["/bin/bash"]

scripts/tests/bootstrapper/patch-lotus.diff

@@ -0,0 +1,12 @@
+diff --git a/node/modules/lp2p/host.go b/node/modules/lp2p/host.go
+index 66256df52..3625b9a31 100644
+--- a/node/modules/lp2p/host.go
++++ b/node/modules/lp2p/host.go
+@@ -93,7 +93,6 @@ func DHTRouting(mode dht.ModeOpt) interface{} {
+ 			dht.Validator(validator),
+ 			dht.ProtocolPrefix(build.DhtProtocolName(nn)),
+ 			dht.QueryFilter(dht.PublicQueryFilter),
+-			dht.RoutingTableFilter(dht.PublicRoutingTableFilter),
+ 			dht.DisableProviders(),
+ 			dht.DisableValues()}
+ 		d, err := dht.New(

scripts/tests/bootstrapper/setup.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# This script is used to set up clean environment for the bootstrapper tests.
+
+set -euxo pipefail
+
+# Path to the directory containing this script.
+PARENT_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+pushd "${PARENT_PATH}"
+source .env
+
+# This should not be needed in GH. It is useful for running locally.
+docker compose down --remove-orphans
+docker compose rm -f
+
+# Run it in the background so we can perform checks on it.
+# Ideally, we could use `--wait` and `--wait-timeout` to wait for services
+# to be up. However, `compose` does not distinct between services and 
+# init containers. See more: https://github.com/docker/compose/issues/10596
+docker compose up --build --force-recreate --detach --timestamps
+
+popd