Introduce riscv64 CI container

A developed version of rust-vmm#91, referenced the idea of environment preparation for qemu-system-riscv64. Add build scripts for v6.10 riscv64 kernel, qemu-system-riscv64 and opensbi required to boot qemu-system. With this approach, we are able to run tests inside qemu-system, while preserving the original output as much as possbile with ssh. Signed-off-by: Ruoqing He <[email protected]>
TimePrinciple · Aug 5, 2024 · 6377783 · 6377783
1 parent 2e2e495
commit 6377783
Show file tree

Hide file tree

Showing 9 changed files with 262 additions and 45 deletions.
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
@@ -3,10 +3,20 @@ name: Docker
 on:
   push:
     branches: [ "main" ]
-    paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
+    paths:
+      - Dockerfile
+      - .github/workflows/docker-publish.yml
+      - build_container.sh
+      - Dockerfile.riscv64
+      - riscv64/*
   pull_request:
     branches: [ "main" ]
-    paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
+    paths:
+      - Dockerfile
+      - .github/workflows/docker-publish.yml
+      - build_container.sh
+      - Dockerfile.riscv64
+      - riscv64/*
 
 jobs:
   build:
@@ -69,6 +79,18 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Build and push Docker image for riscv64
+        id: build-and-push-riscv64
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          file: Dockerfile.riscv64
+          push: ${{ github.event_name != 'pull_request' }}
+          platforms: linux/amd64
+          tags: ${{ env.VERSION }}-riscv
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker
@@ -81,4 +103,6 @@ jobs:
           COSIGN_EXPERIMENTAL: "true"
         # This step uses the identity token to provision an ephemeral certificate
         # against the sigstore community Fulcio instance.
-        run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
+        run: |
+          echo "${{ env.VERSION }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
+          echo "${{ env.VERSION }}-riscv" | xargs -I {} cosign sign {}@${{ steps.build-and-push-riscv.outputs.digest }}
diff --git a/Dockerfile.riscv64 b/Dockerfile.riscv64
@@ -0,0 +1,55 @@
+# Compile QEMU 9.0.2
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS qemu_builder
+
+COPY riscv64/build_qemu_system_riscv64.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Compile kernel 6.10 since we need AIA drivers
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS kernel_builder
+
+COPY riscv64/build_kernel.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Compile OpenSBI
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS opensbi_builder
+
+COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Build rootfs with sshd and Rust related packages ready
+# ---------------------------------------------------------
+FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder
+
+ARG RUST_TOOLCHAIN="1.75.0"
+ENV PATH="$PATH:/root/.cargo/bin"
+COPY build_container.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Finalize
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS final
+
+ARG OUTPUT=/output
+ARG QEMU_DIR=/opt/qemu
+ARG KERNEL_DIR=/opt/kernel
+ARG OPENSBI_DIR=/opt/opensbi
+ARG ROOTFS_DIR=/opt/rootfs
+
+COPY --from=qemu_builder $OUTPUT $QEMU_DIR
+COPY --from=kernel_builder $OUTPUT $KERNEL_DIR
+COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR
+COPY --from=rootfs_builder / $ROOTFS_DIR
+
+COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh
+RUN /opt/src/scripts/finalize.sh
+
+ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \
+    OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \
+    WORKDIR=/workdir
+
+# Start qemu-system-riscv64 as a background process
+COPY riscv64/start_in_qemu.sh /opt/src/scripts/start.sh
+ENTRYPOINT ["/opt/src/scripts/start.sh"]
diff --git a/build_container.sh b/build_container.sh
@@ -22,15 +22,24 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
     libvirglrenderer-dev libvirglrenderer1 \
     debhelper-compat libdbus-1-dev libglib2.0-dev meson ninja-build dbus
 
+# `riscv64` specific
+if [ "$ARCH" == "riscv64" ]; then
+    DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+        openssh-server systemd init ifupdown busybox udev isc-dhcp-client
+fi
+
 # cleanup
 apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # help musl-gcc find linux headers
-pushd /usr/include/$ARCH-linux-musl 
-ln -s ../$ARCH-linux-gnu/asm asm 
-ln -s ../linux linux 
-ln -s ../asm-generic asm-generic
-popd
+# Skip on `riscv64` for now
+if [ "$ARCH" != "riscv64" ]; then
+    pushd /usr/include/$ARCH-linux-musl 
+    ln -s ../$ARCH-linux-gnu/asm asm 
+    ln -s ../linux linux 
+    ln -s ../asm-generic asm-generic
+    popd
+fi
 
 pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip
 
@@ -52,46 +61,67 @@ rustup component add miri rust-src --toolchain nightly
 rustup component add llvm-tools-preview  # needed for coverage
 
 # Install other rust targets.
-rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
+# Skip on `riscv64` for now
+if [ "$ARCH" != "riscv64" ]; then
+    rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
+fi
 
 cargo install cargo-llvm-cov
 
 # Install aemu, gfxstream, libgpiod and libpipewire (required by vhost-device crate)
-pushd /opt
-git clone https://android.googlesource.com/platform/hardware/google/aemu
-pushd aemu
-git checkout v0.1.2-aemu-release
-cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
-       -DAEMU_COMMON_BUILD_CONFIG=gfxstream \
-       -DENABLE_VKCEREAL_TESTS=OFF -B build
-cmake --build build -j
-cmake --install build
-popd
-rm -rf aemu
-git clone https://android.googlesource.com/platform/hardware/google/gfxstream
-pushd gfxstream
-git checkout v0.1.2-gfxstream-release
-meson setup host-build/
-meson install -C host-build/
-popd
-rm -rf gfxstream
-git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
-pushd libgpiod
-./autogen.sh --prefix=/usr && make && make install
-popd
-rm -rf libgpiod
-wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
-tar xzvf pipewire-0.3.71.tar.gz
-pushd pipewire-0.3.71
-meson setup builddir --prefix="/usr" -Dbuildtype=release \
-    -Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
-    -Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
-meson compile -C builddir && \
-meson install -C builddir
-popd
-rm -rf pipewire-0.3.71
-rm pipewire-0.3.71.tar.gz
-popd
+# `aemu` has yet supported `riscv64`, skipping `vhost-device` related
+# dependencies for `riscv64` at the time being
+if [ "$ARCH" != "riscv64" ]; then
+    pushd /opt
+    git clone https://android.googlesource.com/platform/hardware/google/aemu
+    pushd aemu
+    git checkout v0.1.2-aemu-release
+    cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
+        -DAEMU_COMMON_BUILD_CONFIG=gfxstream \
+        -DENABLE_VKCEREAL_TESTS=OFF -B build
+    cmake --build build -j
+    cmake --install build
+    popd
+    rm -rf aemu
+    git clone https://android.googlesource.com/platform/hardware/google/gfxstream
+    pushd gfxstream
+    git checkout v0.1.2-gfxstream-release
+    meson setup host-build/
+    meson install -C host-build/
+    popd
+    rm -rf gfxstream
+    git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
+    pushd libgpiod
+    ./autogen.sh --prefix=/usr && make && make install
+    popd
+    rm -rf libgpiod
+    wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
+    tar xzvf pipewire-0.3.71.tar.gz
+    pushd pipewire-0.3.71
+    meson setup builddir --prefix="/usr" -Dbuildtype=release \
+        -Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
+        -Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
+    meson compile -C builddir && \
+    meson install -C builddir
+    popd
+    rm -rf pipewire-0.3.71
+    rm pipewire-0.3.71.tar.gz
+    popd
+fi
 
 # dbus-daemon expects this folder
 mkdir /run/dbus
+
+# `riscv64` specific
+if [ "$ARCH" == "riscv64" ]; then
+    # Set passwd for debugging
+    echo 'root:rustvmm' | chpasswd
+    # Allow root login
+    sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
+    sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config
+    # Enable ssh
+    systemctl enable ssh
+    mkdir -p /root/.ssh
+    # Setup network
+    echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces
+fi
diff --git a/docker.sh b/docker.sh
@@ -7,11 +7,12 @@ DOCKER_TAG=rustvmm/dev
 
 # Get the latest published version. Returns a number.
 # If latest is v100, returns 100.
+# If latest for riscv64 is v100-riscv, returns 100.
 # This works as long as we have less than 100 tags because we set the page size to 100,
 # once we have more than that this script needs to be updated.
 latest(){
   curl -L -s 'https://registry.hub.docker.com/v2/repositories/rustvmm/dev/tags?page_size=100'| \
-    jq '."results"[]["name"]' |  sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+$" | sort -n | tail -1
+    jq '."results"[]["name"]' |  sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+" | sort -n | tail -1
 }
 
 next_version() {

diff --git a/riscv64/build_finalize.sh b/riscv64/build_finalize.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \
+    libpixman-1-dev netcat
+
+# Setup container ssh config
+yes "" | ssh-keygen -P ""
+cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys
+cat > /root/.ssh/config << EOF
+Host riscv-qemu
+    HostName localhost
+    User root
+    Port 2222
+    StrictHostKeyChecking no
+EOF
+
+# Set `nameserver` for `resolv.conf`
+echo 'nameserver 8.8.8.8' > $ROOTFS_DIR/etc/resolv.conf
diff --git a/riscv64/build_kernel.sh b/riscv64/build_kernel.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+KERNEL_TAG=v6.10
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison gcc-riscv64-linux-gnu
+
+git clone --depth 1 --branch $KERNEL_TAG https://github.com/torvalds/linux.git
+pushd linux
+# Enable kvm module instead of inserting manually
+sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" arch/riscv/configs/defconfig
+make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- defconfig && \
+make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- -j$(nproc)
+mv arch/riscv/boot/Image $OUTPUT
+popd
diff --git a/riscv64/build_opensbi.sh b/riscv64/build_opensbi.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+OPENSBI_TAG=v1.3.1
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison gcc-riscv64-linux-gnu
+
+git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git
+pushd opensbi
+make -j$(nproc) PLATFORM=generic CROSS_COMPILE=riscv64-linux-gnu-
+mv build/platform/generic/firmware/fw_jump.elf $OUTPUT
+popd
diff --git a/riscv64/build_qemu_system_riscv64.sh b/riscv64/build_qemu_system_riscv64.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+QEMU_TAG=v9.0.2
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison
+
+git clone --depth 1 --branch $QEMU_TAG https://gitlab.com/qemu-project/qemu.git
+pushd qemu
+./configure --target-list=riscv64-softmmu --prefix=$OUTPUT && \
+	make -j$(nproc) && make install
+popd
diff --git a/riscv64/start_in_qemu.sh b/riscv64/start_in_qemu.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -ex
+
+$QEMU_DIR/bin/qemu-system-riscv64 \
+    -M virt,aclint=on,aia=aplic-imsic -nographic \
+    -smp 16 -m 32G \
+    -bios $OPENSBI_DIR/fw_jump.elf \
+    -kernel $KERNEL_DIR/Image \
+    -device virtio-net-device,netdev=usernet -netdev user,id=usernet,hostfwd=tcp::2222-:22 \
+    -virtfs local,path=$ROOTFS_DIR,mount_tag=rootfs,security_model=none,id=rootfs \
+    -append "root=rootfs rw rootfstype=9p rootflags=trans=virtio,cache=mmap,msize=512000 console=ttyS0 earlycon=sbi nokaslr rdinit=/sbin/init" 2>&1 &
+
+# Copy WORKDIR to rootfs
+cp -a $WORKDIR /opt/rootfs/root
+
+HOST=riscv-qemu
+
+while ! nc -z localhost 2222; do   
+  echo "Dialing qemu-system-riscv64..."
+  sleep 1
+done
+
+# Issue command
+COMMAND=$@
+ssh $HOST "export PATH=\"\$PATH:/root/.cargo/bin\" && cd workdir && $COMMAND"