From a330bc28dae335ce687934c5d57760e20c03cecd Mon Sep 17 00:00:00 2001 From: Kostis Papazafeiropoulos Date: Mon, 2 Sep 2024 20:35:14 +0100 Subject: [PATCH] virtio-accel: Add Dockerfile and workflow to build and push image Signed-off-by: Kostis Papazafeiropoulos --- .github/workflows/vaccel-build-and-upload.yml | 119 ++++++++++++++++++ subprojects/vaccel/.gitignore | 1 + subprojects/vaccel/docker/Dockerfile | 60 +++++++++ subprojects/vaccel/docker/qemu-ifup | 38 ++++++ subprojects/vaccel/docker/qemu-script.sh | 86 +++++++++++++ subprojects/vaccel/docker/vq-size.patch | 13 ++ 6 files changed, 317 insertions(+) create mode 100644 .github/workflows/vaccel-build-and-upload.yml create mode 100644 subprojects/vaccel/.gitignore create mode 100644 subprojects/vaccel/docker/Dockerfile create mode 100755 subprojects/vaccel/docker/qemu-ifup create mode 100755 subprojects/vaccel/docker/qemu-script.sh create mode 100644 subprojects/vaccel/docker/vq-size.patch diff --git a/.github/workflows/vaccel-build-and-upload.yml b/.github/workflows/vaccel-build-and-upload.yml new file mode 100644 index 00000000000..4ab65692803 --- /dev/null +++ b/.github/workflows/vaccel-build-and-upload.yml @@ -0,0 +1,119 @@ +name: Build QEMU+vAccel docker image + +on: + push: + branches: [ '*\+vaccel' ] + + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + REGISTRY: harbor.nbfc.io/nubificus + IMAGE_NAME: qemu-vaccel + APP: qemu-vaccel + +jobs: + build: + name: Build Docker Image + runs-on: [self-hosted, gcc, lite, "${{ matrix.arch }}"] + strategy: + matrix: + arch: [x86_64, aarch64] + outputs: + digest-x86_64: ${{ steps.set-outputs.outputs.digest-x86_64 }} + digest-aarch64: ${{ steps.set-outputs.outputs.digest-aarch64 }} + + steps: + - name: Cleanup previous jobs + run: | + echo "Cleaning up previous runs" + sudo rm -rf ${{ github.workspace }}/* + sudo rm -rf ${{ github.workspace }}/.??* + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.HARBOR_USER }} + password: ${{ secrets.HARBOR_PASSWD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ matrix.arch }} + type=sha,prefix=${{ matrix.arch }}- + type=sha,format=long,prefix=${{ matrix.arch }}- + type=ref,event=branch,prefix=${{ matrix.arch }}- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v6 + with: + context: ./subprojects/vaccel/docker + no-cache: true + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + ARCHTAG=${{ matrix.arch }} + BRANCH=${{ github.event.ref_name || github.ref_name }} + + - name: Set per-arch outputs + id: set-outputs + run: | + # Workaround for https://github.com/actions/runner/issues/2499 + echo "digest-${{ matrix.arch }}=${{ steps.build-and-push.outputs.digest }}" \ + >> "$GITHUB_OUTPUT" + + sign: + name: Sign Docker Image + runs-on: [self-hosted] + needs: [build] + strategy: + matrix: + arch: [x86_64, aarch64] + permissions: + contents: read + id-token: write + + steps: + - name: Install Cosign + uses: sigstore/cosign-installer@v3.6.0 + + - name: Check install + run: cosign version + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.HARBOR_USER }} + password: ${{ secrets.HARBOR_PASSWD }} + + - name: Sign published Docker image + env: + DIGEST: ${{ needs.build.outputs[format('digest-{0}', matrix.arch)] }} + run: | + cosign sign --yes ${{ env.REGISTRY }}/${{ env.APP }}@${{ env.DIGEST }} \ + -a "repo=${{ github.repository }}" \ + -a "workflow=${{ github.workflow }}" \ + -a "ref=${{ github.sha }}" \ + -a "author=Nubificus LTD" + + - name: Cleanup previous runs + if: ${{ always() }} + run: | + sudo rm -rf ${{ github.workspace }}/* + sudo rm -rf ${{ github.workspace }}/.??* diff --git a/subprojects/vaccel/.gitignore b/subprojects/vaccel/.gitignore new file mode 100644 index 00000000000..69f1bf45309 --- /dev/null +++ b/subprojects/vaccel/.gitignore @@ -0,0 +1 @@ +!*.patch diff --git a/subprojects/vaccel/docker/Dockerfile b/subprojects/vaccel/docker/Dockerfile new file mode 100644 index 00000000000..1a6e80e77ed --- /dev/null +++ b/subprojects/vaccel/docker/Dockerfile @@ -0,0 +1,60 @@ +FROM ubuntu:24.04 + +# Install common build utilities +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + build-essential \ + libglib2.0-dev \ + libfdt-dev \ + libpixman-1-dev \ + zlib1g-dev \ + ninja-build \ + git \ + python3-pip \ + pkg-config \ + iproute2 \ + libcap-ng-dev \ + libattr1-dev \ + $(apt-get -s build-dep qemu | egrep ^Inst | fgrep '[all]' | cut -d\ -f2) \ + libclang-dev \ + vim \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && pip install --break-system-packages meson + +# Build & install vAccel +WORKDIR /vaccel +RUN git clone https://github.com/nubificus/vaccel . && \ + meson setup -Dplugins=enabled -Dexamples=enabled build && \ + meson compile -C build && \ + meson install -C build && \ + ldconfig + +ARG BRANCH=master+vaccel +ARG ARCHTAG=x86_64 +ARG DOCKER_DIR=. +COPY ${DOCKER_DIR}/vq-size.patch /vq-size.patch +# Build & install QEMU w/ vAccel backend +WORKDIR /qemu-vaccel +RUN git clone -b ${BRANCH} --depth 1 \ + https://github.com/cloudkernels/qemu-vaccel.git . && \ + mv /vq-size.patch . && \ + git apply vq-size.patch && \ + mkdir build && cd build && \ + ../configure --target-list=${ARCHTAG}-softmmu --enable-virtfs && \ + make -j$(nproc) && make install + +# Cleanup +WORKDIR / +RUN rm -rf /vaccel /qemu-vaccel + +COPY ${DOCKER_DIR}/qemu-ifup /usr/local/etc/qemu-ifup +COPY ${DOCKER_DIR}/qemu-script.sh /run.sh + +VOLUME /data +WORKDIR /data +ENTRYPOINT ["/run.sh"] diff --git a/subprojects/vaccel/docker/qemu-ifup b/subprojects/vaccel/docker/qemu-ifup new file mode 100755 index 00000000000..6d306aabc7f --- /dev/null +++ b/subprojects/vaccel/docker/qemu-ifup @@ -0,0 +1,38 @@ +#! /bin/sh +# Script to bring a network (tap) device for qemu up. +# The idea is to add the tap device to the same bridge +# as we have default routing to. + +# in order to be able to find brctl +PATH=$PATH:/sbin:/usr/sbin +ip=$(which ip) + +if [ -n "$ip" ]; then + ip link set "$1" up +else + brctl=$(which brctl) + if [ ! "$ip" -o ! "$brctl" ]; then + echo "W: $0: not doing any bridge processing: neither ip nor brctl utility not found" >&2 + exit 0 + fi + ifconfig "$1" 0.0.0.0 up +fi + +switch=virbr0 + +# only add the interface to default-route bridge if we +# have such interface (with default route) and if that +# interface is actually a bridge. +# It is possible to have several default routes too +for br in $switch; do + if [ -d /sys/class/net/$br/bridge/. ]; then + if [ -n "$ip" ]; then + ip link set "$1" master "$br" + else + brctl addif $br "$1" + fi + exit # exit with status of the previous command + fi +done + +echo "W: $0: no bridge for guest interface found" >&2 diff --git a/subprojects/vaccel/docker/qemu-script.sh b/subprojects/vaccel/docker/qemu-script.sh new file mode 100755 index 00000000000..053dbe4c48a --- /dev/null +++ b/subprojects/vaccel/docker/qemu-script.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +export LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} +export QEMU_AUDIO_DRV=none +export VACCEL_BACKENDS=${VACCEL_BACKENDS:=libvaccel-noop.so} +export VACCEL_DEBUG_LEVEL=${VACCEL_DEBUG_LEVEL:=4} + +smp=1 +cpu=host +ram=512 + +machine="pc,accel=kvm" +kernel="-kernel bzImage" +dtb="" +rootfs=rootfs.img +cmdline="rw root=/dev/vda console=ttyS0 " +stderr=run/stderr.log +extra_args= +cid= + +cd /data +mkdir -p run +while getopts 'c:m:r:a:s:v:n:' opt; do + case $opt in + c) + # VM vCPUs + [[ $OPTARG =~ ^[0-9]+$ ]] || error "${opt}: ${OPTARG} is not a number" 1 + smp="${OPTARG}" + ;; + m) + # VM RAM + [[ $OPTARG =~ ^[0-9]+$ ]] || error "${opt}: ${OPTARG} is not a number" 1 + ram="${OPTARG}" + ;; + r) + # VM rootfs + [[ -z "${OPTARG}" ]] && error "${opt}: requires a non-empty string" 1 + rootfs="${OPTARG}" + ;; + a) + # VM kernel command line append + cmdline+="${OPTARG}" + ;; + s) + # QEMU output to socket + [[ -z "${OPTARG}" ]] && error "${opt}: requires a non-empty string" 1 + vm_id="${OPTARG}" + stderr="${vm_id}-stderr.log" + extra_args+="-serial pipe:./run/${vm_id}.serial " + extra_args+="-chardev socket,id=monitor,path=./run/${vm_id}.monitor,server,nowait " + extra_args+="-monitor chardev:monitor " + ;; + n) + # VM w/ network + [[ -z "${OPTARG}" ]] && mac="52:54:00:12:34:01" || mac="${OPTARG}" + extra_args+="-netdev type=tap,id=net0 -device virtio-net,netdev=net0 " + ;; + v) + # VM w/ vsock + [[ $OPTARG =~ ^[0-9]+$ ]] || error "${opt}: ${OPTARG} is not a number" 1 + cid="${OPTARG}" + extra_args+="-device vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid=${cid} " + ;; + :) + error "Option -$OPTARG requires an argument" 1 + ;; + \?) + exit 1 + ;; + esac +done +cmdline+="mem=${ram}M" + +fsck.ext4 -fy $rootfs 1>/dev/null 2>&1 + +TERM=linux qemu-system-"$(uname -m)" \ + -cpu "$cpu" -m "$ram" -smp "$smp" -M "$machine" -nographic \ + $kernel $dtb -append "$cmdline" 2>$stderr \ + -drive if=none,id=rootfs,file=$rootfs,format=raw,cache=none \ + -device virtio-blk,drive=rootfs \ + -fsdev local,id=fsdev0,path=/data/data,security_model=none \ + -device virtio-9p-pci,fsdev=fsdev0,mount_tag=data \ + -device virtio-rng-pci \ + -object acceldev-backend-vaccel,id=rt0 \ + -device virtio-accel-pci,id=accel0,runtime=rt0 \ + $extra_args diff --git a/subprojects/vaccel/docker/vq-size.patch b/subprojects/vaccel/docker/vq-size.patch new file mode 100644 index 00000000000..27741c18f6a --- /dev/null +++ b/subprojects/vaccel/docker/vq-size.patch @@ -0,0 +1,13 @@ +diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h +index b69d517496..64967d7e3d 100644 +--- a/include/hw/virtio/virtio.h ++++ b/include/hw/virtio/virtio.h +@@ -47,7 +47,7 @@ size_t virtio_feature_get_config_size(VirtIOFeature *features, + + typedef struct VirtQueue VirtQueue; + +-#define VIRTQUEUE_MAX_SIZE 1024 ++#define VIRTQUEUE_MAX_SIZE 8192 + + typedef struct VirtQueueElement + {