diff --git a/.github/workflows/vaccel-build-and-upload.yml b/.github/workflows/vaccel-build-and-upload.yml new file mode 100644 index 00000000000..c8ed44ded30 --- /dev/null +++ b/.github/workflows/vaccel-build-and-upload.yml @@ -0,0 +1,118 @@ +name: Build QEMU+vAccel docker image + +on: + push: + branches: [ '*\+vaccel' ] + + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + REGISTRY: harbor.nbfc.io/nubificus + IMAGE_NAME: qemu-vaccel + APP: qemu-vaccel + +jobs: + build: + name: Build Docker Image + runs-on: [self-hosted, gcc, lite, "${{ matrix.arch }}"] + strategy: + matrix: + arch: [x86_64, aarch64] + outputs: + digest-x86_64: ${{ steps.set-outputs.outputs.digest-x86_64 }} + digest-aarch64: ${{ steps.set-outputs.outputs.digest-aarch64 }} + steps: + - name: Cleanup previous jobs + run: | + echo "Cleaning up previous runs" + sudo rm -rf ${{ github.workspace }}/* + sudo rm -rf ${{ github.workspace }}/.??* + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.HARBOR_USER }} + password: ${{ secrets.HARBOR_PASSWD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ matrix.arch }} + type=sha,prefix=${{ matrix.arch }}- + type=sha,format=long,prefix=${{ matrix.arch }}- + type=ref,event=branch,prefix=${{ matrix.arch }}- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v6 + with: + context: ./subprojects/vaccel/docker + no-cache: true + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + ARCHTAG=${{ matrix.arch }} + BRANCH=${{ github.event.ref_name || github.ref_name }} + + - name: Set per-arch outputs + id: set-outputs + run: | + # Workaround for https://github.com/actions/runner/issues/2499 + echo "digest-${{ matrix.arch }}=${{ steps.build-and-push.outputs.digest }}" \ + >> "$GITHUB_OUTPUT" + + sign: + name: Sign Docker Image + runs-on: [self-hosted] + needs: [build] + strategy: + matrix: + arch: [x86_64, aarch64] + permissions: + contents: read + id-token: write + + steps: + - name: Install Cosign + uses: sigstore/cosign-installer@v3.6.0 + + - name: Check install + run: cosign version + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.HARBOR_USER }} + password: ${{ secrets.HARBOR_PASSWD }} + + - name: Sign published Docker image + env: + DIGEST: ${{ needs.build.outputs[format('digest-{0}', matrix.arch)] }} + run: | + cosign sign --yes ${{ env.REGISTRY }}/${{ env.APP }}@${{ env.DIGEST }} \ + -a "repo=${{ github.repository }}" \ + -a "workflow=${{ github.workflow }}" \ + -a "ref=${{ github.sha }}" \ + -a "author=Nubificus LTD" + + - name: Cleanup previous runs + if: ${{ always() }} + run: | + sudo rm -rf ${{ github.workspace }}/* + sudo rm -rf ${{ github.workspace }}/.??* diff --git a/subprojects/vaccel/.gitignore b/subprojects/vaccel/.gitignore new file mode 100644 index 00000000000..69f1bf45309 --- /dev/null +++ b/subprojects/vaccel/.gitignore @@ -0,0 +1 @@ +!*.patch diff --git a/subprojects/vaccel/docker/Dockerfile b/subprojects/vaccel/docker/Dockerfile new file mode 100644 index 00000000000..a7246b802d5 --- /dev/null +++ b/subprojects/vaccel/docker/Dockerfile @@ -0,0 +1,64 @@ +FROM ubuntu:24.04 + +WORKDIR / + +# Install common build utilities +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + build-essential \ + libglib2.0-dev \ + libfdt-dev \ + libpixman-1-dev \ + libslirp-dev \ + zlib1g-dev \ + libcap-ng-dev \ + libattr1-dev \ + ninja-build \ + git \ + python3-pip \ + libclang-dev \ + pkg-config \ + iproute2 \ + openssh-client \ + iputils-ping \ + socat \ + vim \ + less \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && pip install --break-system-packages meson + +# Build & install vAccel +RUN git clone https://github.com/nubificus/vaccel && \ + cd vaccel && \ + meson setup -Dplugins=enabled -Dexamples=enabled build && \ + meson compile -C build && \ + meson install -C build && \ + ldconfig && \ + cd .. && rm -rf vaccel + +ARG BRANCH=master+vaccel +ARG ARCHTAG=x86_64 +ARG DOCKER_DIR=. +COPY ${DOCKER_DIR}/vq-size.patch /vq-size.patch +# Build & install QEMU w/ vAccel backend +RUN git clone -b ${BRANCH} --depth 1 \ + https://github.com/cloudkernels/qemu-vaccel && \ + cd qemu-vaccel && \ + mv /vq-size.patch . && \ + git apply vq-size.patch && \ + mkdir build && cd build && \ + ../configure --target-list=${ARCHTAG}-softmmu --enable-virtfs && \ + make -j$(nproc) && make install && \ + cd ../.. && rm -rf qemu-vaccel + +COPY ${DOCKER_DIR}/qemu-ifup /usr/local/etc/qemu-ifup +COPY ${DOCKER_DIR}/qemu-script.sh /run.sh + +VOLUME /data +WORKDIR /data +ENTRYPOINT ["/run.sh"] diff --git a/subprojects/vaccel/docker/qemu-ifup b/subprojects/vaccel/docker/qemu-ifup new file mode 100755 index 00000000000..6d306aabc7f --- /dev/null +++ b/subprojects/vaccel/docker/qemu-ifup @@ -0,0 +1,38 @@ +#! /bin/sh +# Script to bring a network (tap) device for qemu up. +# The idea is to add the tap device to the same bridge +# as we have default routing to. + +# in order to be able to find brctl +PATH=$PATH:/sbin:/usr/sbin +ip=$(which ip) + +if [ -n "$ip" ]; then + ip link set "$1" up +else + brctl=$(which brctl) + if [ ! "$ip" -o ! "$brctl" ]; then + echo "W: $0: not doing any bridge processing: neither ip nor brctl utility not found" >&2 + exit 0 + fi + ifconfig "$1" 0.0.0.0 up +fi + +switch=virbr0 + +# only add the interface to default-route bridge if we +# have such interface (with default route) and if that +# interface is actually a bridge. +# It is possible to have several default routes too +for br in $switch; do + if [ -d /sys/class/net/$br/bridge/. ]; then + if [ -n "$ip" ]; then + ip link set "$1" master "$br" + else + brctl addif $br "$1" + fi + exit # exit with status of the previous command + fi +done + +echo "W: $0: no bridge for guest interface found" >&2 diff --git a/subprojects/vaccel/docker/qemu-script.sh b/subprojects/vaccel/docker/qemu-script.sh new file mode 100755 index 00000000000..2604fbca908 --- /dev/null +++ b/subprojects/vaccel/docker/qemu-script.sh @@ -0,0 +1,301 @@ +#!/bin/bash + +set -e + +export LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} +export QEMU_AUDIO_DRV=none +export VACCEL_BACKENDS=${VACCEL_BACKENDS:=libvaccel-noop.so} +export VACCEL_DEBUG_LEVEL=${VACCEL_DEBUG_LEVEL:=4} + +SCRIPT_NAME=$(basename "$0") +RUN_PATH=./run +SSH_HOST=localhost +SSH_PORT_LOCAL=60022 +SSH_PORT_VM=22 + +smp=1 +cpu=host +ram=512 + +machine=pc,accel=kvm +device=pci +kernel=bzImage +rootfs=rootfs.img +cmdline='rw root=/dev/vda console=ttyS0 ' +dcache=none +stderr=${RUN_PATH}/stderr.log + +timeout=300 + +log_error() { + local error=${1:-'Unknown error'} + local code=${2:-1} + echo "${SCRIPT_NAME}: ${error} [error ${code}]" >&2 +} + +error() { + local code=${2:-1} + log_error "$1" "${code}" + exit "${code}" +} + +cleanup_log_file() { + [[ -n "$1" ]] && [[ -z "$(cat "$1")" ]] && rm -f "$1" || true +} + +print_log_file() { + [[ -f "$1" ]] && echo "${1}:" && cat "$1" || true +} + +parse_args() { + short_opts=M:m:r:k:s:n::v::c:t: + read -r -d '' long_opts <<-EOF || true + machine:,cpu:,dtb:, + vcpus:,ram:,rootfs:,kernel:,cmdline-append:,output-socket:, + net-tap::,vsock::,cmd:,timeout:,no-pci,no-kvm,drive-cache,skip-fsck + EOF + + if ! getopt=$(getopt -o "${short_opts}" --long "${long_opts}" \ + -n "${SCRIPT_NAME}" -- "$@"); then + echo 'Failed to parse args' >&2 + exit 1 + fi + + eval set -- "$getopt" + unset "$getopt" + + while true; do + case "$1" in + '-M' | '--machine') + # QEMU Machine + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + machine="$2" + shift 2 + ;; + '--cpu') + # QEMU CPU + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + cpu="$2" + shift 2 + ;; + '--dtb') + # QEMU CPU + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + dtb="$2" + extra_args+="-dtb ${dtb} " + shift 2 + ;; + '--vcpus') + # VM vCPUs + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + smp="$2" + shift 2 + ;; + '-m' | '--memory') + # VM RAM + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + ram="$2" + shift 2 + ;; + '-r' | '--rootfs') + # VM rootfs + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + rootfs="$2" + shift 2 + ;; + '-k' | '--kernel') + # VM kernel + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + kernel="$2" + shift 2 + ;; + '--cmdline-append') + # VM kernel command line append + cmdline+="$2 " + shift 2 + ;; + '-s' | '--output-socket') + # QEMU output to socket + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + socket_prefix="$2" + shift 2 + ;; + '-n' | '--net-tap') + # VM w/ network + [[ -z "$2" ]] && mac='52:54:00:12:34:01' || mac="$2" + shift 2 + ;; + '-v' | '--vsock') + # VM w/ vsock + [[ "$2" =~ ^[0-9]+$ ]] || error "'$1' requires a number" + cid="$2" + shift 2 + ;; + '-c' | '--cmd') + # Command to run in the VM + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + cmd="$2" + shift 2 + ;; + '-t' | '--timeout') + # Change default timeout + [[ -z "$2" ]] && error "'$1' requires a non-empty string" + timeout="$2" + shift 2 + ;; + '--no-pci') + # Switch to MMIO devices + device='device' + shift + ;; + '--no-kvm') + # Do not enable KVM + no_kvm=1 + shift + ;; + '--drive-cache') + # Use drive cache (writeback) + dcache=writeback + shift + ;; + '--skip-fsck') + # Skip rootfs image check + skip_fsck=1 + shift + ;; + --) + shift + break + ;; + *) + echo 'Internal error parsing args' >&2 + exit 1 + ;; + esac + done + cmdline+="mem=${ram}M" + [[ -z "${no_kvm}" ]] && extra_args+='-enable-kvm ' || true +} + +setup_qemu_network() { + if [[ -n "${mac}" ]]; then + extra_args+="-nic tap,model=virtio-net-${device},mac=${mac} " + else + extra_args+="-netdev user,id=net0,hostfwd=tcp::${SSH_PORT_LOCAL}-:${SSH_PORT_VM} " + extra_args+="-device virtio-net-${device},netdev=net0 " + fi + + if [[ -n "${cid}" ]]; then + extra_args+="-device vhost-vsock-${device},id=vsock0,guest-cid=${cid} " + fi +} + +setup_qemu_socket() { + [[ -z "$1" ]] && error "'setup_qemu_socket()' requires a non-empty string" + stderr="${RUN_PATH}/${1}.stderr.log" + monitor_socket=${RUN_PATH}/${1}.monitor.sock + serial_socket=${RUN_PATH}/${1}.serial.sock + serial_log=${RUN_PATH}/${1}.serial.log + extra_args+="-chardev socket,id=ser0,path=${serial_socket},logfile=${serial_log},server=on,wait=off " + extra_args+="-serial chardev:ser0 " + extra_args+="-chardev socket,id=mon0,path=${monitor_socket},server=on,wait=off " + extra_args+='-monitor chardev:mon0 ' +} + +cleanup_qemu_socket() { + rm -f "${monitor_socket}" "${serial_socket}" + cleanup_log_file "${serial_log}" +} + +print_qemu_output() { + [[ -n "$1" ]] && print_log_file "${serial_log}" || true +} + +check_rootfs_img() { + [[ -z "$1" ]] && error "'check_rootfs_img()' requires a non-empty string" + fsck.ext4 -fy "$1" 1>/dev/null 2>"${stderr}" || res=$? + if [[ "${res}" -gt 2 ]]; then + log_error "'fsck.ext4' error" "${res}" + print_log_file "${stderr}" + cleanup_log_file "${stderr}" + fi +} + +run_qemu() { + TERM=linux qemu-system-"$(uname -m)" \ + -cpu "${cpu}" -m "${ram}" -smp "${smp}" -M "${machine}" -nographic \ + -kernel "${kernel}" -append "${cmdline}" \ + -drive if=none,id=rootfs,file="${rootfs}",format=raw,cache="${dcache}" \ + -device "virtio-blk-${device}",drive=rootfs \ + -fsdev local,id=fsdev0,path=/data/data,security_model=none \ + -device "virtio-9p-${device}",fsdev=fsdev0,mount_tag=data \ + -device "virtio-rng-${device}" \ + -object acceldev-backend-vaccel,id=rt0 \ + -device "virtio-accel-${device}",id=accel0,runtime=rt0 \ + ${extra_args} 2>"${stderr}" +} + +run_cmd() { + [[ -z "$1" ]] && error "'run_cmd()' requires a non-empty string" + sleep 1 + ssh -o StrictHostKeyChecking=accept-new -o BatchMode=yes -o LogLevel=ERROR \ + "${SSH_HOST}" -p "${SSH_PORT_LOCAL}" \ + "${1}"' || res=$?; poweroff 2>/dev/null; exit "${res:-0}"' +} + +main() { + parse_args "$@" + + cd /data + mkdir -p "${RUN_PATH}" data + chown "$(stat -c %u .)":"$(stat -c %g .)" "${RUN_PATH}" data + setup_qemu_network + [[ -n "${socket_prefix}" ]] && setup_qemu_socket "${socket_prefix}" + [[ -z "${skip_fsck}" ]] && check_rootfs_img "${rootfs}" + + res=0 + + if [[ -z "${cmd}" ]]; then + run_qemu || res=$? + if [[ "${res}" -ne 0 ]]; then + log_error "'run_qemu()' error" "${res}" + print_qemu_output "${socket_prefix}" + print_log_file "${stderr}" + fi + + cleanup_qemu_socket "${socket_prefix}" + cleanup_log_file "${stderr}" + exit "${res}" + fi + + if [[ -z "${socket_prefix}" ]]; then + socket_prefix=qemu-$(date +"%Y%m%d-%H%M%S") + setup_qemu_socket "${socket_prefix}" + fi + + run_qemu & + pid_qemu=$! + if [[ "${timeout}" -ne 0 ]]; then + sleep "${timeout}" && + echo 'Timeout' >>"${stderr}" && + echo 'q' | socat - unix:"${monitor_socket}" & + pid_sleep=$! + disown + fi + + run_cmd "${cmd}" || res=$? + [[ "${res}" -ne 0 ]] && log_error "'run_cmd()' error" "${res}" + wait "${pid_qemu}" || res=$? + [[ -n "${pid_sleep}" ]] && kill -9 "${pid_sleep}" &>/dev/null + if [[ "${res}" -ne 0 ]]; then + log_error "'run_qemu()' error" "${res}" + print_qemu_output "${socket_prefix}" + print_log_file "${stderr}" + fi + + cleanup_qemu_socket "${socket_prefix}" + cleanup_log_file "${stderr}" + exit "${res}" +} + +main "$@" diff --git a/subprojects/vaccel/docker/vq-size.patch b/subprojects/vaccel/docker/vq-size.patch new file mode 100644 index 00000000000..27741c18f6a --- /dev/null +++ b/subprojects/vaccel/docker/vq-size.patch @@ -0,0 +1,13 @@ +diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h +index b69d517496..64967d7e3d 100644 +--- a/include/hw/virtio/virtio.h ++++ b/include/hw/virtio/virtio.h +@@ -47,7 +47,7 @@ size_t virtio_feature_get_config_size(VirtIOFeature *features, + + typedef struct VirtQueue VirtQueue; + +-#define VIRTQUEUE_MAX_SIZE 1024 ++#define VIRTQUEUE_MAX_SIZE 8192 + + typedef struct VirtQueueElement + {