From 642dde94ed4439a4a5a0fbbbb9039066f6c46dcf Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Tue, 8 Oct 2024 18:02:12 +0900 Subject: [PATCH] CI: VM: switch away from Vagrant to Lima Unlike the current version of Vagrant, Lima remains FLOSS, and is easier to debug (at least for myself) Signed-off-by: Akihiro Suda --- .github/workflows/vm.yaml | 59 ++++++++++++------- hack/ci/Vagrantfile | 45 -------------- hack/ci/init-fedora.sh | 18 ++++++ hack/ci/{vagrant-helper.sh => lima-helper.sh} | 10 +--- 4 files changed, 60 insertions(+), 72 deletions(-) delete mode 100644 hack/ci/Vagrantfile create mode 100755 hack/ci/init-fedora.sh rename hack/ci/{vagrant-helper.sh => lima-helper.sh} (70%) diff --git a/.github/workflows/vm.yaml b/.github/workflows/vm.yaml index f784c4ed06..266679ee6c 100644 --- a/.github/workflows/vm.yaml +++ b/.github/workflows/vm.yaml @@ -23,9 +23,10 @@ jobs: provider: [docker, podman] rootless: ["rootful", "rootless"] env: + LIMA_VERSION: "0.23.2" # Fedora 40 KIND_EXPERIMENTAL_PROVIDER: "${{ matrix.provider }}" ROOTLESS: "${{ matrix.rootless }}" - HELPER: "./hack/ci/vagrant-helper.sh" + HELPER: "./hack/ci/lima-helper.sh" JOB_NAME: "cgroup2-${{ matrix.provider }}-${{ matrix.rootless }}" steps: - name: Check out code @@ -43,27 +44,44 @@ jobs: go-version: ${{ steps.golangversion.outputs.go_version }} check-latest: true - # https://github.com/containerd/containerd/blob/420503072e58f27a7192ddea4e6e41dced911cb9/.github/workflows/ci.yml#L569-L581 - - name: Set up vagrant + - name: "Install QEMU" run: | - # Canonical's Vagrant 2.2.19 dpkg cannot download Fedora 38 image: https://bugs.launchpad.net/vagrant/+bug/2017828 - # So we have to install Vagrant >= 2.3.1 from the upstream: https://github.com/opencontainers/runc/blob/v1.1.8/.cirrus.yml#L41-L49 - curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg - echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list sudo apt-get update - sudo apt-get install -y vagrant virtualbox - - name: Boot Fedora + sudo apt-get install -y --no-install-recommends ovmf qemu-system-x86 qemu-utils + sudo modprobe kvm + # `sudo usermod -aG kvm $(whoami)` does not take an effect on GHA + sudo chown $(whoami) /dev/kvm + + - name: "Install Lima" + run: curl -fsSL https://github.com/lima-vm/lima/releases/download/v${LIMA_VERSION}/lima-${LIMA_VERSION}-Linux-x86_64.tar.gz | sudo tar Cxzvf /usr/local - + + - name: "Cache ~/.cache/lima" + uses: actions/cache@v4 + with: + path: ~/.cache/lima + key: lima-${{ env.LIMA_VERSION }} + + - name: "Start Fedora" + # --plain is set to disable file sharing, port forwarding, built-in containerd, etc. + run: limactl start --name=default --plain template://fedora + + - name: "Initialize Fedora" + # plain old rsync and ssh are used for the initialization of the guest, + # so that people who are not familiar with Lima can understand the initialization steps. run: | - ln -sf ./hack/ci/Vagrantfile ./Vagrantfile - # Retry if it fails (download.fedoraproject.org returns 404 sometimes) - # Spend up to 10 seconds on this - for i in {1..4}; do - if vagrant up; then - break - fi - vagrant destroy -f - sleep $i - done + set -eux -o pipefail + # Initialize SSH + mkdir -p -m 0700 ~/.ssh + cat ~/.lima/default/ssh.config >> ~/.ssh/config + # Sync the current directory to /tmp/kind in the guest + rsync -a -e ssh . lima-default:/tmp/kind + # Install packages + ssh lima-default sudo /tmp/kind/hack/ci/init-fedora.sh + # Enable systemd lingering for rootless + ssh lima-default sudo loginctl enable-linger "$USER" + # Install kind + ssh lima-default sudo git config --global --add safe.directory /tmp/kind + ssh lima-default sudo make -C /tmp/kind install INSTALL_DIR=/usr/local/bin - name: Set up Rootless Docker if: ${{ matrix.provider == 'docker' && matrix.rootless == 'rootless' }} @@ -103,7 +121,8 @@ jobs: if: always() run: | "$HELPER" kind export logs /tmp/kind/logs - mkdir -p /tmp/kind/logs + mkdir -p /tmp/kind/logs/lima + cp -a ~/.lima/default/*.log /tmp/kind/logs/lima || true "$HELPER" tar cC /tmp/kind/logs . | tar xC /tmp/kind/logs - name: Upload logs diff --git a/hack/ci/Vagrantfile b/hack/ci/Vagrantfile deleted file mode 100644 index ea5774ebf3..0000000000 --- a/hack/ci/Vagrantfile +++ /dev/null @@ -1,45 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrant box for testing kind with non-Ubuntu -Vagrant.configure("2") do |config| - config.vm.box = "fedora/39-cloud-base" - # https://mirrormanager.fedoraproject.org/mirrors/Fedora/39/x86_64 - config.vm.box_url = "https://gsl-syd.mm.fcix.net/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-Vagrant-39-1.5.x86_64.vagrant-virtualbox.box" - # assume some ram is needed for the host environment but very little CPU - memory = 10240 - cpus = 3 - config.vm.provider :virtualbox do |v| - v.memory = memory - v.cpus = cpus - end - config.vm.provision "install-packages", type: "shell", run: "once" do |sh| - sh.inline = <<~SHELL - set -eux -o pipefail - # Ensure network-related modules to be loaded - modprobe tap ip_tables iptable_nat ip6_tables ip6table_nat - - # The moby-engine package included in Fedora lacks support for rootless, - # So we need to install docker-ce and docker-ce-rootless-extras from the upstream. - curl -fsSL https://get.docker.com | sh - dnf install -y golang-go make kubernetes-client podman docker-ce-rootless-extras - systemctl enable --now docker - - # Configuration for rootless: https://kind.sigs.k8s.io/docs/user/rootless/ - mkdir -p "/etc/systemd/system/user@.service.d" - cat <"/etc/systemd/system/user@.service.d/delegate.conf" - [Service] - Delegate=yes - EOF - systemctl daemon-reload - loginctl enable-linger vagrant - SHELL - end - config.vm.provision "install-kind", type: "shell", run: "once" do |sh| - sh.inline = <<~SHELL - set -eux -o pipefail - git config --global --add safe.directory /vagrant - make -C /vagrant install INSTALL_DIR=/usr/local/bin - SHELL - end -end diff --git a/hack/ci/init-fedora.sh b/hack/ci/init-fedora.sh new file mode 100755 index 0000000000..8574e6a87f --- /dev/null +++ b/hack/ci/init-fedora.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -eux -o pipefail +# Ensure network-related modules to be loaded +modprobe tap ip_tables iptable_nat ip6_tables ip6table_nat + +# The moby-engine package included in Fedora lacks support for rootless, +# So we need to install docker-ce and docker-ce-rootless-extras from the upstream. +curl -fsSL https://get.docker.com | sh +dnf install -y golang-go make kubernetes-client podman docker-ce-rootless-extras +systemctl enable --now docker + +# Configuration for rootless: https://kind.sigs.k8s.io/docs/user/rootless/ +mkdir -p "/etc/systemd/system/user@.service.d" +cat <"/etc/systemd/system/user@.service.d/delegate.conf" +[Service] +Delegate=yes +EOF +systemctl daemon-reload diff --git a/hack/ci/vagrant-helper.sh b/hack/ci/lima-helper.sh similarity index 70% rename from hack/ci/vagrant-helper.sh rename to hack/ci/lima-helper.sh index bd10759fa8..10f7ec7c48 100755 --- a/hack/ci/vagrant-helper.sh +++ b/hack/ci/lima-helper.sh @@ -15,14 +15,10 @@ set -o errexit -o nounset -o pipefail - +: "${LIMA_INSTANCE:=default}" : "${KIND_EXPERIMENTAL_PROVIDER:=docker}" -SSH_CONFIG=".vagrant/ssh-config" -if [ ! -f "$SSH_CONFIG" ]; then - vagrant ssh-config > "$SSH_CONFIG" -fi if [ "$ROOTLESS" = "rootless" ]; then - exec ssh -F "$SSH_CONFIG" default KIND_EXPERIMENTAL_PROVIDER="$KIND_EXPERIMENTAL_PROVIDER" "${@}" + exec ssh "lima-${LIMA_INSTANCE}" KIND_EXPERIMENTAL_PROVIDER="$KIND_EXPERIMENTAL_PROVIDER" "${@}" fi -exec ssh -F "$SSH_CONFIG" default sudo KIND_EXPERIMENTAL_PROVIDER="$KIND_EXPERIMENTAL_PROVIDER" "${@}" +exec ssh "lima-${LIMA_INSTANCE}" sudo KIND_EXPERIMENTAL_PROVIDER="$KIND_EXPERIMENTAL_PROVIDER" "${@}"