CI: VM: switch away from Vagrant to Lima

Unlike the current version of Vagrant, Lima remains FLOSS, and is easier to debug (at least for myself) Signed-off-by: Akihiro Suda <[email protected]>
kubernetes-sigs · Oct 8, 2024 · 642dde9 · 642dde9
1 parent b404897
commit 642dde9
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 72 deletions.
diff --git a/.github/workflows/vm.yaml b/.github/workflows/vm.yaml
@@ -23,9 +23,10 @@ jobs:
         provider: [docker, podman]
         rootless: ["rootful", "rootless"]
     env:
+      LIMA_VERSION: "0.23.2" # Fedora 40
       KIND_EXPERIMENTAL_PROVIDER: "${{ matrix.provider }}"
       ROOTLESS: "${{ matrix.rootless }}"
-      HELPER: "./hack/ci/vagrant-helper.sh"
+      HELPER: "./hack/ci/lima-helper.sh"
       JOB_NAME: "cgroup2-${{ matrix.provider }}-${{ matrix.rootless }}"
     steps:
       - name: Check out code
@@ -43,27 +44,44 @@ jobs:
           go-version: ${{ steps.golangversion.outputs.go_version }}
           check-latest: true
 
-      # https://github.com/containerd/containerd/blob/420503072e58f27a7192ddea4e6e41dced911cb9/.github/workflows/ci.yml#L569-L581
-      - name: Set up vagrant
+      - name: "Install QEMU"
         run: |
-          # Canonical's Vagrant 2.2.19 dpkg cannot download Fedora 38 image: https://bugs.launchpad.net/vagrant/+bug/2017828
-          # So we have to install Vagrant >= 2.3.1 from the upstream: https://github.com/opencontainers/runc/blob/v1.1.8/.cirrus.yml#L41-L49
-          curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
-          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
           sudo apt-get update
-          sudo apt-get install -y vagrant virtualbox
-      - name: Boot Fedora
+          sudo apt-get install -y --no-install-recommends ovmf qemu-system-x86 qemu-utils
+          sudo modprobe kvm
+          # `sudo usermod -aG kvm $(whoami)` does not take an effect on GHA
+          sudo chown $(whoami) /dev/kvm
+
+      - name: "Install Lima"
+        run: curl -fsSL https://github.com/lima-vm/lima/releases/download/v${LIMA_VERSION}/lima-${LIMA_VERSION}-Linux-x86_64.tar.gz | sudo tar Cxzvf /usr/local -
+
+      - name: "Cache ~/.cache/lima"
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/lima
+          key: lima-${{ env.LIMA_VERSION }}
+
+      - name: "Start Fedora"
+        # --plain is set to disable file sharing, port forwarding, built-in containerd, etc.
+        run: limactl start --name=default --plain template://fedora
+
+      - name: "Initialize Fedora"
+        # plain old rsync and ssh are used for the initialization of the guest,
+        # so that people who are not familiar with Lima can understand the initialization steps.
         run: |
-          ln -sf ./hack/ci/Vagrantfile ./Vagrantfile
-          # Retry if it fails (download.fedoraproject.org returns 404 sometimes)
-          # Spend up to 10 seconds on this
-          for i in {1..4}; do
-            if vagrant up; then
-              break
-            fi
-            vagrant destroy -f
-            sleep $i
-          done
+          set -eux -o pipefail
+          # Initialize SSH
+          mkdir -p -m 0700 ~/.ssh
+          cat ~/.lima/default/ssh.config >> ~/.ssh/config
+          # Sync the current directory to /tmp/kind in the guest
+          rsync -a -e ssh . lima-default:/tmp/kind
+          # Install packages
+          ssh lima-default sudo /tmp/kind/hack/ci/init-fedora.sh
+          # Enable systemd lingering for rootless
+          ssh lima-default sudo loginctl enable-linger "$USER"
+          # Install kind
+          ssh lima-default sudo git config --global --add safe.directory /tmp/kind
+          ssh lima-default sudo make -C /tmp/kind install INSTALL_DIR=/usr/local/bin
 
       - name: Set up Rootless Docker
         if: ${{ matrix.provider == 'docker' && matrix.rootless == 'rootless' }}
@@ -103,7 +121,8 @@ jobs:
         if: always()
         run: |
           "$HELPER" kind export logs /tmp/kind/logs
-          mkdir -p /tmp/kind/logs
+          mkdir -p /tmp/kind/logs/lima
+          cp -a ~/.lima/default/*.log /tmp/kind/logs/lima || true
           "$HELPER" tar cC /tmp/kind/logs . | tar xC /tmp/kind/logs
 
       - name: Upload logs

diff --git a/hack/ci/Vagrantfile b/hack/ci/Vagrantfile
diff --git a/hack/ci/init-fedora.sh b/hack/ci/init-fedora.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eux -o pipefail
+# Ensure network-related modules to be loaded
+modprobe tap ip_tables iptable_nat ip6_tables ip6table_nat
+
+# The moby-engine package included in Fedora lacks support for rootless,
+# So we need to install docker-ce and docker-ce-rootless-extras from the upstream.
+curl -fsSL https://get.docker.com | sh
+dnf install -y golang-go make kubernetes-client podman docker-ce-rootless-extras
+systemctl enable --now docker
+
+# Configuration for rootless: https://kind.sigs.k8s.io/docs/user/rootless/
+mkdir -p "/etc/systemd/system/[email protected]"
+cat <<EOF >"/etc/systemd/system/[email protected]/delegate.conf"
+[Service]
+Delegate=yes
+EOF
+systemctl daemon-reload
diff --git a/hack/ci/vagrant-helper.sh → hack/ci/lima-helper.sh b/hack/ci/vagrant-helper.sh → hack/ci/lima-helper.sh
@@ -15,14 +15,10 @@
 
 set -o errexit -o nounset -o pipefail
 
-
+: "${LIMA_INSTANCE:=default}"
 : "${KIND_EXPERIMENTAL_PROVIDER:=docker}"
-SSH_CONFIG=".vagrant/ssh-config"
-if [ ! -f "$SSH_CONFIG" ]; then
-  vagrant ssh-config > "$SSH_CONFIG"
-fi
 
 if [ "$ROOTLESS" = "rootless" ]; then
-  exec ssh -F "$SSH_CONFIG" default KIND_EXPERIMENTAL_PROVIDER="$KIND_EXPERIMENTAL_PROVIDER" "${@}"
+  exec ssh "lima-${LIMA_INSTANCE}" KIND_EXPERIMENTAL_PROVIDER="$KIND_EXPERIMENTAL_PROVIDER" "${@}"
 fi
-exec ssh -F "$SSH_CONFIG" default sudo KIND_EXPERIMENTAL_PROVIDER="$KIND_EXPERIMENTAL_PROVIDER" "${@}"
+exec ssh "lima-${LIMA_INSTANCE}" sudo KIND_EXPERIMENTAL_PROVIDER="$KIND_EXPERIMENTAL_PROVIDER" "${@}"