From d215446998d756718c8016e770493b6e606e2508 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Fri, 23 Sep 2022 20:37:39 +0300 Subject: [PATCH] ceph-windows: Use vstart cluster for Ceph Windows tests Previously, we spawned a minimal cluster with `cephadm` just to have a Ceph API available (without any OSDs configured). New Ceph Windows tests will be available that require actual Ceph store, so a fully functional Ceph cluster is needed. The entire ceph-windows testing is done with libvirt VMs now. This way, we avoid accidentally tainting the CI machines after each job run. Building and running Ceph vstart doesn't drastically impact the job running times, since the CI machines have a lot of CPU cores available. Signed-off-by: Ionut Balutoiu --- ceph-windows-installer-build/build/build | 6 + .../ceph-windows-installer-build.yml | 3 +- .../ceph-windows-pull-requests.yml | 5 +- .../config/definitions/ceph-windows-test.yml | 5 +- scripts/ceph-windows/cleanup | 34 +---- scripts/ceph-windows/run_tests | 55 ++------ scripts/ceph-windows/setup_ceph_vstart | 95 ++++++++++++++ scripts/ceph-windows/setup_libvirt | 102 +++++++++++++++ scripts/ceph-windows/setup_libvirt_ubuntu_vm | 65 ++++++++++ scripts/ceph-windows/setup_libvirt_vm | 118 ------------------ scripts/ceph-windows/setup_libvirt_windows_vm | 38 ++++++ scripts/ceph-windows/win32_build | 1 + wnbd-build/build/build | 6 + wnbd-build/config/definitions/wnbd-build.yml | 3 +- 14 files changed, 337 insertions(+), 199 deletions(-) create mode 100644 scripts/ceph-windows/setup_ceph_vstart create mode 100644 scripts/ceph-windows/setup_libvirt create mode 100644 scripts/ceph-windows/setup_libvirt_ubuntu_vm delete mode 100644 scripts/ceph-windows/setup_libvirt_vm create mode 100644 scripts/ceph-windows/setup_libvirt_windows_vm diff --git a/ceph-windows-installer-build/build/build b/ceph-windows-installer-build/build/build index 4c1c3b63f..b8ddd1b17 100644 --- a/ceph-windows-installer-build/build/build +++ b/ceph-windows-installer-build/build/build @@ -2,6 +2,12 @@ set -o errexit set -o pipefail +if [[ -z $WINDOWS_SSH_USER ]]; then echo "ERROR: The WINDOWS_SSH_USER env variable is not set"; exit 1; fi +if [[ -z $WINDOWS_VM_IP ]]; then echo "ERROR: The WINDOWS_VM_IP env variable is not set"; exit 1; fi + +export SSH_USER=$WINDOWS_SSH_USER +export SSH_ADDRESS=$WINDOWS_VM_IP + BUILD_CONFIGURATION=${BUILD_CONFIGURATION:-"Release"} diff --git a/ceph-windows-installer-build/config/definitions/ceph-windows-installer-build.yml b/ceph-windows-installer-build/config/definitions/ceph-windows-installer-build.yml index b5efaa293..f5fa40d1c 100644 --- a/ceph-windows-installer-build/config/definitions/ceph-windows-installer-build.yml +++ b/ceph-windows-installer-build/config/definitions/ceph-windows-installer-build.yml @@ -62,7 +62,8 @@ !include-raw: - ../../../scripts/build_utils.sh - ../../build/setup - - ../../../scripts/ceph-windows/setup_libvirt_vm + - ../../../scripts/ceph-windows/setup_libvirt + - ../../../scripts/ceph-windows/setup_libvirt_windows_vm - ../../build/build wrappers: diff --git a/ceph-windows-pull-requests/config/definitions/ceph-windows-pull-requests.yml b/ceph-windows-pull-requests/config/definitions/ceph-windows-pull-requests.yml index af6cfb597..3c9278a22 100644 --- a/ceph-windows-pull-requests/config/definitions/ceph-windows-pull-requests.yml +++ b/ceph-windows-pull-requests/config/definitions/ceph-windows-pull-requests.yml @@ -68,7 +68,10 @@ - ../../../scripts/build_utils.sh - ../../build/check_docs_pr_only - ../../../scripts/ceph-windows/win32_build - - ../../../scripts/ceph-windows/setup_libvirt_vm + - ../../../scripts/ceph-windows/setup_libvirt + - ../../../scripts/ceph-windows/setup_libvirt_ubuntu_vm + - ../../../scripts/ceph-windows/setup_libvirt_windows_vm + - ../../../scripts/ceph-windows/setup_ceph_vstart - ../../../scripts/ceph-windows/run_tests publishers: diff --git a/ceph-windows-test/config/definitions/ceph-windows-test.yml b/ceph-windows-test/config/definitions/ceph-windows-test.yml index f11ae6272..6eff5d31c 100644 --- a/ceph-windows-test/config/definitions/ceph-windows-test.yml +++ b/ceph-windows-test/config/definitions/ceph-windows-test.yml @@ -56,7 +56,10 @@ !include-raw: - ../../../scripts/build_utils.sh - ../../../scripts/ceph-windows/win32_build - - ../../../scripts/ceph-windows/setup_libvirt_vm + - ../../../scripts/ceph-windows/setup_libvirt + - ../../../scripts/ceph-windows/setup_libvirt_ubuntu_vm + - ../../../scripts/ceph-windows/setup_libvirt_windows_vm + - ../../../scripts/ceph-windows/setup_ceph_vstart - ../../../scripts/ceph-windows/run_tests wrappers: diff --git a/scripts/ceph-windows/cleanup b/scripts/ceph-windows/cleanup index 5d46063c0..f184260df 100644 --- a/scripts/ceph-windows/cleanup +++ b/scripts/ceph-windows/cleanup @@ -7,37 +7,7 @@ set -o pipefail delete_libvirt_vms clear_libvirt_networks -# Cleanup Ceph clusters spawned via cephadm -if [[ -x $WORKSPACE/cephadm ]]; then - sudo $WORKSPACE/cephadm rm-repo - if [[ -d /var/lib/ceph ]]; then - for FSID in $(sudo ls /var/lib/ceph); do - echo "Removing Ceph cluster $FSID" - sudo $WORKSPACE/cephadm rm-cluster --fsid $FSID --force - done - fi -fi - -# Uninstall packages installed by "cephadm install ceph-common" -sudo apt-get -y purge \ - ceph-common \ - python3-ceph-argparse \ - python3-ceph-common \ - python3-cephfs \ - librbd1 \ - python3-rados \ - python3-rbd \ - python3-rgw \ - libcephfs2 \ - librados2 \ - libradosstriper1 - -# Cleanup repos -sudo rm -f /etc/apt/sources.list.d/docker.list \ - /etc/apt/sources.list.d/ceph.list - # Cleanup remaining files / directories sudo rm -rf \ - $WORKSPACE/ceph.conf $WORKSPACE/keyring $WORKSPACE/cephadm \ - $WORKSPACE/ceph.zip $WORKSPACE/known_hosts \ - /etc/ceph /var/log/ceph /var/lib/ceph /var/run/ceph + $WORKSPACE/ceph $WORKSPACE/ceph_vstart $WORKSPACE/ceph.zip \ + $WORKSPACE/libvirt diff --git a/scripts/ceph-windows/run_tests b/scripts/ceph-windows/run_tests index b9c3aa331..bb5c55177 100644 --- a/scripts/ceph-windows/run_tests +++ b/scripts/ceph-windows/run_tests @@ -2,52 +2,17 @@ set -o errexit set -o pipefail -if [[ ! -f $WORKSPACE/ceph.zip ]]; then - echo "ERROR: The Ceph Windows build zip file doesn't exist at $WORKSPACE/ceph.zip" - exit 1 -fi +if [[ ! -f $WORKSPACE/ceph.zip ]]; then echo "ERROR: The Ceph Windows build zip file doesn't exist at '$WORKSPACE/ceph.zip'"; exit 1; fi +if [[ ! -f $CEPH_WINDOWS_CONF ]]; then echo "ERROR: The Ceph Windows config file doesn't exist at '$CEPH_WINDOWS_CONF'"; exit 1; fi +if [[ ! -f $CEPH_KEYRING ]]; then echo "ERROR: The Ceph keyring file doesn't exist at '$CEPH_KEYRING'"; exit 1; fi -CEPHADM_RELEASE=${CEPHADM_RELEASE:-"quincy"} -WIN_USERSPACE_CRASH_DUMPS=${WIN_USERSPACE_CRASH_DUMPS:-"C:\\userspace_crash_dumps"} - -# -# Install requirements (if needed) -# -if ! sudo docker version &>/dev/null; then - sudo apt-get update - sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io +if [[ -z $WINDOWS_SSH_USER ]]; then echo "ERROR: The WINDOWS_SSH_USER env variable is not set"; exit 1; fi +if [[ -z $WINDOWS_VM_IP ]]; then echo "ERROR: The WINDOWS_VM_IP env variable is not set"; exit 1; fi - sudo usermod -aG docker $USER -fi -if ! which xmllint >/dev/null; then - sudo apt-get update - sudo apt-get install -y libxml2-utils -fi +export SSH_USER=$WINDOWS_SSH_USER +export SSH_ADDRESS=$WINDOWS_VM_IP -# -# Start minimal Ceph development cluster via cephadm -# -curl -L "https://github.com/ceph/ceph/raw/${CEPHADM_RELEASE}/src/cephadm/cephadm" -o $WORKSPACE/cephadm -chmod +x $WORKSPACE/cephadm -sudo virsh net-dumpxml default > $WORKSPACE/default-net.xml -MON_IP=`xmllint --xpath 'string(/network/ip/@address)' $WORKSPACE/default-net.xml` -sudo rm $WORKSPACE/default-net.xml -sudo $WORKSPACE/cephadm bootstrap --allow-fqdn-hostname --single-host-defaults --mon-ip $MON_IP -HOST_DISTRO=$(lsb_release -cs) -curl -s -L https://shaman.ceph.com/api/repos/ceph/main/latest/ubuntu/$HOST_DISTRO/flavors/default/repo?arch=$(arch) | sudo tee /etc/apt/sources.list.d/ceph.list -sudo $WORKSPACE/cephadm install ceph-common - -sudo cp /etc/ceph/ceph.conf $WORKSPACE/ceph.conf -sudo cp /etc/ceph/ceph.client.admin.keyring $WORKSPACE/keyring -sudo chown $USER $WORKSPACE/ceph.conf $WORKSPACE/keyring - -sudo ceph osd pool create rbd +WIN_USERSPACE_CRASH_DUMPS=${WIN_USERSPACE_CRASH_DUMPS:-"C:\\userspace_crash_dumps"} # # Clone ceph-win32-tests repo @@ -63,8 +28,8 @@ ssh_exec powershell.exe /workspace/repos/ceph-win32-tests/test_host/set_userspac # Copy the ceph.conf and keyring to the Windows VM # ssh_exec powershell.exe mkdir -force /ProgramData/ceph/out -scp_upload $WORKSPACE/ceph.conf /ProgramData/ceph/ceph.conf -scp_upload $WORKSPACE/keyring /ProgramData/ceph/keyring +scp_upload $CEPH_WINDOWS_CONF /ProgramData/ceph/ceph.conf +scp_upload $CEPH_KEYRING /ProgramData/ceph/keyring # # Setup the Ceph Windows build in the Windows VM diff --git a/scripts/ceph-windows/setup_ceph_vstart b/scripts/ceph-windows/setup_ceph_vstart new file mode 100644 index 000000000..798e79aa9 --- /dev/null +++ b/scripts/ceph-windows/setup_ceph_vstart @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +if [[ -z $UBUNTU_SSH_USER ]]; then echo "ERROR: The UBUNTU_SSH_USER env variable is not set"; exit 1; fi +if [[ -z $UBUNTU_VM_IP ]]; then echo "ERROR: The UBUNTU_VM_IP env variable is not set"; exit 1; fi + +export VSTART_DIR="$WORKSPACE/ceph_vstart" +export VSTART_MEMSTORE_BYTES="5368709120" # 5GB + +export SSH_USER=$UBUNTU_SSH_USER +export SSH_ADDRESS=$UBUNTU_VM_IP + +mkdir -p $VSTART_DIR + +function rsync_cmd() { + rsync -a --delete -e "ssh -i $CEPH_WIN_CI_KEY -o UserKnownHostsFile=$SSH_KNOWN_HOSTS_FILE" ${@} +} + +# +# Build Ceph vstart +# +cat > ${VSTART_DIR}/build-ceph-vstart.sh << EOF +cd ~/ceph +./install-deps.sh +./do_cmake.sh \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_RADOSGW=OFF \ + -DWITH_MGR_DASHBOARD_FRONTEND=OFF \ + -WITH_MGR=OFF \ + -WITH_LTTNG=OFF \ + -DWITH_TESTS=OFF +cd ./build +ninja vstart +EOF +chmod +x ${VSTART_DIR}/build-ceph-vstart.sh +time rsync_cmd $WORKSPACE/ceph ${VSTART_DIR}/build-ceph-vstart.sh ${UBUNTU_SSH_USER}@${UBUNTU_VM_IP}: + +time SSH_TIMEOUT=1h ssh_exec ./build-ceph-vstart.sh +ssh_exec sudo apt-get install -y python3-prettytable + +# +# Run Ceph vstart +# +cat > ${VSTART_DIR}/ceph-vstart.sh << EOF +mkdir -p \$HOME/ceph-vstart/out + +cd ~/ceph/build +VSTART_DEST=\$HOME/ceph-vstart ../src/vstart.sh \ + -n --memstore -o "memstore_device_bytes=$VSTART_MEMSTORE_BYTES" \ + --without-dashboard -i "$UBUNTU_VM_IP" \ + 2>&1 | tee \$HOME/ceph-vstart/vstart.log + +export CEPH_CONF=\$HOME/ceph-vstart/ceph.conf +export CEPH_KEYRING=\$HOME/ceph-vstart/keyring + +./bin/ceph osd pool create rbd + +./bin/ceph osd pool set cephfs.a.data size 1 --yes-i-really-mean-it +./bin/ceph osd pool set cephfs.a.meta size 1 --yes-i-really-mean-it +./bin/ceph osd pool set rbd size 1 --yes-i-really-mean-it + +./bin/ceph tell mon.\* config set debug_mon 0 +./bin/ceph tell mon.\* config set debug_ms 0 +EOF +chmod +x ${VSTART_DIR}/ceph-vstart.sh + +rsync_cmd ${VSTART_DIR}/ceph-vstart.sh ${UBUNTU_SSH_USER}@${UBUNTU_VM_IP}: +time SSH_TIMEOUT=30m ssh_exec ./ceph-vstart.sh + +ssh_exec sudo mkdir -p /etc/ceph +ssh_exec sudo cp ./ceph-vstart/ceph.conf ./ceph-vstart/keyring /etc/ceph + +rsync_cmd ${UBUNTU_SSH_USER}@${UBUNTU_VM_IP}:./ceph-vstart/ceph.conf ${VSTART_DIR}/ceph.conf +rsync_cmd ${UBUNTU_SSH_USER}@${UBUNTU_VM_IP}:./ceph-vstart/keyring ${VSTART_DIR}/keyring + +export CEPH_CONF="$VSTART_DIR/ceph.conf" +export CEPH_KEYRING="$VSTART_DIR/keyring" +export CEPH_WINDOWS_CONF="$VSTART_DIR/ceph-windows.conf" + +MON_HOST=$(cat $CEPH_CONF | grep -o "mon host \=.*") + +cat > $CEPH_WINDOWS_CONF << EOF +[client] + keyring = C:/ProgramData/ceph/keyring + admin socket = C:/ProgramData/ceph/out/\$name.\$pid.asok + client_mount_uid = 1000 + client_mount_gid = 1000 + client_permissions = true +[global] + log to stderr = true + run dir = C:/ProgramData/ceph/out + crash dir = C:/ProgramData/ceph/out + $MON_HOST +EOF diff --git a/scripts/ceph-windows/setup_libvirt b/scripts/ceph-windows/setup_libvirt new file mode 100644 index 000000000..1c6313dd2 --- /dev/null +++ b/scripts/ceph-windows/setup_libvirt @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +if [[ -z $CEPH_WIN_CI_KEY ]]; then echo "ERROR: The CI SSH private key secret (CEPH_WIN_CI_KEY) is not set"; exit 1; fi + +export LIBVIRT_DIR="$WORKSPACE/libvirt" + +export SSH_KEY="$CEPH_WIN_CI_KEY" +export SSH_KNOWN_HOSTS_FILE="$LIBVIRT_DIR/known_hosts" + +mkdir -p $LIBVIRT_DIR + +function get_libvirt_vm_ssh_address() { + if [[ -z $VM_NAME ]]; then + echo "ERROR: Env variable VM_NAME is not set" + exit 1 + fi + if [[ -z $SSH_USER ]]; then + echo "ERROR: Env variable SSH_USER is not set" + exit 1 + fi + + if ! which xmllint >/dev/null; then + sudo apt-get update + sudo apt-get install -y libxml2-utils + fi + if ! which jq >/dev/null; then + sudo apt-get update + sudo apt-get install -y jq + fi + + sudo virsh dumpxml $VM_NAME > $LIBVIRT_DIR/$VM_NAME.xml + local VM_NIC_MAC_ADDRESS=`xmllint --xpath 'string(/domain/devices/interface/mac/@address)' $LIBVIRT_DIR/$VM_NAME.xml` + rm $LIBVIRT_DIR/$VM_NAME.xml + + local TIMEOUT=${TIMEOUT:-600} + local SLEEP_SECS=${SLEEP_SECS:-10} + + SECONDS=0 + while true; do + if [[ $SECONDS -gt $TIMEOUT ]]; then + >&2 echo "Timeout waiting for the VM to start" + return 1 + fi + # Get the VM NIC IP address from the "default" virsh network + VM_IP=$(sudo virsh qemu-agent-command $VM_NAME '{"execute":"guest-network-get-interfaces"}' | jq -r ".return[] | select(.\"hardware-address\"==\"${VM_NIC_MAC_ADDRESS}\") | .\"ip-addresses\"[] | select(.\"ip-address\" | startswith(\"192.168.122.\")) | .\"ip-address\"") || { + >&2 echo "Retrying in $SLEEP_SECS seconds" + sleep $SLEEP_SECS + continue + } + if [[ -z $VM_IP ]]; then + >&2 echo "Cannot find the VM IP address. Retrying in $SLEEP_SECS seconds" + sleep $SLEEP_SECS + continue + fi + ssh-keyscan -H $VM_IP &> ${LIBVIRT_DIR}/${VM_NAME}_known_hosts || { + >&2 echo "SSH is not reachable yet" + sleep $SLEEP_SECS + continue + } + SSH_ADDRESS=$VM_IP SSH_KNOWN_HOSTS_FILE=${LIBVIRT_DIR}/${VM_NAME}_known_hosts ssh_exec hostname 1>&2 || { + >&2 echo "Cannot execute SSH commands yet" + sleep $SLEEP_SECS + continue + } + break + done + cat ${LIBVIRT_DIR}/${VM_NAME}_known_hosts >> $SSH_KNOWN_HOSTS_FILE + rm ${LIBVIRT_DIR}/${VM_NAME}_known_hosts + echo $VM_IP +} + +# +# Setup requirements (if needed) +# +if ! which virt-install >/dev/null; then + sudo apt-get update + sudo apt-get install -y virtinst +fi +if ! which cloud-localds >/dev/null; then + sudo apt-get update + sudo apt-get install -y cloud-image-utils +fi +if ! sudo virsh net-info default &>/dev/null; then + cat << EOF > $LIBVIRT_DIR/default-net.xml + + default + + + + + + + + +EOF + sudo virsh net-define $LIBVIRT_DIR/default-net.xml + sudo virsh net-start default + sudo virsh net-autostart default + rm $LIBVIRT_DIR/default-net.xml +fi diff --git a/scripts/ceph-windows/setup_libvirt_ubuntu_vm b/scripts/ceph-windows/setup_libvirt_ubuntu_vm new file mode 100644 index 000000000..9a27528a8 --- /dev/null +++ b/scripts/ceph-windows/setup_libvirt_ubuntu_vm @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +if [[ -z $LIBVIRT_DIR ]]; then echo "ERROR: The env variable LIBVIRT_DIR is not set"; exit 1; fi + +export UBUNTU_VM_IMAGE_URL=${UBUNTU_VM_IMAGE_URL:-"https://cloud-images.ubuntu.com/minimal/releases/jammy/release/ubuntu-22.04-minimal-cloudimg-amd64.img"} +export UBUNTU_VM_NAME=${UBUNTU_VM_NAME:-"ceph-ubuntu-vstart-${JOB_NAME}-${BUILD_ID}"} +export UBUNTU_VM_VCPUS="$(nproc)" # use all the CPU cores avaiable on the host +export UBUNTU_VM_MEMORY="32768" # 32 GB +export UBUNTU_SSH_USER="ubuntu" + +# +# Setup the Ubuntu VM to run Ceph vstart +# +mkdir -p $LIBVIRT_DIR +echo "Downloading VM image from $UBUNTU_VM_IMAGE_URL" +curl -s -L $UBUNTU_VM_IMAGE_URL -o ${LIBVIRT_DIR}/ceph-ubuntu-vstart.qcow2 +qemu-img resize ${LIBVIRT_DIR}/ceph-ubuntu-vstart.qcow2 128G + +cat > ${LIBVIRT_DIR}/metadata.yaml << EOF +instance-id: ceph-ubuntu-vstart +local-hostname: ceph-ubuntu-vstart.local +locale: en_US +EOF + +cat > ${LIBVIRT_DIR}/user-data.yaml << EOF +#cloud-config + +ssh_authorized_keys: + - $(ssh-keygen -y -f $SSH_KEY) + +packages_update: true +packages: + - qemu-guest-agent + - locales + - rsync + +runcmd: + - [localedef, -i, en_US, -c, -f, UTF-8, -A, /usr/share/locale/locale.alias, en_US.UTF-8] + - [systemctl, start, qemu-guest-agent] +EOF + +cloud-localds ${LIBVIRT_DIR}/config-drive.img ${LIBVIRT_DIR}/user-data.yaml ${LIBVIRT_DIR}/metadata.yaml + +sudo virt-install \ + --name $UBUNTU_VM_NAME \ + --os-variant ubuntu22.04 \ + --boot hd \ + --virt-type kvm \ + --graphics spice \ + --cpu host \ + --vcpus $UBUNTU_VM_VCPUS \ + --memory $UBUNTU_VM_MEMORY \ + --disk ${LIBVIRT_DIR}/ceph-ubuntu-vstart.qcow2,bus=virtio \ + --disk ${LIBVIRT_DIR}/config-drive.img,bus=virtio \ + --network network=default,model=virtio \ + --controller type=virtio-serial \ + --channel unix,target_type=virtio,name=org.qemu.guest_agent.0 \ + --noautoconsol + +# +# Get the VM SSH address +# +export UBUNTU_VM_IP=$(VM_NAME=$UBUNTU_VM_NAME SSH_USER=$UBUNTU_SSH_USER get_libvirt_vm_ssh_address) diff --git a/scripts/ceph-windows/setup_libvirt_vm b/scripts/ceph-windows/setup_libvirt_vm deleted file mode 100644 index 294b263d3..000000000 --- a/scripts/ceph-windows/setup_libvirt_vm +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -set -o errexit -set -o pipefail - -if [[ -z $CEPH_WIN_CI_KEY ]]; then - echo "ERROR: The CI SSH private key secret (CEPH_WIN_CI_KEY) is not set" - exit 1 -fi - -export VM_IMAGE_URL=${VM_IMAGE_URL:-"https://filedump.ceph.com/windows/ceph-win-ltsc2019-ci-image.qcow2"} -export VM_NAME=${VM_NAME:-"ceph-win-ltsc2019-${JOB_NAME}-${BUILD_ID}"} - -export SSH_USER="administrator" -export SSH_KNOWN_HOSTS_FILE="$WORKSPACE/known_hosts" -export SSH_KEY="$CEPH_WIN_CI_KEY" - - -# -# Install requirements (if needed) -# -if ! which virt-install >/dev/null; then - sudo apt-get update - sudo apt-get install -y virtinst -fi -if ! which xmllint >/dev/null; then - sudo apt-get update - sudo apt-get install -y libxml2-utils -fi -if ! which jq >/dev/null; then - sudo apt-get update - sudo apt-get install -y jq -fi -if ! sudo virsh net-info default &>/dev/null; then - cat << EOF > $WORKSPACE/default-net.xml - - default - - - - - - - - -EOF - sudo virsh net-define $WORKSPACE/default-net.xml - sudo virsh net-start default - sudo virsh net-autostart default - rm $WORKSPACE/default-net.xml -fi - -# -# Download the Windows qcow2 image -# -IMAGE_FILE="$(basename $VM_IMAGE_URL)" -echo "Downloading VM image from $VM_IMAGE_URL" -curl -L -o ${WORKSPACE}/${IMAGE_FILE} $VM_IMAGE_URL - -# -# Start the Windows testing VM -# -sudo virt-install \ - --name $VM_NAME \ - --os-variant win2k19 \ - --boot hd \ - --virt-type kvm \ - --graphics spice \ - --cpu host \ - --vcpus 4 \ - --memory 4096 \ - --disk ${WORKSPACE}/${IMAGE_FILE},bus=virtio \ - --network network=default,model=virtio \ - --controller type=virtio-serial \ - --channel unix,target_type=virtio,name=org.qemu.guest_agent.0 \ - --noautoconsol - -# -# Find the VM NIC MAC address -# -sudo virsh dumpxml $VM_NAME > $WORKSPACE/libvirt_vm.xml -VM_NIC_MAC_ADDRESS=`xmllint --xpath 'string(/domain/devices/interface/mac/@address)' $WORKSPACE/libvirt_vm.xml` -rm $WORKSPACE/libvirt_vm.xml - -# -# Wait until the QEMU agent reports the VM IP, and it's reachable via SSH -# -SECONDS=0 -TIMEOUT=600 -SLEEP_SECS=10 -while true; do - if [[ $SECONDS -gt $TIMEOUT ]]; then - echo "Timeout waiting for the VM to start" - exit 1 - fi - # Get the VM NIC IP address from the "default" virsh network - VM_IP=$(sudo virsh qemu-agent-command $VM_NAME '{"execute":"guest-network-get-interfaces"}' | jq -r ".return[] | select(.\"hardware-address\"==\"${VM_NIC_MAC_ADDRESS}\") | .\"ip-addresses\"[] | select(.\"ip-address\" | startswith(\"192.168.122.\")) | .\"ip-address\"") || { - echo "Retrying in $SLEEP_SECS seconds" - sleep $SLEEP_SECS - continue - } - if [[ -z $VM_IP ]]; then - echo "Cannot find the VM IP address. Retrying in $SLEEP_SECS seconds" - sleep $SLEEP_SECS - continue - fi - ssh-keyscan -H $VM_IP &> ${WORKSPACE}/known_hosts || { - echo "SSH is not reachable yet" - sleep $SLEEP_SECS - continue - } - SSH_ADDRESS=$VM_IP ssh_exec hostname || { - echo "Cannot execute SSH commands yet" - sleep $SLEEP_SECS - continue - } - break -done -export SSH_ADDRESS=$VM_IP diff --git a/scripts/ceph-windows/setup_libvirt_windows_vm b/scripts/ceph-windows/setup_libvirt_windows_vm new file mode 100644 index 000000000..ade873d39 --- /dev/null +++ b/scripts/ceph-windows/setup_libvirt_windows_vm @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +if [[ -z $LIBVIRT_DIR ]]; then echo "ERROR: The env variable LIBVIRT_DIR is not set"; exit 1; fi + +export WINDOWS_VM_IMAGE_URL=${WINDOWS_VM_IMAGE_URL:-"https://filedump.ceph.com/windows/ceph-win-ltsc2019-ci-image.qcow2"} +export WINDOWS_VM_NAME=${WINDOWS_VM_NAME:-"ceph-windows-client-${JOB_NAME}-${BUILD_ID}"} +export WINDOWS_VM_VCPUS="8" +export WINDOWS_VM_MEMORY="8192" # 8GB +export WINDOWS_SSH_USER="administrator" + +# +# Setup the Windows VM to run Ceph client +# +mkdir -p $LIBVIRT_DIR +echo "Downloading VM image from $WINDOWS_VM_IMAGE_URL" +curl -s -L $WINDOWS_VM_IMAGE_URL -o ${LIBVIRT_DIR}/ceph-windows-client.qcow2 + +sudo virt-install \ + --name $WINDOWS_VM_NAME \ + --os-variant win2k19 \ + --boot hd \ + --virt-type kvm \ + --graphics spice \ + --cpu host \ + --vcpus $WINDOWS_VM_VCPUS \ + --memory $WINDOWS_VM_MEMORY \ + --disk ${LIBVIRT_DIR}/ceph-windows-client.qcow2,bus=virtio \ + --network network=default,model=virtio \ + --controller type=virtio-serial \ + --channel unix,target_type=virtio,name=org.qemu.guest_agent.0 \ + --noautoconsol + +# +# Get the VM SSH address +# +export WINDOWS_VM_IP=$(VM_NAME=$WINDOWS_VM_NAME SSH_USER=$WINDOWS_SSH_USER get_libvirt_vm_ssh_address) diff --git a/scripts/ceph-windows/win32_build b/scripts/ceph-windows/win32_build index cb4087a84..aeb619a49 100644 --- a/scripts/ceph-windows/win32_build +++ b/scripts/ceph-windows/win32_build @@ -8,3 +8,4 @@ set -o pipefail cd $WORKSPACE/ceph git submodule update --init --recursive ZIP_DEST=$WORKSPACE/ceph.zip $CEPH_WIN32_BUILD_FLAGS timeout 3h ./win32_build.sh +rm -rf build diff --git a/wnbd-build/build/build b/wnbd-build/build/build index ba7bb7bec..a642114e3 100644 --- a/wnbd-build/build/build +++ b/wnbd-build/build/build @@ -2,6 +2,12 @@ set -o errexit set -o pipefail +if [[ -z $WINDOWS_SSH_USER ]]; then echo "ERROR: The WINDOWS_SSH_USER env variable is not set"; exit 1; fi +if [[ -z $WINDOWS_VM_IP ]]; then echo "ERROR: The WINDOWS_VM_IP env variable is not set"; exit 1; fi + +export SSH_USER=$WINDOWS_SSH_USER +export SSH_ADDRESS=$WINDOWS_VM_IP + BUILD_CONFIGURATION=${BUILD_CONFIGURATION:-"Release"} diff --git a/wnbd-build/config/definitions/wnbd-build.yml b/wnbd-build/config/definitions/wnbd-build.yml index 9e3f8f551..71e3677ec 100644 --- a/wnbd-build/config/definitions/wnbd-build.yml +++ b/wnbd-build/config/definitions/wnbd-build.yml @@ -42,7 +42,8 @@ !include-raw: - ../../../scripts/build_utils.sh - ../../build/setup - - ../../../scripts/ceph-windows/setup_libvirt_vm + - ../../../scripts/ceph-windows/setup_libvirt + - ../../../scripts/ceph-windows/setup_libvirt_windows_vm - ../../build/build wrappers: