fix: ensure new backing images are based on target of symlinks

If the disk image pointed at in the topology file is a symlink to a
different file, copy the target of the symlink for the new backing image
and not the actual symlink.

Also take into account situations where the disk image pointed at in the
topology file is a snapshot (or chain of snapshots) on top of a base
image. If so, rebase the snapshot chain onto the new backing image prior
to rebasing the final minimega snapshot.

src/go/api/vm/util.go

@@ -2,6 +2,7 @@ package vm
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -12,12 +13,19 @@ import (
 	"time"
 
 	"phenix/api/experiment"
+	"phenix/util/common"
+	"phenix/util/mm"
 )
 
+type qemuBackingChain struct {
+	Filename    string `json:"filename"`
+	BackingFile string `json:"backing-filename"`
+}
+
 var diskNameWithTstampRegex = regexp.MustCompile(`(.*)_\d{14}`)
 
 func GetNewDiskName(expName, vmName string) (string, error) {
-	base, err := getBaseImage(expName, vmName)
+	base, err := getVMImage(expName, vmName)
 	if err != nil {
 		return "", fmt.Errorf("getting base disk image: %w", err)
 	}
@@ -39,7 +47,7 @@ func GetNewDiskName(expName, vmName string) (string, error) {
 	return name, nil
 }
 
-func getBaseImage(expName, vmName string) (string, error) {
+func getVMImage(expName, vmName string) (string, error) {
 	exp, err := experiment.Get(expName)
 	if err != nil {
 		return "", fmt.Errorf("getting experiment %s: %w", expName, err)
@@ -50,9 +58,79 @@ func getBaseImage(expName, vmName string) (string, error) {
 		return "", fmt.Errorf("getting vm %s for experiment %s", vmName, expName)
 	}
 
+	// base image from topology
 	return vm.Hardware().Drives()[0].Image(), nil
 }
 
+func getBackingImage(path string) (string, error) {
+	if !filepath.IsAbs(path) {
+		path = common.PhenixBase + "/images/" + path
+	}
+
+	chain, err := getImageBackingChain(path)
+	if err != nil {
+		return "", fmt.Errorf("getting image backing chain for %s: %w", path, err)
+	}
+
+	// backing image should always be last image in chain
+	path = chain[len(chain)-1].Filename
+
+	stats, err := os.Lstat(path)
+	if err != nil {
+		return "", fmt.Errorf("getting file stats for %s: %w", path, err)
+	}
+
+	// base image path is a symlink
+	if stats.Mode()&os.ModeSymlink != 0 {
+		origPath := path
+
+		path, err = os.Readlink(path)
+		if err != nil {
+			return "", fmt.Errorf("getting symlink target for %s: %w", origPath, err)
+		}
+	}
+
+	return path, nil
+}
+
+func getImageSnapshots(path string) ([]string, error) {
+	chain, err := getImageBackingChain(path)
+	if err != nil {
+		return nil, fmt.Errorf("getting image backing chain for %s: %w", path, err)
+	}
+
+	if len(chain) <= 1 {
+		return nil, nil
+	}
+
+	var snapshots []string
+
+	// range chain in reverse to get snapshots in correct order for rebasing
+	// skip last entry since it will be the base image (not a snapshot)
+	for i := len(chain) - 2; i >= 0; i-- {
+		snapshots = append(snapshots, chain[i].Filename)
+	}
+
+	return snapshots, nil
+}
+
+func getImageBackingChain(path string) ([]qemuBackingChain, error) {
+	cmd := fmt.Sprintf("qemu-img info --backing-chain %s --output json", path)
+
+	resp, err := mm.MeshShellResponse("", cmd)
+	if err != nil {
+		return nil, fmt.Errorf("getting image info for %s: %w", path, err)
+	}
+
+	var chain []qemuBackingChain
+
+	if err := json.Unmarshal([]byte(resp), &chain); err != nil {
+		return nil, fmt.Errorf("parsing image info for %s: %w", path, err)
+	}
+
+	return chain, nil
+}
+
 type copier struct {
 	subs []chan float64
 }

src/go/api/vm/vm.go

@@ -940,9 +940,27 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error)
 		}
 	}
 
-	base, err := getBaseImage(expName, vmName)
+	disk, err := getVMImage(expName, vmName)
 	if err != nil {
-		return "", fmt.Errorf("getting base image for VM %s in experiment %s: %w", vmName, expName, err)
+		return "", fmt.Errorf("getting disk image for VM %s in experiment %s: %w", vmName, expName, err)
+	}
+
+	if !filepath.IsAbs(disk) {
+		disk = common.PhenixBase + "/images/" + disk
+	}
+
+	if !filepath.IsAbs(out) {
+		out = common.PhenixBase + "/images/" + out
+	}
+
+	base, err := getBackingImage(disk)
+	if err != nil {
+		return "", fmt.Errorf("getting backing image for VM %s in experiment %s: %w", vmName, expName, err)
+	}
+
+	snapshots, err := getImageSnapshots(disk)
+	if err != nil {
+		return "", fmt.Errorf("getting disk snapshots for VM %s in experiment %s: %w", vmName, expName, err)
 	}
 
 	// Get status of VM (scheduled host, VM state).
@@ -964,14 +982,6 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error)
 		node = status[0]["host"]
 	)
 
-	if !filepath.IsAbs(base) {
-		base = common.PhenixBase + "/images/" + base
-	}
-
-	if !filepath.IsAbs(out) {
-		out = common.PhenixBase + "/images/" + out
-	}
-
 	wait, ctx := errgroup.WithContext(context.Background())
 
 	// Make copy of base image locally on headnode. Using a context here will help
@@ -1007,6 +1017,37 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error)
 		}
 	}
 
+	// Make a copy of any snapshots pointed at by the VM disk image in the
+	// experiment topology so they can be rebased onto the base backing image for
+	// the VM.
+
+	for idx, snapshot := range snapshots {
+		var (
+			id   = idx
+			file = snapshot
+		)
+
+		wait.Go(func() error {
+			copier := newCopier()
+
+			tmp := fmt.Sprintf("%s/images/%s/tmp/snapshot-%d.qc2", common.PhenixBase, expName, id)
+
+			if err := os.MkdirAll(filepath.Dir(tmp), 0755); err != nil {
+				return fmt.Errorf("creating experiment tmp directory: %w", err)
+			}
+
+			if err := copier.copy(ctx, file, tmp); err != nil {
+				os.Remove(tmp)
+				return fmt.Errorf("making copy of snapshot %s: %w", file, err)
+			}
+
+			// update snapshots entry to point at new copy of snapshot
+			snapshots[id] = tmp
+
+			return nil
+		})
+	}
+
 	// Copy minimega snapshot disk on remote machine to a location (still on
 	// remote machine) that can be seen by minimega files. Then use minimega `file
 	// get` to copy it to the headnode.
@@ -1048,13 +1089,27 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error)
 		return "", fmt.Errorf("preparing images for rebase/commit: %w", err)
 	}
 
-	snap = fmt.Sprintf("%s/images/%s/tmp/%s.qc2", common.PhenixBase, expName, vmName)
+	// Add the final boss (the minimega snapshot disk) to the end of the list of
+	// snapshots to rebase onto the copy of the original backing image. If the
+	// image pointed at by the VM config in the topology was not a snapshot, then
+	// this will be the only snapshot in the list.
+	snapshots = append(snapshots, fmt.Sprintf("%s/images/%s/tmp/%s.qc2", common.PhenixBase, expName, vmName))
 
-	shell := exec.Command("qemu-img", "rebase", "-f", "qcow2", "-b", out, "-F", "qcow2", snap)
+	for idx, snapshot := range snapshots {
+		// first parent should be copy of original backing image
+		parent := out
+
+		if idx != 0 {
+			parent = snapshots[idx-1]
+		}
+
+		shell := exec.Command("qemu-img", "rebase", "-f", "qcow2", "-b", parent, "-F", "qcow2", snapshot)
+
+		res, err := shell.CombinedOutput()
+		if err != nil {
+			return "", fmt.Errorf("rebasing snapshot (%s): %w", string(res), err)
+		}
 
-	res, err := shell.CombinedOutput()
-	if err != nil {
-		return "", fmt.Errorf("rebasing snapshot (%s): %w", string(res), err)
 	}
 
 	done := make(chan struct{})
@@ -1091,9 +1146,9 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error)
 		}()
 	}
 
-	shell = exec.Command("qemu-img", "commit", snap)
+	shell := exec.Command("qemu-img", "commit", "-b", out, snapshots[len(snapshots)-1])
 
-	res, err = shell.CombinedOutput()
+	res, err := shell.CombinedOutput()
 	if err != nil {
 		return "", fmt.Errorf("committing snapshot (%s): %w", string(res), err)
 	}
@@ -1477,16 +1532,13 @@ func ChangeOpticalDisc(expName, vmName, isoPath string) error {
 		return fmt.Errorf("no optical disc path provided")
 	}
 
-
-	cmd := mmcli.NewNamespacedCommand(expName)	
-	cmd.Command = fmt.Sprintf("vm cdrom change %s %s",vmName,isoPath)
+	cmd := mmcli.NewNamespacedCommand(expName)
+	cmd.Command = fmt.Sprintf("vm cdrom change %s %s", vmName, isoPath)
 
 	if err := mmcli.ErrorResponse(mmcli.Run(cmd)); err != nil {
 		return fmt.Errorf("changing optical disc for VM %s: %w", vmName, err)
 	}
-
 
-
 	return nil
 }
 
@@ -1501,15 +1553,12 @@ func EjectOpticalDisc(expName, vmName string) error {
 		return fmt.Errorf("no VM name provided")
 	}
 
-
-	cmd := mmcli.NewNamespacedCommand(expName)	
-	cmd.Command = fmt.Sprintf("vm cdrom eject %s",vmName)
+	cmd := mmcli.NewNamespacedCommand(expName)
+	cmd.Command = fmt.Sprintf("vm cdrom eject %s", vmName)
 
 	if err := mmcli.ErrorResponse(mmcli.Run(cmd)); err != nil {
 		return fmt.Errorf("ejecting optical disc for VM %s: %w", vmName, err)
 	}
 
-
 	return nil
 }
-

src/go/util/mm/mm.go

@@ -47,5 +47,6 @@ type MM interface {
 
 	TapVLAN(...TapOption) error
 	MeshShell(string, string) error
+	MeshShellResponse(string, string) (string, error)
 	MeshSend(string, string, string) error
 }

src/go/util/mm/package.go

@@ -140,6 +140,10 @@ func MeshShell(host, cmd string) error {
 	return DefaultMM.MeshShell(host, cmd)
 }
 
+func MeshShellResponse(host, cmd string) (string, error) {
+	return DefaultMM.MeshShellResponse(host, cmd)
+}
+
 func MeshSend(ns, host, command string) error {
 	return DefaultMM.MeshSend(ns, host, command)
 }