Skip to content

Commit

Permalink
machine/linux: Support virtiofs mounts (retain 9p default)
Browse files Browse the repository at this point in the history
I'm hitting a bug with 9p when trying to transfer large files.
In RHEL at least 9p isn't supported because it's known to have a
lot of design flaws; virtiofsd is the supported and recommended
way to share files between a host and guest.

Add a new hidden `PODMAN_MACHINE_VIRTFS` environment
variable  that can be set to `virtiofs` to switch to virtiofsd.

Signed-off-by: Colin Walters <[email protected]>
  • Loading branch information
cgwalters committed Jun 7, 2024
1 parent 406f130 commit bf541c6
Show file tree
Hide file tree
Showing 9 changed files with 229 additions and 14 deletions.
7 changes: 6 additions & 1 deletion contrib/cirrus/setup_environment.sh
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,12 @@ case "$TEST_FLAVOR" in
install_test_configs
;;
machine-linux)
showrun dnf install -y podman-gvproxy*
showrun dnf install -y podman-gvproxy* virtiofsd
# Bootstrap this link if it isn't yet in the package; xref
# https://github.com/containers/podman/pull/22920
if ! test -L /usr/libexec/podman/virtiofsd; then
showrun ln -sfr /usr/libexec/virtiofsd /usr/libexec/podman/virtiofsd
fi
remove_packaged_podman_files
showrun make install PREFIX=/usr ETCDIR=/etc
install_test_configs
Expand Down
2 changes: 1 addition & 1 deletion pkg/machine/apple/apple.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func GenerateSystemDFilesForVirtiofsMounts(mounts []machine.VirtIoFs) ([]ignitio
mountUnit.Add("Mount", "What", "%s")
mountUnit.Add("Mount", "Where", "%s")
mountUnit.Add("Mount", "Type", "virtiofs")
mountUnit.Add("Mount", "Options", "context=\"system_u:object_r:nfs_t:s0\"")
mountUnit.Add("Mount", "Options", fmt.Sprintf("context=\"%s\"", machine.NFSSELinuxContext))
mountUnit.Add("Install", "WantedBy", "multi-user.target")
mountUnitFile, err := mountUnit.ToString()
if err != nil {
Expand Down
27 changes: 27 additions & 0 deletions pkg/machine/e2e/basic_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,33 @@ var _ = Describe("run basic podman commands", func() {
Expect(runAlp).To(Exit(0))
})

It("Volume should be virtiofs", func() {
// In theory this could run on MacOS too, but we know virtiofs works for that now,
// this is just testing linux
skipIfNotVmtype(define.QemuVirt, "This is just adding coverage for virtiofs on linux")

tDir, err := filepath.Abs(GinkgoT().TempDir())
Expect(err).ToNot(HaveOccurred())

err = os.WriteFile(filepath.Join(tDir, "testfile"), []byte("some test contents"), 0o644)
Expect(err).ToNot(HaveOccurred())

name := randomString()
i := new(initMachine).withImage(mb.imagePath).withNow()

// Ensure that this is a volume, it may not be automatically on qemu
i.withVolume(tDir)
session, err := mb.setName(name).setCmd(i).run()
Expect(err).ToNot(HaveOccurred())
Expect(session).To(Exit(0))

ssh := new(sshMachine).withSSHCommand([]string{"findmnt", "-no", "FSTYPE", tDir})
findmnt, err := mb.setName(name).setCmd(ssh).run()
Expect(err).ToNot(HaveOccurred())
Expect(findmnt).To(Exit(0))
Expect(findmnt.outputToString()).To(ContainSubstring("virtiofs"))
})

It("Podman ops with port forwarding and gvproxy", func() {
name := randomString()
i := new(initMachine)
Expand Down
2 changes: 1 addition & 1 deletion pkg/machine/provider/platform.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func Get() (vmconfigs.VMProvider, error) {
logrus.Debugf("Using Podman machine with `%s` virtualization provider", resolvedVMType.String())
switch resolvedVMType {
case define.QemuVirt:
return new(qemu.QEMUStubber), nil
return qemu.NewStubber()
default:
return nil, fmt.Errorf("unsupported virtualization provider: `%s`", resolvedVMType.String())
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/machine/qemu/command/qemu_command_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ func TestQemuCmd(t *testing.T) {

expected := []string{
"/usr/bin/qemu-system-x86_64",
"-object",
"memory-backend-memfd,id=mem,size=2048M,share=on",
"-m", "2048",
"-smp", "4",
"-fw_cfg", fmt.Sprintf("name=opt/com.coreos/config,file=%s", ignPath),
Expand Down
21 changes: 20 additions & 1 deletion pkg/machine/qemu/machine.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,28 @@ import (
)

const (
MountType9p = "9p"
MountType9p = "9p"
MountTypeVirtiofs = "virtiofs"
)

func NewStubber() (*QEMUStubber, error) {
var mountType string
if v, ok := os.LookupEnv("PODMAN_MACHINE_VIRTFS"); ok {
logrus.Debugf("using virtiofs %q", v)
switch v {
case MountType9p, MountTypeVirtiofs:
mountType = v
default:
return nil, fmt.Errorf("failed to parse PODMAN_MACHINE_VIRTFS=%s", v)
}
} else {
mountType = MountType9p
}
return &QEMUStubber{
mountType: mountType,
}, nil
}

// qemuPid returns -1 or the PID of the running QEMU instance.
func qemuPid(pidFile *define.VMFile) (int, error) {
pidData, err := os.ReadFile(pidFile.GetPath())
Expand Down
78 changes: 68 additions & 10 deletions pkg/machine/qemu/stubber.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ type QEMUStubber struct {
vmconfigs.QEMUConfig
// Command describes the final QEMU command line
Command command.QemuCmd

// mountType is one of virtiofs or 9p
mountType string
// virtiofsHelpers are virtiofsd child processes
virtiofsHelpers []virtiofsdHelperCmd
}

var (
Expand Down Expand Up @@ -83,13 +88,6 @@ func (q *QEMUStubber) setQEMUCommandLine(mc *vmconfigs.MachineConfig) error {
}
q.Command.SetSerialPort(*readySocket, *mc.QEMUHypervisor.QEMUPidPath, mc.Name)

// Add volumes to qemu command line
for _, mount := range mc.Mounts {
// the index provided in this case is thrown away
_, _, _, _, securityModel := vmconfigs.SplitVolume(0, mount.OriginalInput)
q.Command.SetVirtfsMount(mount.Source, mount.Tag, securityModel, mount.ReadOnly)
}

q.Command.SetUSBHostPassthrough(mc.Resources.USBs)

return nil
Expand Down Expand Up @@ -168,6 +166,33 @@ func (q *QEMUStubber) StartVM(mc *vmconfigs.MachineConfig) (func() error, func()
defer dnr.Close()
defer dnw.Close()

if q.mountType == MountTypeVirtiofs {
runtime, err := mc.RuntimeDir()
if err != nil {
return nil, nil, err
}
spawner, err := newVirtiofsdSpawner(runtime)
if err != nil {
return nil, nil, err
}

for _, hostmnt := range mc.Mounts {
qemuArgs, virtiofsdHelper, err := spawner.spawnForMount(hostmnt)
if err != nil {
return nil, nil, fmt.Errorf("failed to init virtiofsd for mount %s: %w", hostmnt.Source, err)
}
q.Command = append(q.Command, qemuArgs...)
q.virtiofsHelpers = append(q.virtiofsHelpers, *virtiofsdHelper)
}
} else {
// Add volumes to qemu command line
for _, mount := range mc.Mounts {
// the index provided in this case is thrown away
_, _, _, _, securityModel := vmconfigs.SplitVolume(0, mount.OriginalInput)
q.Command.SetVirtfsMount(mount.Source, mount.Tag, securityModel, mount.ReadOnly)
}
}

cmdLine := q.Command

// Disable graphic window when not in debug mode
Expand Down Expand Up @@ -198,8 +223,20 @@ func (q *QEMUStubber) StartVM(mc *vmconfigs.MachineConfig) (func() error, func()
return waitForReady(readySocket, cmd.Process.Pid, stderrBuf)
}

releaseFunc := func() error {
if err := cmd.Process.Release(); err != nil {
return err
}
for _, virtiofsdCmd := range q.virtiofsHelpers {
if err := virtiofsdCmd.command.Process.Release(); err != nil {
return err
}
}
return nil
}

// if this is not the last line in the func, make it a defer
return cmd.Process.Release, readyFunc, nil
return releaseFunc, readyFunc, nil
}

func waitForReady(readySocket *define.VMFile, pid int, stdErrBuffer *bytes.Buffer) error {
Expand Down Expand Up @@ -325,7 +362,9 @@ func (q *QEMUStubber) MountVolumesToVM(mc *vmconfigs.MachineConfig, quiet bool)
if err != nil {
return err
}
switch mount.Type {
// NOTE: We ignore q.Type here (and it should eventually be dropped from being serialized); we
// want the mount type to be dynamic, not static.
switch q.mountType {
case MountType9p:
mountOptions := []string{"-t", "9p"}
mountOptions = append(mountOptions, []string{"-o", "trans=virtio", mount.Tag, mount.Target}...)
Expand All @@ -337,6 +376,18 @@ func (q *QEMUStubber) MountVolumesToVM(mc *vmconfigs.MachineConfig, quiet bool)
if err != nil {
return err
}
case MountTypeVirtiofs:
mountOptions := []string{"-t", "virtiofs"}
mountOptions = append(mountOptions, []string{mount.Tag, mount.Target}...)
mountFlags := fmt.Sprintf("context=\"%s\"", machine.NFSSELinuxContext)
if mount.ReadOnly {
mountFlags += ",ro"
}
mountOptions = append(mountOptions, "-o", mountFlags)
err = machine.CommonSSH(mc.SSH.RemoteUsername, mc.SSH.IdentityPath, mc.Name, mc.SSH.Port, append([]string{"sudo", "mount"}, mountOptions...))
if err != nil {
return err
}
default:
return fmt.Errorf("unknown mount type: %s", mount.Type)
}
Expand All @@ -345,7 +396,14 @@ func (q *QEMUStubber) MountVolumesToVM(mc *vmconfigs.MachineConfig, quiet bool)
}

func (q *QEMUStubber) MountType() vmconfigs.VolumeMountType {
return vmconfigs.NineP
switch q.mountType {
case MountType9p:
return vmconfigs.NineP
case MountTypeVirtiofs:
return vmconfigs.VirtIOFS
default:
panic(fmt.Errorf("unexpected mount type %q", q.mountType))
}
}

func (q *QEMUStubber) PostStartNetworking(mc *vmconfigs.MachineConfig, noInfo bool) error {
Expand Down
99 changes: 99 additions & 0 deletions pkg/machine/qemu/virtiofsd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
//go:build linux || freebsd

package qemu

import (
"fmt"
"os"
"os/exec"
"time"

"github.com/containers/common/pkg/config"
"github.com/containers/podman/v5/pkg/machine/define"
"github.com/containers/podman/v5/pkg/machine/vmconfigs"
"github.com/containers/storage/pkg/fileutils"
"github.com/sirupsen/logrus"
)

// VirtiofsdSpawner spawns an instance of virtiofsd
type virtiofsdSpawner struct {
runtimeDir *define.VMFile
binaryPath string
mountCount uint
}

func newVirtiofsdSpawner(runtimeDir *define.VMFile) (*virtiofsdSpawner, error) {
var binaryPath string
cfg, err := config.Default()
if err != nil {
return nil, err
}
binaryPath, err = cfg.FindHelperBinary("virtiofsd", true)
if err != nil {
return nil, fmt.Errorf("failed to find virtiofsd: %w", err)
}
return &virtiofsdSpawner{binaryPath: binaryPath, runtimeDir: runtimeDir}, nil
}

// createVirtiofsCmd returns a new command instance configured to launch virtiofsd.
func (v *virtiofsdSpawner) createVirtiofsCmd(directory, socketPath string) *exec.Cmd {
args := []string{"--sandbox", "none", "--socket-path", socketPath, "--shared-dir", "."}
// We don't need seccomp filtering; we trust our workloads. This incidentally
// works around issues like https://gitlab.com/virtio-fs/virtiofsd/-/merge_requests/200.
args = append(args, "--seccomp=none")
cmd := exec.Command(v.binaryPath, args...)
// This sets things up so that the `.` we passed in the arguments is the target directory
cmd.Dir = directory
// Quiet the daemon by default
cmd.Env = append(cmd.Environ(), "RUST_LOG=ERROR")
cmd.Stdin = nil
cmd.Stdout = nil
if logrus.IsLevelEnabled(logrus.DebugLevel) {
cmd.Stderr = os.Stderr
}
return cmd
}

type virtiofsdHelperCmd struct {
command exec.Cmd
socket *define.VMFile
}

// spawnForMount returns on success a combination of qemu commandline and child process for virtiofsd
func (v *virtiofsdSpawner) spawnForMount(hostmnt *vmconfigs.Mount) ([]string, *virtiofsdHelperCmd, error) {
logrus.Debugf("Initializing virtiofsd mount for %s", hostmnt.Source)
// By far the most common failure to spawn virtiofsd will be a typo'd source directory,
// so let's synchronously check that ourselves here.
if err := fileutils.Exists(hostmnt.Source); err != nil {
return nil, nil, fmt.Errorf("failed to access virtiofs source directory %s", hostmnt.Source)
}
virtiofsChar := fmt.Sprintf("virtiofschar%d", v.mountCount)
virtiofsCharPath, err := v.runtimeDir.AppendToNewVMFile(virtiofsChar, nil)
if err != nil {
return nil, nil, err
}

qemuCommand := []string{}

qemuCommand = append(qemuCommand, "-chardev", fmt.Sprintf("socket,id=%s,path=%s", virtiofsChar, virtiofsCharPath.Path))
qemuCommand = append(qemuCommand, "-device", fmt.Sprintf("vhost-user-fs-pci,queue-size=1024,chardev=%s,tag=%s", virtiofsChar, hostmnt.Tag))
// TODO: Honor hostmnt.readonly somehow here (add an option to virtiofsd)
virtiofsdCmd := v.createVirtiofsCmd(hostmnt.Source, virtiofsCharPath.Path)
if err := virtiofsdCmd.Start(); err != nil {
return nil, nil, fmt.Errorf("failed to start virtiofsd")
}
// Wait for the socket
for {
if err := fileutils.Exists(virtiofsCharPath.Path); err == nil {
break
}
logrus.Debugf("waiting for virtiofsd socket %q", virtiofsCharPath.Path)
time.Sleep(time.Millisecond * 100)
}
// Increment our count of mounts which are used to create unique names for the devices
v.mountCount += 1
return qemuCommand, &virtiofsdHelperCmd{
command: *virtiofsdCmd,
socket: virtiofsCharPath,
}, nil
}
5 changes: 5 additions & 0 deletions pkg/machine/volumes.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ import (
"github.com/containers/podman/v5/pkg/machine/vmconfigs"
)

// NFSSELinuxContext is what is used by NFS mounts, which is allowed
// access by container_t. We need to fix the Fedora selinux policy
// to just allow access to virtiofs_t.
const NFSSELinuxContext = "system_u:object_r:nfs_t:s0"

type Volume interface {
Kind() VolumeKind
}
Expand Down

0 comments on commit bf541c6

Please sign in to comment.