Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ci: Test virtiofsd as root #3623

Closed
wants to merge 3 commits into from
Closed

ci: Test virtiofsd as root #3623

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1fe40f4
devshell: Don't output status if not on a tty
cgwalters Nov 30, 2022
47c73b4
ci: Test virtiofsd as root
cgwalters Sep 18, 2023
bcc6080
qemu: Work around virtiofsd as root in pod
cgwalters Sep 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 53 additions & 41 deletions .cci.jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,58 +6,70 @@ def cpuCount = 6
def cpuCount_s = cpuCount.toString()
def imageName = buildImage(env: [ENABLE_GO_RACE_DETECTOR: "1", GOMAXPROCS: cpuCount_s], cpu: cpuCount_s)

def memory = (cpuCount * 1536) as Integer
pod(image: imageName + ":latest", kvm: true, cpu: "${cpuCount}", memory: "${memory}Mi") {
pod(runAsUser: 0, image: imageName + ":latest", kvm: true, cpu: 2, memory: "2Gi") {
checkout scm

stage("Unit tests") {
shwrap("make check")
shwrap("make unittest")
stage("Test running as root with virtiofs") {
shwrap("""
coreos-installer download -p qemu -f qcow2.xz --decompress
echo blah > blah.txt
cosa run --qemu-image ./*.qcow2 --bind-ro .,/run/hostqemu/workdir -x "cat /run/hostqemu/workdir/blah.txt" > blah2.txt
diff -u blah.txt blah2.txt
""")
}
}
// def memory = (cpuCount * 1536) as Integer
// pod(image: imageName + ":latest", kvm: true, cpu: "${cpuCount}", memory: "${memory}Mi") {
// checkout scm

shwrap("rpm -qa | sort > rpmdb.txt")
archiveArtifacts artifacts: 'rpmdb.txt'
// stage("Unit tests") {
// shwrap("make check")
// shwrap("make unittest")
// }

// Run stage Build FCOS (init, fetch and build)
cosaBuild(skipKola: 1, cosaDir: "/srv", noForce: true)
// shwrap("rpm -qa | sort > rpmdb.txt")
// archiveArtifacts artifacts: 'rpmdb.txt'

// Run stage Kola QEMU (basic-qemu-scenarios, upgrade and self tests)
kola(cosaDir: "/srv", addExtTests: ["${env.WORKSPACE}/ci/run-kola-self-tests"])
// // Run stage Build FCOS (init, fetch and build)
// cosaBuild(skipKola: 1, cosaDir: "/srv", noForce: true)

stage("Build Metal") {
cosaParallelCmds(cosaDir: "/srv", commands: ["metal", "metal4k"])
}
// // Run stage Kola QEMU (basic-qemu-scenarios, upgrade and self tests)
// kola(cosaDir: "/srv", addExtTests: ["${env.WORKSPACE}/ci/run-kola-self-tests"])

stage("Build Live Images") {
// Explicitly test re-importing the ostree repo
shwrap("cd /srv && rm tmp/repo -rf")
utils.cosaCmd(cosaDir: "/srv", args: "buildextend-live --fast")
}
// stage("Build Metal") {
// cosaParallelCmds(cosaDir: "/srv", commands: ["metal", "metal4k"])
// }

kolaTestIso(cosaDir: "/srv")
// stage("Build Live Images") {
// // Explicitly test re-importing the ostree repo
// shwrap("cd /srv && rm tmp/repo -rf")
// utils.cosaCmd(cosaDir: "/srv", args: "buildextend-live --fast")
// }

stage("Build Cloud Images") {
cosaParallelCmds(cosaDir: "/srv", commands: ["Aliyun", "AWS", "Azure", "DigitalOcean", "Exoscale", "GCP",
"IBMCloud", "OpenStack", "VMware", "Vultr"])
// kolaTestIso(cosaDir: "/srv")

// quick schema validation
utils.cosaCmd(cosaDir: "/srv", args: "meta --get name")
}
// stage("Build Cloud Images") {
// cosaParallelCmds(cosaDir: "/srv", commands: ["Aliyun", "AWS", "Azure", "DigitalOcean", "Exoscale", "GCP",
// "IBMCloud", "OpenStack", "VMware", "Vultr"])

stage("Compress") {
utils.cosaCmd(cosaDir: "/srv", args: "compress --fast")
}
// // quick schema validation
// utils.cosaCmd(cosaDir: "/srv", args: "meta --get name")
// }

stage("Upload Dry Run") {
utils.cosaCmd(cosaDir: "/srv", args: "buildupload --dry-run s3 --acl=public-read my-nonexistent-bucket/my/prefix")
}
// stage("Compress") {
// utils.cosaCmd(cosaDir: "/srv", args: "compress --fast")
// }

// Random other tests that aren't about building. XXX: These should be part of `make
// check` or something and use dummy cosa builds.
stage("CLI Tests") {
shwrap("""
cd /srv
${env.WORKSPACE}/tests/test_pruning.sh
""")
}
}
// stage("Upload Dry Run") {
// utils.cosaCmd(cosaDir: "/srv", args: "buildupload --dry-run s3 --acl=public-read my-nonexistent-bucket/my/prefix")
// }

// // Random other tests that aren't about building. XXX: These should be part of `make
// // check` or something and use dummy cosa builds.
// stage("CLI Tests") {
// shwrap("""
// cd /srv
// ${env.WORKSPACE}/tests/test_pruning.sh
// """)
// }
// }
34 changes: 23 additions & 11 deletions mantle/cmd/kola/devshell.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,10 @@ func stripControlCharacters(s string) string {
}, s)
}

func displayStatusMsg(status, msg string, termMaxWidth int) {
func displayStatusMsg(ontty bool, status, msg string, termMaxWidth int) {
if !ontty {
return
}
s := strings.TrimSpace(msg)
if s == "" {
return
Expand All @@ -63,8 +66,11 @@ func displayStatusMsg(status, msg string, termMaxWidth int) {
}

func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *conf.Conf, sshCommand string) error {
if !term.IsTerminal(0) {
return fmt.Errorf("stdin is not a tty")
ontty := term.IsTerminal(0)
if sshCommand == "" {
if !ontty {
return fmt.Errorf("stdin is not a tty")
}
}
termMaxWidth, _, err := term.GetSize(0)
if err != nil {
Expand Down Expand Up @@ -170,6 +176,7 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co

// Start the SSH client
sc := newSshClient(ip, agent.Socket, sshCommand)
sc.ontty = ontty
go sc.controlStartStop()

ready := false
Expand All @@ -187,7 +194,7 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co
// a status message on the console.
case serialMsg := <-serialChan:
if !ready {
displayStatusMsg(statusMsg, serialMsg, termMaxWidth)
displayStatusMsg(ontty, statusMsg, serialMsg, termMaxWidth)
}
lastMsg = serialMsg
// monitor the err channel
Expand All @@ -201,7 +208,9 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co

// monitor the instance state
case <-qemuWaitChan:
displayStatusMsg("DONE", "QEMU instance terminated", termMaxWidth)
if ontty {
displayStatusMsg(ontty, "DONE", "QEMU instance terminated", termMaxWidth)
}
return nil

// monitor the machine state events from console/serial logs
Expand Down Expand Up @@ -232,17 +241,17 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co
statusMsg = "QEMU guest is booting"
}
}
displayStatusMsg(fmt.Sprintf("EVENT | %s", statusMsg), lastMsg, termMaxWidth)
displayStatusMsg(ontty, fmt.Sprintf("EVENT | %s", statusMsg), lastMsg, termMaxWidth)

// monitor the SSH connection
case err := <-sc.errChan:
if err == nil {
sc.controlChan <- sshNotReady
displayStatusMsg("SESSION", "Clean exit from SSH, terminating instance", termMaxWidth)
displayStatusMsg(ontty, "SESSION", "Clean exit from SSH, terminating instance", termMaxWidth)
return nil
} else if sshCommand != "" {
sc.controlChan <- sshNotReady
displayStatusMsg("SESSION", "SSH command exited, terminating instance", termMaxWidth)
displayStatusMsg(ontty, "SESSION", "SSH command exited, terminating instance", termMaxWidth)
return err
}
if ready {
Expand Down Expand Up @@ -455,6 +464,7 @@ type sshClient struct {
port string
agent string
cmd string
ontty bool
controlChan chan sshControlMessage
errChan chan error
sshCmd *exec.Cmd
Expand Down Expand Up @@ -511,8 +521,10 @@ func (sc *sshClient) start() {
if sc.cmd != "" {
sshArgs = append(sshArgs, "--", sc.cmd)
}
fmt.Printf("\033[2K\r") // clear serial console line
fmt.Printf("[SESSION] Starting SSH\r") // and stage a status msg which will be erased
if sc.ontty {
fmt.Printf("\033[2K\r") // clear serial console line
fmt.Printf("[SESSION] Starting SSH\r") // and stage a status msg which will be erased
}
sshCmd := exec.Command(sshArgs[0], sshArgs[1:]...)
sshCmd.Stdin = os.Stdin
sshCmd.Stdout = os.Stdout
Expand All @@ -531,7 +543,7 @@ func (sc *sshClient) start() {
for scanner.Scan() {
msg := scanner.Text()
if strings.Contains(msg, "Connection to 127.0.0.1 closed") {
displayStatusMsg("SSH", "connection closed", 0)
displayStatusMsg(sc.ontty, "SSH", "connection closed", 0)
}
}
}()
Expand Down
30 changes: 22 additions & 8 deletions mantle/platform/qemu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1587,6 +1587,27 @@ func (builder *QemuBuilder) VirtioJournal(config *conf.Conf, queryArguments stri
return stream, nil
}

// createVirtiofsCmd returns a new command instance configured to launch virtiofsd.
func createVirtiofsCmd(directory, socketPath string) exec.Cmd {
args := []string{"--sandbox", "none", "--socket-path", socketPath, "--shared-dir", "."}
// Work around https://gitlab.com/virtio-fs/virtiofsd/-/merge_requests/197
if os.Getuid() == 0 {
args = append(args, "--modcaps=-mknod:-setfcap")
}
cmd := exec.Command("/usr/libexec/virtiofsd", args...)
// This sets things up so that the `.` we passed in the arguments is the target directory
cmd.Dir = directory
// Quiet the daemon by default
cmd.Env = append(cmd.Env, "RUST_LOG=ERROR")
// But we do want to see errors
cmd.Stderr = os.Stderr
// Like other processes, "lifecycle bind" it to us
cmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGTERM,
}
return cmd
}

// Exec tries to run a QEMU instance with the given settings.
func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
builder.finalize()
Expand Down Expand Up @@ -1805,14 +1826,7 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
builder.Append("-device", fmt.Sprintf("vhost-user-fs-pci,queue-size=1024,chardev=%s,tag=%s", virtiofsChar, hostmnt.dest))
plog.Debugf("creating virtiofs helper for %s", hostmnt.src)
// TODO: Honor hostmnt.readonly somehow here (add an option to virtiofsd)
p := exec.Command("/usr/libexec/virtiofsd", "--sandbox", "none", "--socket-path", virtiofsdSocket, "--shared-dir", ".")
p.Dir = hostmnt.src
// Quiet the daemon by default
p.Env = append(p.Env, "RUST_LOG=ERROR")
p.Stderr = os.Stderr
p.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGTERM,
}
p := createVirtiofsCmd(hostmnt.src, virtiofsdSocket)
if err := p.Start(); err != nil {
return nil, fmt.Errorf("failed to start virtiofsd")
}
Expand Down
Loading