Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/pr/647'
Browse files Browse the repository at this point in the history
* origin/pr/647:
  tests: add qrexec performance tests

Pull request description:

Add simple connection latency, and throughput tests. Run them with
different type of services (scripts, socket, via fork-server or not).
They print a test run time for comparison - the lower the better.

QubesOS/qubes-issues#5740

This will be especially useful to measure impact of QubesOS/qubes-core-qrexec#141 and similar changes.
marmarek committed Jan 26, 2025

Verified

This commit was signed with the committer’s verified signature.
marmarek Marek Marczykowski-Górecki
2 parents a784c37 + 6c06362 commit 1f2dbf5
Showing 5 changed files with 376 additions and 0 deletions.
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -235,6 +235,8 @@ endif
mkdir -p "$(DESTDIR)$(FILESDIR)"
cp -r templates "$(DESTDIR)$(FILESDIR)/templates"
cp -r tests-data "$(DESTDIR)$(FILESDIR)/tests-data"
mkdir -p "$(DESTDIR)/usr/lib/qubes"
cp -r tests "$(DESTDIR)/usr/lib/qubes/"
rm -f "$(DESTDIR)$(FILESDIR)/templates/README"

mkdir -p "$(DESTDIR)$(DOCDIR)"
1 change: 1 addition & 0 deletions qubes/tests/__init__.py
Original file line number Diff line number Diff line change
@@ -1825,6 +1825,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument
"qubes.tests.integ.devices_block",
"qubes.tests.integ.devices_pci",
"qubes.tests.integ.qrexec",
"qubes.tests.integ.qrexec_perf",
"qubes.tests.integ.dom0_update",
"qubes.tests.integ.vm_update",
"qubes.tests.integ.network",
117 changes: 117 additions & 0 deletions qubes/tests/integ/qrexec_perf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
#
# The Qubes OS Project, https://www.qubes-os.org/
#
# Copyright (C) 2025 Marek Marczykowski-Górecki
# <[email protected]>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.

import asyncio
import os
import subprocess
import sys
import time

import qubes.tests


class TC_00_QrexecPerfMixin:
def setUp(self: qubes.tests.SystemTestCase):
super().setUp()
self.vm1 = self.app.add_new_vm(
"AppVM",
name=self.make_vm_name("vm1"),
label="red",
)
self.vm2 = self.app.add_new_vm(
"AppVM",
name=self.make_vm_name("vm2"),
label="red",
)
self.loop.run_until_complete(
asyncio.gather(
self.vm1.create_on_disk(),
self.vm2.create_on_disk(),
)
)
self.loop.run_until_complete(
asyncio.gather(
self.vm1.start(),
self.vm2.start(),
)
)

def run_test(self, name):
cmd = [
"/usr/lib/qubes/tests/qrexec_perf.py",
f"--vm1={self.vm1.name}",
f"--vm2={self.vm2.name}",
name,
]
p = self.loop.run_until_complete(asyncio.create_subprocess_exec(*cmd))
self.loop.run_until_complete(p.wait())
if p.returncode:
self.fail(f"'{' '.join(cmd)}' failed: {p.returncode}")

def test_000_simple(self):
"""Measure simple exec-based vm-vm calls latency"""
self.loop.run_until_complete(self.wait_for_session(self.vm2))
self.run_test("exec")

def test_010_simple_root(self):
"""Measure simple exec-based vm-vm calls latency, use root to
bypass qrexec-fork-server"""
self.run_test("exec-root")

def test_020_socket(self):
"""Measure simple socket-based vm-vm calls latency"""
self.run_test("socket")

def test_030_socket_root(self):
"""Measure simple socket-based vm-vm calls latency, use root to
bypass qrexec-fork-server"""
self.run_test("socket-root")

def test_100_simple_data_simplex(self):
"""Measure simple exec-based vm-vm calls throughput"""
self.run_test("exec-data-simplex")

def test_110_simple_data_duplex(self):
"""Measure simple exec-based vm-vm calls throughput"""
self.run_test("exec-data-duplex")

def test_120_simple_data_duplex_root(self):
"""Measure simple exec-based vm-vm calls throughput"""
self.run_test("exec-data-duplex-root")

def test_130_socket_data_duplex(self):
"""Measure simple socket-based vm-vm calls throughput"""
self.run_test("socket-data-duplex")


def create_testcases_for_templates():
return qubes.tests.create_testcases_for_templates(
"TC_00_QrexecPerf",
TC_00_QrexecPerfMixin,
qubes.tests.SystemTestCase,
module=sys.modules[__name__],
)


def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(create_testcases_for_templates()))
return tests


qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)
2 changes: 2 additions & 0 deletions rpm_spec/core-dom0.spec.in
Original file line number Diff line number Diff line change
@@ -526,6 +526,7 @@ done
%{python3_sitelib}/qubes/tests/integ/grub.py
%{python3_sitelib}/qubes/tests/integ/salt.py
%{python3_sitelib}/qubes/tests/integ/qrexec.py
%{python3_sitelib}/qubes/tests/integ/qrexec_perf.py
%{python3_sitelib}/qubes/tests/integ/storage.py
%{python3_sitelib}/qubes/tests/integ/vm_qrexec_gui.py

@@ -547,6 +548,7 @@ done
/usr/lib/qubes/cleanup-dispvms
/usr/lib/qubes/fix-dir-perms.sh
/usr/lib/qubes/startup-misc.sh
/usr/lib/qubes/tests/qrexec_perf.py
%{_unitdir}/[email protected]/30_qubes.conf
%{_unitdir}/qubes-core.service
%{_unitdir}/qubes-qmemman.service
254 changes: 254 additions & 0 deletions tests/qrexec_perf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,254 @@
#!/usr/bin/python3
#
# The Qubes OS Project, https://www.qubes-os.org/
#
# Copyright (C) 2025 Marek Marczykowski-Górecki
# <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.

import argparse
import contextlib
import os
import time
import subprocess
import dataclasses
from typing import Union

import qubesadmin


@dataclasses.dataclass
class TestConfig:
name: str
#: socket or exec
socket: bool
#: latency or throughput test
throughput: bool
#: user (via fork-server) or root
root: bool
#: for throughput test: simplex/duplex
duplex: bool
#: service file content or socat command
service: Union[bytes, list[str]]


latency_service_exec = b"#!/bin/sh\necho test"
latency_service_socat = [
"socat",
"UNIX-LISTEN:/etc/qubes-rpc/test.Echo,mode=0666,fork",
"EXEC:/bin/echo test",
]
throughput_service_exec_simplex = b"#!/bin/sh\nhead -c 100000000 /dev/zero\n"
throughput_service_exec_duplex = b"#!/bin/sh\ncat\n"
throughput_service_socat_duplex = [
"socat",
"UNIX-LISTEN:/etc/qubes-rpc/test.Echo,mode=0666,fork",
"EXEC:/bin/cat",
]

all_tests = [
TestConfig("exec", False, False, False, False, latency_service_exec),
TestConfig("exec-root", False, False, True, False, latency_service_exec),
TestConfig("socket", True, False, False, False, latency_service_socat),
TestConfig("socket-root", True, False, True, False, latency_service_socat),
TestConfig(
"exec-data-simplex",
False,
True,
False,
False,
throughput_service_exec_simplex,
),
TestConfig(
"exec-data-duplex",
False,
True,
False,
True,
throughput_service_exec_simplex,
),
TestConfig(
"exec-data-duplex-root",
False,
True,
True,
True,
throughput_service_exec_duplex,
),
TestConfig(
"socket-data-duplex",
True,
True,
False,
True,
throughput_service_socat_duplex,
),
]

policy_file = "/run/qubes/policy.d/10-test-qrexec.policy"

parser = argparse.ArgumentParser(
epilog="You can set QUBES_TEST_PERF_FILE env variable to a path where "
"machine-readable results should be saved."
)
parser.add_argument("--vm1", required=True)
parser.add_argument("--vm2", required=True)
parser.add_argument(
"--iterations",
default=os.environ.get("QUBES_TEST_ITERATIONS", "500"),
type=int,
)
parser.add_argument("test", choices=[t.name for t in all_tests] + ["all"])


class TestRun:
def __init__(self, vm1, vm2):
self.vm1 = vm1
self.vm2 = vm2
self.iterations = 500

def run_latency_calls(self):
start_time = time.clock_gettime(time.CLOCK_MONOTONIC)
try:
self.vm1.run(
f"set -e;"
f"for i in $(seq {self.iterations}); do "
f" out=$(qrexec-client-vm {self.vm2.name} test.Echo);"
f" test \"$out\" = 'test';"
f"done"
)
except subprocess.CalledProcessError as e:
raise Exception(
f"test.Echo service failed ({e.returncode}):"
f" {e.stdout},"
f" {e.stderr}"
)
end_time = time.clock_gettime(time.CLOCK_MONOTONIC)
return end_time - start_time

def run_throughput_calls(self, duplex=False):
prefix = ""
if duplex:
prefix = "head -c 100000000 /dev/zero | "
start_time = time.clock_gettime(time.CLOCK_MONOTONIC)
try:
self.vm1.run(
f"set -e;"
f"for i in $(seq {self.iterations//2}); do "
f" out=$({prefix}qrexec-client-vm {self.vm2.name} test.Echo "
f"| wc -c);"
f' test "$out" = \'100000000\' || {{ echo "failed iteration $i:'
f" '$out'\"; exit 1; }};"
f"done"
)
except subprocess.CalledProcessError as e:
raise Exception(
f"test.Echo service failed ({e.returncode}):"
f" {e.stdout},"
f" {e.stderr}"
)
end_time = time.clock_gettime(time.CLOCK_MONOTONIC)
return end_time - start_time

def report_result(self, test_name, result):
print(f"Run time ({test_name}): {result}s")
results_file = os.environ.get("QUBES_TEST_PERF_FILE")
if results_file:
try:
if self.vm1.template != self.vm2.template:
name_prefix = (
f"{self.vm1.template!s}_" f"{self.vm2.template!s}_"
)
else:
name_prefix = f"{self.vm1.template!s}_"
except AttributeError:
name_prefix = f"{self.vm1!s}_{self.vm2!s}_"
with open(results_file, "a") as f:
f.write(name_prefix + test_name + " " + str(result) + "\n")

def run_test(self, test: TestConfig):
if test.root:
policy_action = "allow user=root"
else:
policy_action = "allow"

service_proc = None
if not test.socket:
self.vm2.run_with_args(
"rm", "-f", "/etc/qubes-rpc/test.Echo", user="root"
)
self.vm2.run_with_args(
"tee",
"/etc/qubes-rpc/test.Echo",
input=test.service,
user="root",
)
self.vm2.run_with_args(
"chmod", "+x", "/etc/qubes-rpc/test.Echo", user="root"
)
else:
self.vm2.run_with_args(
"tee",
"/etc/qubes/rpc-config/test.Echo",
user="root",
input=b"skip-service-descriptor=true\n",
)
cmd = qubesadmin.utils.encode_for_vmexec(test.service)
service_proc = self.vm2.run_service(
"qubes.VMExec+" + cmd, user="root"
)
# wait for socat startup
self.vm2.run(
"while ! test -e /etc/qubes-rpc/test.Echo; do sleep 0.1; done"
)

with open(policy_file, "w") as p:
p.write(
f"test.Echo + {self.vm1.name} {self.vm2.name} {policy_action}\n"
)
try:
if test.throughput:
result = self.run_throughput_calls(test.duplex)
else:
result = self.run_latency_calls()
self.report_result(test.name, result)
finally:
os.unlink(policy_file)
if service_proc:
with contextlib.suppress(subprocess.CalledProcessError):
self.vm2.run_with_args("pkill", "socat", user="root")
service_proc.wait()


def main():
args = parser.parse_args()

if args.test == "all":
tests = all_tests
else:
tests = [t for t in all_tests if t.name == args.test]

app = qubesadmin.Qubes()

run = TestRun(app.domains[args.vm1], app.domains[args.vm2])
if args.iterations:
run.iterations = args.iterations

for test in tests:
run.run_test(test)


if __name__ == "__main__":
main()

0 comments on commit 1f2dbf5

Please sign in to comment.