Introduce bsd-user-qemu and run-{shell,user}-riscv64-purecap targets.

README.md

@@ -185,6 +185,8 @@ For the `cheribsd`, `disk-image` and `run` targets the hybrid vs purecap distinc
 - `freebsd-<architecture>` builds and installs [freebsd/freebsd](https://github.com/freebsd/freebsd).
 - `disk-image-freebsd-<architecture>` creates a FreeBSD disk-image.
 - `run-freebsd-<architecture>` launches QEMU with the FreeBSD disk image.
+- `run-user-<architecture>` launches a command with the QEMU user mode in a CheriBSD sysroot.
+- `run-user-shell-<architecture>` launches a shell with the QEMU user mode in a CheriBSD sysroot.
 - `cmake` builds and installs latest [CMake](https://github.com/Kitware/CMake)
 - `cherios` builds and installs [CTSRD-CHERI/cherios](https://github.com/CTSRD-CHERI/cherios)
 - `cheritrace` builds and installs [CTSRD-CHERI/cheritrace](https://github.com/CTSRD-CHERI/cheritrace)

pycheribuild/boot_cheribsd/__init__.py

@@ -772,13 +772,14 @@ def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kerne
         bios_args = riscv_bios_arguments(qemu_options.xtarget, None)
     else:
         bios_args = []
-    qemu_args = qemu_options.get_commandline(qemu_command=qemu_command, kernel_file=kernel_image, disk_image=disk_image,
-                                             bios_args=bios_args, user_network_args=user_network_args,
-                                             write_disk_image_changes=write_disk_image_changes,
-                                             add_network_device=True,
-                                             trap_on_unrepresentable=trap_on_unrepresentable,  # For debugging
-                                             add_virtio_rng=True  # faster entropy gathering
-                                             )
+    qemu_args = qemu_options.get_system_commandline(qemu_command=qemu_command, kernel_file=kernel_image,
+                                                    disk_image=disk_image, bios_args=bios_args,
+                                                    user_network_args=user_network_args,
+                                                    write_disk_image_changes=write_disk_image_changes,
+                                                    add_network_device=True,
+                                                    trap_on_unrepresentable=trap_on_unrepresentable,  # For debugging
+                                                    add_virtio_rng=True  # faster entropy gathering
+                                                    )
     qemu_args.extend(smp_args)
     kernel_commandline = []
     if kernel_init_only:

pycheribuild/config/chericonfig.py

@@ -325,6 +325,7 @@ def __init__(self, loader, action_class) -> None:
         self.build_root = None  # type: Optional[Path]
         # Path to kernel/disk images (this is the same as output_root by default but different in Jenkins)
         self.cheribsd_image_root = None  # type: Optional[Path]
+        self.bsd_user_sdk_dir = None  # type: Optional[Path]
         self.cheri_sdk_dir = None  # type: Optional[Path]
         self.morello_sdk_dir = None  # type: Optional[Path]
         self.other_tools_dir = None  # type: Optional[Path]
@@ -529,6 +530,10 @@ def make_j_flag(self):
     def mips_cheri_bits_str(self):
         return str(self.mips_cheri_bits)
 
+    @property
+    def default_bsd_user_sdk_directory_name(self) -> str:
+        return "bsd-user-sdk"
+
     @property
     def default_cheri_sdk_directory_name(self) -> str:
         return "sdk"
@@ -537,6 +542,10 @@ def default_cheri_sdk_directory_name(self) -> str:
     def default_morello_sdk_directory_name(self) -> str:
         return "morello-sdk"
 
+    @property
+    def bsd_user_sdk_bindir(self):
+        return self.bsd_user_sdk_dir / "bin"
+
     @property
     def cheri_sdk_bindir(self):
         return self.cheri_sdk_dir / "bin"
@@ -549,6 +558,10 @@ def morello_sdk_bindir(self):
     def qemu_bindir(self):
         return self.cheri_sdk_bindir
 
+    @property
+    def bsd_user_qemu_bindir(self):
+        return self.bsd_user_sdk_bindir
+
     @property
     def test_ssh_key(self) -> Path:
         if self._test_ssh_key is not None:

pycheribuild/config/defaultconfig.py

@@ -193,6 +193,7 @@ def load(self) -> None:
         super().load()
         # now set some generic derived config options
         self.cheri_sdk_dir = self.tools_root / self.default_cheri_sdk_directory_name
+        self.bsd_user_sdk_dir = self.tools_root / self.default_bsd_user_sdk_directory_name
         self.other_tools_dir = self.tools_root / "bootstrap"
         self.cheribsd_image_root = self.output_root  # TODO: allow this to be different?
 

pycheribuild/config/jenkinsconfig.py

@@ -141,6 +141,9 @@ def __init__(self, loader: ConfigLoaderBase, available_targets: list) -> None:
             help="Don't use the CHERI SDK -> only /usr (for native builds)")
         self.strip_elf_files = loader.add_commandline_only_bool_option(
             "strip-elf-files", help="Strip ELF files before creating the tarball", default=True, negatable=True)
+        self._bsd_user_sdk_dir_override = loader.add_commandline_only_option(
+            "bsd-user-sdk-path", default=None, type=Path,
+            help="Override the path to the BSD user mode SDK (default is $WORKSPACE/bsd-user-sdk)")  # type: Path
         self._cheri_sdk_dir_override = loader.add_commandline_only_option(
             "cheri-sdk-path", default=None, type=Path,
             help="Override the path to the CHERI SDK (default is $WORKSPACE/cherisdk)")
@@ -207,6 +210,23 @@ def qemu_bindir(self):
             os_suffix = "unknown-os"
         return self.workspace / ("qemu-" + os_suffix) / "bin"
 
+    @property
+    def bsd_user_qemu_bindir(self):
+        for i in self.bsd_user_sdk_bindir.glob("qemu-*"):
+            if self.verbose:
+                print("Found QEMU binary", i, "in BSD user-mode SDK dir -> using that for BSD user-mode QEMU binaries")
+            # If one qemu-foo exists in the bsd_user_sdk_bindir use that instead of $WORKSPACE/qemu-<OS>
+            return self.bsd_user_sdk_bindir
+        if OSInfo.IS_LINUX:
+            os_suffix = "linux"
+        elif OSInfo.IS_FREEBSD:
+            os_suffix = "freebsd"
+        elif OSInfo.IS_MAC:
+            os_suffix = "mac"
+        else:
+            os_suffix = "unknown-os"
+        return self.workspace / ("qemu-" + os_suffix) / "bin"
+
     def load(self) -> None:
         super().load()
 
@@ -230,6 +250,11 @@ def load(self) -> None:
 
         self.other_tools_dir = self.workspace / "bootstrap"
 
+        if self._bsd_user_sdk_dir_override is not None:
+            self.bsd_user_sdk_dir = self._bsd_user_sdk_dir_override
+        else:
+            self.bsd_user_sdk_dir = self.workspace / self.default_bsd_user_sdk_directory_name
+
         if self._cheri_sdk_dir_override is not None:
             self.cheri_sdk_dir = self._cheri_sdk_dir_override
         elif Path("/cheri-sdk/bin/clang").exists():  # check for ctsrd/cheri-sdk docker image
@@ -280,6 +305,8 @@ def load(self) -> None:
                 self.clang_plusplus_path = compiler_dir_override / "clang++"
                 self.clang_cpp_path = compiler_dir_override / "clang-cpp"
 
+        if self._bsd_user_sdk_dir_override is not None:
+            assert self.bsd_user_sdk_bindir == self._bsd_user_sdk_dir_override / "bin"
         if self._cheri_sdk_dir_override is not None:
             assert self.cheri_sdk_bindir == self._cheri_sdk_dir_override / "bin"
         if self._morello_sdk_dir_override is not None:

pycheribuild/config/target_info.py

@@ -131,6 +131,7 @@ class DefaultInstallDir(Enum):
     ROOTFS_LOCALBASE = "The sysroot for this target (<rootfs>/usr/local/<arch> by default)"
     KDE_PREFIX = "The sysroot for this target (<rootfs>/opt/<arch>/kde by default)"
     CHERI_SDK = "The CHERI SDK directory"
+    BSD_USER_SDK = "The QEMU BSD user mode SDK directory"
     MORELLO_SDK = "The Morello SDK directory"
     BOOTSTRAP_TOOLS = "The bootstap tools directory"
     CUSTOM_INSTALL_DIR = "Custom install directory"
@@ -541,6 +542,8 @@ def default_install_dir(self, install_dir: DefaultInstallDir) -> Path:
             if self._is_libcompat_target:
                 return config.output_root / ("local" + self._compat_abi_suffix)
             return config.output_root / "local"
+        elif install_dir == DefaultInstallDir.BSD_USER_SDK:
+            return config.bsd_user_sdk_dir
         elif install_dir == DefaultInstallDir.CHERI_SDK:
             return config.cheri_sdk_dir
         elif install_dir == DefaultInstallDir.MORELLO_SDK:

pycheribuild/projects/build_qemu.py

@@ -60,6 +60,7 @@ class BuildQEMUBase(AutotoolsProject):
     can_build_with_asan = True
     default_targets: str = "some-invalid-target"
     default_build_type = BuildType.RELEASE
+    default_use_smbd = True
     lto_by_default = True
     use_smbd: bool
     smbd_path: Optional[Path]
@@ -78,7 +79,7 @@ def _build_type_basic_compiler_flags(self):
     @classmethod
     def setup_config_options(cls, **kwargs):
         super().setup_config_options(**kwargs)
-        cls.use_smbd = cls.add_bool_option("use-smbd", show_help=False, default=True,
+        cls.use_smbd = cls.add_bool_option("use-smbd", show_help=False, default=cls.default_use_smbd,
                                            help="Don't require SMB support when building QEMU (warning: most --test "
                                                 "targets will fail without smbd support)")
 
@@ -394,3 +395,38 @@ def install(self, **kwargs):
                                 *self.config.morello_sdk_dir.rglob("share/icons/**/qemu.png"),
                                 *self.config.morello_sdk_dir.rglob("share/icons/**/qemu.bmp"),
                                 *self.config.morello_sdk_dir.rglob("share/icons/**/qemu.svg"))
+
+
+class BuildBsdUserQEMU(BuildQEMUBase):
+    repository = GitRepository("https://github.com/CTSRD-CHERI/qemu.git",
+                               default_branch="qemu-cheri-bsd-user",
+                               force_branch=True)
+    native_install_dir = DefaultInstallDir.BSD_USER_SDK
+    default_targets = "aarch64-bsd-user,morello-bsd-user,riscv64-bsd-user,riscv64cheri-bsd-user"
+    default_use_smbd = False
+    target = "bsd-user-qemu"
+    hide_options_from_help = True
+
+    @classmethod
+    def qemu_cheri_binary(cls, caller: SimpleProject, xtarget: CrossCompileTarget = None, absolute_path=True):
+        if xtarget is None:
+            xtarget = caller.get_crosscompile_target()
+        if xtarget.is_riscv(include_purecap=True):
+            binary_name = "qemu-riscv64cheri"
+        else:
+            raise ValueError("Invalid xtarget" + str(xtarget))
+        if absolute_path:
+            return caller.config.bsd_user_qemu_bindir / os.getenv("QEMU_BSD_USER_PATH", binary_name)
+        else:
+            return binary_name
+
+    def setup(self):
+        super().setup()
+        # Disable capstone disassembler unsupporting CHERI instructions.
+        self.configure_args.append("--disable-capstone")
+        # Disable RVFI-DDI unsupported in the user mode.
+        self.configure_args.append("--disable-rvfi-dii")
+        # Enable to build BSD user mode targets.
+        self.configure_args.append("--enable-bsd-user")
+        # Build a static binary that can be easily included in a guest jail.
+        self.configure_args.append("--static")

pycheribuild/projects/run_qemu.py

@@ -37,7 +37,7 @@
 from pathlib import Path
 from typing import Optional
 
-from .build_qemu import BuildQEMU, BuildQEMUBase, BuildUpstreamQEMU
+from .build_qemu import BuildBsdUserQEMU, BuildQEMU, BuildQEMUBase, BuildUpstreamQEMU
 from .cherios import BuildCheriOS
 from .cross.cheribsd import BuildCHERIBSD, BuildCheriBsdMfsKernel, BuildFreeBSD, ConfigPlatform, KernelABI
 from .cross.gdb import BuildGDB
@@ -447,16 +447,12 @@ def add_smb_or_9p_dir(directory, target, share_name=None, readonly=False):
             self._after_disk_options += ["-snapshot"]
 
         # input("Press enter to continue")
-        qemu_command = self.qemu_options.get_commandline(qemu_command=self.chosen_qemu.binary,
-                                                         kernel_file=qemu_loader_or_kernel,
-                                                         disk_image=self.disk_image,
-                                                         disk_image_format=self.disk_image_format,
-                                                         add_network_device=self.qemu_user_networking,
-                                                         bios_args=self.bios_flags,
-                                                         user_network_args=user_network_options,
-                                                         trap_on_unrepresentable=self.config.trap_on_unrepresentable,
-                                                         debugger_on_cheri_trap=self.config.debugger_on_cheri_trap,
-                                                         add_virtio_rng=self._add_virtio_rng)
+        qemu_command = self.qemu_options.get_system_commandline(
+            qemu_command=self.chosen_qemu.binary, kernel_file=qemu_loader_or_kernel, disk_image=self.disk_image,
+            disk_image_format=self.disk_image_format, add_network_device=self.qemu_user_networking,
+            bios_args=self.bios_flags, user_network_args=user_network_options,
+            trap_on_unrepresentable=self.config.trap_on_unrepresentable,
+            debugger_on_cheri_trap=self.config.debugger_on_cheri_trap, add_virtio_rng=self._add_virtio_rng)
         qemu_command += self._project_specific_options + self._after_disk_options + monitor_options
         qemu_command += logfile_options + self.extra_qemu_options + virtfs_args
         if self.disk_image is None:
@@ -585,6 +581,78 @@ def is_port_available(port: int):
             return False
 
 
+class LaunchBsdUserQEMUBase(SimpleProject):
+    do_not_add_to_targets = True
+    _source_class = BuildCHERIBSD
+    _freebsd_class = BuildCHERIBSD
+    _always_add_suffixed_targets = True
+    supported_architectures = [CompilationTargets.CHERIBSD_RISCV_PURECAP]
+
+    @classmethod
+    def setup_config_options(cls, **kwargs):
+        super().setup_config_options(**kwargs)
+        cls.chroot = cls.add_bool_option("chroot", default=False, show_help=True,
+                                         help="Change the root directory to a sysroot before executing a command.")
+        cls.interpreter_path = cls.add_path_option("interpreter", metavar="PATH", default="/libexec/ld-elf.so.1",
+                                              show_help=True, help="ELF interpreter path relative to a sysroot")
+        cls.jail = cls.add_bool_option("jail", default=False, show_help=True,
+                                       help="Enter a jail with a sysroot before executing a command.")
+        cls.jail_extra_args = cls.add_config_option("jail-extra-args", metavar="ARGS", kind=list, show_help=True,
+                                                    help="Extra jail parameters to pass to jail(8).")
+
+    @classmethod
+    def dependencies(cls: "typing.Type[_RunMultiArchFreeBSDImage]", config: CheriConfig) -> "list[str]":
+        xtarget = cls.get_crosscompile_target()
+        result = ["bsd-user-qemu", cls._source_class.get_class_for_target(xtarget).target]
+        return result
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.qemu_binary = None  # type: typing.Optional[Path]
+        self.qemu_options = QemuOptions(self.crosscompile_target)
+        self.rootfs_path = self._source_class.get_rootfs_dir(self)
+
+    def setup(self):
+        super().setup()
+        absolute_path = not self.chroot and not self.jail
+        xtarget = self.crosscompile_target
+        if xtarget.is_riscv(include_purecap=True):
+            self.qemu_binary = BuildBsdUserQEMU.qemu_cheri_binary(self, absolute_path=absolute_path)
+        else:
+            assert False, "Unknown target " + str(xtarget)
+
+    def process(self):
+        assert self.qemu_binary is not None
+        assert self.rootfs_path is not None
+        assert self.interpreter_path is not None
+        assert self.command is not None
+
+        if self.chroot and self.jail:
+            self.fatal("Chroot and jail options are mutually exclusive.")
+        elif self.chroot:
+            command = ["chroot", self.rootfs_path]
+            qemu_command = self.qemu_binary
+            rootfs_path = None
+        elif self.jail:
+            command = ["jail", "-c", "path={}".format(self.rootfs_path)]
+            if self.jail_extra_args:
+                command.extend(self.jail_extra_args)
+            qemu_command = "command={}".format(self.qemu_binary)
+            rootfs_path = None
+        else:
+            command = []
+            qemu_command = self.qemu_binary
+            rootfs_path = self.rootfs_path
+
+        command.extend(self.qemu_options.get_user_commandline(qemu_command=qemu_command,
+                       rootfs_path=rootfs_path, interpreter_path=self.interpreter_path,
+                       user_command=self.command))
+
+        self.info("About to run '{}' with the QEMU user mode".format(" ".join(self.command)))
+
+        self.run_cmd(command, stdout=sys.stdout, stderr=sys.stderr, give_tty_control=True)
+
+
 class AbstractLaunchFreeBSD(LaunchQEMUBase):
     do_not_add_to_targets = True
     kernel_project: Optional[BuildFreeBSD]
@@ -757,6 +825,31 @@ def dependencies(cls, config: CheriConfig) -> "list[str]":
         return result
 
 
+class LaunchUserShell(LaunchBsdUserQEMUBase):
+    target = "run-user-shell"
+
+    def process(self):
+        assert self.rootfs_path is not None
+
+        command = "/bin/sh"
+        if self.chroot or self.jail:
+            self.command = [command]
+        else:
+            self.command = [str(self.rootfs_path) + command]
+        super().process()
+
+
+class LaunchUser(LaunchBsdUserQEMUBase):
+    target = "run-user"
+
+    @classmethod
+    def setup_config_options(cls, **kwargs):
+        super().setup_config_options(**kwargs)
+        cls.command = cls.add_config_option("command", metavar="COMMAND", show_help=True, kind=list,
+                                            help="Command to execute (default: '<ROOTFS>/bin/sh').",
+                                            default=lambda _, p: str(p.rootfs_path) + "/bin/sh")
+
+
 class LaunchCheriOSQEMU(LaunchQEMUBase):
     target = "run-cherios"
     dependencies = ["qemu", "cherios"]