Allow dynamically linked main VM for storage example

Author: fwsGonzo

guest/rust_storage/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-png = "0.15.3"
 libc = "0.2"
 
 [profile.release]

guest/rust_storage/build.sh

@@ -8,4 +8,5 @@ objcopy -w --extract-symbol --strip-symbol=!remote* --strip-symbol=* $storage_bi
 gcc-12 -static -O2 symbol_offset.c -o symbol_offset
 ./symbol_offset storage.syms 0x44000000
 
-RUSTFLAGS="-Ctarget-feature=+crt-static -Zexport-executable-symbols -C link_arg=-Wl,--just-symbols=storage.syms" cargo build --target x86_64-unknown-linux-gnu --release
+rm -rf target/
+RUSTFLAGS="-Zexport-executable-symbols -C link_arg=-Wl,--undefined=do_calculation,--just-symbols=$PWD/storage.syms" cargo build --release

guest/rust_storage/main

@@ -1,7 +1,7 @@
+use std::arch::asm;
 extern "C" { fn remote_function(arg: extern "C" fn(i32) -> i32, value: i32) -> i32; }
 use std::process::ExitCode;
 
-#[no_mangle]
 extern "C" fn do_calculation(input: i32) -> i32 {
     return unsafe { remote_function(double_int, input) };
 }
@@ -10,10 +10,26 @@ extern "C" fn double_int(input: i32) -> i32 {
     return input * 2;
 }
 
+// Don't exactly know how else to call a named function
+// in a dynamic ELF w/interpreter. We don't know the
+// base address of the binary, so we can't adjust the
+// symbol address.
+fn set_callback(_cb: extern "C" fn(i32) -> i32) {
+	let sysnum = 0x10001; // Set callback syscall
+	unsafe {
+		asm!(
+			"out 0, eax",
+			in("eax") sysnum,
+			in("rdi") _cb
+		);
+	}
+}
+
 fn main() -> ExitCode
 {
 	println!("Hello, world!");
 	let result = unsafe { remote_function(double_int, 21) };
 	println!("Result from remote function: {}", result);
+	set_callback(do_calculation);
 	return ExitCode::from(result as u8);
 }

guest/rust_storage/src/main.rs

@@ -7,7 +7,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-png = "0.15.3"
 libc = "0.2"
 
 [profile.release]

guest/rust_storage/storage_program/Cargo.toml

@@ -180,7 +180,8 @@ struct Machine
 	void do_mmap_callback(vCPU&, address_t, size_t, int, int, int, address_t);
 	void set_mmap_callback(mmap_func_t f) { m_mmap_func = std::move(f); }
 
-	uint64_t address_of(const char*) const;
+	uint64_t address_of(std::string_view symbol, const std::vector<uint8_t>&) const;
+	uint64_t address_of(std::string_view symbol, std::string_view binary = {}) const;
 	std::string resolve(uint64_t rip, std::string_view binary = {}) const;
 
 	bool smp_active() const noexcept;

lib/tinykvm/machine.hpp

@@ -164,7 +164,9 @@ void Machine::elf_loader(std::string_view binary, const MachineOptions& options)
 	   stack being above the image base. */
 	const uint32_t STACK_SIZE = (options.stack_size + PageMask()) & ~PageMask();
 	this->m_stack_address = this->m_heap_address + STACK_SIZE;
-	this->m_heap_address = this->m_stack_address;
+	this->m_brk_address   = this->m_stack_address;
+	this->m_brk_end_address = this->m_stack_address + BRK_MAX;
+	this->m_heap_address = this->m_brk_end_address;
 
 	/* Make sure mmap starts at a sane offset */
 	this->mmap_cache().current() = this->m_heap_address;
@@ -268,11 +270,18 @@ static const Elf64_Sym* resolve_symbol(std::string_view binary, const char* name
 	return nullptr;
 }
 
-uint64_t Machine::address_of(const char* name) const
+uint64_t Machine::address_of(std::string_view name, std::string_view binary) const
 {
-	const auto* sym = resolve_symbol(m_binary, name);
+	if (binary.empty())
+		binary = this->m_binary;
+	const auto* sym = resolve_symbol(binary, name.data());
 	return (sym) ? this->m_image_base + sym->st_value : 0x0;
 }
+uint64_t Machine::address_of(std::string_view name, const std::vector<uint8_t>& binary) const
+{
+	return this->address_of(name, std::string_view{
+		reinterpret_cast<const char*>(binary.data()), binary.size()});
+}
 std::string Machine::resolve(uint64_t rip, std::string_view binary) const
 {
 	if (binary.empty())

lib/tinykvm/machine_elf.cpp

@@ -273,6 +273,9 @@ long vCPU::run_once()
 				} else if (UNLIKELY(addr < 0x2000)) {
 					/* Kernel space page fault */
 					this->handle_exception(intr);
+					if (machine().is_remote_connected()) {
+						machine().remote_disconnect();
+					}
 					Machine::machine_exception("Kernel or zero page fault", intr);
 				} else if (addr >= machine().remote_base_address()) {
 					if (this->remote_original_tls_base != 0) {
@@ -432,7 +435,7 @@ void vCPU::handle_exception(uint8_t intr)
 			"*** %s on address 0x%llX\n",
 			amd64_exception_name(intr), sregs.cr2);
 		uint64_t code;
-		machine().unsafe_copy_from_guest(&code, regs.rsp+8,  8);
+		machine().unsafe_copy_from_guest(&code, regs.rsp+16,  8);
 		PRINTER(printer, buffer,
 			"Error code: 0x%lX (%s)\n", code,
 			(code & 0x02) ? "memory write" : "memory read");
@@ -471,7 +474,7 @@ void vCPU::handle_exception(uint8_t intr)
 
 	try {
 		uint64_t off = (has_code) ? (regs.rsp+8) : (regs.rsp+0);
-		if (intr == 14) off += 8;
+		if (intr == 14) off += 16;
 		uint64_t rip, rfl, cs = 0x0, rsp, ss;
 		try {
 			machine().unsafe_copy_from_guest(&rip, off+0,  8);

lib/tinykvm/vcpu_run.cpp

@@ -1,13 +1,15 @@
 #include <tinykvm/machine.hpp>
 #include <cstring>
 #include <cstdio>
+#include <unistd.h>
 #include "assert.hpp"
 #include "load_file.hpp"
 #include "timing.hpp"
 
 #include <tinykvm/rsp_client.hpp>
 #define GUEST_MEMORY   1024UL * 1024 * 1024  /* 1024MB main memory */
 #define GUEST_WORK_MEM 256UL * 1024 * 1024 /* 256MB working memory */
+static const std::string ld_linux_so = "/lib64/ld-linux-x86-64.so.2";
 
 static double timed_action(std::function<void()> action)
 {
@@ -30,9 +32,17 @@ int main(int argc, char** argv)
 		fprintf(stderr, "%s  [guest ELF] [storage ELF]\n", argv[0]);
 		exit(1);
 	}
-	const auto guest_binary   = load_file(argv[1]);
-	const auto storage_binary = load_file(argv[2]);
-	printf(">>> Guest: %s  >>> Storage: %s\n", argv[1], argv[2]);
+	const std::string guest_binary_path = argv[1];
+	auto original_guest_binary = load_file(guest_binary_path);
+	const std::string storage_binary_path = argv[2];
+	const auto storage_binary = load_file(storage_binary_path);
+	printf(">>> Guest: %s  >>> Storage: %s\n", guest_binary_path.c_str(), storage_binary_path.c_str());
+	std::string cwd;
+	{
+		char buf[PATH_MAX];
+		if (getcwd(buf, sizeof(buf)) != nullptr)
+			cwd = buf;
+	}
 
 	tinykvm::Machine::init();
 
@@ -54,33 +64,83 @@ int main(int argc, char** argv)
 		}
 	});
 
+	std::vector<std::string> guest_args;
+	std::vector<uint8_t> guest_binary;
+	const tinykvm::DynamicElf guest_dyn_elf = tinykvm::is_dynamic_elf(
+		std::string_view{(const char*)original_guest_binary.data(), original_guest_binary.size()});
+	if (guest_dyn_elf.is_dynamic) {
+		// Add ld-linux.so.2 as first argument
+		guest_binary = load_file(ld_linux_so);
+		guest_args.push_back(ld_linux_so);
+	} else {
+		guest_binary = original_guest_binary;
+	}
+	guest_args.push_back(guest_binary_path);
+	guest_args.push_back("Hello Main World!");
+
 	/* Setup */
 	const tinykvm::MachineOptions options {
 		.max_mem = GUEST_MEMORY,
 		.max_cow_mem = GUEST_WORK_MEM,
-		.verbose_loader = false
+		.dylink_address_hint = 0x400000, // 4MB
+		.verbose_loader = false,
+		.executable_heap = guest_dyn_elf.is_dynamic,
+		.mmap_backed_files = true,
 	};
 	tinykvm::Machine master_vm {guest_binary, options};
 	master_vm.setup_linux(
-		{"main", "Hello Main World!"},
+		guest_args,
 		{"LC_TYPE=C", "LC_ALL=C", "USER=root"});
 	//master_vm.print_pagetables();
+	master_vm.fds().set_open_readable_callback(
+		[] (std::string&) -> bool {
+		return true;
+	});
+	master_vm.set_verbose_system_calls(getenv("VERBOSE") != nullptr);
+
+	std::vector<std::string> storage_args;
+	storage_args.push_back(storage_binary_path);
+	storage_args.push_back("Hello Storage World!");
 
 	/* Create storage VM */
 	const tinykvm::MachineOptions storage_options {
 		.max_mem = 256ULL << 20, // MB
 		.dylink_address_hint = 0x44000000, // 1GB + 64MB
 		.vmem_base_address = 1ULL << 30, // 1GB
 		.verbose_loader = false,
+		.mmap_backed_files = true,
 	};
 	tinykvm::Machine storage_vm{storage_binary, storage_options};
+	storage_vm.set_verbose_system_calls(getenv("VERBOSE") != nullptr);
+	storage_vm.set_verbose_mmap_syscalls(getenv("VERBOSE") != nullptr);
+	storage_vm.set_verbose_thread_syscalls(getenv("VERBOSE") != nullptr);
+	storage_vm.fds().set_open_readable_callback(
+		[] (std::string&) -> bool {
+		return true;
+	});
+	storage_vm.fds().set_current_working_directory(cwd.c_str());
 	storage_vm.setup_linux(
-		{"storage", "Hello Storage World!"},
+		storage_args,
 		{"LC_TYPE=C", "LC_ALL=C", "USER=root"});
 	storage_vm.run(5.0f);
 
 	master_vm.remote_connect(storage_vm, false);
 
+	static uint64_t callback_address = 0x0;
+	master_vm.install_unhandled_syscall_handler(
+	[] (tinykvm::vCPU& cpu, unsigned sysnum) {
+		auto& regs = cpu.registers();
+		switch (sysnum) {
+			case 0x10001: // Set callback address
+				callback_address = regs.rdi;
+				return;
+			default:
+				printf("Unhandled master VM syscall: %u\n", sysnum);
+				regs.rax = -ENOSYS;
+				cpu.set_registers(regs);
+		}
+	});
+
 	auto tdiff = timed_action([&] {
 		try {
 			master_vm.run();
@@ -107,15 +167,15 @@ int main(int argc, char** argv)
 	assert(vm.is_remote_connected());
 
 	/* Call 'do_calculation' with 21 as argument */
-	const auto call_addr = vm.address_of("do_calculation");
-	if (call_addr == 0) {
+	if (callback_address == 0) {
 		fprintf(stderr, "Error: no do_calculation() in guest\n");
 		exit(1);
 	}
+	printf("Calling do_calculation() @ 0x%lX\n", callback_address);
 	for (int i = 0; i < 100; i++)
-		vm.timed_vmcall(call_addr, 5.0f, 21);
+		vm.timed_vmcall(callback_address, 5.0f, 21);
 	auto fork_tdiff = timed_action([&] {
-		vm.timed_vmcall(call_addr, 5.0f, 21);
+		vm.timed_vmcall(callback_address, 5.0f, 21);
 	});
 	printf("Fork call time: %.2fus Return value: %ld\n", fork_tdiff*1e6, vm.return_value());
 }