IH-533: Remove usage of forkexecd daemon to execute processes

Forkexecd was written to avoid some issues with Ocaml and
multi-threading.
Instead use C code to launch processes and avoid these issues.
Interface remains unchanged from Ocaml side but implementation rely
entirely on C code.
vfork() is used to avoid performance memory issue.
Reap of the processes are done directly.
Code automatically reap child processes to avoid zombies.
One small helper is used to better separate Ocaml and C code and
handling syslog redirection. This allows to better debug in
case of issues.
Syslog handling is done in a separate process allowing to restart
the toolstack and keep launched programs running;
note that even with forkexecd daemon one process was used for this
purpose.
Code tries to keep compatibility with forkexecd, in particular:
- SIGPIPE is ignored in the parent;
- /dev/null is open with O_WRONLY even for stdin;
- file descriptors are limited to 1024.
We use close_range (if available) to reduce system calls to close
file descriptors.
Cgroup is set to avoid systemd closing processes on toolstack restart.
There's a fuzzer program to check file remapping algorithm; for this
reason the algorithm is in a separate file.

To turn internal debug on you need to set FORKEXECD_DEBUG_LOGS C
preprocessor macro to 1.

Signed-off-by: Frediano Ziglio <[email protected]>

Makefile

@@ -254,6 +254,8 @@ install: build doc sdk doc-json
 	install -m 644 _build/default/ocaml/networkd/bin/xcp-networkd.1 $(DESTDIR)/usr/share/man/man1/xcp-networkd.1
 # wsproxy
 	install -m 755 _build/install/default/bin/wsproxy $(DESTDIR)$(LIBEXECDIR)/wsproxy
+# forkexecd
+	install -m 755 _build/default/ocaml/forkexecd/helper/vfork_helper $(DESTDIR)/usr/libexec/xapi/vfork_helper
 # dune can install libraries and several other files into the right locations
 	dune install --destdir=$(DESTDIR) --prefix=$(PREFIX) --libdir=$(LIBDIR) --mandir=$(MANDIR) \
 		xapi-client xapi-schema xapi-consts xapi-cli-protocol xapi-datamodel xapi-types \

ocaml/forkexecd/helper/Makefile

@@ -0,0 +1,35 @@
+## Set some macro but not override environment ones
+CFLAGS ?= -O2 -g -Wall -Werror
+LDFLAGS ?=
+
+all:: vfork_helper
+
+clean::
+	rm -f vfork_helper *.o
+
+%.o: %.c
+	gcc $(CFLAGS) -c -o $@ $<
+
+vfork_helper: vfork_helper.o close_from.o syslog.o
+	gcc $(CFLAGS) $(LDFLAGS) -o $@ $^ -pthread
+
+close_from.o: close_from.h Makefile
+syslog.o: syslog.h Makefile
+vfork_helper.o: redirect_algo.h Makefile
+
+## Fuzzer uses AFL (American Fuzzy Lop).
+##
+## Use "make fuzz" to build and launch the fuzzer
+##
+## Use "make show" to look at the first failures (if found).
+
+fuzz::
+	afl-gcc $(CFLAGS) -Wall -Werror -o algo_fuzzer algo_fuzzer.c
+	rm -rf testcase_dir
+	mkdir testcase_dir
+	echo maomaoamaoaoao > testcase_dir/test1
+	rm -rf findings_dir/
+	afl-fuzz -i testcase_dir -o findings_dir -D -- ./algo_fuzzer
+
+show::
+	cat "$$(ls -1 findings_dir/default/crashes/id* | head -1)" | ./algo_fuzzer

ocaml/forkexecd/helper/algo_fuzzer.c

@@ -0,0 +1,246 @@
+
+/*
+ * Copyright (C) Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#undef NDEBUG
+#define DEBUG 1
+
+#if DEBUG
+#define log(fmt, ...) printf(fmt "\n", ##__VA_ARGS__)
+#else
+#define log(fmt, ...) do {} while(0)
+#endif
+
+// include as first file to make sure header is self container
+#include "redirect_algo.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <assert.h>
+
+static int fake_close(int fd);
+
+typedef struct {
+    bool open;
+    bool cloexec;
+    char *name;
+} fd;
+
+#define NUM_FDS 4096
+static fd fds[NUM_FDS];
+
+static bool
+fake_close_fds_from(int fd_from)
+{
+    for (int fd = fd_from; fd < NUM_FDS; ++fd)
+        fake_close(fd);
+
+    return true;
+}
+
+#define O_WRONLY 1
+static int
+fake_open(const char *fn, int dummy)
+{
+    for (int i = 0; i < NUM_FDS; ++i)
+        if (!fds[i].open) {
+            assert(fds[i].name == NULL);
+            fds[i].name = strdup(fn);
+            fds[i].open = true;
+            fds[i].cloexec = false;
+            return i;
+        }
+    assert(0);
+    return -1;
+}
+
+static int
+fake_close(int fd)
+{
+    assert(fd >= 0);
+    assert(fd < NUM_FDS);
+    if (!fds[fd].open) {
+        errno = EBADF;
+        return -1;
+    }
+    fds[fd].open = false;
+    free(fds[fd].name);
+    fds[fd].name = NULL;
+    return 0;
+}
+
+static int
+fake_dup2(int from, int to)
+{
+    assert(from >= 0 && from < NUM_FDS);
+    assert(to >= 0 && to < NUM_FDS);
+    assert(fds[from].open);
+    assert(from != to);
+    free(fds[to].name);
+    fds[to].open = true;
+    fds[to].name = strdup(fds[from].name);
+    fds[to].cloexec = false;
+    return 0;
+}
+
+static int
+fake_fcntl(int fd)
+{
+    assert(fd >= 0 && fd < NUM_FDS);
+    assert(fds[fd].open);
+    fds[fd].cloexec = false;
+    return 0;
+}
+
+int main(int argc, char **argv)
+{
+    // Input where a given FD goes??
+    // No, not enough, can be duplicated.
+    // Numbers >4096 in 2 bytes not file descriptor,
+    // (-1 for standard, skip for normal).
+    // We should add some random fds.
+    enum { MAX_FILE_BUF = 2048 };
+    uint16_t file_buf[MAX_FILE_BUF];
+    size_t read = fread(file_buf, 2, MAX_FILE_BUF, stdin);
+    if (read < 3)
+        return 0;
+
+    static const char standard_names[][8] = {
+        "stdin", "stdout", "stderr"
+    };
+    int num_mappings = 0;
+    uint16_t *num = file_buf;
+    mapping mappings[MAX_FILE_BUF];
+    int i = 0;
+    for (i = 0; i < 3; ++i) {
+        mapping *m = &mappings[num_mappings++];
+        m->uuid = standard_names[i];
+        uint16_t n = *num++;
+        m->current_fd = n < NUM_FDS ? n : -1;
+        m->wanted_fd = i;
+    }
+    for (; i < read; ++i) {
+        uint16_t n = *num++;
+        if (n >= NUM_FDS)
+            continue;
+
+        mapping *m = &mappings[num_mappings++];
+        m->current_fd = n;
+        m->wanted_fd = -1;
+        char buf[64];
+        sprintf(buf, "file%d", i);
+        m->uuid = strdup(buf);
+    }
+    if (num_mappings > MAX_TOTAL_MAPPINGS)
+        return 0;
+
+    for (unsigned n = 0; n < num_mappings; ++n) {
+        mapping *m = &mappings[n];
+        int fd = m->current_fd;
+        if (fd < 0)
+            continue;
+        fake_close(fd);
+        fds[fd].open = true;
+        fds[fd].name = strdup(m->uuid);
+        fds[fd].cloexec = true;
+    }
+
+    // Check in the final file mapping all valid mappings
+    // have an open file descriptor.
+    // There should be no duplicate numbers in current_fd.
+    // current_fd must be in a range.
+    // Only if wanted_fd >= 0 current_fd can be -1.
+    // There should be a correspondance between input and output names.
+    // If current_fd was -1 it will still be -1.
+    // If wanted_fd >= 0 current_fd should be the same.
+
+    fd_operation operations[MAX_OPERATIONS];
+    int num_operations =
+        redirect_mappings(mappings, num_mappings, operations);
+    assert(num_operations > 0);
+    assert(num_operations <= MAX_OPERATIONS);
+
+    for (int i = 0; i < num_operations; ++i) {
+        const fd_operation* op = &operations[i];
+        log("op %d %d %d", op->fd_from, op->fd_to, op->operation);
+        switch (op->operation) {
+        case FD_OP_DUP:
+            if (op->fd_from == op->fd_to)
+                fake_fcntl(op->fd_from);
+            else
+                fake_dup2(op->fd_from, op->fd_to);
+            break;
+        case FD_OP_MOVE:
+            assert(op->fd_from != op->fd_to);
+            fake_dup2(op->fd_from, op->fd_to);
+            fake_close(op->fd_from);
+            break;
+        case FD_OP_DEVNULL:
+            // first close old, then create new one
+            fake_close(op->fd_to);
+            // TODO ideally we want read only for input for Ocaml did the same...
+            assert(fake_open("/dev/null", O_WRONLY) == op->fd_to);
+            break;
+        case FD_OP_CLOSE_FROM:
+            fake_close_fds_from(op->fd_from);
+            break;
+        default:
+            assert(0);
+        }
+    }
+
+    // check files opened
+    for (int fd = 0; fd < NUM_FDS; ++fd)
+        assert(fds[fd].open == (fd < num_mappings));
+
+    for (int fd = 0; fd < num_mappings; ++fd) {
+        assert(fds[fd].cloexec == false);
+        log("file %d %s", fd, fds[fd].name);
+    }
+
+    // Check in the final file mapping all valid mappings
+    // has an open file descriptor.
+    bool already_found[NUM_FDS] = { false, };
+    for (unsigned n = 0; n < num_mappings; ++n) {
+        const int fd = mappings[n].current_fd;
+        const int wanted = mappings[n].wanted_fd;
+        if (fd >= 0) {
+            assert(fd < NUM_FDS);
+            assert(fds[fd].open);
+
+            // There should be no duplicate numbers in current_fd.
+            assert(!already_found[fd]);
+            already_found[fd] = true;
+        } else {
+            // Only if wanted_fd >= 0 current_fd can be -1.
+            assert(mappings[n].wanted_fd >= 0);
+            assert(fd == -1);
+        }
+
+        // If wanted_fd >= 0 current_fd should be the same.
+        if (wanted >= 0)
+            assert(wanted == fd || fd == -1);
+
+        // current_fd must be in a range.
+        assert(fd >= -1);
+        assert(fd < num_mappings);
+    }
+
+    // There should be a correspondance between input and output names.
+    // If current_fd was -1 it will still be -1.
+}

ocaml/forkexecd/helper/close_from.c

@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "close_from.h"
+
+#include <stdlib.h>
+#include <errno.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/resource.h>
+
+#ifdef __linux__
+#include <sys/syscall.h>
+#endif
+
+// try to use close_range on Linux even if not defined by headers
+#if defined(__linux__) && !defined(SYS_close_range)
+#  if defined(__alpha__)
+#    define SYS_close_range 546
+#  elif defined(__amd64__) || defined(__x86_64__) || defined(__arm__) || \
+        defined(__aarch64__) || defined(__hppa__) || defined(__i386__) || \
+        defined(__ia64__) || defined(__m68k__) || defined(__mips__) || \
+        defined(__powerpc__) || defined(__powerpc64__) || defined(__sparc__) || \
+        defined(__s390x__)
+#    define SYS_close_range 436
+#  endif
+#endif
+
+bool
+close_fds_from(int fd_from)
+{
+    // first method, use close_range
+#if (defined(__linux__) && defined(SYS_close_range)) \
+    || (defined(__FreeBSD__) && defined(CLOSE_RANGE_CLOEXEC))
+    static bool close_range_supported = true;
+    if (close_range_supported) {
+#if defined(__linux__)
+        if (syscall(SYS_close_range, fd_from, ~0U, 0) == 0)
+#else
+        if (close_range(fd_from, ~0U, 0) == 0)
+#endif
+            return true;
+
+        if (errno == ENOSYS)
+            close_range_supported = false;
+    }
+#endif
+
+    // second method, read fds list from /proc
+    DIR *dir = opendir("/proc/self/fd");
+    if (dir) {
+        const int dir_fd = dirfd(dir);
+        struct dirent *ent;
+        while ((ent = readdir(dir)) != NULL) {
+            char *end = NULL;
+            unsigned long fd = strtoul(ent->d_name, &end, 10);
+            if (end == NULL || *end)
+                continue;
+            if (fd >= fd_from && fd != dir_fd)
+                close(fd);
+        }
+        closedir(dir);
+        return true;
+    }
+
+    // third method, use just a loop
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_NOFILE, &limit) < 0)
+        return false;
+    for (int fd = fd_from; fd < limit.rlim_cur; ++ fd)
+        close(fd);
+
+    return true;
+}

ocaml/forkexecd/helper/close_from.h

@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+
+bool close_fds_from(int fd);