test open with multiple procs

t/Makefile.am

@@ -51,6 +51,7 @@ libexec_PROGRAMS = \
 	std/stdio-static.t \
 	sys/sysio-gotcha.t \
 	sys/sysio-static.t \
+	multi/stdio-static.t \
 	unifyfs_unmount.t
 
 
@@ -153,6 +154,12 @@ std_stdio_static_t_CPPFLAGS = $(test_cppflags)
 std_stdio_static_t_LDADD = $(test_static_ldadd)
 std_stdio_static_t_LDFLAGS = $(test_static_ldflags)
 
+multi_stdio_static_t_SOURCES = multi/open_multi.c
+
+multi_stdio_static_t_CPPFLAGS = $(test_cppflags)
+multi_stdio_static_t_LDADD = $(test_static_ldadd)
+multi_stdio_static_t_LDFLAGS = $(test_static_ldflags)
+
 unifyfs_unmount_t_SOURCES = unifyfs_unmount.c
 unifyfs_unmount_t_CPPFLAGS = $(test_cppflags)
 unifyfs_unmount_t_LDADD = $(test_static_ldadd)

t/multi/open_multi.c

@@ -0,0 +1,191 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include "t/lib/tap.h"
+#include "t/lib/testutil.h"
+
+#include "mpi.h"
+#include "unifyfs.h"
+
+#define all_ok(condition, comm, ...) \
+    do { \
+        int input = (int) condition; \
+        int output; \
+        MPI_Allreduce(&input, &output, 1, MPI_INT, MPI_LAND, comm); \
+        int rank; \
+        MPI_Comm_rank(comm, &rank); \
+        if (rank == 0) { \
+            ok_at_loc(__FILE__, __LINE__, output, __VA_ARGS__, NULL); \
+        } \
+    } while (0)
+
+int mpi_sum(int val)
+{
+    int sum;
+    MPI_Allreduce(&val, &sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+    return sum;
+}
+
+off_t getsize(char* file)
+{
+    off_t size = (off_t)-1;
+    struct stat buf;
+    int rc = stat(file, &buf);
+    if (rc == 0) {
+        size = buf.st_size;
+    }
+    return size;
+}
+
+int main(int argc, char* argv[])
+{
+    char mountpoint[] = "/unifyfs";
+
+    MPI_Init(&argc, &argv);
+
+    int rank, ranks;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &ranks);
+
+    unifyfs_mount(mountpoint, rank, ranks, 0);
+
+    char file[256];
+    sprintf(file, "%s/testfile", mountpoint);
+
+    size_t bufsize = 1024*1024;
+    char* buf = (char*) malloc(bufsize);
+    memset(buf, rank, bufsize);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    /* create a file in exclusive mode,
+     * one rank should win, all others should get EEXIST */
+    errno = 0;
+    int success = 0;
+    int eexist  = 0;
+    int fd = open(file, O_WRONLY | O_CREAT | O_EXCL, 0700);
+    if (fd >= 0) {
+        success = 1;
+        close(fd);
+    } else if (errno == EEXIST) {
+        eexist = 1;
+    }
+
+    /* one rank should win */
+    int sum = mpi_sum(success);
+    all_ok(sum == 1, MPI_COMM_WORLD,
+        "More than one process opened file in exclusive mode");
+
+    /* all others should get EEXIST */
+    sum = mpi_sum(eexist);
+    all_ok(sum == (ranks - 1), MPI_COMM_WORLD,
+        "All but one process should get EEXIST when opening file in exclusive mode");
+
+    /* all delete,
+     * one rank should win, all others should get ENOENT */
+    errno = 0;
+    success = 0;
+    int enoent = 0;
+    int rc = unlink(file);
+    if (rc == 0) {
+        success = 1;
+    } else if (errno == ENOENT) {
+        enoent = 1;
+    }
+
+    /* one winner */
+    sum = mpi_sum(success);
+    all_ok(sum == 1, MPI_COMM_WORLD,
+        "More than one process got success on unlink of the same file");
+
+    /* everyone else should get ENOENT */
+    sum = mpi_sum(enoent);
+    all_ok(sum == (ranks - 1), MPI_COMM_WORLD,
+        "All but one process should get ENOENT when unlinking the same file");
+
+    /* all create, this time not exclusive */
+    errno = 0;
+    success = 0;
+    fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0700);
+    if (fd >= 0) {
+        success = 1;
+        close(fd);
+    }
+
+    /* all should succeed */
+    all_ok(success, MPI_COMM_WORLD,
+        "All processes should open file with O_CREAT and not O_EXCL");
+
+    /* open file for writing */
+    errno = 0;
+    success = 0;
+    fd = open(file, O_WRONLY);
+    if (fd >= 0) {
+        success = 1;
+        close(fd);
+    }
+
+    /* all should succeeed */
+    all_ok(success, MPI_COMM_WORLD,
+        "All processes should open file with O_CREAT for writing");
+
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    unlink(file);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    /* have all ranks write to a different section of the file,
+     * then open file with truncate on one rank to verify size change */
+    success = 0;
+    fd = open(file, O_WRONLY | O_CREAT, 0700);
+    if (fd >= 0) {
+        success = 1;
+        off_t offset = (off_t) (rank * bufsize);
+        ssize_t nwritten = pwrite(fd, buf, bufsize, offset);
+        if (nwritten != bufsize) {
+            success = 0;
+        }
+        fsync(fd);
+        close(fd);
+    }
+    MPI_Barrier(MPI_COMM_WORLD);
+    if (rank == 0) {
+        /* all ranks should have written some data */
+        off_t size = getsize(file);
+        ok(size == bufsize * ranks,
+            "File size %lu does not match expected size %lu",
+            size, bufsize * ranks);
+
+        /* create file with truncate */
+        fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0700);
+        if (fd >= 0) {
+            close(fd);
+        }
+
+        /* now file should be 0 length again */
+        size = getsize(file);
+        ok(size == 0,
+            "File size %lu does not match expected size %lu",
+            size, 0);
+    }
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    unlink(file);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    free(buf);
+
+    unifyfs_unmount();
+
+    MPI_Finalize();
+
+    return 0;
+}