From 9c947f15388dbe7679f0d403702aa3049371b023 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 10 Dec 2020 17:34:15 -0800 Subject: [PATCH 1/2] test open with multiple procs --- open_multi.c | 176 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 open_multi.c diff --git a/open_multi.c b/open_multi.c new file mode 100644 index 000000000..921eee1f9 --- /dev/null +++ b/open_multi.c @@ -0,0 +1,176 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "mpi.h" +#include "unifyfs.h" + +int mpi_sum(int val) +{ + int sum; + MPI_Allreduce(&val, &sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + return sum; +} + +off_t getsize(char* file) +{ + off_t size = (off_t)-1; + struct stat buf; + int rc = stat(file, &buf); + if (rc == 0) { + size = buf.st_size; + } + return size; +} + +int main(int argc, char* argv[]) +{ + char mountpoint[] = "/unifyfs"; + + MPI_Init(&argc, &argv); + + int rank, ranks; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &ranks); + + int ret = unifyfs_mount(mountpoint, rank, ranks, 0); + + char file[256]; + sprintf(file, "%s/testfile", mountpoint); + + size_t bufsize = 1024*1024; + char* buf = (char*) malloc(bufsize); + memset(buf, rank, bufsize); + + MPI_Barrier(MPI_COMM_WORLD); + + /* create a file in exclusive mode, + * one rank should win, all others should get EEXIST */ + errno = 0; + int success = 0; + int eexist = 0; + int fd = open(file, O_WRONLY | O_CREAT | O_EXCL); + if (fd >= 0) { + success = 1; + close(fd); + } else if (errno == EEXIST) { + eexist = 1; + } + + /* one rank should win */ + int sum = mpi_sum(success); + if (sum != 1) { + } + + /* all others should get EEXIST */ + sum = mpi_sum(eexist); + if (sum != (ranks - 1)) { + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* all delete, + * one rank should win, all others should get ENOENT */ + errno = 0; + success = 0; + int enoent = 0; + int rc = unlink(file); + if (rc == 0) { + success = 1; + } else if (errno == ENOENT) { + enoent = 1; + } + + /* one winner */ + sum = mpi_sum(success); + if (sum != 1) { + } + + /* everyone else should get ENOENT */ + sum = mpi_sum(enoent); + if (sum != (ranks - 1)) { + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* all create, this time not exclusive */ + errno = 0; + success = 0; + fd = open(file, O_WRONLY | O_CREAT | O_TRUNC); + if (fd >= 0) { + success = 1; + close(fd); + } + + /* all should succeed */ + sum = mpi_sum(success); + if (sum != ranks) { + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* open file for writing */ + errno = 0; + success = 0; + fd = open(file, O_WRONLY); + if (fd >= 0) { + success = 1; + close(fd); + } + + /* all should succeeed */ + sum = mpi_sum(success); + if (sum != ranks) { + } + + MPI_Barrier(MPI_COMM_WORLD); + + unlink(file); + + MPI_Barrier(MPI_COMM_WORLD); + + /* have all ranks write to a different section of the file, + * then open file with truncate on one rank to verify size change */ + fd = open(file, O_WRONLY | O_CREAT); + if (fd >= 0) { + off_t offset = (off_t) (rank * bufsize); + pwrite(fd, buf, bufsize, offset); + fsync(fd); + close(fd); + } + MPI_Barrier(MPI_COMM_WORLD); + if (rank == 0) { + /* all ranks should have written some data */ + off_t size = getsize(file); + if (size != bufsize * ranks) { + } + + /* create file with truncate */ + fd = open(file, O_WRONLY | O_CREAT | O_TRUNC); + if (fd >= 0) { + close(fd); + } + + /* now file should be 0 length again */ + size = getsize(file); + if (size != 0) { + } + } + MPI_Barrier(MPI_COMM_WORLD); + + unlink(file); + + MPI_Barrier(MPI_COMM_WORLD); + + free(buf); + + unifyfs_unmount(); + + MPI_Finalize(); + + return 0; +} From b87b5a5d81f3f729d750edc2f59e62447fca891d Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 14 Dec 2020 13:59:47 -0800 Subject: [PATCH 2/2] move multi open test to new t/multi directory --- t/Makefile.am | 7 +++ open_multi.c => t/multi/open_multi.c | 75 +++++++++++++++++----------- 2 files changed, 52 insertions(+), 30 deletions(-) rename open_multi.c => t/multi/open_multi.c (61%) diff --git a/t/Makefile.am b/t/Makefile.am index abfe4491e..bd9e1152f 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -51,6 +51,7 @@ libexec_PROGRAMS = \ std/stdio-static.t \ sys/sysio-gotcha.t \ sys/sysio-static.t \ + multi/stdio-static.t \ unifyfs_unmount.t @@ -153,6 +154,12 @@ std_stdio_static_t_CPPFLAGS = $(test_cppflags) std_stdio_static_t_LDADD = $(test_static_ldadd) std_stdio_static_t_LDFLAGS = $(test_static_ldflags) +multi_stdio_static_t_SOURCES = multi/open_multi.c + +multi_stdio_static_t_CPPFLAGS = $(test_cppflags) +multi_stdio_static_t_LDADD = $(test_static_ldadd) +multi_stdio_static_t_LDFLAGS = $(test_static_ldflags) + unifyfs_unmount_t_SOURCES = unifyfs_unmount.c unifyfs_unmount_t_CPPFLAGS = $(test_cppflags) unifyfs_unmount_t_LDADD = $(test_static_ldadd) diff --git a/open_multi.c b/t/multi/open_multi.c similarity index 61% rename from open_multi.c rename to t/multi/open_multi.c index 921eee1f9..e0413df6d 100644 --- a/open_multi.c +++ b/t/multi/open_multi.c @@ -1,14 +1,30 @@ #include #include +#include #include #include #include #include #include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + #include "mpi.h" #include "unifyfs.h" +#define all_ok(condition, comm, ...) \ + do { \ + int input = (int) condition; \ + int output; \ + MPI_Allreduce(&input, &output, 1, MPI_INT, MPI_LAND, comm); \ + int rank; \ + MPI_Comm_rank(comm, &rank); \ + if (rank == 0) { \ + ok_at_loc(__FILE__, __LINE__, output, __VA_ARGS__, NULL); \ + } \ + } while (0) + int mpi_sum(int val) { int sum; @@ -37,7 +53,7 @@ int main(int argc, char* argv[]) MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); - int ret = unifyfs_mount(mountpoint, rank, ranks, 0); + unifyfs_mount(mountpoint, rank, ranks, 0); char file[256]; sprintf(file, "%s/testfile", mountpoint); @@ -53,7 +69,7 @@ int main(int argc, char* argv[]) errno = 0; int success = 0; int eexist = 0; - int fd = open(file, O_WRONLY | O_CREAT | O_EXCL); + int fd = open(file, O_WRONLY | O_CREAT | O_EXCL, 0700); if (fd >= 0) { success = 1; close(fd); @@ -63,15 +79,13 @@ int main(int argc, char* argv[]) /* one rank should win */ int sum = mpi_sum(success); - if (sum != 1) { - } + all_ok(sum == 1, MPI_COMM_WORLD, + "More than one process opened file in exclusive mode"); /* all others should get EEXIST */ sum = mpi_sum(eexist); - if (sum != (ranks - 1)) { - } - - MPI_Barrier(MPI_COMM_WORLD); + all_ok(sum == (ranks - 1), MPI_COMM_WORLD, + "All but one process should get EEXIST when opening file in exclusive mode"); /* all delete, * one rank should win, all others should get ENOENT */ @@ -87,31 +101,26 @@ int main(int argc, char* argv[]) /* one winner */ sum = mpi_sum(success); - if (sum != 1) { - } + all_ok(sum == 1, MPI_COMM_WORLD, + "More than one process got success on unlink of the same file"); /* everyone else should get ENOENT */ sum = mpi_sum(enoent); - if (sum != (ranks - 1)) { - } - - MPI_Barrier(MPI_COMM_WORLD); + all_ok(sum == (ranks - 1), MPI_COMM_WORLD, + "All but one process should get ENOENT when unlinking the same file"); /* all create, this time not exclusive */ errno = 0; success = 0; - fd = open(file, O_WRONLY | O_CREAT | O_TRUNC); + fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0700); if (fd >= 0) { success = 1; close(fd); } /* all should succeed */ - sum = mpi_sum(success); - if (sum != ranks) { - } - - MPI_Barrier(MPI_COMM_WORLD); + all_ok(success, MPI_COMM_WORLD, + "All processes should open file with O_CREAT and not O_EXCL"); /* open file for writing */ errno = 0; @@ -123,9 +132,8 @@ int main(int argc, char* argv[]) } /* all should succeeed */ - sum = mpi_sum(success); - if (sum != ranks) { - } + all_ok(success, MPI_COMM_WORLD, + "All processes should open file with O_CREAT for writing"); MPI_Barrier(MPI_COMM_WORLD); @@ -135,10 +143,15 @@ int main(int argc, char* argv[]) /* have all ranks write to a different section of the file, * then open file with truncate on one rank to verify size change */ - fd = open(file, O_WRONLY | O_CREAT); + success = 0; + fd = open(file, O_WRONLY | O_CREAT, 0700); if (fd >= 0) { + success = 1; off_t offset = (off_t) (rank * bufsize); - pwrite(fd, buf, bufsize, offset); + ssize_t nwritten = pwrite(fd, buf, bufsize, offset); + if (nwritten != bufsize) { + success = 0; + } fsync(fd); close(fd); } @@ -146,19 +159,21 @@ int main(int argc, char* argv[]) if (rank == 0) { /* all ranks should have written some data */ off_t size = getsize(file); - if (size != bufsize * ranks) { - } + ok(size == bufsize * ranks, + "File size %lu does not match expected size %lu", + size, bufsize * ranks); /* create file with truncate */ - fd = open(file, O_WRONLY | O_CREAT | O_TRUNC); + fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0700); if (fd >= 0) { close(fd); } /* now file should be 0 length again */ size = getsize(file); - if (size != 0) { - } + ok(size == 0, + "File size %lu does not match expected size %lu", + size, 0); } MPI_Barrier(MPI_COMM_WORLD);