From 4a37dff50e94246a73d6c81ff58777420cc85697 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Mon, 30 Dec 2019 17:27:37 -0800 Subject: [PATCH] Add bootstrap.sh, and selectable AXL transfer types - Add bootstrap.sh script to download and build dependencies - Add an optional axl_xfer_str argument to Filo_Fetch()/ Filo_Flush()/Filo_Flush_start(). This allows you to set the AXL transfer type on a per-stream basis. - Remove some trailing whitespace - Update travis.yml per @CamStan's instructions to get rid of an error. --- .travis.yml | 2 - README.md | 37 ++-------- bootstrap.sh | 65 +++++++++++++++++ cmake/FindAXL.cmake | 4 +- cmake/FindKVTREE.cmake | 4 +- cmake/FindSPATH.cmake | 4 +- src/filo.c | 127 +++++++++++++++++++++++++++------ src/filo.h | 14 ++-- test/test_filo.c | 7 +- test/test_filo_async.c | 4 +- test/test_filo_corrupt_fetch.c | 4 +- 11 files changed, 198 insertions(+), 74 deletions(-) create mode 100755 bootstrap.sh diff --git a/.travis.yml b/.travis.yml index 8f35bf0..6bfe4ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,6 @@ langauge: c dist: trusty compiler: - - gcc - - clang matrix: include: diff --git a/README.md b/README.md index 83b225a..a5283fb 100644 --- a/README.md +++ b/README.md @@ -15,42 +15,13 @@ For usage, see [src/filo.h](src/filo.h), [test/test\_filo.c](test/test_filo.c), To build dependencies: - git clone git@github.com:ECP-VeloC/KVTree.git KVTree.git - git clone git@github.com:ECP-VeloC/AXL.git AXL.git - git clone git@github.com:ECP-VeloC/spath.git spath.git + ./bootstrap.sh - mkdir install - - mkdir build - cd build - cmake -DCMAKE_INSTALL_PREFIX=../install -DMPI=ON ../KVTree.git - make clean - make - make install - make test - cd .. - - rm -rf build - mkdir build - cd build - cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=../install ../AXL.git - make clean - make - make install - cd .. +To build filo: - rm -rf build mkdir build cd build - cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=../install ../spath.git - make clean - make - make install - cd .. - -To build filo: - - cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=./install -DWITH_KVTREE_PREFIX=`pwd`/install -DWITH_AXL_PREFIX=`pwd`/install -DWITH_SPATH_PREFIX=`pwd`/install . + cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=../install .. make make install @@ -63,7 +34,7 @@ All the tests can be run with: To build a test for the filo API: - mpicc -g -O0 -o test_filo test_filo.c -I../install/include -L../install/lib64 -I../src -L../src -lkvtree -laxl -lspath -lfilo + mpicc -g -O0 -o test_filo ../test/test_filo.c -I../install/include -L../install/lib64 -I../src -L../src -lkvtree -laxl -lspath -lfilo ## Release diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000..e1b340b --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# This is an easy-bake script to download and build all FILO's external +# dependencies. The dependencies will be built in filo/deps/ and +# installed to filo/install/ +# +# Usage: +# +# ./bootstrap.sh + +ROOT="$(pwd)" + +mkdir -p deps +mkdir -p install +INSTALL_DIR=$ROOT/install + +cd deps + +repos=(https://github.com/ECP-Veloc/KVTree.git + https://github.com/ECP-Veloc/AXL.git + https://github.com/ECP-Veloc/spath.git +) + +for i in "${repos[@]}" ; do + # Get just the name of the project (like "mercury") + name=$(basename $i | sed 's/\.git//g') + if [ -d $name ] ; then + echo "$name already exists, skipping it" + else + git clone $i + fi +done + +cd KVTree +mkdir -p build && cd build +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DMPI=ON .. +make -j `nproc` +make install +cd ../.. + +cd AXL +mkdir -p build && cd build +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DAXL_ASYNC_DAEMON=OFF -DMPI=ON .. +make -j `nproc` +make install +cd ../.. + +# spath +cd spath +mkdir -p build && cd build +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DMPI=ON .. +make -j `nproc` +make install +#make test + +cd "$ROOT" + +echo "*************************************************************************" +echo "Dependencies are all built. You can now build FILO with:" +echo "" +echo " mkdir -p build && cd build" +echo " cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR ../" +echo " make" +echo "" +echo "*************************************************************************" diff --git a/cmake/FindAXL.cmake b/cmake/FindAXL.cmake index 41d2a86..e2fee1b 100644 --- a/cmake/FindAXL.cmake +++ b/cmake/FindAXL.cmake @@ -10,12 +10,12 @@ FIND_PATH(WITH_AXL_PREFIX FIND_LIBRARY(AXL_LIBRARIES NAMES axl - HINTS ${WITH_AXL_PREFIX}/lib + HINTS ${WITH_AXL_PREFIX}/lib install/lib64 ) FIND_PATH(AXL_INCLUDE_DIRS NAMES axl.h - HINTS ${WITH_AXL_PREFIX}/include + HINTS ${WITH_AXL_PREFIX}/include install/include ) INCLUDE(FindPackageHandleStandardArgs) diff --git a/cmake/FindKVTREE.cmake b/cmake/FindKVTREE.cmake index 7f30ad5..17a3172 100644 --- a/cmake/FindKVTREE.cmake +++ b/cmake/FindKVTREE.cmake @@ -10,12 +10,12 @@ FIND_PATH(WITH_KVTREE_PREFIX FIND_LIBRARY(KVTREE_LIBRARIES NAMES kvtree - HINTS ${WITH_KVTREE_PREFIX}/lib + HINTS ${WITH_KVTREE_PREFIX}/lib install/lib64 ) FIND_PATH(KVTREE_INCLUDE_DIRS NAMES kvtree.h - HINTS ${WITH_KVTREE_PREFIX}/include + HINTS ${WITH_KVTREE_PREFIX}/include install/include ) INCLUDE(FindPackageHandleStandardArgs) diff --git a/cmake/FindSPATH.cmake b/cmake/FindSPATH.cmake index a916b7f..6e1ccbb 100644 --- a/cmake/FindSPATH.cmake +++ b/cmake/FindSPATH.cmake @@ -10,12 +10,12 @@ FIND_PATH(WITH_SPATH_PREFIX FIND_LIBRARY(SPATH_LIBRARIES NAMES spath - HINTS ${WITH_SPATH_PREFIX}/lib + HINTS ${WITH_SPATH_PREFIX}/lib install/lib64 ) FIND_PATH(SPATH_INCLUDE_DIRS NAMES spath.h - HINTS ${WITH_SPATH_PREFIX}/include + HINTS ${WITH_SPATH_PREFIX}/include install/include ) INCLUDE(FindPackageHandleStandardArgs) diff --git a/src/filo.c b/src/filo.c index 686902b..1941e13 100644 --- a/src/filo.c +++ b/src/filo.c @@ -360,14 +360,62 @@ static int filo_fetch_files_list( } #endif -static int filo_axl(int num_files, const char** src_filelist, const char** dest_filelist) +/* + * Given an AXL transfer string (like "bbapi") return a axl_xfer_t. + * + * Default to "pthread" if axl_xfer_str = NULL, since it has good performance + * and works across all filesystems. + */ +static axl_xfer_t +axl_xfer_str_to_type(const char *axl_xfer_str) +{ + int i; + struct { + char *str; + axl_xfer_t type; + } axl_str_to_type[] = { + {"default", AXL_XFER_DEFAULT}, + {"native", AXL_XFER_NATIVE}, + {"pthread", AXL_XFER_PTHREAD}, + {"sync", AXL_XFER_SYNC}, + {"dw", AXL_XFER_ASYNC_DW}, + {"bbapi", AXL_XFER_ASYNC_BBAPI}, + {"cprr", AXL_XFER_ASYNC_CPPR}, + {"AXL_XFER_SYNC", AXL_XFER_SYNC}, + {"AXL_XFER_PTHREAD", AXL_XFER_PTHREAD}, + {"AXL_XFER_ASYNC_DW", AXL_XFER_ASYNC_DW}, + {"AXL_XFER_ASYNC_BBAPI", AXL_XFER_ASYNC_BBAPI}, + {"AXL_XFER_ASYNC_CPPR", AXL_XFER_ASYNC_CPPR}, + }; + axl_xfer_t type; + + if (!axl_xfer_str) + axl_xfer_str = "pthread"; + + type = AXL_XFER_NULL; + for (i = 0; i < sizeof (axl_str_to_type); i++) { + if (strncmp(axl_xfer_str, axl_str_to_type[i].str, + strlen(axl_str_to_type[i].str)) == 0) { + /* Match */ + type = axl_str_to_type[i].type; + break; + } + } + return type; +} + +static int filo_axl(int num_files, const char** src_filelist, + const char** dest_filelist, const char *axl_xfer_str) { int rc = FILO_SUCCESS; + axl_xfer_t type; /* TODO: allow user to name this transfer */ + + type = axl_xfer_str_to_type(axl_xfer_str); /* define a transfer handle */ - int id = AXL_Create(AXL_XFER_SYNC, "transfer"); + int id = AXL_Create(type, "transfer"); if (id < 0) { filo_err("Failed to create AXL transfer handle @ %s:%d", __FILE__, __LINE__ @@ -417,16 +465,20 @@ static int filo_axl(int num_files, const char** src_filelist, const char** dest_ return rc; } -static int filo_axl_start(const char* name, int num_files, const char** src_filelist, const char** dest_filelist, MPI_Comm comm) +static int filo_axl_start(const char* name, int num_files, + const char** src_filelist, const char** dest_filelist, MPI_Comm comm, + const char* axl_xfer_str) { int rc = FILO_SUCCESS; + axl_xfer_t type; /* create record for this transfer in outstanding list */ kvtree* name_hash = kvtree_set_kv(filo_outstanding, FILO_KEY_OUT_NAME, name); /* define a transfer handle */ //int id = AXL_Create(AXL_XFER_ASYNC_DAEMON, "transfer"); - int id = AXL_Create(AXL_XFER_SYNC, "transfer"); + type = axl_xfer_str_to_type(axl_xfer_str); + int id = AXL_Create(type, "transfer"); if (id < 0) { filo_err("Failed to create AXL transfer handle @ %s:%d", __FILE__, __LINE__ @@ -526,13 +578,20 @@ static int filo_axl_stop(MPI_Comm comm) static int filo_window_width = 256; -/* fetch files specified in file_list into specified dir and update - * filemap */ +/* + * Fetch files specified in file_list into specified dir and update + * filemap. + * + * axl_xfer_str: Optional AXL transfer string to tell what kind of transfer + * type to use ("sync", "pthread", "bbapi", etc..). Defaults to + * "default" if NULL. + */ static int filo_axl_sliding_window( int num_files, const char** src_filelist, const char** dest_filelist, - MPI_Comm comm) + MPI_Comm comm, + const char *axl_xfer_str) { int success = FILO_SUCCESS; @@ -541,10 +600,14 @@ static int filo_axl_sliding_window( MPI_Comm_rank(comm, &rank_world); MPI_Comm_size(comm, &ranks_world); + if (!axl_xfer_str) { + axl_xfer_str = "default"; + } + /* flow control rate of file reads from rank 0 */ if (rank_world == 0) { /* fetch these files into the directory */ - if (filo_axl(num_files, src_filelist, dest_filelist) != FILO_SUCCESS) { + if (filo_axl(num_files, src_filelist, dest_filelist, axl_xfer_str) != FILO_SUCCESS) { success = FILO_FAILURE; } @@ -606,7 +669,8 @@ static int filo_axl_sliding_window( /* if rank 0 hasn't seen a failure, try to read in our files */ if (success == FILO_SUCCESS) { /* fetch these files into the directory */ - if (filo_axl(num_files, src_filelist, dest_filelist) != FILO_SUCCESS) { + if (filo_axl(num_files, src_filelist, dest_filelist, axl_xfer_str) + != FILO_SUCCESS) { success = FILO_FAILURE; } } @@ -622,7 +686,14 @@ static int filo_axl_sliding_window( return FILO_FAILURE; } -/* fetch files from parallel file system */ + +/* + * Fetch files from parallel file system + * + * axl_xfer_str: Optional AXL transfer string to tell what kind of transfer + * type to use ("sync", "pthread", "bbapi", etc..). Defaults to + * "default" if NULL. + */ int Filo_Fetch( const char* filopath, const char* basepath, @@ -630,7 +701,8 @@ int Filo_Fetch( int* out_num_files, char*** out_src_filelist, char*** out_dest_filelist, - MPI_Comm comm) + MPI_Comm comm, + const char *axl_xfer_str) { int rc = FILO_SUCCESS; @@ -687,7 +759,8 @@ int Filo_Fetch( /* now we can finally fetch the actual files */ int success = 1; - if (filo_axl_sliding_window(count, src_filelist, dest_filelist, comm) != FILO_SUCCESS) { + if (filo_axl_sliding_window(count, src_filelist, dest_filelist, comm, + axl_xfer_str) != FILO_SUCCESS) { success = 0; } @@ -804,13 +877,20 @@ static int filo_create_dirs(const char* basepath, int count, const char** dest_f return FILO_SUCCESS; } +/* + * Flush files to the parallel file system + * + * axl_xfer_str: The AXL transfer type you want to use for your transfer + * (like "default", "sync", "pthread", "bbapi", etc). + */ int Filo_Flush( const char* filopath, const char* basepath, int num_files, const char** src_filelist, const char** dest_filelist, - MPI_Comm comm) + MPI_Comm comm, + const char *axl_xfer_str) { int rc = FILO_SUCCESS; @@ -828,9 +908,9 @@ int Filo_Flush( spath* dest = spath_from_str(filename); spath* rel = spath_relative(base, dest); char* relfile = spath_strdup(rel); - + kvtree_set_kv(filelist, FILO_KEY_FILE, relfile); - + filo_free(&relfile); spath_delete(&rel); spath_delete(&dest); @@ -849,7 +929,8 @@ int Filo_Flush( /* write files (via AXL) */ int success = 1; - if (filo_axl_sliding_window(num_files, src_filelist, dest_filelist, comm) != FILO_SUCCESS) { + if (filo_axl_sliding_window(num_files, src_filelist, dest_filelist, comm, + axl_xfer_str) != FILO_SUCCESS) { success = 0; } @@ -871,7 +952,8 @@ int Filo_Flush_start( int num_files, const char** src_filelist, const char** dest_filelist, - MPI_Comm comm) + MPI_Comm comm, + const char *axl_xfer_str) { int rc = FILO_SUCCESS; @@ -889,9 +971,9 @@ int Filo_Flush_start( spath* dest = spath_from_str(filename); spath* rel = spath_relative(base, dest); char* relfile = spath_strdup(rel); - + kvtree_set_kv(filelist, FILO_KEY_FILE, relfile); - + filo_free(&relfile); spath_delete(&rel); spath_delete(&dest); @@ -913,7 +995,8 @@ int Filo_Flush_start( /* write files (via AXL) */ int success = 1; - if (filo_axl_start(filopath, num_files, src_filelist, dest_filelist, comm) != FILO_SUCCESS) { + if (filo_axl_start(filopath, num_files, src_filelist, dest_filelist, comm, + axl_xfer_str) != FILO_SUCCESS) { success = 0; } @@ -1060,10 +1143,10 @@ int Filo_Set_write(Filo_Set* set, const char* filopath, const char* basepath) /* generate relative path to destination file */ spath* rel = spath_relative(base, dest); char* relfile = spath_strdup(rel); - + /* add to our list */ kvtree_set_kv(newfiles_hash, FILO_KEY_FILE, relfile); - + /* free destination and relative paths */ filo_free(&relfile); spath_delete(&rel); diff --git a/src/filo.h b/src/filo.h index 431249a..59da145 100644 --- a/src/filo.h +++ b/src/filo.h @@ -26,12 +26,15 @@ extern "C" { #define FILO_SUCCESS (0) #define FILO_FAILURE (1) -/** intialize the library */ +/** initialize the library */ int Filo_Init(); /** shut down the library */ int Filo_Finalize(); +/** Set a FILO config parameter */ +int Filo_Config(char *config_str); + /** given a pointer to a filo file at filopath that was written by * filo_flush, copy files from their current location to path, * returns the number of files copied and the full path to the @@ -58,7 +61,8 @@ int Filo_Fetch( int* num_files, /**< [OUT] - number of files copied */ char*** src_filelist, /**< [OUT] - full source path of each file copied */ char*** dest_filelist, /**< [OUT] - full destination path of each file copied */ - MPI_Comm comm /**< [IN] - communicator used for coordination and flow control */ + MPI_Comm comm, /**< [IN] - communicator used for coordination and flow control */ + const char *axl_xfer_str /**< [IN] - AXL transfer type ("sync", "pthread", "bbapi", etc) */ ); /** copies the list of source files to their corresponding @@ -72,7 +76,8 @@ int Filo_Flush( int num_files, /**< [IN] - number of files in source and dest lists */ const char** src_filelist, /**< [IN] - list of source paths */ const char** dest_filelist, /**< [IN] - list of destination paths */ - MPI_Comm comm /**< [IN] - communicator used for coordination and flow control */ + MPI_Comm comm, /**< [IN] - communicator used for coordination and flow control */ + const char *axl_xfer_str /**< [IN] - AXL transfer type ("sync", "pthread", "bbapi", etc) */ ); int Filo_Flush_start( @@ -81,7 +86,8 @@ int Filo_Flush_start( int num_files, /**< [IN] - number of files in source and dest lists */ const char** src_filelist, /**< [IN] - list of source paths */ const char** dest_filelist, /**< [IN] - list of destination paths */ - MPI_Comm comm /**< [IN] - communicator used for coordination and flow control */ + MPI_Comm comm, /**< [IN] - communicator used for coordination and flow control */ + const char *axl_xfer_str /**< [IN] - AXL transfer type ("sync", "pthread", "bbapi", etc) */ ); int Filo_Flush_test( diff --git a/test/test_filo.c b/test/test_filo.c index 06b41b8..dc8b6f1 100644 --- a/test/test_filo.c +++ b/test/test_filo.c @@ -40,12 +40,12 @@ int main(int argc, char* argv[]) sprintf(dest_filename, "./testfile_%d.out", rank); rc = Filo_Init(); - const char* filelist[1] = { filename }; const char* dest_filelist[1] = { dest_filename }; /* base path for storage is NULL, so destination files will be written to the local dir*/ - rc = Filo_Flush("mapfile", NULL, 1, filelist, dest_filelist, MPI_COMM_WORLD); + rc = Filo_Flush("mapfile", NULL, 1, filelist, dest_filelist, + MPI_COMM_WORLD, "pthread"); unlink(filename); @@ -54,7 +54,8 @@ int main(int argc, char* argv[]) char** src_filelist; char** dst_filelist; /* src base path is still NULL (consistent with Filo_Flush), but the dest base path is /dev/shm*/ - rc = Filo_Fetch("mapfile", NULL, "/dev/shm", &num_files, &src_filelist, &dst_filelist, MPI_COMM_WORLD); + rc = Filo_Fetch("mapfile", NULL, "/dev/shm", &num_files, &src_filelist, + &dst_filelist, MPI_COMM_WORLD, "pthread"); /* free file list returned by fetch */ int i; diff --git a/test/test_filo_async.c b/test/test_filo_async.c index 620c7d4..b847c3f 100644 --- a/test/test_filo_async.c +++ b/test/test_filo_async.c @@ -45,7 +45,7 @@ int main(int argc, char* argv[]) const char* dest_filelist[1] = { dest_filename }; /* base path for storage is NULL, so destination files will be written to the local dir*/ - rc = Filo_Flush_start("mapfile", NULL, 1, filelist, dest_filelist, MPI_COMM_WORLD); + rc = Filo_Flush_start("mapfile", NULL, 1, filelist, dest_filelist, MPI_COMM_WORLD, "pthread"); rc = Filo_Flush_test("mapfile", MPI_COMM_WORLD); rc = Filo_Flush_wait("mapfile", MPI_COMM_WORLD); @@ -56,7 +56,7 @@ int main(int argc, char* argv[]) char** src_filelist; char** dst_filelist; /* src base path is still NULL (consistent with Filo_Flush), but the dest base path is /dev/shm*/ - rc = Filo_Fetch("mapfile", NULL, "/dev/shm", &num_files, &src_filelist, &dst_filelist, MPI_COMM_WORLD); + rc = Filo_Fetch("mapfile", NULL, "/dev/shm", &num_files, &src_filelist, &dst_filelist, MPI_COMM_WORLD, NULL); /* free file list returned by fetch */ int i; diff --git a/test/test_filo_corrupt_fetch.c b/test/test_filo_corrupt_fetch.c index ad41834..b5373f8 100644 --- a/test/test_filo_corrupt_fetch.c +++ b/test/test_filo_corrupt_fetch.c @@ -45,7 +45,7 @@ int main(int argc, char* argv[]) const char* dest_filelist[1] = { dest_filename }; /* base path for storage is NULL, so destination files will be written to the local dir*/ - rc = Filo_Flush("mapfile", NULL, 1, filelist, dest_filelist, MPI_COMM_WORLD); + rc = Filo_Flush("mapfile", NULL, 1, filelist, dest_filelist, MPI_COMM_WORLD, NULL); //remove one of the flush destination files. This should result in Filo_Fetch returning error on ALL processes. if(rank == 1) unlink(dest_filename); @@ -57,7 +57,7 @@ int main(int argc, char* argv[]) char** src_filelist; char** dst_filelist; /* src base path is still NULL (consistent with Filo_Flush), but the dest base path is /dev/shm*/ - int filo_ret = Filo_Fetch("mapfile", NULL, "/dev/shm", &num_files, &src_filelist, &dst_filelist, MPI_COMM_WORLD); + int filo_ret = Filo_Fetch("mapfile", NULL, "/dev/shm", &num_files, &src_filelist, &dst_filelist, MPI_COMM_WORLD, NULL); //check if the return value is error -- which is the correct behavior -- otherwise return fail if(filo_ret ==0){ printf("Error: Filo_Fetch should fail because rank 1 destination file was removed pre- fetch. rank = %d, filo_ret = %d\n", rank, filo_ret);