add scr_flush_async_usleep for polling async flush

LLNL · Mar 11, 2022 · ff168c1 · ff168c1
1 parent 25c83fb
commit ff168c1
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 5 deletions.
diff --git a/src/scr.c b/src/scr.c
@@ -1169,6 +1169,14 @@ static int scr_get_params()
     scr_dbg(1, "SCR_FLUSH_ASYNC_PERCENT=%f", scr_flush_async_percent);
   }
 
+  /* number of microseconds to sleep between polling async flush status */
+  if ((value = scr_param_get("SCR_FLUSH_ASYNC_USLEEP")) != NULL) {
+    scr_flush_async_usleep = atoi(value);
+  }
+  if (scr_my_rank_world == 0) {
+    scr_dbg(1, "SCR_FLUSH_ASYNC_USLEEP=%d", scr_flush_async_usleep);
+  }
+
   /* set file copy buffer size (file chunk size) */
   if ((value = scr_param_get("SCR_FILE_BUF_SIZE")) != NULL) {
     if (scr_abtoull(value, &ull) == SCR_SUCCESS) {

diff --git a/src/scr_conf.h b/src/scr_conf.h
@@ -236,6 +236,11 @@
 #define SCR_FLUSH_ASYNC_PERCENT (0.0) /* TODO: the fsync complicates this throttling, disable it for now */
 #endif
 
+/* sleep time when polling for an async flush to complete */
+#ifndef SCR_FLUSH_ASYNC_USLEEP
+#define SCR_FLUSH_ASYNC_USLEEP (1000)
+#endif
+
 /* max number of checkpoints to keep in prefix (0 disables) */
 #ifndef SCR_PREFIX_SIZE
 #define SCR_PREFIX_SIZE (0)

diff --git a/src/scr_flush_async.c b/src/scr_flush_async.c
@@ -590,8 +590,8 @@ int scr_flush_async_wait(scr_cache_index* cindex, int id)
         /* complete the flush */
         scr_flush_async_complete(cindex, id);
       } else {
-        /* otherwise, sleep to get out of the way */
-        usleep(10*1000*1000);
+        /* otherwise, sleep for a bit to get out of the way */
+        usleep(scr_flush_async_usleep);
       }
     }
   }

diff --git a/src/scr_globals.c b/src/scr_globals.c
@@ -128,9 +128,10 @@ int   scr_flush_on_restart = SCR_FLUSH_ON_RESTART; /* specify whether to flush c
 int   scr_global_restart   = SCR_GLOBAL_RESTART;   /* set if code must be restarted from parallel file system */
 int   scr_drop_after_current = 0;                  /* whether to drop datasets from index that come after dataset named in SCR_Current */
 
-int    scr_flush_async             = SCR_FLUSH_ASYNC;         /* whether to use asynchronous flush */
-double scr_flush_async_bw          = SCR_FLUSH_ASYNC_BW;      /* bandwidth limit imposed during async flush */
-double scr_flush_async_percent     = SCR_FLUSH_ASYNC_PERCENT; /* runtime limit imposed during async flush */
+int    scr_flush_async         = SCR_FLUSH_ASYNC;         /* whether to use asynchronous flush */
+double scr_flush_async_bw      = SCR_FLUSH_ASYNC_BW;      /* bandwidth limit imposed during async flush */
+double scr_flush_async_percent = SCR_FLUSH_ASYNC_PERCENT; /* runtime limit imposed during async flush */
+int    scr_flush_async_usleep  = SCR_FLUSH_ASYNC_USLEEP;  /* number of microsecs to sleep between polling async transfer */
 
 int scr_flush_poststage = SCR_FLUSH_POSTSTAGE; /* Use scr_poststage to finalize transfers */
 

diff --git a/src/scr_globals.h b/src/scr_globals.h
@@ -188,6 +188,7 @@ extern int scr_prefix_purge; /* whether to delete all datasets listed in index f
 extern int scr_flush_async;            /* whether to use asynchronous flush */
 extern double scr_flush_async_bw;      /* bandwidth limit imposed during async flush */
 extern double scr_flush_async_percent; /* runtime limit imposed during async flush */
+extern int scr_flush_async_usleep;     /* number of microsecs to sleep between polling async transfer */
 
 extern int scr_flush_poststage; /* whether to use scr_poststage.sh to finalize transfers */