Merge pull request #147 from discoverygarden/fix/ulimit-passing

Account for defining and passing around ulimit.

modules/dgi_migrate_foxml_standard_mods/migrations/dgis_nodes.yml

@@ -259,6 +259,9 @@ process:
       missing_behavior: skip_process
     - plugin: dgi_migrate.method
       method: getUri
+    - plugin: skip_on_empty
+      method: row
+      message: 'Failed to fetch the MODS XML.'
     # XXX: An issue in the passing off of paths/URIs to libxml prevents the use
     # of "dgi_migrate.process.xml.domfile"
     - plugin: callback

scripts/env.sample

@@ -59,6 +59,13 @@
 # ---
 #PROCESSES=1
 
+# ===
+# ULIMIT_FOR_PROCESSING: The ulimit to set for the migration processes.
+# ---
+# NOTE: This may need to be tweaked in multiprocessing scenarios.
+#ULIMIT_FOR_PROCESSING=-1
+
+
 # ===
 # SKIP_STATUS: Suppress dumping of migration status before/after operations.
 # ---

scripts/util.in

@@ -65,6 +65,7 @@ function init_vars () {
   declare -g TIME=${TIME:-/usr/bin/time}
   declare -g LOG_DIR=${LOG_DIR:-$CONFIG_DIR}
   declare -g PROCESSES=${PROCESSES:-1}
+  declare -g ULIMIT_FOR_PROCESSING=${ULIMIT_FOR_PROCESSING:--1}
   declare -g SKIP_STATUS=${SKIP_STATUS:-false}
   declare -g -a MULTIPROCESS_SKIP_MIGRATIONS=(${MULTIPROCESS_SKIP_MIGRATIONS[@]})
   declare -g -a MULTIPROCESS_PRE_ENQUEUE_PAUSE=(${MULTIPROCESS_PRE_ENQUEUE_PAUSE[@]})
@@ -86,6 +87,9 @@ function init_vars () {
     export DGI_MIGRATE__DO_MIGRATION_LOOKUP_LOCKING="FALSE"
   fi
 
+  if [ $ULIMIT_FOR_PROCESSING -gt -1 ]; then
+    ulimit -n $ULIMIT_FOR_PROCESSING
+  fi
   set -u
 }