diff --git a/modules/dgi_migrate_foxml_standard_mods/migrations/dgis_nodes.yml b/modules/dgi_migrate_foxml_standard_mods/migrations/dgis_nodes.yml index 1cbafcef..91f4f2f1 100644 --- a/modules/dgi_migrate_foxml_standard_mods/migrations/dgis_nodes.yml +++ b/modules/dgi_migrate_foxml_standard_mods/migrations/dgis_nodes.yml @@ -259,6 +259,9 @@ process: missing_behavior: skip_process - plugin: dgi_migrate.method method: getUri + - plugin: skip_on_empty + method: row + message: 'Failed to fetch the MODS XML.' # XXX: An issue in the passing off of paths/URIs to libxml prevents the use # of "dgi_migrate.process.xml.domfile" - plugin: callback diff --git a/scripts/env.sample b/scripts/env.sample index 1292b265..c511cdcf 100644 --- a/scripts/env.sample +++ b/scripts/env.sample @@ -59,6 +59,13 @@ # --- #PROCESSES=1 +# === +# ULIMIT_FOR_PROCESSING: The ulimit to set for the migration processes. +# --- +# NOTE: This may need to be tweaked in multiprocessing scenarios. +#ULIMIT_FOR_PROCESSING=-1 + + # === # SKIP_STATUS: Suppress dumping of migration status before/after operations. # --- diff --git a/scripts/util.in b/scripts/util.in index ccac2ca6..c606abde 100644 --- a/scripts/util.in +++ b/scripts/util.in @@ -65,6 +65,7 @@ function init_vars () { declare -g TIME=${TIME:-/usr/bin/time} declare -g LOG_DIR=${LOG_DIR:-$CONFIG_DIR} declare -g PROCESSES=${PROCESSES:-1} + declare -g ULIMIT_FOR_PROCESSING=${ULIMIT_FOR_PROCESSING:--1} declare -g SKIP_STATUS=${SKIP_STATUS:-false} declare -g -a MULTIPROCESS_SKIP_MIGRATIONS=(${MULTIPROCESS_SKIP_MIGRATIONS[@]}) declare -g -a MULTIPROCESS_PRE_ENQUEUE_PAUSE=(${MULTIPROCESS_PRE_ENQUEUE_PAUSE[@]}) @@ -86,6 +87,9 @@ function init_vars () { export DGI_MIGRATE__DO_MIGRATION_LOOKUP_LOCKING="FALSE" fi + if [ $ULIMIT_FOR_PROCESSING -gt -1 ]; then + ulimit -n $ULIMIT_FOR_PROCESSING + fi set -u }