Investigate argument list too long errors

[nikhil]

Traceback (most recent call last):$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 240, in run$
    while self._runStep():$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 230, in _runStep$
    activity |= self.checkOnJobs()$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 190, in checkOnJobs$
    statuses = self.boss.with_retries(self.coalesceJobExitCodes, batchJobIDList)$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 454, in with_retries$
    return operation(*args, **kwargs)$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/lsf.py", line 123, in coalesceJobExitCodes$
    stderr=subprocess.STDOUT)$
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 769, in __init__$
    restore_signals, start_new_session)$
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 1516, in _execute_child$
    raise child_exception_type(errno_num, err_msg, err_filename)$
OSError: [Errno 7] Argument list too long: 'bjobs'

Traceback (most recent call last):$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 240, in run$
    while self._runStep():$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 229, in _runStep$
    activity |= self.createJobs(newJob)$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 115, in createJobs$
    batchJobID = self.boss.with_retries(self.submitJob, subLine)$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 454, in with_retries$
    return operation(*args, **kwargs)$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/batchSystems/lsf.py", line 91, in submitJob$
    stdout = call_command(subLine, env=combinedEnv)$
  File "/work/ci/vurals/run_CWL_in_command_line/common_ops/venv/lib/python3.7/site-packages/toil/lib/misc.py", line 62, in call_command$
    encoding='utf-8', errors="replace", env=env)$
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 769, in __init__$
    restore_signals, start_new_session)$
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 1516, in _execute_child$
    raise child_exception_type(errno_num, err_msg, err_filename)$
OSError: [Errno 7] Argument list too long: 'bsub'

Traceback (most recent call last):
  File "/juno/work/ci/prod_ridgeback/venv/lib/python3.7/site-packages/toil/worker.py", line 393, in workerScript
    job._runner(jobGraph=None, jobStore=jobStore, fileStore=fileStore, defer=defer)
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/contextlib.py", line 119, in __exit__
    next(self.gen)
  File "/juno/work/ci/prod_ridgeback/venv/lib/python3.7/site-packages/toil/fileStores/nonCachingFileStore.py", line 59, in open
    disk = getDirSizeRecursively(self.localTempDir)
  File "/juno/work/ci/prod_ridgeback/venv/lib/python3.7/site-packages/toil/common.py", line 1432, in getDirSizeRecursively
    env=dict(os.environ, BLOCKSIZE='512')).decode('utf-8').split()[0]) * 512
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 389, in check_output
    **kwargs).stdout
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 466, in run
    with Popen(*popenargs, **kwargs) as process:
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 769, in __init__
    restore_signals, start_new_session)
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 1516, in _execute_child
    raise child_exception_type(errno_num, err_msg, err_filename)
OSError: [Errno 7] Argument list too long: 'du'

[2021-08-05T11:27:10-0400] [MainThread] [I] [toil.worker] Loaded body Job('CWLJob' tail kind-CWLJob/instance-pt8_n0ot) from description 'CWLJob' tail kind-CWLJob/instance-pt8_n0ot
[2021-08-05T11:27:10-0400] [MainThread] [I] [cwltool] ['singularity', 'pull', '--force', '--name', 'mskcc_helix_filters_01:21.3.3.sif', 'docker://mskcc/helix_filters_01:21.3.3']
[2021-08-05T11:27:10-0400] [MainThread] [W] [toil.fileStores.abstractFileStore] Failed job accessed files:
[2021-08-05T11:27:10-0400] [MainThread] [W] [toil.fileStores.abstractFileStore] Downloaded file 'files/for-job/kind-CWLJob/instance-8fhrwsgc/file-a3609922dea24ad9b9c2796d7c5a0d5e/output.txt.strip.txt' to path '/fscratch/voyager/work/fde4651eda1159668ea16320a663c122/e217/b229/tmp7qhyddwz.tmp'
Traceback (most recent call last):
  File "/fscratch/voyager/venv/lib/python3.7/site-packages/cwltool/job.py", line 768, in run
    runtimeContext.tmp_outdir_prefix,
  File "/fscratch/voyager/venv/lib/python3.7/site-packages/cwltool/singularity.py", line 274, in get_from_requirements
    if not self.get_image(cast(Dict[str, str], r), pull_image, force_pull):
  File "/fscratch/voyager/venv/lib/python3.7/site-packages/cwltool/singularity.py", line 217, in get_image
    check_call(cmd, stdout=sys.stderr)  # nosec
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 336, in check_call
    retcode = call(*popenargs, **kwargs)
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 317, in call
    with Popen(*popenargs, **kwargs) as p:
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 769, in __init__
    restore_signals, start_new_session)
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 1516, in _execute_child
    raise child_exception_type(errno_num, err_msg, err_filename)
OSError: [Errno 7] Argument list too long: 'singularity'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/fscratch/voyager/toil/src/toil/fileStores/cachingFileStore.py", line 989, in open
    yield
  File "/fscratch/voyager/toil/src/toil/worker.py", line 393, in workerScript
    job._runner(jobGraph=None, jobStore=jobStore, fileStore=fileStore, defer=defer)
  File "/fscratch/voyager/toil/src/toil/job.py", line 2362, in _runner
    returnValues = self._run(jobGraph=None, fileStore=fileStore)
  File "/fscratch/voyager/toil/src/toil/job.py", line 2281, in _run
    return self.run(fileStore)
  File "/fscratch/voyager/toil/src/toil/cwl/cwltoil.py", line 1224, in run
    logger=cwllogger,
  File "/fscratch/voyager/venv/lib/python3.7/site-packages/cwltool/executors.py", line 150, in execute
    self.run_jobs(process, job_order_object, logger, runtime_context)
  File "/fscratch/voyager/venv/lib/python3.7/site-packages/cwltool/executors.py", line 257, in run_jobs
    job.run(runtime_context)
  File "/fscratch/voyager/venv/lib/python3.7/site-packages/cwltool/job.py", line 811, in run
    ) from err
cwltool.errors.UnsupportedRequirement: Singularity is required to run this tool: [Errno 7] Argument list too long: 'singularity'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/fscratch/voyager/toil/src/toil/worker.py", line 393, in workerScript
    job._runner(jobGraph=None, jobStore=jobStore, fileStore=fileStore, defer=defer)
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/contextlib.py", line 130, in __exit__
    self.gen.throw(type, value, traceback)
  File "/fscratch/voyager/toil/src/toil/fileStores/cachingFileStore.py", line 994, in open
    disk: int = getDirSizeRecursively(self.localTempDir)
  File "/fscratch/voyager/toil/src/toil/common.py", line 1432, in getDirSizeRecursively
    env=dict(os.environ, BLOCKSIZE='512')).decode('utf-8').split()[0]) * 512
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 389, in check_output
    **kwargs).stdout
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 466, in run
    with Popen(*popenargs, **kwargs) as process:
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 769, in __init__
    restore_signals, start_new_session)
  File "/opt/common/CentOS_7/python/python-3.7.1/lib/python3.7/subprocess.py", line 1516, in _execute_child
    raise child_exception_type(errno_num, err_msg, err_filename)
OSError: [Errno 7] Argument list too long: 'du'