From d0f3e99d91c39bbd35a16966f7aa535974bc8e9c Mon Sep 17 00:00:00 2001 From: Johannes Lange Date: Thu, 11 Aug 2022 11:57:11 +0200 Subject: [PATCH] use the stored env_extra parameter from the base class for htcondor (#572) - no need to reparse the parameter and config in HTCondorCluster - moved the joining to a string to the only place where it is needed - renamed _env_header to _env_extra, because it now contains exactly the parameter --- dask_jobqueue/core.py | 4 ++-- dask_jobqueue/htcondor.py | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/dask_jobqueue/core.py b/dask_jobqueue/core.py index fb1ac994..dbf7a5ad 100644 --- a/dask_jobqueue/core.py +++ b/dask_jobqueue/core.py @@ -245,7 +245,7 @@ def __init__( self.shebang = shebang - self._env_header = "\n".join(filter(None, env_extra)) + self._env_extra = env_extra self.header_skip = set(header_skip) # dask-worker command line build @@ -299,7 +299,7 @@ def job_script(self): pieces = { "shebang": self.shebang, "job_header": header, - "env_header": self._env_header, + "env_header": "\n".join(filter(None, self._env_extra)), "worker_command": self._command_template, } return self._script_template % pieces diff --git a/dask_jobqueue/htcondor.py b/dask_jobqueue/htcondor.py index 77ded3d1..b3dcfd07 100644 --- a/dask_jobqueue/htcondor.py +++ b/dask_jobqueue/htcondor.py @@ -61,17 +61,13 @@ def __init__( else: self.job_extra = job_extra - env_extra = base_class_kwargs.get("env_extra", None) - if env_extra is None: - env_extra = dask.config.get( - "jobqueue.%s.env-extra" % self.config_name, default=[] - ) - - if env_extra is not None: + if self._env_extra is not None: # Overwrite command template: prepend commands from env_extra separated by semicolon. # This is special for HTCondor, because lines to execute on the worker node cannot be # simply added to the submit script like for other batch systems. - self._command_template = "; ".join(env_extra + [self._command_template]) + self._command_template = "; ".join( + self._env_extra + [self._command_template] + ) self.job_header_dict = { "MY.DaskWorkerName": '"htcondor--$F(MY.JobId)--"',