Make method get_parsl_config() to return a suitable default Parsl con…

…figuration

and support Parsl configuration attributes to be specified in submission file.

python/lsst/ctrl/bps/parsl/site.py

@@ -199,18 +199,23 @@ def get_monitor(self) -> MonitoringHub | None:
             + get_bps_config_value(self.site, "monitorFilename", str, "monitor.sqlite"),
         )
 
-    def get_parsl_config(self) -> parsl.config.Config | None:
+    def get_parsl_config(self) -> parsl.config.Config:
         """Get Parsl configuration for this site.
 
-        This method allows concrete subclasses to override this method to
-        provide a a Parsl configuration specific for the site. If not
-        implemented by the subclasses a default configuration is built and
-        provided to Parsl.
+        Subclasses can overwrite this method to build a more specific Parsl
+        configuration, if required.
+
+        The retries are set from the ``site.<computeSite>.retries`` value
+        found in the BPS configuration file.
 
         Returns
         -------
-        config : `parsl.config.Config` or `None`
-            The configuration to be used for Parsl. If `None` a default
-            configuration with sensible values will be passed to Parsl.
+        config : `parsl.config.Config`
+            The configuration to be used for Parsl.
         """
-        return None
+        executors = self.get_executors()
+        monitor = self.get_monitor()
+        retries = get_bps_config_value(self.site, "retries", int, 1)
+        return parsl.config.Config(
+            executors=executors, monitoring=monitor, retries=retries, checkpoint_mode="task_exit"
+        )

python/lsst/ctrl/bps/parsl/sites/ccin2p3.py

@@ -165,7 +165,7 @@ def get_executors(self) -> List[ParslExecutor]:
                     # '#SBATCH' directives to prepend to the Slurm submission
                     # script.
                     scheduler_options=f"#SBATCH --qos={qos} --licenses=sps",
-                    # Set the number of file descriptors and process to
+                    # Set the number of file descriptors and processes to
                     # the maximum allowed.
                     worker_init="ulimit -n hard && ulimit -u hard",
                     # Requests nodes which are not shared with other running
@@ -224,16 +224,22 @@ def get_parsl_config(self) -> parsl.config.Config:
         config : `parsl.config.Config`
             The configuration to be used to initialize Parsl for this site.
         """
+        executors = self.get_executors()
+        monitor = self.get_monitor()
+        retries = get_bps_config_value(self.site, "retries", int, 1)
+        # Path to Parsl run directory. The default set by Parsl is
+        # 'runinfo' which is not explicit enough for end users given that
+        # we are using BPS + Parsl + Slurm to execute a workflow.
+        run_dir = get_bps_config_value(self.site, "run_dir", str, "parsl_runinfo")
+        # Strategy for scaling blocks according to workflow needs.
+        # Use a strategy that allows for scaling in and out Parsl
+        # workers.
+        strategy = get_bps_config_value(self.site, "strategy", str, "htex_auto_scale")
         return parsl.config.Config(
-            executors=self.get_executors(),
-            monitoring=self.get_monitor(),
-            retries=get_bps_config_value(self.site, "retries", int, 1),
+            executors=executors,
+            monitoring=monitor,
+            retries=retries,
             checkpoint_mode="task_exit",
-            # The default is 'runinfo' which is not explicit enough for end
-            # users given that we are using BPS + Parsl + Slurm to execute
-            # a workflow.
-            run_dir="parsl_runinfo",
-            # This allocation strategy allows for scaling in and out Parsl
-            # workers.
-            strategy="htex_auto_scale",
+            run_dir=run_dir,
+            strategy=strategy,
         )

python/lsst/ctrl/bps/parsl/workflow.py

@@ -37,7 +37,7 @@
 from parsl.app.bash import BashApp
 from parsl.app.futures import Future
 
-from .configuration import get_bps_config_value, get_workflow_filename, set_parsl_logging
+from .configuration import get_workflow_filename, set_parsl_logging
 from .job import ParslJob, get_file_paths
 from .site import SiteConfig
 
@@ -52,11 +52,10 @@ def get_parsl_config(config: BpsConfig) -> parsl.config.Config:
     For details on the site configuration, see `SiteConfig`. For details on the
     monitor configuration, see ``get_parsl_monitor``.
 
-    Subclasses of `SiteConfig` can override their method ``get_parsl_config``
-    to configure Parsl for the specific context of the site. A default
-    Parsl configuration is returned if the subclass did not provide a config.
-
-    The retries are set from the ``site.<computeSite>.retries`` value.
+    `SiteConfig` provides an implementation of the method ``get_parsl_config``
+    which returns a Parsl configuration with sensible defaults. Subclasses
+    of `SiteConfig` can overwrite that method to configure Parsl in a
+    way specific to the site's configuration.
 
     Parameters
     ----------
@@ -69,15 +68,7 @@ def get_parsl_config(config: BpsConfig) -> parsl.config.Config:
         Parsl configuration.
     """
     site = SiteConfig.from_config(config)
-    if parsl_config := site.get_parsl_config():
-        return parsl_config
-
-    return parsl.config.Config(
-        executors=site.get_executors(),
-        monitoring=site.get_monitor(),
-        retries=get_bps_config_value(site.site, "retries", int, 1),
-        checkpoint_mode="task_exit",
-    )
+    return site.get_parsl_config()
 
 
 class ParslWorkflow(BaseWmsWorkflow):