Fix formatting issues

bin.src/rucio-integration-register

@@ -14,39 +14,44 @@ logger = logging.getLogger(__name__)
 RUCIO_REGISTER_CONFIG = "RUCIO_REGISTER_CONFIG"
 _MSG = "environment variable not set, and no configuration was specified on the command line"
 
+
 def main():
     ap = AppParser(sys.argv)
-        
+
     if RUCIO_REGISTER_CONFIG in os.environ:
         config_file = os.environ[RUCIO_REGISTER_CONFIG]
     elif ap.register_config is None:
         print("bailed")
         sys.exit(100)
     else:
         config_file = ap.register_config
-    
+
     config = RucioRegisterConfig(config_file)
-    
+
     rucio_rse = config.rucio_rse
     scope = config.scope
     rse_root = config.rse_root
     dtn_url = config.dtn_url
-    
+
     butler = Butler(ap.butler_repo)
-
-    ri = RucioInterface(butler=butler,
-                        rucio_rse=rucio_rse,
-                        scope=scope,
-                        rse_root=rse_root,
-                        dtn_url=dtn_url
-                       )
-
-    dataset_refs = butler.registry.queryDatasets(ap.dataset_type, collections=ap.collections)
-
+
+    ri = RucioInterface(
+        butler=butler,
+        rucio_rse=rucio_rse,
+        scope=scope,
+        rse_root=rse_root,
+        dtn_url=dtn_url,
+    )
+
+    dataset_refs = butler.registry.queryDatasets(
+        ap.dataset_type, collections=ap.collections
+    )
+
     cnt = ri.register_as_replicas(ap.rucio_dataset, dataset_refs)
     if cnt == 0:
         print("no datasets to register")
         sys.exit(200)
 
+
 if __name__ == "__main__":
     main()

python/lsst/dm/rucio/register/app_parser.py

@@ -33,82 +33,96 @@ class AppParser:
     argv: `list`
         list of program name and arguments
     """
+
     def __init__(self, argv):
         parser = argparse.ArgumentParser(prog=argv[0])
-        parser.add_argument("-r",
-                            "--repo",
-                            action="store",
-                            default=None,
-                            dest="repo",
-                            help="Butler repository",
-                            type=str,
-                            required=True
-                            )
-        parser.add_argument("-c",
-                            "--collections",
-                            action="store",
-                            default=None,
-                            dest="collections",
-                            help="collections for lookup",
-                            type=str,
-                            required=True
-                            )
-
-        parser.add_argument("-t",
-                            "--dataset-type",
-                            action="store",
-                            default=None,
-                            dest="dataset_type",
-                            help="dataset type for lookup",
-                            type=str,
-                            required=True
-                            )
-
-        parser.add_argument("-d",
-                            "--rucio-dataset",
-                            action="store",
-                            default=None,
-                            dest="rucio_dataset",
-                            help="rucio dataset to register files to",
-                            type=str,
-                            required=True
-                            )
-        parser.add_argument("-C",
-                            "--rucio-register-config",
-                            action="store",
-                            default=None,
-                            dest="register_config",
-                            help="configuration file used for registration",
-                            type=str,
-                            required=False,
-                            )
+        parser.add_argument(
+            "-r",
+            "--repo",
+            action="store",
+            default=None,
+            dest="repo",
+            help="Butler repository",
+            type=str,
+            required=True,
+        )
+        parser.add_argument(
+            "-c",
+            "--collections",
+            action="store",
+            default=None,
+            dest="collections",
+            help="collections for lookup",
+            type=str,
+            required=True,
+        )
+
+        parser.add_argument(
+            "-t",
+            "--dataset-type",
+            action="store",
+            default=None,
+            dest="dataset_type",
+            help="dataset type for lookup",
+            type=str,
+            required=True,
+        )
+
+        parser.add_argument(
+            "-d",
+            "--rucio-dataset",
+            action="store",
+            default=None,
+            dest="rucio_dataset",
+            help="rucio dataset to register files to",
+            type=str,
+            required=True,
+        )
+        parser.add_argument(
+            "-C",
+            "--rucio-register-config",
+            action="store",
+            default=None,
+            dest="register_config",
+            help="configuration file used for registration",
+            type=str,
+            required=False,
+        )
         # the following arguments are for logging;
         # defaults to WARNING
         # -v sets to INFO
         # -D sets to DEBUG
 
         group = parser.add_mutually_exclusive_group()
-        group.add_argument('-v',
-                            '--verbose',
-                            help="set loglevel to INFO",
-                            action="store_const", dest="loglevel", const=logging.INFO,
-                            default=None,                            
-                          )
-        group.add_argument('-D',
-                            '--debug',
-                            help="set loglevel to DEBUG",
-                            action="store_const", dest="loglevel", const=logging.DEBUG,
-                            default=None,                            
-                            )
-
+        group.add_argument(
+            "-v",
+            "--verbose",
+            help="set loglevel to INFO",
+            action="store_const",
+            dest="loglevel",
+            const=logging.INFO,
+            default=None,
+        )
+        group.add_argument(
+            "-D",
+            "--debug",
+            help="set loglevel to DEBUG",
+            action="store_const",
+            dest="loglevel",
+            const=logging.DEBUG,
+            default=None,
+        )
+
         args = parser.parse_args(argv[1:])
 
         if args.loglevel is None:
             loglevel = logging.WARNING
         else:
             loglevel = args.loglevel
 
-        logging.basicConfig(level=logging.INFO, format=(_FORMAT), datefmt="%Y-%m-%d %H:%M:%S")
+        logging.basicConfig(
+            level=logging.INFO, format=(_FORMAT), datefmt="%Y-%m-%d %H:%M:%S"
+        )
 
         self.butler_repo = args.repo
         self.collections = args.collections

python/lsst/dm/rucio/register/dataset_map.py

@@ -32,12 +32,13 @@ class DatasetMap:
     mapping_yaml: `str`
         name of YAML file containing mapping
     """
+
     def __init__(self, mapping_yaml):
         with open(mapping_yaml) as f:
             self.config = yaml.safe_load(f)
 
     def get_dataset_template(self, typename) -> str:
-        """ Get the dataset_template associated with a dataset type
+        """Get the dataset_template associated with a dataset type
 
         Parameters
         ----------

python/lsst/dm/rucio/register/resource_bundle.py

@@ -25,6 +25,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class ResourceBundle:
     def __init__(self, dataset_id: str, did: dict):
         """Create a resource bundle of a dataset_id and its metadata

python/lsst/dm/rucio/register/rucio_interface.py

@@ -39,6 +39,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class RucioInterface:
     """Add files as replicas in Rucio, along with metadata, and attach them to datasets.
 
@@ -56,7 +57,14 @@ class RucioInterface:
         Base URL of the data transfer node for the Rucio physical filename.
     """
 
-    def __init__(self, butler: lsst.daf.butler.Butler, rucio_rse: str, scope: str, rse_root:str, dtn_url: str):
+    def __init__(
+        self,
+        butler: lsst.daf.butler.Butler,
+        rucio_rse: str,
+        scope: str,
+        rse_root: str,
+        dtn_url: str,
+    ):
         self.butler = butler
         self.rse = rucio_rse
         self.scope = scope
@@ -104,10 +112,18 @@ def _make_did(self, resource_path, metadata: str) -> dict[str, str | int]:
         path = resource_path.path.removeprefix(self.rse_root)
         pfn = self.pfn_base + path
         logging.debug(f"pfn = {pfn}")
-        name = path.removeprefix("/"+self.scope+"/")
+        name = path.removeprefix("/" + self.scope + "/")
 
-        meta = { 'rubin_butler': 1, 'rubin_sidecar': metadata }
-        d = dict(pfn=pfn, bytes=size, adler32=adler32, md5=md5, name=name, scope=self.scope, meta=meta)
+        meta = {"rubin_butler": 1, "rubin_sidecar": metadata}
+        d = dict(
+            pfn=pfn,
+            bytes=size,
+            adler32=adler32,
+            md5=md5,
+            name=name,
+            scope=self.scope,
+            meta=meta,
+        )
 
         return d
 
@@ -121,11 +137,13 @@ def _add_replicas(self, bundles: list[ResourceBundle]) -> None:
         """
         logger.info(f"rse={self.rse}, bundles={bundles}")
         dids = [bundle.get_did() for bundle in bundles]
-        logger.info(f'{dids[0]=}')
+        logger.info(f"{dids[0]=}")
         self.replica_client = ReplicaClient()
         self.replica_client.add_replicas(rse=self.rse, files=dids)
 
-    def _add_files_to_dataset(self, did_client, dataset_id: str, dids: list[dict]) -> None:
+    def _add_files_to_dataset(
+        self, did_client, dataset_id: str, dids: list[dict]
+    ) -> None:
         """Attach a list of files specified by Rucio DIDs to a Rucio dataset.
 
         Ignores already-attached files for idempotency.
@@ -195,7 +213,9 @@ def register_to_dataset(self, bundles) -> None:
                 dids = []
                 for rb in bundles:
                     dids.append(rb.get_did())
-                logger.info("Registering %s in dataset %s, RSE %s", dids, dataset_id, self.rse)
+                logger.info(
+                    "Registering %s in dataset %s, RSE %s", dids, dataset_id, self.rse
+                )
                 self._add_files_to_dataset(did_client, dataset_id, dids)
             except rucio.common.exception.DataIdentifierNotFound:
                 # No such dataset, so create it
@@ -217,7 +237,7 @@ def register_to_dataset(self, bundles) -> None:
         logger.debug("Done with Rucio for %s", bundles)
 
     def register_as_replicas(self, dataset_id, dataset_refs) -> None:
-        """Register a list of DatasetRefs to a Rucio dataset 
+        """Register a list of DatasetRefs to a Rucio dataset
 
         Parameters
         ----------

python/lsst/dm/rucio/register/rucio_register_config.py

@@ -34,10 +34,11 @@ class RucioRegisterConfig:
     config_file: `str`
        path to configuration file
     """
+
     def __init__(self, config_file: str):
         with open(config_file) as f:
             config = yaml.safe_load(f)
-            
+
         self.rucio_rse = config["rucio_rse"]
         self.scope = config["scope"]
         self.rse_root = config["rse_root"]

tests/test_app_parser.py

@@ -31,7 +31,20 @@
 class AppParserTestCase(unittest.TestCase):
 
     def testAppParser(self):
-        argv = ["unittest", "-r", "repo", "-c", "collections", "-t", "type", "-d", "dataset", "-C", "config", "-v"]
+        argv = [
+            "unittest",
+            "-r",
+            "repo",
+            "-c",
+            "collections",
+            "-t",
+            "type",
+            "-d",
+            "dataset",
+            "-C",
+            "config",
+            "-v",
+        ]
         ap = AppParser(argv)
 
         self.assertEqual(ap.butler_repo, "repo")
@@ -49,9 +62,10 @@ def testMissingArguments(self):
         except SystemExit:
             pass
 
+
 class MemoryTester(lsst.utils.tests.MemoryTestCase):
     pass
-        
+
 
 def setup_module(module):
     lsst.utils.tests.init()

tests/test_dataset_map.py

@@ -34,18 +34,19 @@ def testDatasetMap(self):
         map_file = os.path.join(test_dir, "data", "map.yaml")
         td = DatasetMap(map_file)
 
-        s = td.get_dataset_template('visitSummary')
+        s = td.get_dataset_template("visitSummary")
         self.assertEqual(s, "{visit}/one/two")
 
-        s = td.get_dataset_template('isolated_star_cat')
+        s = td.get_dataset_template("isolated_star_cat")
         self.assertEqual(s, "three/four/{tract}")
 
-        s = td.get_dataset_template('isolated_star_sources')
+        s = td.get_dataset_template("isolated_star_sources")
         self.assertEqual(s, "five/six/{tract}")
 
+
 class MemoryTester(lsst.utils.tests.MemoryTestCase):
     pass
-        
+
 
 def setup_module(module):
     lsst.utils.tests.init()