From 423c5aa0f829d2c8eb739befb9dd407e2bb99e1f Mon Sep 17 00:00:00 2001 From: Stefan Piperov Date: Wed, 28 Feb 2024 08:39:58 -0500 Subject: [PATCH] add single file access mode via xrootd --- af_benchmark/data_access/loader.py | 5 +++++ .../example-configs/example-config-dbs-file.yaml | 12 ++++++++++++ af_benchmark/executor/sequential.py | 7 ++++++- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 af_benchmark/example-configs/example-config-dbs-file.yaml diff --git a/af_benchmark/data_access/loader.py b/af_benchmark/data_access/loader.py index 62abfb9..8745661 100644 --- a/af_benchmark/data_access/loader.py +++ b/af_benchmark/data_access/loader.py @@ -20,6 +20,11 @@ def get_file_list(cls): xrootdserver = cls.config.get('data-access.xrootdserver', 'eos.cms.rcac.purdue.edu:1094') dbs = DbsApi('https://cmsweb.cern.ch/dbs/prod/global/DBSReader') file_list = ["root://"+xrootdserver+"/"+file['logical_file_name'] for file in dbs.listFiles(block_name=dbsblock)] + elif mode == 'dbs_file': + dbsfile = cls.config.get('data-access.file', "") + xrootdserver = cls.config.get('data-access.xrootdserver', 'cms-xcache.rcac.purdue.edu:1094') + dbs = DbsApi('https://cmsweb.cern.ch/dbs/prod/global/DBSReader') + file_list = ["root://"+xrootdserver+"/"+dbsfile] else: raise NotImplementedError( f"Data access modes other than 'local' and 'local_dir' are not yet implemented" diff --git a/af_benchmark/example-configs/example-config-dbs-file.yaml b/af_benchmark/example-configs/example-config-dbs-file.yaml new file mode 100644 index 0000000..a2ab66d --- /dev/null +++ b/af_benchmark/example-configs/example-config-dbs-file.yaml @@ -0,0 +1,12 @@ +executor: + backend: sequential +data-access: + mode: dbs_file +# xrootdserver: eos.cms.rcac.purdue.edu:1094 + xrootdserver: cms-xcache.rcac.purdue.edu:1094 +# xrootdserver: cmsxrootd.hep.wisc.edu:1094 + file: /store/data/Run2016B/SingleMuon/NANOAOD/02Apr2020_ver2-v1/20000/90322FC2-4027-0E47-92E4-22307EC8EAD2.root +processor: + parallelize_over: files + columns: 5 + operation: sum diff --git a/af_benchmark/executor/sequential.py b/af_benchmark/executor/sequential.py index aacefdb..7842b57 100644 --- a/af_benchmark/executor/sequential.py +++ b/af_benchmark/executor/sequential.py @@ -12,7 +12,12 @@ def _execute(self, func, args, **kwargs): :meta public: """ + first = [func(arg, **kwargs) for arg in args] return [func(arg, **kwargs) for arg in args] + print(first) + second=2 + print(second) + def get_n_workers(self): - return 1 \ No newline at end of file + return 1