Allow users to specify a list of files to process as the primary input.

dmwm · Feb 11, 2014 · 6d82203 · 6d82203
1 parent 3c334b3
commit 6d82203
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 4 deletions.
diff --git a/src/python/CRABClient/ClientMapping.py b/src/python/CRABClient/ClientMapping.py
@@ -40,9 +40,9 @@
                          },
                   'other-config-params' : ["General.serverUrl", "General.requestName", "General.workArea",
                                            "JobType.pluginName", "JobType.externalPluginFile", "JobType.psetName",
-                                           "JobType.inputFiles", "JobType.pyCfgParams", "Data.primaryDataset"
+                                           "JobType.inputFiles", "JobType.pyCfgParams", "Data.primaryDataset",
                                            "Data.unitsPerJob", "Data.splitting", "Data.inputDataset", "Data.lumiMask", "Data.runRange",
-                                           "User.email", "General.instance", "Debug.oneEventMode"],
+                                           "User.email", "General.instance", "Debug.oneEventMode", "Data.userInputFile"],
                   'requiresTaskOption' : False,
                 },
     'getlog': {'requiresTaskOption' : True},

diff --git a/src/python/CRABClient/Commands/submit.py b/src/python/CRABClient/Commands/submit.py
@@ -213,7 +213,7 @@ def _encodeRequest( self, configreq):
             cmsweb, e.g.:   adduserfiles = ['file1','file2']  ===>  [...]adduserfiles=file1&adduserfiles=file2[...]
         """
         listParams = ['adduserfiles', 'addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist',
-                      'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis'] #TODO automate this using ClientMapping
+                      'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles']
         encodedLists = ''
         for lparam in listParams:
             if lparam in configreq:

diff --git a/src/python/CRABClient/JobType/Analysis.py b/src/python/CRABClient/JobType/Analysis.py
@@ -86,6 +86,28 @@ def run(self, requestConfig):
         configArguments['cacheurl'] = uploadResults[0]
         isbchecksum = uploadResults[2]
 
+        # Upload list of user-defined input files to process as the primary input
+        userFileName = getattr(self.config.Data, 'userInputFile', None)
+        if userFileName:
+            self.logger.debug("Attaching a list of user-specified primary input files from %s." % userFileName)
+            fnames = []
+            for fname in open(userFileName).readlines():
+                fnames.append(fname.strip())
+            configArguments['userfiles'] = fnames
+            print fnames
+
+            primDS = getattr(self.config.Data, 'primaryDataset', None)
+            if primDS:
+                # Normalizes "foo/bar" and "/foo/bar" to "/foo/bar"
+                primDS = "/" + os.path.join(*primDS.split("/"))
+                if not re.match("/%(primDS)s.*" % WMCore.Lexicon.lfnParts, primDS):
+                    self.logger.warning("Invalid primary dataset name %s for private MC; publishing may fail" % primDS)
+                configArguments['inputdata'] = primDS
+            elif getattr(self.config.Data, 'inputDataset', None):
+                configArguments['inputdata'] = self.config.Data.inputDataset
+            else:
+                configArguments['inputdata'] = "/CRAB_UserFiles"
+
         # Upload lumi mask if it exists
         lumiMaskName = getattr(self.config.Data, 'lumiMask', None)
         if lumiMaskName:
@@ -113,7 +135,7 @@ def validateConfig(self, config):
         if not valid:
             return (valid, reason)
 
-        if not getattr(config.Data, 'inputDataset', None):
+        if not getattr(config.Data, 'inputDataset', None) and not getattr(config.Data, 'userInputFile', None):
             valid = False
             reason += 'Crab configuration problem: missing or null input dataset name. '
 

diff --git a/src/python/CRABClient/JobType/PrivateMC.py b/src/python/CRABClient/JobType/PrivateMC.py
@@ -2,6 +2,7 @@
 PrivateMC job type plug-in
 """
 
+import os
 import re
 
 from CRABClient.JobType.Analysis import Analysis