From 8c223d60a2f7a373ce37b06e5f8d1dabb87ae609 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Mon, 1 Jun 2015 13:24:31 -0500 Subject: [PATCH 01/14] Adding suffix to processor Adding suffix to processor (variable in the init) to allow user to run a same spider with different parameters as two different assessors. --- dax/processors.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/dax/processors.py b/dax/processors.py index c5361fe7..0c592f95 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -11,12 +11,17 @@ class Processor(object): """ Base class for processor """ def __init__(self, walltime_str, memreq_mb, spider_path, - version=None, ppn=1, xsitype='proc:genProcData'): + version=None, ppn=1, xsitype='proc:genProcData', + suffix_proc=''): """ init function """ self.walltime_str = walltime_str # 00:00:00 format self.memreq_mb = memreq_mb # memory required in megabytes #default values - self.version = '1.0.0' + self.version = '1_0_0' + if suffix_proc and suffix_proc[0] != '_': + self.suffix_proc = suffix_proc + else: + self.suffix_proc = suffix_proc self.name = None self.spider_path = spider_path #getting name and version from spider_path @@ -34,9 +39,12 @@ def set_spider_settings(self, spider_path, version): proc_name = re.split("/*_v[0-9]/*", proc_name)[0] #setting the version and name of the spider self.version = version - self.name = proc_name+'_v'+self.version.split('.')[0] - self.spider_path = os.path.join(os.path.dirname(spider_path), - 'Spider_'+proc_name+'_v'+version+'.py') + self.name = '''{procname}_v{version}{suffix}'''.format(procname=proc_name, + version=self.version.split('.')[0], + suffix=self.suffix_proc) + spider_name = '''Spider_{procname}_v{version}.p'''.format(procname=proc_name, + version=version.replace('.', '_')) + self.spider_path = os.path.join(os.path.dirname(spider_path), spider_name) else: self.default_settings_spider(spider_path) @@ -48,9 +56,11 @@ def default_settings_spider(self, spider_path): if len(re.split("/*_v[0-9]/*", spider_path)) > 1: self.version = os.path.basename(spider_path)[7:-3].split('_v')[-1] spidername = os.path.basename(spider_path)[7:-3] - self.name = re.split("/*_v[0-9]/*", spidername)[0]+'_v'+self.version.split('.')[0] + self.name = '''{procname}_v{version}{suffix}'''.format(procname=re.split("/*_v[0-9]/*", spidername)[0], + version=self.version.split('.')[0], + suffix=self.suffix_proc) else: - self.name = os.path.basename(spider_path)[7:-3] + self.name = os.path.basename(spider_path)[7:-3]+self.suffix_proc # has_inputs - does this object have the required inputs? # e.g. NIFTI format of the required scan type and quality and are there no conflicting inputs. From 236f80cfdeb17266342accea062253e20a1ee923 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Mon, 1 Jun 2015 13:26:57 -0500 Subject: [PATCH 02/14] Update processors.py Setting default self.version to arguments version --- dax/processors.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dax/processors.py b/dax/processors.py index 0c592f95..5d30ac01 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -16,18 +16,18 @@ def __init__(self, walltime_str, memreq_mb, spider_path, """ init function """ self.walltime_str = walltime_str # 00:00:00 format self.memreq_mb = memreq_mb # memory required in megabytes - #default values - self.version = '1_0_0' + #default values: + self.version = version if suffix_proc and suffix_proc[0] != '_': self.suffix_proc = suffix_proc else: self.suffix_proc = suffix_proc self.name = None self.spider_path = spider_path - #getting name and version from spider_path - self.set_spider_settings(spider_path, version) self.ppn = ppn self.xsitype = xsitype + #getting name and version from spider_path + self.set_spider_settings(spider_path, version) #get the spider_path right with the version: def set_spider_settings(self, spider_path, version): From 386799eda5847071a0e0945da4b7c001c91f1e9e Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Mon, 1 Jun 2015 13:29:08 -0500 Subject: [PATCH 03/14] Update processors.py setting default value of self.version to "1.0.0" --- dax/processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dax/processors.py b/dax/processors.py index 5d30ac01..fdfed068 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -17,7 +17,7 @@ def __init__(self, walltime_str, memreq_mb, spider_path, self.walltime_str = walltime_str # 00:00:00 format self.memreq_mb = memreq_mb # memory required in megabytes #default values: - self.version = version + self.version = "1.0.0" if suffix_proc and suffix_proc[0] != '_': self.suffix_proc = suffix_proc else: From 853a9ea6052d8c3171ab0614cb2c9c302a215c2f Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Mon, 1 Jun 2015 13:32:29 -0500 Subject: [PATCH 04/14] adding suffix_proc to sub class Adding suffix_proc to Scan and Session processor class --- dax/processors.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dax/processors.py b/dax/processors.py index fdfed068..8c1c192f 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -89,9 +89,9 @@ def has_inputs(self): """ raise NotImplementedError() - def __init__(self, scan_types, walltime_str, memreq_mb, spider_path, version=None, ppn=1): + def __init__(self, scan_types, walltime_str, memreq_mb, spider_path, version=None, ppn=1, suffix_proc=''): """ init function overridden from base class """ - super(ScanProcessor, self).__init__(walltime_str, memreq_mb, spider_path, version, ppn) + super(ScanProcessor, self).__init__(walltime_str, memreq_mb, spider_path, version, ppn, suffix_proc) if isinstance(scan_types, list): self.scan_types = scan_types elif isinstance(scan_types, str): @@ -137,9 +137,9 @@ def has_inputs(self): """ raise NotImplementedError() - def __init__(self, walltime_str, memreq_mb, spider_path, version=None, ppn=1): + def __init__(self, walltime_str, memreq_mb, spider_path, version=None, ppn=1, suffix_proc=''): """ init function overridden from base class """ - super(SessionProcessor, self).__init__(walltime_str, memreq_mb, spider_path, version, ppn) + super(SessionProcessor, self).__init__(walltime_str, memreq_mb, spider_path, version, ppn, suffix_proc) def should_run(self, session_dict): """ return if the assessor should exist. Always true on a session """ From 7ae6d4590eb62d6489870b63adf027b094d5ad6a Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Mon, 1 Jun 2015 14:26:53 -0500 Subject: [PATCH 05/14] Adding suffix to spiderHandler Adding suffix_proc to the spiderhandler --- dax/XnatUtils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dax/XnatUtils.py b/dax/XnatUtils.py index b3658782..35c5254f 100644 --- a/dax/XnatUtils.py +++ b/dax/XnatUtils.py @@ -14,6 +14,8 @@ 5) Other Methods 6) Cached Class for DAX + + 7) Old download functions still used in some spiders """ import re @@ -124,7 +126,7 @@ def select_assessor(self, intf): class SpiderProcessHandler: """ Handle the results of a spider """ - def __init__(self, script_name, project, subject, experiment, scan=None): + def __init__(self, script_name, suffix, project, subject, experiment, scan=None): """ initialization """ #Variables: self.error = 0 @@ -145,6 +147,12 @@ def __init__(self, script_name, project, subject, experiment, scan=None): self.version = '1.0.0' proctype = script_name + #if suffix: + if suffix: + if suffix[0] !='_': #check that it starts with an underscore + suffix = '_'+suffix + proctype = proctype + suffix + #Create the assessor handler if not scan: assessor_label = project+'-x-'+subject+'-x-'+experiment+'-x-'+proctype From 027f80f41addf30c3790177c92d4ea1b3fc42984 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Wed, 3 Jun 2015 13:52:13 -0500 Subject: [PATCH 06/14] Update dax_manager Adding suffix_proc to the check_number_process --- bin/dax_tools/dax_manager | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/dax_tools/dax_manager b/bin/dax_tools/dax_manager index 02f72307..f7304105 100644 --- a/bin/dax_tools/dax_manager +++ b/bin/dax_tools/dax_manager @@ -503,8 +503,9 @@ def check_number_process(project_settings, proc_name): """ return the number of processes if more than just one set """ proc_name_tags = project_settings[proc_name[8:]+'_proc_name'].split(';') spider_tags = project_settings[proc_name[8:]+'_spider_path'].split(';') + suffix_tags = project_settings[proc_name[8:]+'_suffix_proc'].split(';') version_tags = project_settings[proc_name[8:]+'_version'].split(';') - return max(len(l) for l in [proc_name_tags, spider_tags, version_tags]) + return max(len(l) for l in [proc_name_tags, spider_tags, suffix_tags, version_tags]) def read_procname(project_settings, index, field_name, field_note): """ return procname from REDCap""" From 5e0b762653f4acd1564c89c0309e29c59be738d1 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Wed, 3 Jun 2015 14:32:52 -0500 Subject: [PATCH 07/14] Update processors.py replacing '_' by '.' for the version when reading from the spider path --- dax/processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dax/processors.py b/dax/processors.py index 8c1c192f..00a67d12 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -54,7 +54,7 @@ def default_settings_spider(self, spider_path): self.spider_path = spider_path #set the name and the version of the spider if len(re.split("/*_v[0-9]/*", spider_path)) > 1: - self.version = os.path.basename(spider_path)[7:-3].split('_v')[-1] + self.version = os.path.basename(spider_path)[7:-3].split('_v')[-1].replace('_','.') spidername = os.path.basename(spider_path)[7:-3] self.name = '''{procname}_v{version}{suffix}'''.format(procname=re.split("/*_v[0-9]/*", spidername)[0], version=self.version.split('.')[0], From 3410ff69f667fb4db0feb4cc9d0470e3a2f0e80c Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Wed, 3 Jun 2015 15:30:34 -0500 Subject: [PATCH 08/14] Update processors.py adding '_' in front of suffix_proc for suffix_proc. fixing issue with order of parameters. --- dax/processors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dax/processors.py b/dax/processors.py index 00a67d12..e450d5b8 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -11,15 +11,15 @@ class Processor(object): """ Base class for processor """ def __init__(self, walltime_str, memreq_mb, spider_path, - version=None, ppn=1, xsitype='proc:genProcData', - suffix_proc=''): + version=None, ppn=1, suffix_proc='', + xsitype='proc:genProcData'): """ init function """ self.walltime_str = walltime_str # 00:00:00 format self.memreq_mb = memreq_mb # memory required in megabytes #default values: self.version = "1.0.0" if suffix_proc and suffix_proc[0] != '_': - self.suffix_proc = suffix_proc + self.suffix_proc = '_'+suffix_proc else: self.suffix_proc = suffix_proc self.name = None From 7db7a272bedde9bf4c777f506e0d299a236c0604 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Thu, 4 Jun 2015 11:39:27 -0500 Subject: [PATCH 09/14] Update XnatUtils.py --- dax/XnatUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dax/XnatUtils.py b/dax/XnatUtils.py index 35c5254f..cb7b88bb 100644 --- a/dax/XnatUtils.py +++ b/dax/XnatUtils.py @@ -141,7 +141,7 @@ def __init__(self, script_name, suffix, project, subject, experiment, scan=None) #ge the processname from spider if len(re.split("/*_v[0-9]/*", script_name)) > 1: - self.version = script_name.split('_v')[-1] + self.version = re.split("/*_v[0-9]/*", script_name)[1].replace('_', '.') proctype = re.split("/*_v[0-9]/*", script_name)[0]+'_v'+self.version.split('.')[0] else: self.version = '1.0.0' From e7ef569033be698e5b2a2bbe1225c6364e205e96 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Fri, 5 Jun 2015 15:30:40 -0500 Subject: [PATCH 10/14] Update XnatUtils.py --- dax/XnatUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dax/XnatUtils.py b/dax/XnatUtils.py index cb7b88bb..8f1e84d4 100644 --- a/dax/XnatUtils.py +++ b/dax/XnatUtils.py @@ -141,7 +141,7 @@ def __init__(self, script_name, suffix, project, subject, experiment, scan=None) #ge the processname from spider if len(re.split("/*_v[0-9]/*", script_name)) > 1: - self.version = re.split("/*_v[0-9]/*", script_name)[1].replace('_', '.') + self.version = script_name.split('_v')[-1].replace('_','.') proctype = re.split("/*_v[0-9]/*", script_name)[0]+'_v'+self.version.split('.')[0] else: self.version = '1.0.0' From d206a13977dfad1cf3b51b589c21c12efaa41fd8 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Tue, 9 Jun 2015 14:34:01 -0500 Subject: [PATCH 11/14] Update processors.py --- dax/processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dax/processors.py b/dax/processors.py index e450d5b8..903bc7ff 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -42,7 +42,7 @@ def set_spider_settings(self, spider_path, version): self.name = '''{procname}_v{version}{suffix}'''.format(procname=proc_name, version=self.version.split('.')[0], suffix=self.suffix_proc) - spider_name = '''Spider_{procname}_v{version}.p'''.format(procname=proc_name, + spider_name = '''Spider_{procname}_v{version}.py'''.format(procname=proc_name, version=version.replace('.', '_')) self.spider_path = os.path.join(os.path.dirname(spider_path), spider_name) else: From 594949b2437344f9982b41a0655701c8d7536fb4 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Wed, 17 Jun 2015 15:53:22 -0500 Subject: [PATCH 12/14] Update processors.py --- dax/processors.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dax/processors.py b/dax/processors.py index 903bc7ff..a111747e 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -17,11 +17,16 @@ def __init__(self, walltime_str, memreq_mb, spider_path, self.walltime_str = walltime_str # 00:00:00 format self.memreq_mb = memreq_mb # memory required in megabytes #default values: - self.version = "1.0.0" + self.version = "1.0.0" if suffix_proc and suffix_proc[0] != '_': self.suffix_proc = '_'+suffix_proc else: self.suffix_proc = suffix_proc + self.suffix_proc = self.suffix_proc.strip().replace(" ","")\ + .replace('/','_').replace('*','_')\ + .replace('.','_').replace(',','_')\ + .replace('?','_').replace('!','_')\ + .replace(';','_').replace(':','_') self.name = None self.spider_path = spider_path self.ppn = ppn From 270f85b727ace557dea9796683398bee059d6cbc Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Wed, 17 Jun 2015 15:54:41 -0500 Subject: [PATCH 13/14] Update XnatUtils.py --- dax/XnatUtils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dax/XnatUtils.py b/dax/XnatUtils.py index 8f1e84d4..425be80f 100644 --- a/dax/XnatUtils.py +++ b/dax/XnatUtils.py @@ -151,6 +151,11 @@ def __init__(self, script_name, suffix, project, subject, experiment, scan=None) if suffix: if suffix[0] !='_': #check that it starts with an underscore suffix = '_'+suffix + suffix = suffix.strip().replace(" ","")\ + .replace('/','_').replace('*','_')\ + .replace('.','_').replace(',','_')\ + .replace('?','_').replace('!','_')\ + .replace(';','_').replace(':','_') proctype = proctype + suffix #Create the assessor handler From b25ac7f1c5137879c2cef7907c129fc9c2d53ea3 Mon Sep 17 00:00:00 2001 From: Benjamin Yvernault Date: Mon, 22 Jun 2015 13:43:31 -0500 Subject: [PATCH 14/14] Update processors.py Updated the init function in processor. --- dax/processors.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dax/processors.py b/dax/processors.py index a111747e..0eaf4402 100644 --- a/dax/processors.py +++ b/dax/processors.py @@ -18,10 +18,14 @@ def __init__(self, walltime_str, memreq_mb, spider_path, self.memreq_mb = memreq_mb # memory required in megabytes #default values: self.version = "1.0.0" - if suffix_proc and suffix_proc[0] != '_': - self.suffix_proc = '_'+suffix_proc + if not suffix_proc: + self.suffix_proc='' else: - self.suffix_proc = suffix_proc + if suffix_proc and suffix_proc[0] != '_': + self.suffix_proc = '_'+suffix_proc + else: + self.suffix_proc = suffix_proc + self.suffix_proc = self.suffix_proc.strip().replace(" ","")\ .replace('/','_').replace('*','_')\ .replace('.','_').replace(',','_')\ @@ -33,6 +37,9 @@ def __init__(self, walltime_str, memreq_mb, spider_path, self.xsitype = xsitype #getting name and version from spider_path self.set_spider_settings(spider_path, version) + #if suffix_proc is empty, set it to "" for the spider call: + if not suffix_proc: + self.suffix_proc = '""' #get the spider_path right with the version: def set_spider_settings(self, spider_path, version):