Merge pull request #21 from VUIIS/spider_class

Spider class

bin/Xnat_tools/Xnatcheck

@@ -265,7 +265,7 @@ def setup_info_logger(name,logfile):
     if logfile:
         handler=logging.FileHandler(logfile,'w')
     else:
-        handler=logging.StreamHandler()
+        handler=logging.StreamHandler(sys.stdout)
 
     logger = logging.getLogger(name)
     logger.setLevel(logging.INFO)

bin/Xnat_tools/Xnatupload

@@ -409,7 +409,7 @@ def upload_data_xnat(xnat,options,scan_list,assessor_list):
     for index,ps in enumerate(sorted(proj_subj_list)):
         project=ps.split('-x-')[0]
         subject=ps.split('-x-')[1]
-        logger.info(' * '+str(index+1)+'/'+str(number)+' Project/Subject : '+project+'/'+subject)
+        logger.info(' * '+str(index+1)+'/'+str(number)+' subjects -- Pair Project/Subject : '+project+'/'+subject)
         ##SCAN##
         for scan in extract_obj_one_subject(project,subject,scan_list): 
             scan_obj,_=get_xnat_obj(xnat,scan,options.session_type)

bin/dax_tools/GenerateModuleTemplate

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 """
     Title: GenerateModuleTemplate
     Author: Benjamin Yvernault
@@ -6,16 +9,14 @@
         Generate your module.py following the template for module describe in this file.
 """
 
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 __author__ = 'Benjamin Yvernault'
 __email__ = '[email protected]'
 __purpose__ = "Generate your module.py following the template for module describe in this file."
 __version__ = '1.0.0'
 __modifications__ = '24 August 2015 - Original write'
 
 import os
+import re
 from datetime import datetime
 
 DEFAULT_TEMPLATE = """'''
@@ -275,10 +276,18 @@ def parse_args():
 if __name__ == '__main__':
     ARGS = parse_args()
 
-    # Removing Spider if present in name
+    ## Get a proper name from the input
+    # remove .py if present at the end of the file
+    if ARGS.name.endswith('.py'):
+        ARGS.name = ARGS.name[:-3]
+    # remove settings if present in name
     if "module" in ARGS.name.lower():
-        ERR = "wrong name for the module. Remove 'module' from the name."
-        raise ValueError(ERR)
+        module_search = re.compile(re.escape('module'), re.IGNORECASE)
+        ARGS.name = module_search.sub('', ARGS.name)
+    # remove any particular character and change it by an underscore
+    ARGS.name = re.sub('[^a-zA-Z0-9]', '_', ARGS.name)
+    if ARGS.name[-1] == '_':
+        ARGS.name = ARGS.name[:-1]
 
     MODULE_NAME = """Module_{name}.py""".format(name=ARGS.name)
     if ARGS.directory and os.path.exists(ARGS.directory):

bin/dax_tools/GenerateProcessorTemplate

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 """
     Title: GenerateProcessorTemplate
     Author: Benjamin Yvernault
@@ -6,16 +9,14 @@
         Generate your processor.py following the template for processor describe in this file.
 """
 
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 __author__ = 'Benjamin Yvernault'
 __email__ = '[email protected]'
 __purpose__ = "Generate your processor.py following the template for processor describe in this file."
 __version__ = '1.0.0'
 __modifications__ = '24 August 2015 - Original write'
 
 import os
+import re
 from datetime import datetime
 
 DEFAULT_TEMPLATE = """'''
@@ -35,7 +36,6 @@ __modifications__ = "{now} - Original write"
 # Python packages import
 import os
 import logging
-from dax_spiders import PathSettings
 from dax import XnatUtils, ScanProcessor, SessionProcessor
 
 # set-up logger for printing statements
@@ -46,13 +46,14 @@ LOGGER = logging.getLogger('dax')
 DEFAULT_SPIDER_PATH = os.path.join(SPIDER_PATH, 'Spider_{name}_1_0_0.py')
 DEFAULT_WALLTIME = '01:00:00'
 DEFAULT_MEM = 2048
-DEFAULT_SCAN_TYPES = [] # ADD SCAN TYPES
 """
 
 SCAN_LEVEL_TEMPLATE = DEFAULT_TEMPLATE+"""
+DEFAULT_SCAN_TYPES = [] # ADD SCAN TYPES
+
 # Format for the spider command line
-SPIDER_FORMAT = '''python {{spider}} -v --spm {{spm}} \
--p {{proj}} -s {{subj}} -e {{sess}} -c {{scan}} -d {{dir}} --suffix "{{suffix_proc}}"'''
+SPIDER_FORMAT = '''python {{spider}} -p {{proj}} -s {{subj}} -e {{sess}} -c {{scan}} \
+-d {{dir}} --suffix "{{suffix_proc}}"'''
 
 class Processor_{name}(ScanProcessor):
     '''
@@ -63,18 +64,18 @@ class Processor_{name}(ScanProcessor):
     :param walltime: walltime required by the spider
     :param mem_mb: memory in Mb required by the spider
     :param scan_types: scan types on XNAT that the spider should run on
-    :param spm_path: path to SPM
+
     #
     # ADD MORE PARAMETERS AS NEEDED HERE
     #
+
     :param suffix: suffix to the spider
     '''
     def __init__(self, spider_path=DEFAULT_SPIDER_PATH, version=None,
                  walltime=DEFAULT_WALLTIME, mem_mb=DEFAULT_MEM,
-                 scan_types=DEFAULT_SCAN_TYPES, spm_path=PathSettings.SPM_PATH, suffix_proc=''):
+                 scan_types=DEFAULT_SCAN_TYPES, suffix_proc=''):
         super(Processor_{name}, self).__init__(scan_types, walltime, mem_mb, spider_path,
                                                version, suffix_proc=suffix_proc)
-        self.spm_path = spm_path
         #
         # ADD MORE PARAMETERS AS NEEDED HERE LIKE self.param = param
         #
@@ -90,6 +91,7 @@ class Processor_{name}(ScanProcessor):
         :param cscan: object cscan define in dax.XnatUtils (see XnatUtils in dax for information)
         :return: status, qcstatus
         '''
+
         #
         # CODE TO CHECK IF THE PROCESS HAS THE INPUTS NEEDED FROM XNAT
         # CHECK FUNCTION FROM XnatUtils IN dax:
@@ -128,7 +130,6 @@ class Processor_{name}(ScanProcessor):
         #
 
         cmd = SPIDER_FORMAT.format(spider=self.spider_path,
-                                   spm=self.spm_path,
                                    proj=proj_label,
                                    subj=subj_label,
                                    sess=sess_label,
@@ -141,8 +142,8 @@ class Processor_{name}(ScanProcessor):
 
 SESSION_LEVEL_TEMPLATE = DEFAULT_TEMPLATE+"""
 # Format for the spider command line
-SPIDER_FORMAT = '''python {{spider}} -v --spm {{spm}} \
--p {{proj}} -s {{subj}} -e {{sess}} -d {{dir}} --suffix "{{suffix_proc}}"'''
+SPIDER_FORMAT = '''python {{spider}} -p {{proj}} -s {{subj}} -e {{sess}} \
+-d {{dir}} --suffix "{{suffix_proc}}"'''
 
 class {name}_Processor(SessionProcessor):
     '''
@@ -152,19 +153,16 @@ class {name}_Processor(SessionProcessor):
     :param version: version of the spider
     :param walltime: walltime required by the spider
     :param mem_mb: memory in Mb required by the spider
-    :param scan_types: scan types on XNAT that the spider should run on
-    :param spm_path: path to SPM
     #
     # ADD MORE PARAMETERS AS NEEDED HERE
     #
     :param suffix: suffix to the spider
     '''
     def __init__(self, spider_path=DEFAULT_SPIDER_PATH, version=None,
                  walltime=DEFAULT_WALLTIME, mem_mb=DEFAULT_MEM,
-                 spm_path=PathSettings.SPM_PATH, suffix_proc=''):
+                 suffix_proc=''):
         super({name}_Processor, self).__init__(walltime, mem_mb, spider_path, version,
                                                suffix_proc=suffix_proc)
-        self.spm_path = spm_path
         #
         # ADD MORE PARAMETERS AS NEEDED HERE LIKE self.param = param
         #
@@ -180,6 +178,7 @@ class {name}_Processor(SessionProcessor):
         :param csess: object csess define in dax.XnatUtils (see XnatUtils in dax for information)
         :return: status, qcstatus
         '''
+
         #
         # CODE TO CHECK IF THE PROCESS HAS THE INPUTS NEEDED FROM XNAT
         # CHECK FUNCTION FROM XnatUtils IN dax:
@@ -209,7 +208,6 @@ class {name}_Processor(SessionProcessor):
                                    proj=proj_label,
                                    subj=subj_label,
                                    sess=sess_label,
-                                   spm=self.spm_path,
                                    dir=jobdir)
 
         return [cmd]
@@ -251,10 +249,18 @@ def parse_args():
 if __name__ == '__main__':
     ARGS = parse_args()
 
-    # Removing Spider if present in name
+    ## Get a proper name from the input
+    # remove .py if present at the end of the file
+    if ARGS.name.endswith('.py'):
+        ARGS.name = ARGS.name[:-3]
+    # remove settings if present in name
     if "processor" in ARGS.name.lower():
-        ERR = "wrong name for the processor. Remove 'processor' from the name"
-        raise ValueError(ERR)
+        processor_search = re.compile(re.escape('processor'), re.IGNORECASE)
+        ARGS.name = processor_search.sub('', ARGS.name)
+    # remove any particular character and change it by an underscore
+    ARGS.name = re.sub('[^a-zA-Z0-9]', '_', ARGS.name)
+    if ARGS.name[-1] == '_':
+        ARGS.name = ARGS.name[:-1]
 
     PROCESSOR_NAME = """Processor_{name}.py""".format(name=ARGS.name)
     if ARGS.directory and os.path.exists(ARGS.directory):

bin/dax_tools/GenerateSettingsTemplate

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    Title: GenerateSettingsTemplate
+    Author: Benjamin Yvernault
+    contact: [email protected]
+    Purpose:
+        Generate your ProjectSettingsFile.py following the template describe in this file.
+"""
+
+__author__ = 'Benjamin Yvernault'
+__email__ = '[email protected]'
+__purpose__ = 'Generate your ProjectSettingsFile.py following the template describe in this file.'
+__version__ = '1.0.0'
+__modifications__ = '29 Septembre 2015 - Original write'
+
+import os
+import re
+from datetime import datetime
+from dax import dax_settings
+
+DEFAULT_TEMPLATE = """'''
+    Author:                     {author}
+    contact:                    {email_addr}
+    Project Settings File name: {name}
+    Creation date:              {now}
+    Purpose:                    Settings file to describe the modules/processors for the project(s)
+'''
+
+## header:
+__author__ = "{author}"
+__email__ = "{email_addr}"
+__purpose__ = "Settings file to describe the modules/processors for the project(s)"
+__processor_name__ = "{name}"
+__modifications__ = "{now} - Original write"
+
+## Python packages import:
+import os
+from dax import Launcher
+from Xnat_process_importer import *
+from Xnat_module_importer import *
+
+## Arguments for Launcher:
+EMAIL = "{email_addr}"
+QUEUE_LIMIT = {q_limit}
+PRIORITY_ORDER = {p_order}  # CHANGE THE ORDER OF PRIORITY FOR YOUR PROJECT(S)
+EMAIL_OPTS = {e_opts}
+
+## Modules
+
+#
+# DEFINE THE MODULES YOU WANT TO RUN FOR YOUR PROJECT(S).
+# E.G: VUSTP_Module_dcm2nii = Module_dcm2nii(directory="/tmp/dcm2nii_phillips")
+#
+
+## Processors
+
+#
+# DEFINE THE PROCESSORS YOU WANT TO RUN FOR YOUR PROJECT(S).
+# E.G: VUSTP_1_Processor_fMRIQA = Processor_fMRIQA(mem_mb="4096",version="2.0.0")
+#
+
+## Associate Project with modules/processors
+# modules
+# ADD THE MODULES FOR EACH PROJECT OR LEAVE IT EMPTY
+proj_mod = {p_mod}
+
+#processors
+# ADD THE PROCESSORS FOR EACH PROJECT OR LEAVE IT EMPTY
+proj_proc = {p_proc}
+
+## Launcher
+myLauncher = Launcher(proj_proc, proj_mod, priority_project=PRIORITY_ORDER,
+                      job_email=EMAIL, job_email_options=EMAIL_OPTS,
+                      queue_limit=QUEUE_LIMIT)
+
+"""
+
+def write_settings(settings_fpath, args):
+    """
+    Write the ProjectSettingsFile.py from the template
+
+    :param settings_fpath: path where the settings script will be saved
+    :param args: arguments parser
+    """
+    priority = args.p_order.split(",")
+    p_mod = '{'
+    p_proc = '{'
+    for ind,  project in enumerate(priority):
+        if ind != 0:
+            p_mod += ' '*12+'"'+project+'": [],\n' # 12 = length of proj_mod = {
+            p_proc += ' '*13+'"'+project+'": [],\n' # 13 = length of proj_proc = {
+        else:
+            p_mod += '"'+project+'": [],\n' # 12 = length of proj_mod = {
+            p_proc += '"'+project+'": [],\n' # 13 = length of proj_proc = {
+    p_mod = p_mod[:-2]+'}'
+    p_proc = p_proc[:-2]+'}'
+
+    settings_code = DEFAULT_TEMPLATE.format(author=args.author,
+                                            email_addr=args.email,
+                                            name=args.name,
+                                            now=str(datetime.now()),
+                                            q_limit=args.q_limit,
+                                            p_order=priority,
+                                            e_opts=args.e_opts,
+                                            p_mod=p_mod,
+                                            p_proc=p_proc)
+    f_obj = open(settings_fpath, "w")
+    f_obj.writelines(settings_code)
+    f_obj.close()
+
+def parse_args():
+    """
+    Parser for arguments
+    """
+    from argparse import ArgumentParser
+    usage = "Generate a ProjectSettingsFile.py file from the dax template for settings file."
+    argp = ArgumentParser(prog='GenerateSettingsTemplate', description=usage)
+    argp.add_argument('-n', dest='name', help='Name for ProjectSettingsFile.', required=True)
+    argp.add_argument('-a', dest='author', help='Author name.', required=True)
+    argp.add_argument('-e', dest='email', help='Author email address.', required=True)
+    argp.add_argument('-p', dest='p_order', help='Projects to process in order separate by a coma. E.G: ADNI,Test,Project3', required=True)
+    argp.add_argument('--Qlimit', dest='q_limit', help='Queue limit on cluster to submit jobs. Default: '+str(dax_settings.DEFAULT_QUEUE_LIMIT),
+                      default=dax_settings.DEFAULT_QUEUE_LIMIT)
+    argp.add_argument('--Eopts', dest='e_opts', help='Options for email in the job. Default= '+dax_settings.DEFAULT_EMAIL_OPTS,
+                      default=dax_settings.DEFAULT_EMAIL_OPTS)
+    argp.add_argument('-d', dest='directory', help='Directory where the processor file will be generated. Default: current directory.', default=None)
+    return argp.parse_args()
+
+if __name__ == '__main__':
+    ARGS = parse_args()
+
+    ## Get a proper name from the input
+    # remove .py if present at the end of the file
+    if ARGS.name.endswith('.py'):
+        ARGS.name = ARGS.name[:-3]
+    # remove settings if present in name
+    if "settings" in ARGS.name.lower():
+        settings_search = re.compile(re.escape('settings'), re.IGNORECASE)
+        ARGS.name = settings_search.sub('', ARGS.name)
+    # remove any particular character and change it by an underscore
+    ARGS.name = re.sub('[^a-zA-Z0-9]', '_', ARGS.name)
+    if ARGS.name[-1] == '_':
+        ARGS.name = ARGS.name[:-1]
+
+    SETTINGS_NAME = """Settings_{name}.py""".format(name=ARGS.name)
+    if ARGS.directory and os.path.exists(ARGS.directory):
+        SETTINGS_FPATH = os.path.join(ARGS.directory, SETTINGS_NAME)
+    else:
+        SETTINGS_FPATH = os.path.join(os.getcwd(), SETTINGS_NAME)
+
+    print "Generating file %s for scan processor %s ..." % (SETTINGS_FPATH, ARGS.name)
+    write_settings(SETTINGS_FPATH, ARGS)