Bio 2360 organise from config

config/organise_config/organise_project.yml

@@ -0,0 +1,31 @@
+# Analysis results, readme and runfolder/runfolders.
+#Input: PROJECTID
+variables:
+  inputkey: "projectid"
+  rootpath: "/proj/ngi2016001/nobackup/NGI"
+  runfolderpath: "/proj/ngi2016001/incoming"
+  analysispath: "{rootpath}/ANALYSIS/{inputkey}"
+  deliverypath: "{rootpath}/DELIVERY/{inputkey}"
+files_to_organise:
+- source: "{analysispath}/results"
+  destination: "{deliverypath}/results"
+  options: 
+    required: True
+    link_type: "softlink" 
+- source: "{rootpath}/DELIVERY/softlinks/DELIVERY.README.RNASEQ.md"
+  destination: "{deliverypath}/DELIVERY.README.RNASEQ.md"
+  options: 
+    required: True
+    link_type: "copy"
+- source: "{runfolderpath}/*/Projects/{inputkey}/*"
+  destination: "{deliverypath}/fastq/{runfoldername}/{samplename}/"
+  options: 
+    required: True
+    link_type: "softlink"
+    filter: "{runfolderpath}/(?P<runfoldername>[^/]+)/Projects/{inputkey}/(?P=runfoldername)/(?P<samplename>[^/]+)/"
+- source: "{runfolderpath}/*/Projects/{inputkey}/*"
+  destination: "{deliverypath}/fastq/{runfoldername}/"
+  options: 
+    required: True
+    link_type: "softlink"
+    filter: "{runfolderpath}/(?P<runfoldername>[^/]+)/Projects/{inputkey}/(?P=runfoldername)/(?P=runfoldername)_{inputkey}_multiqc_report[\\w.-]+"

config/organise_config/organise_runfolder.yml

@@ -2,29 +2,31 @@
 #Input: runfolder_name
 
 #Currently named runfolderpath and runfolder to harmonize with rnaseq config by Monika, but for specific runfolder delivery, technically, these could be "merged" to one.
-runfolderpath: /proj/ngi2016001/incoming
-runfolder: <RUNFOLDERPATH>/<RUNFOLDER_NAME>
-organised: <RUNFOLDER>/Projects/
+variables:
+  inputkey: "runfolder_name"  
+  runfolderpath: "/proj/ngi2016001/incoming"
+  runfolder: "{runfolderpath}/{inputkey}"
+  organised: "{runfolder}/Projects"
 
 #current path is /proj/ngi2016001/incoming/<RUNFOLDER>/Projects/<PROJECT>/<RUNFOLDER>/
 #The following is based on that we are to keep the same directory structure as above.
-files_to_organize:
-
-  #The fastq files
-  - source: <RUNFOLDER>/Unaligned/*  
-    destination: <ORGANISED>/(?P=projectid)/<RUNFOLDER_NAME>/Sample_(?P=samplename)/
-    options:
-      required: True
-      symlink: True
-      regexp: (?P<projectid>[\w-]+)/Sample_(?P<samplename>[\w-]+)/(?P=samplename)_S(?P<samplenumber>\d+)_L(?P<lanes>\d+)_R(?P<read>\d)_001.fastq.gz
-
-   #The MultiQC files
-   - source: <RUNFOLDER>/seqreports/project/*
-     destination: <ORGANISED>/(?P=projectid)/<RUNFOLDER_NAME>/
-     options:
-       required: True
-       symlink: True
-       regexp: (?P<projectid>[\w-]+)/(?P<RUNFOLDER_NAME>\w+)_(?P=projectid)_multiqc_report[\w.-]+
+files_to_organise:
+
+#The fastq files
+- source: "{runfolder}/Unaligned/*"  
+  destination: "{organised}/{projectid}/{inputkey}/Sample_{samplename}/"
+  options:
+    required: True
+    link_type: "softlink"
+    filter: "(?P<projectid>[\\w-]+)/Sample_(?P<samplename>[\\w-]+)/(?P=samplename)_S(?P<samplenumber>\\d+)_L(?P<lanes>\\d+)_R(?P<read>\\d)_001.fastq.gz"
+
+#The MultiQC files
+- source: "{runfolder}/seqreports/projects/*"
+  destination: "{organised}/{projectid}/{inputkey}/"
+  options:
+    required: True
+    link_type: "softlink"
+    filter: "(?P<projectid>[\\w-]+)/{inputkey}_(?P=projectid)_multiqc_report[\\w.-]+"
 
 
 

delivery/exceptions.py

@@ -59,13 +59,6 @@ class ProjectAlreadyDeliveredException(Exception):
     pass
 
 
-class ProjectAlreadyOrganisedException(Exception):
-    """
-    Should be raised when a project has already been organised.
-    """
-    pass
-
-
 class FileNameParsingException(Exception):
     pass
 

delivery/handlers/organise_handlers.py

@@ -3,7 +3,7 @@
 
 from arteria.web.handlers import BaseRestHandler
 from delivery.exceptions import ProjectsDirNotfoundException, ChecksumFileNotFoundException, FileNameParsingException, \
-    SamplesheetNotFoundException, ProjectReportNotFoundException, ProjectAlreadyOrganisedException
+    SamplesheetNotFoundException, ProjectReportNotFoundException
 from delivery.handlers import OK, NOT_FOUND, INTERNAL_SERVER_ERROR, FORBIDDEN
 
 log = logging.getLogger(__name__)
@@ -72,12 +72,14 @@ def post(self, runfolder_id):
         except (ProjectsDirNotfoundException,
                 ChecksumFileNotFoundException,
                 SamplesheetNotFoundException,
-                ProjectReportNotFoundException) as e:
+                ProjectReportNotFoundException,
+                FileNotFoundError) as e:
             log.error(str(e), exc_info=e)
             self.set_status(NOT_FOUND, reason=str(e))
-        except ProjectAlreadyOrganisedException as e:
+        except PermissionError as e:
             log.error(str(e), exc_info=e)
             self.set_status(FORBIDDEN, reason=str(e))
-        except FileNameParsingException as e:
+        except (FileNameParsingException,
+                RuntimeError) as e:
             log.error(str(e), exc_info=e)
             self.set_status(INTERNAL_SERVER_ERROR, reason=str(e))

delivery/services/file_system_service.py

@@ -1,6 +1,10 @@
 
-import os
+import glob
 import logging
+import os
+import pathlib
+import shutil
+from contextlib import contextmanager
 
 log = logging.getLogger(__name__)
 
@@ -11,6 +15,19 @@ class FileSystemService(object):
     easily be mocked out in testing.
     """
 
+    @staticmethod
+    @contextmanager
+    def change_directory(new_dir):
+        """
+        A context manager for changing directory and changing back when leaving the context
+        """
+        current_dir = os.getcwd()
+        try:
+            os.chdir(new_dir)
+            yield
+        finally:
+            os.chdir(current_dir)
+
     @staticmethod
     def list_directories(base_path):
         """
@@ -81,15 +98,46 @@ def abspath(path):
         """
         return os.path.abspath(path)
 
+    def create_parent_dirs(self, child_path):
+        """
+        Create the parent directory structure for a child path
+        :param child_path: path to child
+        :return: None
+        """
+        self.makedirs(self.dirname(child_path), exist_ok=True)
+
     def symlink(self, source, link_name):
         """
         Shadows os.symlink
         :param source: of link
         :param link_name: the name of the link to create
         :return: None
         """
-        self.makedirs(self.dirname(link_name), exist_ok=True)
-        return os.symlink(source, link_name)
+        self.create_parent_dirs(link_name)
+        return pathlib.Path(link_name).symlink_to(source)
+
+    def hardlink(self, source, link_name):
+        """
+        Shadows os.symlink
+        :param source: of link
+        :param link_name: the name of the link to create
+        :return: None
+        """
+        self.create_parent_dirs(link_name)
+        return pathlib.Path(link_name).hardlink_to(source)
+
+    def copy(self, source, dest):
+        """
+        Shadows shutil.copyfile
+        :param source:
+        :param dest:
+        :return: None
+        """
+        self.create_parent_dirs(dest)
+        try:
+            return shutil.copyfile(source, dest)
+        except IsADirectoryError:
+            return shutil.copytree(source, dest, symlinks=True)
 
     @staticmethod
     def mkdir(path):
@@ -124,3 +172,10 @@ def rename(src, dst):
     @staticmethod
     def relpath(path, start):
         return os.path.relpath(path, start)
+
+    @staticmethod
+    def glob(pattern, root_dir=None):
+        if root_dir:
+            with FileSystemService.change_directory(root_dir):
+                return FileSystemService.glob(pattern, root_dir=None)
+        return glob.glob(pattern, recursive=True)