'archive' refuses to perform final archiving if empty 'Visium_images' dirs are present

auto_process_ngs/commands/archive_cmd.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 #
 #     archive_cmd.py: implement auto process archive command
-#     Copyright (C) University of Manchester 2017-2023 Peter Briggs
+#     Copyright (C) University of Manchester 2017-2024 Peter Briggs
 #
 #########################################################################
 
@@ -200,9 +200,27 @@ def archive(ap,archive_dir=None,platform=None,year=None,
         try:
             projects = ap.get_analysis_projects()
         except Exception as ex:
-            logging.warning("Error trying to fetch analysis projects: "
-                            "%s" % ex)
+            logger.warning("Error trying to fetch analysis projects: "
+                           "%s" % ex)
             projects = []
+        # Check projects
+        empty_visium_dirs = False
+        for project in projects:
+            # Check for empty 'Visium_images' subdirs
+            visium_images = os.path.join(project.dirn,"Visium_images")
+            if os.path.isdir(visium_images):
+                if not os.listdir(visium_images):
+                    logger.warning("'%s': project contains an empty "
+                                   "'Visium_images' subdirectory" %
+                                   project.name)
+                    empty_visium_dirs = True
+        # Check for problems before final archiving
+        if final and empty_visium_dirs:
+            if not force:
+                raise Exception("Empty 'Visium_images' subdirectories "
+                                "detected in one or more projects; "
+                                "either populate or remove (or use "
+                                "--force)")
         if not projects:
             if not force:
                 raise Exception("No project directories found, nothing "
@@ -213,9 +231,9 @@ def archive(ap,archive_dir=None,platform=None,year=None,
                 unaligned_dir = os.path.join(ap.analysis_dir,
                                              unaligned_dir)
             if os.path.exists(unaligned_dir):
-                logging.warning("No project directories found, forcing "
-                                "archiving of bcl2fastq output directory "
-                                "'%s' instead" % ap.params.unaligned_dir)
+                logger.warning("No project directories found, forcing "
+                               "archiving of bcl2fastq output directory "
+                               "'%s' instead" % ap.params.unaligned_dir)
                 include_bcl2fastq = True
             else:
                 raise Exception("No project directories or bcl2fastq "

auto_process_ngs/test/commands/test_archive_cmd.py

@@ -1356,3 +1356,99 @@ def test_archive_excludes_unwanted_content(self):
             self.assertFalse(os.path.exists(os.path.join(final_archive_dir,d)))
         for f in exclude_files:
             self.assertFalse(os.path.exists(os.path.join(final_archive_dir,f)))
+
+    def test_archive_to_final_checks_visium_images_dir(self):
+        """archive: test checking 'Visium_images' dir
+        """
+        # Make a mock auto-process directory
+        mockdir = MockAnalysisDirFactory.bcl2fastq2(
+            '170901_M00879_0087_000000000-AGEW9',
+            'miseq',
+            metadata={ "instrument_datestamp": "170901" },
+            top_dir=self.dirn)
+        mockdir.create()
+        # Add empty 'Visium_images' subdirectory in 'AB' project
+        os.mkdir(os.path.join(mockdir.dirn,"AB","Visium_images"))
+        # Make a mock archive directory
+        archive_dir = os.path.join(self.dirn,"archive")
+        final_dir = os.path.join(archive_dir,
+                                 "2017",
+                                 "miseq")
+        os.makedirs(final_dir)
+        self.assertTrue(os.path.isdir(final_dir))
+        self.assertEqual(len(os.listdir(final_dir)),0)
+        # Make autoprocess instance and set required metadata
+        ap = AutoProcess(analysis_dir=mockdir.dirn,
+                         settings=self.settings)
+        ap.set_metadata("source","testing")
+        ap.set_metadata("run_number","87")
+        # Do archiving op to staging
+        status = archive(ap,
+                         archive_dir=archive_dir,
+                         year='2017',platform='miseq',
+                         final=False)
+        self.assertEqual(status,0)
+        # Attempt to finalise archive
+        # Should fail as 'Visium_images' is empty
+        self.assertRaises(Exception,
+                          archive,
+                          ap,
+                          archive_dir=archive_dir,
+                          year='2017',platform='miseq',
+                          final=True)
+        # Add fake image file to 'Visium_images'
+        with open(os.path.join(mockdir.dirn,"AB","Visium_images",
+                               "image1.tff"),'wt') as fp:
+            fp.write("image data")
+        # Attempt archive to final again
+        status = archive(ap,
+                         archive_dir=archive_dir,
+                         year='2017',platform='miseq',
+                         final=True)
+        self.assertEqual(status,0)
+        # Check that final dir exists
+        final_archive_dir = os.path.join(
+            final_dir,
+            "170901_M00879_0087_000000000-AGEW9_analysis")
+        self.assertTrue(os.path.exists(final_archive_dir))
+        self.assertEqual(len(os.listdir(final_dir)),1)
+
+    def test_archive_to_final_force_ignores_empty_visium_images_dir(self):
+        """archive: force final archiving with empty 'Visium_images' dir
+        """
+        # Make a mock auto-process directory
+        mockdir = MockAnalysisDirFactory.bcl2fastq2(
+            '170901_M00879_0087_000000000-AGEW9',
+            'miseq',
+            metadata={ "instrument_datestamp": "170901" },
+            top_dir=self.dirn)
+        mockdir.create()
+        # Add empty 'Visium_images' subdirectory in 'AB' project
+        os.mkdir(os.path.join(mockdir.dirn,"AB","Visium_images"))
+        # Make a mock archive directory
+        archive_dir = os.path.join(self.dirn,"archive")
+        final_dir = os.path.join(archive_dir,
+                                 "2017",
+                                 "miseq")
+        os.makedirs(final_dir)
+        self.assertTrue(os.path.isdir(final_dir))
+        self.assertEqual(len(os.listdir(final_dir)),0)
+        # Make autoprocess instance and set required metadata
+        ap = AutoProcess(analysis_dir=mockdir.dirn,
+                         settings=self.settings)
+        ap.set_metadata("source","testing")
+        ap.set_metadata("run_number","87")
+        # Do final archiving op with force
+        # Should ignore empty 'Visium_images'
+        status = archive(ap,
+                         archive_dir=archive_dir,
+                         year='2017',platform='miseq',
+                         final=True,
+                         force=True)
+        self.assertEqual(status,0)
+        # Check that final dir exists
+        final_archive_dir = os.path.join(
+            final_dir,
+            "170901_M00879_0087_000000000-AGEW9_analysis")
+        self.assertTrue(os.path.exists(final_archive_dir))
+        self.assertEqual(len(os.listdir(final_dir)),1)