🔧 patch for v3.24.0 latest for python 2.7

setup.py

@@ -59,6 +59,7 @@ def runSetup():
     addict = 'addict<=2.2.0'
     sphinx = 'sphinx==1.7.5'
     pathlib2 = 'pathlib2==2.3.2'
+    enum34 = 'enum34==1.1.10'
 
     core_reqs = [
         dill,
@@ -71,7 +72,8 @@ def runSetup():
         subprocess32,
         addict,
         sphinx,
-        pathlib2]
+        pathlib2,
+        enum34]
 
     aws_reqs = [
         boto,

src/toil/batchSystems/abstractBatchSystem.py

@@ -16,6 +16,7 @@
 standard_library.install_aliases()
 from future.utils import with_metaclass
 from builtins import object
+import enum
 import os
 import shutil
 import logging
@@ -37,6 +38,44 @@
 
 logger = logging.getLogger(__name__)
 
+# Value to use as exitStatus in UpdatedBatchJobInfo.exitStatus when status is not available.
+EXIT_STATUS_UNAVAILABLE_VALUE = 255
+
+class BatchJobExitReason(enum.IntEnum):
+    FINISHED = 1
+    """Successfully finished."""
+    FAILED = 2
+    """Job finished, but failed."""
+    LOST = 3
+    """Preemptable failure (job's executing host went away)."""
+    KILLED = 4
+    """Job killed before finishing."""
+    ERROR = 5
+    """Internal error."""
+    MEMLIMIT = 6
+    """Job hit batch system imposed memory limit."""
+    MISSING = 7
+    """Job disappeared from the scheduler without actually stopping, so Toil killed it."""
+    MAXJOBDURATION = 8
+    """Job ran longer than --maxJobDuration, so Toil killed it."""
+    PARTITION = 9
+    """Job was not able to talk to the leader via the job store, so Toil declared it failed."""
+
+
+    @classmethod
+    def to_string(cls, value):
+        """
+        Convert to human-readable string.
+
+        Given an int that may be or may be equal to a value from the enum,
+        produce the string value of its matching enum entry, or a stringified
+        int.
+        """
+        try:
+            return cls(value).name
+        except ValueError:
+            return str(value)
+
 
 # A class containing the information required for worker cleanup on shutdown of the batch system.
 WorkerCleanupInfo = namedtuple('WorkerCleanupInfo', (

src/toil/batchSystems/abstractGridEngineBatchSystem.py

@@ -27,24 +27,31 @@
 
 from toil import subprocess
 from toil.lib.objects import abstractclassmethod
+from toil.lib.misc import CalledProcessErrorStderr
 
 from toil.batchSystems.abstractBatchSystem import BatchSystemLocalSupport
 
 logger = logging.getLogger(__name__)
 
 
 def with_retries(operation, *args, **kwargs):
-    retries = 3
-    latest_err = None
-    while retries:
-        retries -= 1
+    """Add an incremental sleep after each retry."""
+    latest_err = Exception
+
+    for i in [1, 5, 10, 60, 90, 120]:
         try:
             return operation(*args, **kwargs)
-        except subprocess.CalledProcessError as err:
+        except CalledProcessErrorStderr as err:
             latest_err = err
             logger.error(
                 "Operation %s failed with code %d: %s",
-                operation, err.returncode, err.output)
+                operation,
+                err.returncode,
+                err.output,
+            )
+            logger.error("Retrying in %s", str(i))
+            time.sleep(i)
+
     raise latest_err
 
 
@@ -234,7 +241,7 @@ def run(self):
                 activity |= self.createJobs(newJob)
                 activity |= self.checkOnJobs()
                 if not activity:
-                    logger.debug('No activity, sleeping for %is', self.boss.sleepSeconds())
+                   pass # logger.debug('No activity, sleeping for %is', self.boss.sleepSeconds())
 
         @abstractmethod
         def prepareSubmission(self, cpu, memory, jobID, command, jobName):