Always kill oldest instances first (#35)

Right now the order of task killing is "unhealthy" first.
For very big applications, killing unhealthy and then oldest tasks first
is a good heuristic. It favors killing first the instances that may have
have leaked.

Of course, we could find arguments for sorting in reverse (oldest
instacne have better caches) but at least we are specifying the order
instead of letting it random.

Change-Id: Ia4e7ec7751585f337d740a1246cac379d4871995
JIRA: MESOS-4343

src/main/scala/mesosphere/marathon/core/deployment/impl/TaskReplaceActor.scala

@@ -74,15 +74,25 @@ class TaskReplaceActor(
     deploymentManagerActor ! HealthStatusRequest(pathId)
   }
 
+  // sort instances with instances having oldest tasks first. Instances with no task are at the end.
+  def sortByOldestFirst(instances: mutable.Queue[Instance]): mutable.Queue[Instance] = {
+    val instancesByIncarnation = instances.partition(_.tasksMap.size != 0)
+    instancesByIncarnation._1.sortBy(_.appTask.status.stagedAt) ++ instancesByIncarnation._2
+  }
+
   def step(health: Map[Instance.Id, Seq[Health]]): Unit = {
     logger.debug(s"---=== DEPLOYMENT STEP FOR ${pathId} ===---")
     val current_instances = instanceTracker.specInstancesSync(pathId).partition(_.runSpecVersion == runSpec.version)
 
     val new_instances = current_instances._1.partition(isHealthy(_, health))
     val old_instances = current_instances._2.partition(isHealthy(_, health))
 
+    // make sure we kill in order:
+    // - instances with unhealthy tasks
+    // - instances with oldest tasks
+    // - other instances
     val toKill = old_instances._2.to[mutable.Queue]
-    toKill ++= old_instances._1
+    toKill ++= sortByOldestFirst(old_instances._1.to[mutable.Queue])
 
     val state = new TransitionState(new_instances._1.size, new_instances._2.size, old_instances._1.size, old_instances._2.size)