batch-deleted state

design/data.md

@@ -21,7 +21,7 @@
 | /batches/BID/submission | json | profile_name<br/>submitter<br/>payload_filename<br/><br/>erc_what<br/>erc_who<br/>erc_when<br/>erc_where<br/>type<br/>submission_mode | creation | none | |
 | /batches/BID/status | json | status<br/>last_modified<br/>message | creation | all jobs done | message is optional| 
 | /batches/BID/status-report | json | failed_jobs | failure | failure | last status report sent to user | 
-| /batches/BID/states/STATE/JID | none | - | | | STATE = batch-processing / batch-failed / batch-completed<br/>Create watcher to watch for states/processing to be empty| 
+| /batches/BID/states/STATE/JID | none | - | | | STATE = batch-processing / batch-failed / batch-completed / batch-deleted<br/>Create watcher to watch for states/processing to be empty| 
 | /batches/BID/lock | none | - | Several states | - | **Ephemeral node** to lock a job, deleted by the thread that creates the node |
 | /batch-uuids/UUID | string | - | Pending | none | batchID uuid is minted by ingest, used for lookup |
 | /jobs/JID/bid | string | batch_id | creation | none | |

src/main/java/org/cdlib/mrt/zk/Batch.java

@@ -194,6 +194,9 @@ public List<Job> getCompletedJobs(ZooKeeper client) throws KeeperException, Inte
   public List<Job> getFailedJobs(ZooKeeper client) throws KeeperException, InterruptedException {
     return getJobs(client, BatchJobStates.Failed);
   }
+  public List<Job> getDeletedJobs(ZooKeeper client) throws KeeperException, InterruptedException {
+    return getJobs(client, BatchJobStates.Deleted);
+  }
   public List<Job> getJobs(ZooKeeper client, BatchJobStates state) throws KeeperException, InterruptedException {
     ArrayList<Job> jobs = new ArrayList<>();
     String p = String.format("%s/states/%s", path(), state.path);

src/main/java/org/cdlib/mrt/zk/Job.java

@@ -300,6 +300,8 @@ public String batchStateSubpath() {
       return BatchJobStates.Failed.path;
     } else if (this.status() == JobState.Completed) {
       return BatchJobStates.Completed.path;
+    } else if (this.status() == JobState.Deleted) {
+      return BatchJobStates.Deleted.path;
     }
     return BatchJobStates.Processing.path;
   }
@@ -317,7 +319,7 @@ public void setBatchStatePath(ZooKeeper client) throws MerrittZKNodeInvalid, Kee
     if (!QueueItemHelper.exists(client, bs)) {
       String p = Paths.get(bs).getParent().toString();
       if (!QueueItemHelper.exists(client, p)) {
-        QueueItemHelper.create(client, p, QueueItemHelper.empty);
+        QueueItemHelper.createIfNeeded(client, p);
       }
       QueueItemHelper.createIfNeeded(client, bs);
     }

src/main/java/org/cdlib/mrt/zk/QueueItem.java

@@ -41,6 +41,7 @@ public static enum ZkPaths {
   public static enum BatchJobStates {
     Processing("batch-processing"),
     Completed("batch-completed"),
+    Deleted("batch-deleted"),
     Failed("batch-failed");
     public String path;
     BatchJobStates(String path) {

src/main/ruby/lib/merritt_zk_batch.rb

@@ -133,6 +133,10 @@ def get_completed_jobs(zk)
       get_jobs(zk, 'batch-completed')
     end
 
+    def get_deleted_jobs(zk)
+      get_jobs(zk, 'batch-deleted')
+    end
+
     def get_failed_jobs(zk)
       get_jobs(zk, 'batch-failed')
     end

src/main/ruby/lib/merritt_zk_job.rb

@@ -82,6 +82,7 @@ def set_status_with_priority(zk, status, priority)
     def batch_state_subpath
       return 'batch-failed' if @status.status == :Failed
       return 'batch-completed' if @status.status == :Completed
+      return 'batch-deleted' if @status.status == :Deleted
 
       'batch-processing'
     end

src/main/ruby/show_all.rb

@@ -188,16 +188,16 @@ def show(zk, arr)
 
 if ARGV.include?('-debug')
   puts '===> DEBUG'
-  MerrittZK::Job.list_jobs_as_json(zk).each do |j|
-    puts j.fetch(:path, '')
-    puts JSON.pretty_generate(j)
-  end
-  exit
-  # MerrittZK::Access.list_jobs_as_json(zk).each do |j|
+  # MerrittZK::Job.list_jobs_as_json(zk).each do |j|
   #   puts j.fetch(:path, '')
   #   puts JSON.pretty_generate(j)
   # end
   # exit
+  MerrittZK::Access.list_jobs_as_json(zk).each do |j|
+    puts j.fetch(:path, '')
+    puts JSON.pretty_generate(j)
+  end
+  exit
 end
 
 if ARGV.include?('-clear')
@@ -221,6 +221,15 @@ def show(zk, arr)
   zk.create('/migration/m1', data: nil)
 end
 
+if ARGV.include?('-m3')
+  LIST.each do |p|
+    zk.create(p, data: nil) unless zk.exists?(p)
+  end
+  zk.rm_rf('/migration')
+  zk.create('/migration', data: nil)
+  zk.create('/migration/m3', data: nil)
+end
+
 if ARGV.include?('-m13')
   LIST.each do |p|
     zk.create(p, data: nil) unless zk.exists?(p)

src/main/ruby/spec/zk_spec.rb

@@ -446,6 +446,8 @@ def make_batch_json(s = 'bar', u = 'bid-uuid')
       expect(arr.length).to eq(1)
       expect(arr[0].id).to eq(@remap['jid1'])
 
+      jj.set_status(@zk, MerrittZK::JobState::Deleted)
+
       bbbb.set_status(@zk, MerrittZK::BatchState::Deleted)
       expect(bbbb.status.status).to eq(:Deleted)
       expect(bbbb.status.deletable?).to be(true)

src/test/java/org/cdlib/mrt/zk/ZKTestIT.java

@@ -766,17 +766,21 @@ public void batchFailure() throws KeeperException, InterruptedException, Merritt
 
       assertEquals(bbb.status(), BatchState.Failed);
       assertFalse(bbb.status().isDeletable());
+      assertTrue(bbb.hasFailure());
+
+      jj.setStatus(zk, JobState.Deleted);
 
       Batch bbbb = new Batch(bbb.id());
       bbbb.load(zk);
       assertEquals(bbbb.status(), BatchState.Failed);
-      assertTrue(bbbb.hasFailure());
 
       List<Job> jobs = bb.getProcessingJobs(zk);
       assertEquals(jobs.size(), 0);
       jobs = bb.getCompletedJobs(zk);
       assertEquals(jobs.size(), 0);
       jobs = bb.getFailedJobs(zk);
+      assertEquals(jobs.size(), 0);
+      jobs = bb.getDeletedJobs(zk);
       assertEquals(jobs.size(), 1);
 
       bbbb.setStatus(zk, BatchState.Deleted);

test-cases.yml

@@ -351,8 +351,9 @@ batch_failure:
   input:
   output:
     /batch-uuids/bid-uuid: bid0
-    /batches/bid0/states/batch-failed/jid1: null
+    /batches/bid0/states/batch-deleted/jid1: null
     /batches/bid0/states/batch-processing: null
+    /batches/bid0/states/batch-failed: null
     /batches/bid0/submission: 
       foo: bar
       batchID: bid-uuid
@@ -367,11 +368,11 @@ batch_failure:
     /jobs/jid1/space_needed: 0
     /jobs/jid1/status: 
       last_successful_status: null
-      status: Failed
+      status: Deleted
       last_modified: now
       retry_count: 0
-      message: Sample Failure Message
-    /jobs/states/failed/02-jid1: null
+      message: ''
+    /jobs/states/deleted/02-jid1: null
 batch_recovery:
   input:
   output: