Recover from _some_ missing files scenarios more gracefully, improve …

…testing #913

src/main/java/net/openhft/chronicle/queue/impl/single/MissingStoreFileException.java

@@ -0,0 +1,10 @@
+package net.openhft.chronicle.queue.impl.single;
+
+/**
+ * Thrown when a store file we expect to be present is missing (probably because it was deleted)
+ */
+public class MissingStoreFileException extends IllegalStateException {
+    public MissingStoreFileException(String s) {
+        super(s);
+    }
+}

src/main/java/net/openhft/chronicle/queue/impl/single/StoreTailer.java

@@ -687,9 +687,7 @@ ScanResult moveToIndexResult(final long index) {
         return scanResult;
     }
 
-    @NotNull
-    @Override
-    public final ExcerptTailer toStart() {
+    private ExcerptTailer doToStart() {
         assert direction != BACKWARD;
         final int firstCycle = queue.firstCycle();
         if (firstCycle == Integer.MAX_VALUE) {
@@ -700,9 +698,10 @@ public final ExcerptTailer toStart() {
         if (firstCycle != this.cycle) {
             // moves to the expected cycle
             final boolean found = cycle(firstCycle);
-            assert found || store == null;
             if (found)
                 state = FOUND_IN_CYCLE;
+            else if (store != null)
+                throw new MissingStoreFileException("Missing first store file cycle=" + firstCycle);
         }
         index(queue.rollCycle().toIndex(cycle, 0));
 
@@ -715,6 +714,17 @@ public final ExcerptTailer toStart() {
         return this;
     }
 
+    @NotNull
+    @Override
+    public final ExcerptTailer toStart() {
+        try {
+            return doToStart();
+        } catch (MissingStoreFileException e) {
+            queue.refreshDirectoryListing();
+            return doToStart();
+        }
+    }
+
     private boolean moveToIndexInternal(final long index) {
         moveToState.indexMoveCount++;
         final ScanResult scanResult = moveToIndexResult0(index);
@@ -749,7 +759,7 @@ private long approximateLastCycle2(int lastCycle) throws StreamCorruptedExceptio
                 lastCycle, queue.epoch(), false, this.store);
         this.setCycle(lastCycle);
         if (wireStore == null)
-            throw new IllegalStateException("Store not found for cycle " + Long.toHexString(lastCycle) + ". Probably the files were removed? queue=" + queue.fileAbsolutePath());
+            throw new MissingStoreFileException("Store not found for cycle " + Long.toHexString(lastCycle) + ". Probably the files were removed? queue=" + queue.fileAbsolutePath());
 
         if (this.store != wireStore) {
             releaseStore();
@@ -769,7 +779,7 @@ private long approximateLastCycle2(int lastCycle) throws StreamCorruptedExceptio
                 lastCycle--;
                 try {
                     return approximateLastCycle2(lastCycle);
-                } catch (IllegalStateException e) {
+                } catch (MissingStoreFileException e) {
                     // try again.
                 }
             }
@@ -837,7 +847,16 @@ public ExcerptTailer toEnd() {
             return callOriginalToEnd();
         }
 
-        return optimizedToEnd();
+        return callOptimizedToEnd();
+    }
+
+    private ExcerptTailer callOptimizedToEnd() {
+        try {
+            return optimizedToEnd();
+        } catch (MissingStoreFileException e) {
+            queue.refreshDirectoryListing();
+            return optimizedToEnd();
+        }
     }
 
     @NotNull
@@ -888,7 +907,7 @@ private ExcerptTailer optimizedToEnd() {
                     lastCycle, queue.epoch(), false, this.store);
             this.setCycle(lastCycle);
             if (wireStore == null)
-                throw new IllegalStateException("Store not found for cycle " + Long.toHexString(lastCycle) + ". Probably the files were removed? queue=" + queue.fileAbsolutePath());
+                throw new MissingStoreFileException("Store not found for cycle " + Long.toHexString(lastCycle) + ". Probably the files were removed? queue=" + queue.fileAbsolutePath());
 
             if (this.store != wireStore) {
                 releaseStore();
@@ -919,7 +938,6 @@ private ExcerptTailer optimizedToEnd() {
     }
 
     @NotNull
-
     public ExcerptTailer originalToEnd() {
         throwExceptionIfClosed();