pool: make pool_read only if fs is ro

Motivation

this is the second patch after the ` https://rb.dcache.org/r/14174/`
 which addes changes to introduce new READ_ONLY type.

the idea is that once the fs will changed to ro the checkHealthTask should take this into account.

Result

faultOccurred() method now should call case READ_ONLY

this patch has not been tested yet

Acked-by: Tigran
Patch: https://rb.dcache.org/r/14199/
Target: master
Require-book: no

modules/dcache/src/main/java/org/dcache/pool/repository/FlatFileStore.java

@@ -107,7 +107,11 @@ public FileStoreState isOk() {
             Files.createFile(tmp);
             return FileStoreState.OK;
         } catch (IOException e) {
-            return FileStoreState.FAILED;
+            if (e.getMessage().contains("Read-only file system")) {
+                return FileStoreState.READ_ONLY;
+            } else {
+                return FileStoreState.FAILED;
+            }
         }
     }
 }

modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/AbstractBerkeleyDBReplicaStore.java

@@ -35,6 +35,7 @@
 import java.nio.file.Files;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.ReadOnlyFileSystemException;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
@@ -154,14 +155,20 @@ public synchronized FileStoreState isOk() {
             Files.deleteIfExists(tmp);
             Files.createFile(tmp);
 
+            //TODO check
             if (database.isFailed()) {
                 return FileStoreState.FAILED;
             }
 
             return FileStoreState.OK;
         } catch (IOException e) {
-            LOGGER.error("Failed to touch {}: {}", tmp, messageOrClassName(e));
-            return FileStoreState.FAILED;
+            if (e.getMessage().contains("Read-only file system")) {
+                LOGGER.error("Filesystem in read-only mode {}: {}", tmp, messageOrClassName(e));
+                return FileStoreState.READ_ONLY;
+            } else {
+                LOGGER.error("Failed to touch {}: {}", tmp, messageOrClassName(e));
+                return FileStoreState.FAILED;
+            }
         }
     }
 

modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/BerkeleyDBMetaDataRepository.java

@@ -178,6 +178,9 @@ public void remove(PnfsId id) throws CacheException {
     public FileStoreState isOk() {
         if (_fileStore.isOk() == FileStoreState.OK && super.isOk() == FileStoreState.OK){
             return FileStoreState.OK;
+        } else if (_fileStore.isOk() == FileStoreState.READ_ONLY && super.isOk() == FileStoreState.READ_ONLY){
+            return FileStoreState.READ_ONLY;
+
         }
       else {
             return FileStoreState.FAILED;

modules/dcache/src/main/java/org/dcache/pool/repository/meta/file/FileMetaDataRepository.java

@@ -14,6 +14,7 @@
 import java.nio.file.Files;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.ReadOnlyFileSystemException;
 import java.nio.file.StandardOpenOption;
 import java.util.List;
 import java.util.Set;
@@ -191,8 +192,13 @@ public synchronized FileStoreState isOk() {
             Files.createFile(tmp);
             return FileStoreState.OK;
         } catch (IOException e) {
-            LOGGER.error("Failed to touch " + tmp + ": " + messageOrClassName(e), e);
-            return FileStoreState.FAILED;
+            if (e.getMessage().contains("Read-only file system")) {
+                LOGGER.error("Filesystem in read-only mode {}: {}", tmp, messageOrClassName(e));
+                return FileStoreState.READ_ONLY;
+            } else {
+                LOGGER.error("Failed to touch {}: {}", tmp, messageOrClassName(e));
+                return FileStoreState.FAILED;
+            }
         }
     }
 

modules/dcache/src/main/java/org/dcache/pool/repository/v5/CheckHealthTask.java

@@ -71,6 +71,11 @@ public void run() {
             case OPEN:
                 if (_replicaStore.isOk() == FileStoreState.FAILED) {
                     _repository.fail(FaultAction.DISABLED, "I/O test failed");
+                }else if (_replicaStore.isOk() == FileStoreState.READ_ONLY){
+                    LOGGER.error(
+                          "Read-only file system");
+
+                    _repository.fail(FaultAction.READONLY, "I/O test failed, READ_ONLY Error");
                 }
 
                 if (!checkSpaceAccounting()) {