From 862963ec7e49de126500aa1c8ed87ae9e56a59f8 Mon Sep 17 00:00:00 2001 From: Marina Sahakyan Date: Fri, 8 Dec 2023 17:41:54 +0100 Subject: [PATCH] pool:make readonly pool READONLY Motivation When there is a hardware issue for a disk and it is not possibale to write files to that disk anymore. Nonthenless in dCache world we still write files, and we test the state of the disk by creating a file and etc. This is a first patch that should help to fix this issue. Modification Add enum describing 3 states ok, READ_ONLY and Failed Result no results yet and this should be tested Acked-by: Tigran Target: master Require-book: no Require-notes: yes --- .../pool/repository/ConsistentReplicaStore.java | 3 +-- .../dcache/pool/repository/DummyFileStore.java | 4 ++-- .../org/dcache/pool/repository/FileStore.java | 2 +- .../dcache/pool/repository/FileStoreState.java | 5 +++++ .../org/dcache/pool/repository/FlatFileStore.java | 6 +++--- .../pool/repository/ForwardingReplicaStore.java | 2 +- .../org/dcache/pool/repository/ReplicaStore.java | 2 +- .../dcache/pool/repository/ReplicaStoreCache.java | 2 +- .../meta/db/AbstractBerkeleyDBReplicaStore.java | 9 +++++---- .../meta/db/BerkeleyDBMetaDataRepository.java | 10 ++++++++-- .../meta/file/FileMetaDataRepository.java | 11 ++++++----- .../meta/mongo/MongoDbMetadataRepository.java | 15 ++++++++------- .../pool/repository/v5/CheckHealthTask.java | 3 ++- .../tests/repository/ReplicaStoreHelper.java | 5 +++-- 14 files changed, 47 insertions(+), 32 deletions(-) create mode 100644 modules/dcache/src/main/java/org/dcache/pool/repository/FileStoreState.java diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/ConsistentReplicaStore.java b/modules/dcache/src/main/java/org/dcache/pool/repository/ConsistentReplicaStore.java index 11ba464b786..481798163c7 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/ConsistentReplicaStore.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/ConsistentReplicaStore.java @@ -9,7 +9,6 @@ import static org.dcache.namespace.FileAttribute.STORAGEINFO; import static org.dcache.util.Exceptions.messageOrClassName; -import com.google.common.collect.Sets; import diskCacheV111.util.AccessLatency; import diskCacheV111.util.CacheException; import diskCacheV111.util.DiskErrorCacheException; @@ -306,7 +305,7 @@ public void remove(PnfsId id) throws CacheException { * Calls through to the wrapped meta data store. */ @Override - public boolean isOk() { + public FileStoreState isOk() { return _replicaStore.isOk(); } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/DummyFileStore.java b/modules/dcache/src/main/java/org/dcache/pool/repository/DummyFileStore.java index 9af16e2701c..6d9824696db 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/DummyFileStore.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/DummyFileStore.java @@ -94,7 +94,7 @@ public long getTotalSpace() { } @Override - public boolean isOk() { - return false; + public FileStoreState isOk() { + return FileStoreState.FAILED; } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/FileStore.java b/modules/dcache/src/main/java/org/dcache/pool/repository/FileStore.java index 772101792e1..f3e8c24f6f5 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/FileStore.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/FileStore.java @@ -75,5 +75,5 @@ public interface FileStore { * Returns whether the store appears healthy. How this is determined is up to the * implementation. */ - boolean isOk(); + FileStoreState isOk(); } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/FileStoreState.java b/modules/dcache/src/main/java/org/dcache/pool/repository/FileStoreState.java new file mode 100644 index 00000000000..cde8e84a080 --- /dev/null +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/FileStoreState.java @@ -0,0 +1,5 @@ +package org.dcache.pool.repository; + +public enum FileStoreState { + OK, READ_ONLY, FAILED +} diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/FlatFileStore.java b/modules/dcache/src/main/java/org/dcache/pool/repository/FlatFileStore.java index 8ea947271fa..6410191bbad 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/FlatFileStore.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/FlatFileStore.java @@ -100,14 +100,14 @@ public long getTotalSpace() throws IOException { } @Override - public boolean isOk() { + public FileStoreState isOk() { try { Path tmp = _dataDir.resolve(".repository_is_ok"); Files.deleteIfExists(tmp); Files.createFile(tmp); - return true; + return FileStoreState.OK; } catch (IOException e) { - return false; + return FileStoreState.FAILED; } } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/ForwardingReplicaStore.java b/modules/dcache/src/main/java/org/dcache/pool/repository/ForwardingReplicaStore.java index f66dbc94199..971b1f0e74d 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/ForwardingReplicaStore.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/ForwardingReplicaStore.java @@ -59,7 +59,7 @@ public void remove(PnfsId id) throws CacheException { } @Override - public boolean isOk() { + public FileStoreState isOk() { return delegate().isOk(); } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/ReplicaStore.java b/modules/dcache/src/main/java/org/dcache/pool/repository/ReplicaStore.java index db0d7777543..295d4e210d2 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/ReplicaStore.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/ReplicaStore.java @@ -64,7 +64,7 @@ void remove(PnfsId id) * Returns whether the store appears healthy. How this is determined is up to the * implementation. */ - boolean isOk(); + FileStoreState isOk(); /** * Closes the store and frees any associated resources. diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/ReplicaStoreCache.java b/modules/dcache/src/main/java/org/dcache/pool/repository/ReplicaStoreCache.java index 4f605e33418..26f90424db5 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/ReplicaStoreCache.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/ReplicaStoreCache.java @@ -443,7 +443,7 @@ public Set index(IndexOption... options) { } @Override - public boolean isOk() { + public FileStoreState isOk() { return _inner.isOk(); } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/AbstractBerkeleyDBReplicaStore.java b/modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/AbstractBerkeleyDBReplicaStore.java index fd7ea347350..81514929778 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/AbstractBerkeleyDBReplicaStore.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/AbstractBerkeleyDBReplicaStore.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import org.dcache.pool.repository.FileStoreState; import org.dcache.pool.repository.ReplicaStore; import org.dcache.pool.repository.RepositoryChannel; import org.dcache.util.configuration.ConfigurationMapFactoryBean; @@ -147,20 +148,20 @@ public void close() { } @Override - public synchronized boolean isOk() { + public synchronized FileStoreState isOk() { Path tmp = dir.resolve(".repository_is_ok"); try { Files.deleteIfExists(tmp); Files.createFile(tmp); if (database.isFailed()) { - return false; + return FileStoreState.FAILED; } - return true; + return FileStoreState.OK; } catch (IOException e) { LOGGER.error("Failed to touch {}: {}", tmp, messageOrClassName(e)); - return false; + return FileStoreState.FAILED; } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/BerkeleyDBMetaDataRepository.java b/modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/BerkeleyDBMetaDataRepository.java index 391a52e30b7..644fea0ec14 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/BerkeleyDBMetaDataRepository.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/meta/db/BerkeleyDBMetaDataRepository.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.dcache.pool.repository.FileStoreState; import org.dcache.pool.repository.DuplicateEntryException; import org.dcache.pool.repository.FileStore; import org.dcache.pool.repository.ReplicaRecord; @@ -174,8 +175,13 @@ public void remove(PnfsId id) throws CacheException { } @Override - public boolean isOk() { - return _fileStore.isOk() && super.isOk(); + public FileStoreState isOk() { + if (_fileStore.isOk() == FileStoreState.OK && super.isOk() == FileStoreState.OK){ + return FileStoreState.OK; + } + else { + return FileStoreState.FAILED; + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/meta/file/FileMetaDataRepository.java b/modules/dcache/src/main/java/org/dcache/pool/repository/meta/file/FileMetaDataRepository.java index 2a3ee2fd697..3277e8b91c5 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/meta/file/FileMetaDataRepository.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/meta/file/FileMetaDataRepository.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Stream; +import org.dcache.pool.repository.FileStoreState; import org.dcache.pool.repository.DuplicateEntryException; import org.dcache.pool.repository.FileStore; import org.dcache.pool.repository.ReplicaRecord; @@ -180,18 +181,18 @@ protected void deleteIfExists(Path path) throws DiskErrorCacheException { } @Override - public synchronized boolean isOk() { - if (!_fileStore.isOk()) { - return false; + public synchronized FileStoreState isOk() { + if (_fileStore.isOk() == FileStoreState.FAILED) { + return FileStoreState.FAILED; } Path tmp = _metadir.resolve(".repository_is_ok"); try { Files.deleteIfExists(tmp); Files.createFile(tmp); - return true; + return FileStoreState.OK; } catch (IOException e) { LOGGER.error("Failed to touch " + tmp + ": " + messageOrClassName(e), e); - return false; + return FileStoreState.FAILED; } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/meta/mongo/MongoDbMetadataRepository.java b/modules/dcache/src/main/java/org/dcache/pool/repository/meta/mongo/MongoDbMetadataRepository.java index 75e414ae0ea..0a07154dd55 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/meta/mongo/MongoDbMetadataRepository.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/meta/mongo/MongoDbMetadataRepository.java @@ -30,6 +30,7 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.bson.Document; +import org.dcache.pool.repository.FileStoreState; import org.dcache.pool.repository.DuplicateEntryException; import org.dcache.pool.repository.FileStore; import org.dcache.pool.repository.ReplicaRecord; @@ -111,7 +112,7 @@ public class MongoDbMetadataRepository implements ReplicaStore, EnvironmentAware /** * Flag, which indicates client connection status. */ - private volatile boolean isOk; + private volatile FileStoreState dIskFailerState; /** * {@link FileStore} used to store data files. @@ -126,7 +127,7 @@ public MongoDbMetadataRepository(FileStore fileStore, Path ignored, String poolN boolean isReadOnly) { this.fileStore = fileStore; this.pool = poolName; - this.isOk = false; + this.dIskFailerState = FileStoreState.FAILED; } @Override @@ -223,7 +224,7 @@ public void remove(PnfsId id) throws CacheException { deleteIfExists(id); } catch (IOException | MongoException e) { - isOk = false; + dIskFailerState = FileStoreState.FAILED; throw new DiskErrorCacheException( "Failed to remove " + id + ": " + messageOrClassName(e), e); } @@ -247,8 +248,8 @@ private void deleteIfExists(PnfsId id) { } @Override - public boolean isOk() { - return isOk; + public FileStoreState isOk() { + return dIskFailerState; } @Override @@ -291,13 +292,13 @@ public void setEnvironment(Map environment) { @Override public void serverOpening(ServerOpeningEvent event) { LOGGER.info("Connected to server {}", event.getServerId()); - isOk = true; + dIskFailerState = FileStoreState.OK; } @Override public void serverClosed(ServerClosedEvent event) { LOGGER.info("Lost connection with server {}", event.getServerId()); - isOk = false; + dIskFailerState = FileStoreState.FAILED; } @Override diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CheckHealthTask.java b/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CheckHealthTask.java index ae2967eeaae..6067ca6ca27 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CheckHealthTask.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CheckHealthTask.java @@ -14,6 +14,7 @@ import org.dcache.alarms.PredefinedAlarm; import org.dcache.pool.FaultAction; import org.dcache.pool.repository.Account; +import org.dcache.pool.repository.FileStoreState; import org.dcache.pool.repository.ReplicaStore; import org.dcache.pool.repository.SpaceRecord; import org.slf4j.Logger; @@ -68,7 +69,7 @@ public void run() { case CLOSED: break; case OPEN: - if (!_replicaStore.isOk()) { + if (_replicaStore.isOk() == FileStoreState.FAILED) { _repository.fail(FaultAction.DISABLED, "I/O test failed"); } diff --git a/modules/dcache/src/test/java/org/dcache/tests/repository/ReplicaStoreHelper.java b/modules/dcache/src/test/java/org/dcache/tests/repository/ReplicaStoreHelper.java index abdc68a46b5..bc5d213d9c0 100644 --- a/modules/dcache/src/test/java/org/dcache/tests/repository/ReplicaStoreHelper.java +++ b/modules/dcache/src/test/java/org/dcache/tests/repository/ReplicaStoreHelper.java @@ -18,6 +18,7 @@ import java.util.Set; import java.util.stream.Stream; import org.dcache.namespace.FileAttribute; +import org.dcache.pool.repository.FileStoreState; import org.dcache.pool.repository.DuplicateEntryException; import org.dcache.pool.repository.FileStore; import org.dcache.pool.repository.ReplicaRecord; @@ -235,8 +236,8 @@ public Set index(IndexOption... options) { } @Override - public boolean isOk() { - return true; + public FileStoreState isOk() { + return FileStoreState.OK; } @Override