From 1891d81e85e69940f6876f46bd3a4f3123aec82f Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sat, 21 Oct 2023 17:12:53 +0530 Subject: [PATCH 01/10] Make index and global metadata upload wait time dynamic Signed-off-by: Rahul Karajgikar --- .../common/settings/ClusterSettings.java | 2 + .../remote/RemoteClusterStateService.java | 47 +++++++++++++++++-- .../RemoteClusterStateServiceTests.java | 46 ++++++++++++++++++ 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 7ac7da819b215..e4745e1ffbec9 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -682,6 +682,8 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, + RemoteClusterStateService.INDEX_METADATA_UPLOAD_WAIT_TIME_SETTINGS, + RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTINGS, RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING, IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 96ce2fc779ea0..d39602f909aae 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -84,8 +84,27 @@ public class RemoteClusterStateService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); // TODO make this two variable as dynamic setting [issue: #10688] - public static final int INDEX_METADATA_UPLOAD_WAIT_MILLIS = 20000; - public static final int GLOBAL_METADATA_UPLOAD_WAIT_MILLIS = 20000; + // public static final int INDEX_METADATA_UPLOAD_WAIT_MILLIS = 20000; + // public static final int GLOBAL_METADATA_UPLOAD_WAIT_MILLIS = 20000; + + // default value for index metadata upload wait is 20s + static volatile TimeValue indexMetadataUploadWaitTime = TimeValue.timeValueMillis(20000); + // default value for index metadata upload wait is 20s + static volatile TimeValue globalMetadataUploadWaitTime = TimeValue.timeValueMillis(20000); + + public static final Setting INDEX_METADATA_UPLOAD_WAIT_TIME_SETTINGS = Setting.timeSetting( + "cluster.remote_store.index_metadata.upload_wait_time", + indexMetadataUploadWaitTime, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + public static final Setting GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTINGS = Setting.timeSetting( + "cluster.remote_store.global_metadata.upload_wait_time", + globalMetadataUploadWaitTime, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( "index-metadata", @@ -172,6 +191,10 @@ public RemoteClusterStateService( this.threadpool = threadPool; this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); + clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_WAIT_TIME_SETTINGS, this::setIndexMetadataUploadWaitTime); + clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTINGS, this::setGlobalMetadataUploadWaitTime); + setIndexMetadataUploadWaitTime(INDEX_METADATA_UPLOAD_WAIT_TIME_SETTINGS.get(settings)); + setGlobalMetadataUploadWaitTime(GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTINGS.get(settings)); } private BlobStoreTransferService getBlobStoreTransferService() { @@ -367,7 +390,7 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException ); try { - if (latch.await(GLOBAL_METADATA_UPLOAD_WAIT_MILLIS, TimeUnit.MILLISECONDS) == false) { + if (latch.await(getGlobalMetadataUploadWaitTime().millis(), TimeUnit.MILLISECONDS) == false) { // TODO: We should add metrics where transfer is timing out. [Issue: #10687] GlobalMetadataTransferException ex = new GlobalMetadataTransferException( String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete") @@ -422,7 +445,7 @@ private List writeIndexMetadataParallel(ClusterState clus } try { - if (latch.await(INDEX_METADATA_UPLOAD_WAIT_MILLIS, TimeUnit.MILLISECONDS) == false) { + if (latch.await(getIndexMetadataUploadWaitTime().millis(), TimeUnit.MILLISECONDS) == false) { IndexMetadataTransferException ex = new IndexMetadataTransferException( String.format( Locale.ROOT, @@ -615,6 +638,22 @@ private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) { this.slowWriteLoggingThreshold = slowWriteLoggingThreshold; } + private void setIndexMetadataUploadWaitTime(TimeValue newIndexMetadataUploadWaitTime) { + indexMetadataUploadWaitTime = newIndexMetadataUploadWaitTime; + } + + private void setGlobalMetadataUploadWaitTime(TimeValue newGlobalMetadataUploadWaitTime) { + globalMetadataUploadWaitTime = newGlobalMetadataUploadWaitTime; + } + + public static TimeValue getIndexMetadataUploadWaitTime() { + return indexMetadataUploadWaitTime; + } + + public static TimeValue getGlobalMetadataUploadWaitTime() { + return globalMetadataUploadWaitTime; + } + static String getManifestFileName(long term, long version, boolean committed) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ return String.join( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 5202f31c514ed..dde1468a8e8e9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -1039,6 +1039,52 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti assertBusy(() -> assertEquals(1, callCount.get())); } + public void testIndexMetadataUploadWaitTimeSetting() { + ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + // verify defaults + assertEquals(remoteClusterStateService.indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime()); + assertEquals(remoteClusterStateService.globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime()); + + // verify update index metadata upload wait time + int indexMetadataUploadWaitTime = randomIntBetween(1, 10); + Settings newSettings = Settings.builder() + .put("cluster.remote_store.index_metadata.upload_wait_time", indexMetadataUploadWaitTime + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime().seconds()); + + // verify update global metadata upload wait time + int globalMetadataUploadWaitTime = randomIntBetween(1, 10); + newSettings = Settings.builder() + .put("cluster.remote_store.global_metadata.upload_wait_time", globalMetadataUploadWaitTime + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime().seconds()); + } + + public void testIndexMetadataUploadWaitTimeSetting() { + ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + // verify defaults + assertEquals(remoteClusterStateService.indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime()); + assertEquals(remoteClusterStateService.globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime()); + + // verify update index metadata upload wait time + int indexMetadataUploadWaitTime = randomIntBetween(1, 10); + Settings newSettings = Settings.builder() + .put("cluster.remote_store.index_metadata.upload_wait_time", indexMetadataUploadWaitTime + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime().seconds()); + + // verify update global metadata upload wait time + int globalMetadataUploadWaitTime = randomIntBetween(1, 10); + newSettings = Settings.builder() + .put("cluster.remote_store.global_metadata.upload_wait_time", globalMetadataUploadWaitTime + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime().seconds()); + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { final BlobPath blobPath = mock(BlobPath.class); when((blobStoreRepository.basePath())).thenReturn(blobPath); From 8fd9acbb9798a1128d8e37c30acd8e4de4709c21 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sat, 21 Oct 2023 17:19:30 +0530 Subject: [PATCH 02/10] Fix minor issues Signed-off-by: Rahul Karajgikar --- .../common/settings/ClusterSettings.java | 4 ++-- .../remote/RemoteClusterStateService.java | 16 ++++++---------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index e4745e1ffbec9..1538fdf36efd5 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -682,8 +682,8 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, - RemoteClusterStateService.INDEX_METADATA_UPLOAD_WAIT_TIME_SETTINGS, - RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTINGS, + RemoteClusterStateService.INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING, + RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTING, RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING, IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index d39602f909aae..e7b04d4c96215 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -83,23 +83,19 @@ public class RemoteClusterStateService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); - // TODO make this two variable as dynamic setting [issue: #10688] - // public static final int INDEX_METADATA_UPLOAD_WAIT_MILLIS = 20000; - // public static final int GLOBAL_METADATA_UPLOAD_WAIT_MILLIS = 20000; - // default value for index metadata upload wait is 20s static volatile TimeValue indexMetadataUploadWaitTime = TimeValue.timeValueMillis(20000); // default value for index metadata upload wait is 20s static volatile TimeValue globalMetadataUploadWaitTime = TimeValue.timeValueMillis(20000); - public static final Setting INDEX_METADATA_UPLOAD_WAIT_TIME_SETTINGS = Setting.timeSetting( + public static final Setting INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING = Setting.timeSetting( "cluster.remote_store.index_metadata.upload_wait_time", indexMetadataUploadWaitTime, Setting.Property.Dynamic, Setting.Property.NodeScope ); - public static final Setting GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTINGS = Setting.timeSetting( + public static final Setting GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTING = Setting.timeSetting( "cluster.remote_store.global_metadata.upload_wait_time", globalMetadataUploadWaitTime, Setting.Property.Dynamic, @@ -191,10 +187,10 @@ public RemoteClusterStateService( this.threadpool = threadPool; this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); - clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_WAIT_TIME_SETTINGS, this::setIndexMetadataUploadWaitTime); - clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTINGS, this::setGlobalMetadataUploadWaitTime); - setIndexMetadataUploadWaitTime(INDEX_METADATA_UPLOAD_WAIT_TIME_SETTINGS.get(settings)); - setGlobalMetadataUploadWaitTime(GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTINGS.get(settings)); + clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING, this::setIndexMetadataUploadWaitTime); + clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTING, this::setGlobalMetadataUploadWaitTime); + setIndexMetadataUploadWaitTime(INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING.get(settings)); + setGlobalMetadataUploadWaitTime(GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTING.get(settings)); } private BlobStoreTransferService getBlobStoreTransferService() { From 59d6971d368821156d652307c51a0e07447c2489 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sat, 21 Oct 2023 18:30:46 +0530 Subject: [PATCH 03/10] Fix UTs Signed-off-by: Rahul Karajgikar --- .../RemoteClusterStateServiceTests.java | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index dde1468a8e8e9..ad045bd54f3d0 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -96,6 +96,7 @@ public class RemoteClusterStateServiceTests extends OpenSearchTestCase { private RemoteClusterStateService remoteClusterStateService; + private ClusterSettings clusterSettings; private Supplier repositoriesServiceSupplier; private RepositoriesService repositoriesService; private BlobStoreRepository blobStoreRepository; @@ -126,6 +127,7 @@ public void setup() { .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) .build(); + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); blobStore = mock(BlobStore.class); when(blobStoreRepository.blobStore()).thenReturn(blobStore); @@ -135,7 +137,7 @@ public void setup() { "test-node-id", repositoriesServiceSupplier, settings, - new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + clusterSettings, () -> 0L, threadPool ); @@ -1040,10 +1042,8 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti } public void testIndexMetadataUploadWaitTimeSetting() { - ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - // verify defaults - assertEquals(remoteClusterStateService.indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime()); - assertEquals(remoteClusterStateService.globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime()); + // verify default value + assertEquals(RemoteClusterStateService.indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime()); // verify update index metadata upload wait time int indexMetadataUploadWaitTime = randomIntBetween(1, 10); @@ -1053,32 +1053,15 @@ public void testIndexMetadataUploadWaitTimeSetting() { clusterSettings.applySettings(newSettings); assertEquals(indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime().seconds()); - // verify update global metadata upload wait time - int globalMetadataUploadWaitTime = randomIntBetween(1, 10); - newSettings = Settings.builder() - .put("cluster.remote_store.global_metadata.upload_wait_time", globalMetadataUploadWaitTime + "s") - .build(); - clusterSettings.applySettings(newSettings); - assertEquals(globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime().seconds()); } - public void testIndexMetadataUploadWaitTimeSetting() { - ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - // verify defaults - assertEquals(remoteClusterStateService.indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime()); - assertEquals(remoteClusterStateService.globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime()); - - // verify update index metadata upload wait time - int indexMetadataUploadWaitTime = randomIntBetween(1, 10); - Settings newSettings = Settings.builder() - .put("cluster.remote_store.index_metadata.upload_wait_time", indexMetadataUploadWaitTime + "s") - .build(); - clusterSettings.applySettings(newSettings); - assertEquals(indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime().seconds()); + public void testGlobalMetadataUploadWaitTimeSetting() { + // verify default value + assertEquals(RemoteClusterStateService.globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime()); // verify update global metadata upload wait time int globalMetadataUploadWaitTime = randomIntBetween(1, 10); - newSettings = Settings.builder() + Settings newSettings = Settings.builder() .put("cluster.remote_store.global_metadata.upload_wait_time", globalMetadataUploadWaitTime + "s") .build(); clusterSettings.applySettings(newSettings); From ee83dc133f55a337eec6739e0079a0da96c5bdb4 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sat, 21 Oct 2023 19:27:10 +0530 Subject: [PATCH 04/10] Fix typo Signed-off-by: Rahul Karajgikar --- .../opensearch/gateway/remote/RemoteClusterStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index e7b04d4c96215..b969bd1a2d5bc 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -85,7 +85,7 @@ public class RemoteClusterStateService implements Closeable { // default value for index metadata upload wait is 20s static volatile TimeValue indexMetadataUploadWaitTime = TimeValue.timeValueMillis(20000); - // default value for index metadata upload wait is 20s + // default value for global metadata upload wait is 20s static volatile TimeValue globalMetadataUploadWaitTime = TimeValue.timeValueMillis(20000); public static final Setting INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING = Setting.timeSetting( From ad0fa70d8bbc31e52bfc74d305ad720dab38d67c Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sat, 21 Oct 2023 23:16:16 +0530 Subject: [PATCH 05/10] Update name from wait time to timeout, standardize convention Signed-off-by: Rahul Karajgikar --- .../common/settings/ClusterSettings.java | 4 +- .../remote/RemoteClusterStateService.java | 44 +++++++++---------- .../RemoteClusterStateServiceTests.java | 20 ++++----- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 1538fdf36efd5..a0fca4f0a2ff0 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -682,8 +682,8 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, - RemoteClusterStateService.INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING, - RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTING, + RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, + RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING, IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index b969bd1a2d5bc..e0bc99cda45ae 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -84,20 +84,20 @@ public class RemoteClusterStateService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); // default value for index metadata upload wait is 20s - static volatile TimeValue indexMetadataUploadWaitTime = TimeValue.timeValueMillis(20000); + static volatile TimeValue INDEX_METADATA_UPLOAD_TIMEOUT = TimeValue.timeValueMillis(20000); // default value for global metadata upload wait is 20s - static volatile TimeValue globalMetadataUploadWaitTime = TimeValue.timeValueMillis(20000); + static volatile TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT = TimeValue.timeValueMillis(20000); - public static final Setting INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING = Setting.timeSetting( - "cluster.remote_store.index_metadata.upload_wait_time", - indexMetadataUploadWaitTime, + public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( + "cluster.remote_store.index_metadata.upload_timeout", + INDEX_METADATA_UPLOAD_TIMEOUT, Setting.Property.Dynamic, Setting.Property.NodeScope ); - public static final Setting GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTING = Setting.timeSetting( - "cluster.remote_store.global_metadata.upload_wait_time", - globalMetadataUploadWaitTime, + public static final Setting GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( + "cluster.remote_store.global_metadata.upload_timeout", + GLOBAL_METADATA_UPLOAD_TIMEOUT, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -187,10 +187,10 @@ public RemoteClusterStateService( this.threadpool = threadPool; this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); - clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING, this::setIndexMetadataUploadWaitTime); - clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTING, this::setGlobalMetadataUploadWaitTime); - setIndexMetadataUploadWaitTime(INDEX_METADATA_UPLOAD_WAIT_TIME_SETTING.get(settings)); - setGlobalMetadataUploadWaitTime(GLOBAL_METADATA_UPLOAD_WAIT_TIME_SETTING.get(settings)); + clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); + clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); + setIndexMetadataUploadTimeout(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.get(settings)); + setGlobalMetadataUploadTimeout(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING.get(settings)); } private BlobStoreTransferService getBlobStoreTransferService() { @@ -386,7 +386,7 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException ); try { - if (latch.await(getGlobalMetadataUploadWaitTime().millis(), TimeUnit.MILLISECONDS) == false) { + if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { // TODO: We should add metrics where transfer is timing out. [Issue: #10687] GlobalMetadataTransferException ex = new GlobalMetadataTransferException( String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete") @@ -441,7 +441,7 @@ private List writeIndexMetadataParallel(ClusterState clus } try { - if (latch.await(getIndexMetadataUploadWaitTime().millis(), TimeUnit.MILLISECONDS) == false) { + if (latch.await(getIndexMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { IndexMetadataTransferException ex = new IndexMetadataTransferException( String.format( Locale.ROOT, @@ -634,20 +634,20 @@ private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) { this.slowWriteLoggingThreshold = slowWriteLoggingThreshold; } - private void setIndexMetadataUploadWaitTime(TimeValue newIndexMetadataUploadWaitTime) { - indexMetadataUploadWaitTime = newIndexMetadataUploadWaitTime; + private void setIndexMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeout) { + INDEX_METADATA_UPLOAD_TIMEOUT = newIndexMetadataUploadTimeout; } - private void setGlobalMetadataUploadWaitTime(TimeValue newGlobalMetadataUploadWaitTime) { - globalMetadataUploadWaitTime = newGlobalMetadataUploadWaitTime; + private void setGlobalMetadataUploadTimeout(TimeValue newGlobalMetadataUploadTimeout) { + GLOBAL_METADATA_UPLOAD_TIMEOUT = newGlobalMetadataUploadTimeout; } - public static TimeValue getIndexMetadataUploadWaitTime() { - return indexMetadataUploadWaitTime; + public static TimeValue getIndexMetadataUploadTimeout() { + return INDEX_METADATA_UPLOAD_TIMEOUT; } - public static TimeValue getGlobalMetadataUploadWaitTime() { - return globalMetadataUploadWaitTime; + public static TimeValue getGlobalMetadataUploadTimeout() { + return GLOBAL_METADATA_UPLOAD_TIMEOUT; } static String getManifestFileName(long term, long version, boolean committed) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index ad045bd54f3d0..bea2602f24ba3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -1043,29 +1043,29 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti public void testIndexMetadataUploadWaitTimeSetting() { // verify default value - assertEquals(RemoteClusterStateService.indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime()); + assertEquals(RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT, RemoteClusterStateService.getIndexMetadataUploadTimeout()); - // verify update index metadata upload wait time - int indexMetadataUploadWaitTime = randomIntBetween(1, 10); + // verify update index metadata upload timeout + int indexMetadataUploadTimeout = randomIntBetween(1, 10); Settings newSettings = Settings.builder() - .put("cluster.remote_store.index_metadata.upload_wait_time", indexMetadataUploadWaitTime + "s") + .put("cluster.remote_store.index_metadata.upload_timeout", indexMetadataUploadTimeout + "s") .build(); clusterSettings.applySettings(newSettings); - assertEquals(indexMetadataUploadWaitTime, RemoteClusterStateService.getIndexMetadataUploadWaitTime().seconds()); + assertEquals(indexMetadataUploadTimeout, RemoteClusterStateService.getIndexMetadataUploadTimeout().seconds()); } public void testGlobalMetadataUploadWaitTimeSetting() { // verify default value - assertEquals(RemoteClusterStateService.globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime()); + assertEquals(RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT, RemoteClusterStateService.getGlobalMetadataUploadTimeout()); - // verify update global metadata upload wait time - int globalMetadataUploadWaitTime = randomIntBetween(1, 10); + // verify update global metadata upload timeout + int globalMetadataUploadTimeout = randomIntBetween(1, 10); Settings newSettings = Settings.builder() - .put("cluster.remote_store.global_metadata.upload_wait_time", globalMetadataUploadWaitTime + "s") + .put("cluster.remote_store.global_metadata.upload_timeout", globalMetadataUploadTimeout + "s") .build(); clusterSettings.applySettings(newSettings); - assertEquals(globalMetadataUploadWaitTime, RemoteClusterStateService.getGlobalMetadataUploadWaitTime().seconds()); + assertEquals(globalMetadataUploadTimeout, RemoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { From a66e150560f1ce4e0ebceae5372ffa82e7819253 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sun, 22 Oct 2023 10:55:29 +0530 Subject: [PATCH 06/10] empty commit Signed-off-by: Rahul Karajgikar From baa3118ab39cbf4d963538771275f78dedc9c0ab Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sun, 22 Oct 2023 16:07:09 +0530 Subject: [PATCH 07/10] Change based on comments Signed-off-by: Rahul Karajgikar --- .../remote/RemoteClusterStateService.java | 34 ++++++++++--------- .../RemoteClusterStateServiceTests.java | 19 +++++++---- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index e0bc99cda45ae..14dfaefd0d5be 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -83,21 +83,20 @@ public class RemoteClusterStateService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); - // default value for index metadata upload wait is 20s - static volatile TimeValue INDEX_METADATA_UPLOAD_TIMEOUT = TimeValue.timeValueMillis(20000); - // default value for global metadata upload wait is 20s - static volatile TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT = TimeValue.timeValueMillis(20000); + public static final TimeValue INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); + + public static final TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( - "cluster.remote_store.index_metadata.upload_timeout", - INDEX_METADATA_UPLOAD_TIMEOUT, + "cluster.remote_store.state.index_metadata.upload_timeout", + INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT, Setting.Property.Dynamic, Setting.Property.NodeScope ); public static final Setting GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( - "cluster.remote_store.global_metadata.upload_timeout", - GLOBAL_METADATA_UPLOAD_TIMEOUT, + "cluster.remote_store.state.global_metadata.upload_timeout", + GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -156,6 +155,9 @@ public class RemoteClusterStateService implements Closeable { private BlobStoreTransferService blobStoreTransferService; private volatile TimeValue slowWriteLoggingThreshold; + private volatile TimeValue indexMetadataUploadTimeout; + private volatile TimeValue globalMetadataUploadTimeout; + private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; @@ -186,11 +188,11 @@ public RemoteClusterStateService( this.relativeTimeNanosSupplier = relativeTimeNanosSupplier; this.threadpool = threadPool; this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); + this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); + this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); - setIndexMetadataUploadTimeout(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.get(settings)); - setGlobalMetadataUploadTimeout(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING.get(settings)); } private BlobStoreTransferService getBlobStoreTransferService() { @@ -635,19 +637,19 @@ private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) { } private void setIndexMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeout) { - INDEX_METADATA_UPLOAD_TIMEOUT = newIndexMetadataUploadTimeout; + this.indexMetadataUploadTimeout = newIndexMetadataUploadTimeout; } private void setGlobalMetadataUploadTimeout(TimeValue newGlobalMetadataUploadTimeout) { - GLOBAL_METADATA_UPLOAD_TIMEOUT = newGlobalMetadataUploadTimeout; + this.globalMetadataUploadTimeout = newGlobalMetadataUploadTimeout; } - public static TimeValue getIndexMetadataUploadTimeout() { - return INDEX_METADATA_UPLOAD_TIMEOUT; + public TimeValue getIndexMetadataUploadTimeout() { + return this.indexMetadataUploadTimeout; } - public static TimeValue getGlobalMetadataUploadTimeout() { - return GLOBAL_METADATA_UPLOAD_TIMEOUT; + public TimeValue getGlobalMetadataUploadTimeout() { + return this.globalMetadataUploadTimeout; } static String getManifestFileName(long term, long version, boolean committed) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index bea2602f24ba3..e665bb92b7142 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -1043,29 +1043,34 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti public void testIndexMetadataUploadWaitTimeSetting() { // verify default value - assertEquals(RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT, RemoteClusterStateService.getIndexMetadataUploadTimeout()); + assertEquals( + RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT, + remoteClusterStateService.getIndexMetadataUploadTimeout() + ); // verify update index metadata upload timeout int indexMetadataUploadTimeout = randomIntBetween(1, 10); Settings newSettings = Settings.builder() - .put("cluster.remote_store.index_metadata.upload_timeout", indexMetadataUploadTimeout + "s") + .put("cluster.remote_store.state.index_metadata.upload_timeout", indexMetadataUploadTimeout + "s") .build(); clusterSettings.applySettings(newSettings); - assertEquals(indexMetadataUploadTimeout, RemoteClusterStateService.getIndexMetadataUploadTimeout().seconds()); - + assertEquals(indexMetadataUploadTimeout, remoteClusterStateService.getIndexMetadataUploadTimeout().seconds()); } public void testGlobalMetadataUploadWaitTimeSetting() { // verify default value - assertEquals(RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT, RemoteClusterStateService.getGlobalMetadataUploadTimeout()); + assertEquals( + RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT, + remoteClusterStateService.getGlobalMetadataUploadTimeout() + ); // verify update global metadata upload timeout int globalMetadataUploadTimeout = randomIntBetween(1, 10); Settings newSettings = Settings.builder() - .put("cluster.remote_store.global_metadata.upload_timeout", globalMetadataUploadTimeout + "s") + .put("cluster.remote_store.state.global_metadata.upload_timeout", globalMetadataUploadTimeout + "s") .build(); clusterSettings.applySettings(newSettings); - assertEquals(globalMetadataUploadTimeout, RemoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); + assertEquals(globalMetadataUploadTimeout, remoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { From 82027c43f527d19e056219e3aad2859916510509 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sun, 22 Oct 2023 18:04:33 +0530 Subject: [PATCH 08/10] empty commit Signed-off-by: Rahul Karajgikar From cb68ea82e0f3597a876dae6b5e9c120ad772973f Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Sun, 22 Oct 2023 19:39:32 +0530 Subject: [PATCH 09/10] empty commit Signed-off-by: Rahul Karajgikar From f08c371b2f0a6b50678dec54cfde49294bbe0e8e Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar Date: Mon, 23 Oct 2023 14:05:45 +0530 Subject: [PATCH 10/10] Add changelog Signed-off-by: Rahul Karajgikar --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 374dd4ab57ee6..9874e7c431b7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Store] Add repository stats for remote store([#10567](https://github.com/opensearch-project/OpenSearch/pull/10567)) - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255)) - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352)) +- [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814)) ### Dependencies - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))