Merge branch 'main' into snap_restore_search_replica

Signed-off-by: Vinay Krishna Pudyodu <[email protected]>

CHANGELOG.md

@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641))
 - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993))
 - Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383))
+- Add support for renaming aliases during snapshot restore ([#16292](https://github.com/opensearch-project/OpenSearch/pull/16292))
 - Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387))
 - URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938))
 - [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289))
@@ -68,6 +69,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273))
 - Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296))
 - Update last seen cluster state in the commit phase ([#16215](https://github.com/opensearch-project/OpenSearch/pull/16215))
+- Make multiple settings dynamic for tuning on larger clusters([#16347](https://github.com/opensearch-project/OpenSearch/pull/16347))
 
 ### Deprecated
 
@@ -85,14 +87,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158))
 - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194))
 - Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254))
+- Fix get index settings API doesn't show `number_of_routing_shards` setting when it was explicitly set ([#16294](https://github.com/opensearch-project/OpenSearch/pull/16294))
 - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265))
 - [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337))
 - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331))
-- [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370)
+- [Workload Management] Make query groups persistent across process restarts ([#16370](https://github.com/opensearch-project/OpenSearch/pull/16370))
+- [Workload Management] Enhance rejection mechanism in workload management ([#16417](https://github.com/opensearch-project/OpenSearch/pull/16417))
 - Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386))
 - Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378))
 - Fix missing fields in task index mapping to ensure proper task result storage ([#16201](https://github.com/opensearch-project/OpenSearch/pull/16201))
 - Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362))
+- [Workload Management] Fixing Create/Update QueryGroup TransportActions to execute from non-cluster manager nodes ([16422](https://github.com/opensearch-project/OpenSearch/pull/16422))
+- Fix flaky test in `testApproximateRangeWithSizeOverDefault` by adjusting totalHits assertion logic ([#16434](https://github.com/opensearch-project/OpenSearch/pull/16434#pullrequestreview-2386999409))
 
 ### Security
 

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "23+37";
+    private static final String SYSTEM_JDK_VERSION = "23.0.1+11";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "23+37";
+    private static final String GRADLE_JDK_VERSION = "23.0.1+11";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -2,7 +2,7 @@ opensearch        = 3.0.0
 lucene            = 9.12.0
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 23+37
+bundled_jdk = 23.0.1+11
 
 # optional dependencies
 spatial4j         = 0.7

modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java

@@ -146,6 +146,7 @@ protected void initChannel(Channel ch) throws Exception {
                 .map(SecureTransportSettingsProvider.SecureTransportParameters::dualModeEnabled)
                 .orElse(false);
             if (dualModeEnabled) {
+                logger.info("SSL Dual mode enabled, using port unification handler");
                 final ChannelHandler portUnificationHandler = new DualModeSslHandler(
                     settings,
                     secureTransportSettingsProvider,

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java

@@ -8,8 +8,8 @@
 
 package org.opensearch.plugin.wlm.action;
 
-import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.cluster.metadata.QueryGroup;
 import org.opensearch.common.UUIDs;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -33,7 +33,7 @@
  *
  * @opensearch.experimental
  */
-public class CreateQueryGroupRequest extends ActionRequest {
+public class CreateQueryGroupRequest extends ClusterManagerNodeRequest<CreateQueryGroupRequest> {
     private final QueryGroup queryGroup;
 
     /**

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportCreateQueryGroupAction.java

@@ -9,43 +9,82 @@
 package org.opensearch.plugin.wlm.action;
 
 import org.opensearch.action.support.ActionFilters;
-import org.opensearch.action.support.HandledTransportAction;
+import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.block.ClusterBlockException;
+import org.opensearch.cluster.block.ClusterBlockLevel;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService;
-import org.opensearch.tasks.Task;
+import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
+import java.io.IOException;
+
+import static org.opensearch.threadpool.ThreadPool.Names.SAME;
+
 /**
  * Transport action to create QueryGroup
  *
  * @opensearch.experimental
  */
-public class TransportCreateQueryGroupAction extends HandledTransportAction<CreateQueryGroupRequest, CreateQueryGroupResponse> {
+public class TransportCreateQueryGroupAction extends TransportClusterManagerNodeAction<CreateQueryGroupRequest, CreateQueryGroupResponse> {
 
     private final QueryGroupPersistenceService queryGroupPersistenceService;
 
     /**
      * Constructor for TransportCreateQueryGroupAction
      *
-     * @param actionName - action name
+     * @param threadPool - {@link ThreadPool} object
      * @param transportService - a {@link TransportService} object
      * @param actionFilters - a {@link ActionFilters} object
+     * @param indexNameExpressionResolver - {@link IndexNameExpressionResolver} object
      * @param queryGroupPersistenceService - a {@link QueryGroupPersistenceService} object
      */
     @Inject
     public TransportCreateQueryGroupAction(
-        String actionName,
+        ThreadPool threadPool,
         TransportService transportService,
         ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
         QueryGroupPersistenceService queryGroupPersistenceService
     ) {
-        super(CreateQueryGroupAction.NAME, transportService, actionFilters, CreateQueryGroupRequest::new);
+        super(
+            CreateQueryGroupAction.NAME,
+            transportService,
+            queryGroupPersistenceService.getClusterService(),
+            threadPool,
+            actionFilters,
+            CreateQueryGroupRequest::new,
+            indexNameExpressionResolver
+        );
         this.queryGroupPersistenceService = queryGroupPersistenceService;
     }
 
     @Override
-    protected void doExecute(Task task, CreateQueryGroupRequest request, ActionListener<CreateQueryGroupResponse> listener) {
+    protected void clusterManagerOperation(
+        CreateQueryGroupRequest request,
+        ClusterState clusterState,
+        ActionListener<CreateQueryGroupResponse> listener
+    ) {
         queryGroupPersistenceService.persistInClusterStateMetadata(request.getQueryGroup(), listener);
     }
+
+    @Override
+    protected String executor() {
+        return SAME;
+    }
+
+    @Override
+    protected CreateQueryGroupResponse read(StreamInput in) throws IOException {
+        return new CreateQueryGroupResponse(in);
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(CreateQueryGroupRequest request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
 }

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java

@@ -9,43 +9,81 @@
 package org.opensearch.plugin.wlm.action;
 
 import org.opensearch.action.support.ActionFilters;
-import org.opensearch.action.support.HandledTransportAction;
+import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.block.ClusterBlockException;
+import org.opensearch.cluster.block.ClusterBlockLevel;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService;
-import org.opensearch.tasks.Task;
+import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
+import java.io.IOException;
+
+import static org.opensearch.threadpool.ThreadPool.Names.SAME;
+
 /**
  * Transport action to update QueryGroup
  *
  * @opensearch.experimental
  */
-public class TransportUpdateQueryGroupAction extends HandledTransportAction<UpdateQueryGroupRequest, UpdateQueryGroupResponse> {
+public class TransportUpdateQueryGroupAction extends TransportClusterManagerNodeAction<UpdateQueryGroupRequest, UpdateQueryGroupResponse> {
 
     private final QueryGroupPersistenceService queryGroupPersistenceService;
 
     /**
      * Constructor for TransportUpdateQueryGroupAction
      *
-     * @param actionName - action name
+     * @param threadPool - {@link ThreadPool} object
      * @param transportService - a {@link TransportService} object
      * @param actionFilters - a {@link ActionFilters} object
+     * @param indexNameExpressionResolver - {@link IndexNameExpressionResolver} object
      * @param queryGroupPersistenceService - a {@link QueryGroupPersistenceService} object
      */
     @Inject
     public TransportUpdateQueryGroupAction(
-        String actionName,
+        ThreadPool threadPool,
         TransportService transportService,
         ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
         QueryGroupPersistenceService queryGroupPersistenceService
     ) {
-        super(UpdateQueryGroupAction.NAME, transportService, actionFilters, UpdateQueryGroupRequest::new);
+        super(
+            UpdateQueryGroupAction.NAME,
+            transportService,
+            queryGroupPersistenceService.getClusterService(),
+            threadPool,
+            actionFilters,
+            UpdateQueryGroupRequest::new,
+            indexNameExpressionResolver
+        );
         this.queryGroupPersistenceService = queryGroupPersistenceService;
     }
 
     @Override
-    protected void doExecute(Task task, UpdateQueryGroupRequest request, ActionListener<UpdateQueryGroupResponse> listener) {
+    protected void clusterManagerOperation(
+        UpdateQueryGroupRequest request,
+        ClusterState clusterState,
+        ActionListener<UpdateQueryGroupResponse> listener
+    ) {
         queryGroupPersistenceService.updateInClusterStateMetadata(request, listener);
     }
+
+    @Override
+    protected String executor() {
+        return SAME;
+    }
+
+    @Override
+    protected UpdateQueryGroupResponse read(StreamInput in) throws IOException {
+        return new UpdateQueryGroupResponse(in);
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(UpdateQueryGroupRequest request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
 }

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java

@@ -8,8 +8,8 @@
 
 package org.opensearch.plugin.wlm.action;
 
-import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.cluster.metadata.QueryGroup;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -23,7 +23,7 @@
  *
  * @opensearch.experimental
  */
-public class UpdateQueryGroupRequest extends ActionRequest {
+public class UpdateQueryGroupRequest extends ClusterManagerNodeRequest<UpdateQueryGroupRequest> {
     private final String name;
     private final MutableQueryGroupFragment mutableQueryGroupFragment;
 

qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java

@@ -26,8 +26,8 @@ public class ClusterStatsIT extends AbstractRollingTestCase {
     public void testClusterStats() throws IOException {
         Response response = client().performRequest(new Request("GET", "/_cluster/stats"));
         validateClusterStatsWithFilterResponse(response, nodeStatsMetrics, indicesStatsMetrics);
-        if (AbstractRollingTestCase.UPGRADE_FROM_VERSION.onOrAfter(Version.V_3_0_0) || (
-            CLUSTER_TYPE == ClusterType.UPGRADED && Version.CURRENT.onOrAfter(Version.V_3_0_0))) {
+        if (AbstractRollingTestCase.UPGRADE_FROM_VERSION.onOrAfter(Version.V_2_18_0) || (
+            CLUSTER_TYPE == ClusterType.UPGRADED && Version.CURRENT.onOrAfter(Version.V_2_18_0))) {
             response = client().performRequest(new Request("GET", "/_cluster/stats/os/nodes/_all"));
             validateClusterStatsWithFilterResponse(response, List.of("os"), Collections.emptyList());
             response = client().performRequest(new Request("GET", "/_cluster/stats/indices/mappings/nodes/_all"));

rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml

@@ -0,0 +1,41 @@
+---
+setup:
+  - do:
+      indices.create:
+        body:
+          settings:
+            index:
+              number_of_routing_shards: 4
+              number_of_shards: 2
+              number_of_replicas: 1
+        index: test-index
+
+  - do:
+      indices.create:
+        body:
+          settings:
+            index:
+              number_of_shards: 2
+              number_of_replicas: 1
+        index: test-index1
+
+---
+Test retrieval of number_routing_shards settings:
+  - skip:
+      version: " - 2.99.99"
+      reason: "introduced in 3.0.0" # TODO: change it to 2.18.0 after backport to 2.x branch
+  - do:
+      indices.get_settings:
+        flat_settings: true
+        index: test-index
+  # show `index.number_of_routing_shards` if it was explicitly set when creating
+  - match:
+      test-index.settings.index\.number_of_routing_shards: "4"
+
+  - do:
+      indices.get_settings:
+        flat_settings: true
+        index: test-index1
+  # do not show `index.number_of_routing_shards` if it was not explicitly set when creating
+  - match:
+      test-index1.settings.index\.number_of_routing_shards: null

server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java

@@ -529,14 +529,13 @@ public void testDeleteWhileV2CreateOngoing() throws Exception {
         awaitNumberOfSnapshotsInProgress(1);
 
         ActionFuture<AcknowledgedResponse> a = startDeleteSnapshot(repoName, "snapshot-v1");
+        expectThrows(ConcurrentSnapshotExecutionException.class, a::actionGet);
 
         unblockNode(repoName, clusterManagerName);
         CreateSnapshotResponse csr = snapshotFuture.actionGet();
         assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0);
-        assertTrue(a.actionGet().isAcknowledged());
         List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
-        assertEquals(1, snapInfo.size());
-        assertThat(snapInfo, contains(csr.getSnapshotInfo()));
+        assertEquals(2, snapInfo.size());
     }
 
     @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/16205")

server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java

@@ -108,6 +108,12 @@ public void onResponse(ClusterStateResponse clusterStateResponse) {
                                 : paginationStrategy.getRequestedEntities()
                         );
                         catShardsResponse.setPageToken(Objects.isNull(paginationStrategy) ? null : paginationStrategy.getResponseToken());
+                        // For paginated queries, if strategy outputs no shards to be returned, avoid fetching IndicesStats.
+                        if (shouldSkipIndicesStatsRequest(paginationStrategy)) {
+                            catShardsResponse.setIndicesStatsResponse(IndicesStatsResponse.getEmptyResponse());
+                            cancellableListener.onResponse(catShardsResponse);
+                            return;
+                        }
                         IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
                         indicesStatsRequest.setShouldCancelOnTimeout(true);
                         indicesStatsRequest.all();
@@ -159,4 +165,8 @@ private void validateRequestLimit(
             }
         }
     }
+
+    private boolean shouldSkipIndicesStatsRequest(ShardPaginationStrategy paginationStrategy) {
+        return Objects.nonNull(paginationStrategy) && paginationStrategy.getRequestedEntities().isEmpty();
+    }
 }