Merge branch 'main' into main-telemetry-metrics-framework

Signed-off-by: Gagan Juneja <[email protected]>

.github/pull_request_template.md

@@ -19,6 +19,7 @@ Resolves #[Issue number to be closed when this PR is merged]
   - [ ] New functionality has javadoc added
 - [ ] Commits are signed per the DCO using --signoff
 - [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
+- [ ] GitHub issue/PR created in [OpenSearch documentation repo](https://github.com/opensearch-project/documentation-website) for the required public documentation changes (#[Issue/PR number])
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
 For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/lucene-snapshots.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Set hash
         working-directory: ./lucene
         run: |
-          echo "::set-output name=REVISION::$(git rev-parse --short HEAD)"
+          echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
         id: version
 
       - name: Initialize gradle settings

.github/workflows/version.yml

@@ -59,7 +59,7 @@ jobs:
           sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ steps.github_app_token.outputs.token }}
           base: ${{ env.BASE }}
@@ -86,7 +86,7 @@ jobs:
           sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ steps.github_app_token.outputs.token }}
           base: ${{ env.BASE_X }}
@@ -113,7 +113,7 @@ jobs:
           sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ steps.github_app_token.outputs.token }}
           base: main

CHANGELOG.md

@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
 - Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110))
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
+- Configurable merge policy for index with an option to choose from LogByteSize and Tiered merge policy ([#9992](https://github.com/opensearch-project/OpenSearch/pull/9992))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -88,6 +89,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131))
 - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189))
 - Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562))
+- Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204))
 
 ### Dependencies
 - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575))
@@ -104,11 +106,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206))
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208))
 - Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209))
-- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
-- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
-- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
-- Bump Lucene from 9.7.0 to 9.8.0 ([#10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
+- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
+- Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295))
 - Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302))
+- Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303))
+- Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301))
 
 ### Changed
 - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
@@ -118,19 +123,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042))
 - [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122))
 - Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246))
+- Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200))
 - [Metrics Framework] Add Metrics framework. ([#10241](https://github.com/opensearch-project/OpenSearch/pull/10241))
 
 ### Deprecated
 
 ### Removed
+- Remove spurious SGID bit on directories ([#9447](https://github.com/opensearch-project/OpenSearch/pull/9447))
 
 ### Fixed
 - Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725))
 - Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045))
 - Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082))
 - Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089))
+- Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194))
+- Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256))
 
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x

buildSrc/version.properties

@@ -28,7 +28,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.97.Final
+netty             = 4.1.99.Final
 joda              = 2.12.2
 
 # client dependencies

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.4.0"
+  id "com.netflix.nebula.ospackage-base" version "11.5.0"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {
@@ -213,7 +213,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
     configurationFile '/etc/opensearch/jvm.options'
     configurationFile '/etc/opensearch/log4j2.properties'
     from("${packagingFiles}") {
-      dirMode 02750
+      dirMode 0750
       into('/etc')
       permissionGroup 'opensearch'
       includeEmptyDirs true
@@ -223,7 +223,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
     }
     from("${packagingFiles}/etc/opensearch") {
       into('/etc/opensearch')
-      dirMode 02750
+      dirMode 0750
       fileMode 0660
       permissionGroup 'opensearch'
       includeEmptyDirs true
@@ -281,8 +281,8 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
         dirMode mode
       }
     }
-    copyEmptyDir('/var/log/opensearch', 'opensearch', 'opensearch', 02750)
-    copyEmptyDir('/var/lib/opensearch', 'opensearch', 'opensearch', 02750)
+    copyEmptyDir('/var/log/opensearch', 'opensearch', 'opensearch', 0750)
+    copyEmptyDir('/var/lib/opensearch', 'opensearch', 'opensearch', 0750)
     copyEmptyDir('/usr/share/opensearch/plugins', 'root', 'root', 0755)
 
     into '/usr/share/opensearch'

distribution/packages/src/deb/lintian/opensearch

@@ -15,11 +15,11 @@ missing-dep-on-jarwrapper
 
 # we prefer to not make our config and log files world readable
 non-standard-file-perm etc/default/opensearch 0660 != 0644
-non-standard-dir-perm etc/opensearch/ 2750 != 0755
-non-standard-dir-perm etc/opensearch/jvm.options.d/ 2750 != 0755
+non-standard-dir-perm etc/opensearch/ 0750 != 0755
+non-standard-dir-perm etc/opensearch/jvm.options.d/ 0750 != 0755
 non-standard-file-perm etc/opensearch/*
-non-standard-dir-perm var/lib/opensearch/ 2750 != 0755
-non-standard-dir-perm var/log/opensearch/ 2750 != 0755
+non-standard-dir-perm var/lib/opensearch/ 0750 != 0755
+non-standard-dir-perm var/log/opensearch/ 0750 != 0755
 executable-is-not-world-readable etc/init.d/opensearch 0750
 non-standard-file-permissions-for-etc-init.d-script etc/init.d/opensearch 0750 != 0755
 

libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java

@@ -25,7 +25,10 @@
  */
 @InternalApi
 class DefaultTracer implements Tracer {
-    static final String THREAD_NAME = "th_name";
+    /**
+     * Current thread name.
+     */
+    static final String THREAD_NAME = "thread.name";
 
     private final TracingTelemetry tracingTelemetry;
     private final TracerContextStorage<String, Span> tracerContextStorage;

modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9f02dcb9b15a647a56af210dffdc294a57922fb0

modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9984cbd6e5d55c768f198e975d8aaf7fd42a4602

modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+7142095066eaebd5f29b88c41af7b383b6a953f6

modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+c5a3481c4bb9732a3a94fb63cf916141a1a14669

modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+278f6dfa49d6bd75c40ae1470eb165716f87dce0

modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+742693761d7ea4c038bccfda96bb38194720b80d

modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+080e45397d9d5b134477de3ffd0f94283b908621

modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9ca2e3ae19a6713b749df154622115f480b6716c

modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+cb0fc6c31c387404212949c57950b5d72ce908b9

plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+21c76a42a468faafac6c84f8aca775073fc8e345

plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+c5a3481c4bb9732a3a94fb63cf916141a1a14669

plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+259bf1c5178c3e23bb89a2fab59b6d22846e3fa6

plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+8c8a89ea89b06e120c57bdb3db14b9a47ca30bb3

plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+804d8b752847923d3bb81f24de604597047c9b2e

plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+cb0fc6c31c387404212949c57950b5d72ce908b9

plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+a45aa70bc50d0500da5cdcd595cc838d87ada987

plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9f02dcb9b15a647a56af210dffdc294a57922fb0

plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9984cbd6e5d55c768f198e975d8aaf7fd42a4602

plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+7142095066eaebd5f29b88c41af7b383b6a953f6

plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+c5a3481c4bb9732a3a94fb63cf916141a1a14669

plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+278f6dfa49d6bd75c40ae1470eb165716f87dce0

plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+742693761d7ea4c038bccfda96bb38194720b80d

plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+080e45397d9d5b134477de3ffd0f94283b908621

plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9ca2e3ae19a6713b749df154622115f480b6716c

plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+961bd5b8d97ea6a07168176462f398089a24b5c8

plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+cb0fc6c31c387404212949c57950b5d72ce908b9

plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9f02dcb9b15a647a56af210dffdc294a57922fb0

plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9984cbd6e5d55c768f198e975d8aaf7fd42a4602

plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+7142095066eaebd5f29b88c41af7b383b6a953f6

plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+278f6dfa49d6bd75c40ae1470eb165716f87dce0

plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+742693761d7ea4c038bccfda96bb38194720b80d

plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+080e45397d9d5b134477de3ffd0f94283b908621

plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1

@@ -0,0 +1 @@
+9ca2e3ae19a6713b749df154622115f480b6716c

qa/os/src/test/java/org/opensearch/packaging/util/Packages.java

@@ -194,11 +194,11 @@ private static void verifyInstallation(Installation opensearch, Distribution dis
 
         // we shell out here because java's posix file permission view doesn't support special modes
         assertThat(opensearch.config, file(Directory, "root", "opensearch", p750));
-        assertThat(sh.run("find \"" + opensearch.config + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750"));
+        assertThat(sh.run("find \"" + opensearch.config + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("750"));
 
         final Path jvmOptionsDirectory = opensearch.config.resolve("jvm.options.d");
         assertThat(jvmOptionsDirectory, file(Directory, "root", "opensearch", p750));
-        assertThat(sh.run("find \"" + jvmOptionsDirectory + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750"));
+        assertThat(sh.run("find \"" + jvmOptionsDirectory + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("750"));
 
         Stream.of("opensearch.keystore", "opensearch.yml", "jvm.options", "log4j2.properties")
             .forEach(configFile -> assertThat(opensearch.config(configFile), file(File, "root", "opensearch", p660)));