[WIP] Update to Apache Lucene 10 for 3.0.0

.github/workflows/assemble.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21, 23 ]
         os: [ubuntu-latest, windows-latest, macos-13]
     steps:
       - uses: actions/checkout@v4
@@ -16,17 +16,6 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Setup docker (missing on MacOS)
         id: setup_docker
         if: runner.os == 'macos'
@@ -47,8 +36,8 @@ jobs:
         shell: bash
         if: runner.os != 'macos'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} 
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE 
       - name: Run Gradle (assemble)
         if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE

.github/workflows/precommit.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21, 23 ]
+        java: [ 21, 23 ]
         os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
     steps:
       - uses: actions/checkout@v4
@@ -17,18 +17,7 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Run Gradle (precommit)
         shell: bash
         run: |
-          ./gradlew javadoc precommit --parallel -Druntime.java=${{ matrix.java }}
+          ./gradlew javadoc precommit --parallel

CHANGELOG-3.0.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
 
 ### Dependencies
+- Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))
 
 ### Changed
 - Changed locale provider from COMPAT to CLDR  ([#14345](https://github.com/opensearch-project/OpenSearch/pull/14345))

benchmarks/build.gradle

@@ -85,44 +85,5 @@ spotless {
   }
 }
 
-if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_20) {
-  // Add support for incubator modules on supported Java versions.
-  run.jvmArgs += ['--add-modules=jdk.incubator.vector']
-  run.classpath += files(jar.archiveFile)
-  run.classpath -= sourceSets.main.output
-  evaluationDependsOn(':libs:opensearch-common')
-
-  sourceSets {
-    java20 {
-      java {
-        srcDirs = ['src/main/java20']
-      }
-    }
-  }
-
-  configurations {
-    java20Implementation.extendsFrom(implementation)
-  }
-
-  dependencies {
-    java20Implementation sourceSets.main.output
-    java20Implementation project(':libs:opensearch-common').sourceSets.java20.output
-    java20AnnotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
-  }
-
-  compileJava20Java {
-    targetCompatibility = JavaVersion.VERSION_20
-    options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
-  }
-
-  jar {
-    metaInf {
-      into 'versions/20'
-      from sourceSets.java20.output
-    }
-    manifest.attributes('Multi-Release': 'true')
-  }
-
-  // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
-  disableTasks('forbiddenApisJava20')
-}
+// Add support for incubator modules on supported Java versions.
+run.jvmArgs += ['--add-modules=jdk.incubator.vector']

benchmarks/src/main/java/org/opensearch/common/round/RoundableSupplier.java

@@ -13,7 +13,7 @@
 public class RoundableSupplier implements Supplier<Roundable> {
     private final Supplier<Roundable> delegate;
 
-    RoundableSupplier(String type, long[] values, int size) throws ClassNotFoundException {
+    RoundableSupplier(String type, long[] values, int size) {
         switch (type) {
             case "binary":
                 delegate = () -> new BinarySearcher(values, size);
@@ -22,7 +22,8 @@ public class RoundableSupplier implements Supplier<Roundable> {
                 delegate = () -> new BidirectionalLinearSearcher(values, size);
                 break;
             case "btree":
-                throw new ClassNotFoundException("BtreeSearcher is not supported below JDK 20");
+                delegate = () -> new BtreeSearcher(values, size);
+                break;
             default:
                 throw new IllegalArgumentException("invalid type: " + type);
         }

buildSrc/build.gradle

@@ -74,8 +74,8 @@ processResources {
  *         Java version                                                      *
  *****************************************************************************/
 
-if (JavaVersion.current() < JavaVersion.VERSION_11) {
-  throw new GradleException('At least Java 11 is required to build opensearch gradle tools')
+if (JavaVersion.current() < JavaVersion.VERSION_21) {
+  throw new GradleException('At least Java 21 is required to build opensearch gradle tools')
 }
 
 sourceSets {
@@ -111,7 +111,7 @@ dependencies {
   api 'org.apache.rat:apache-rat:0.15'
   api "commons-io:commons-io:${props.getProperty('commonsio')}"
   api "net.java.dev.jna:jna:5.14.0"
-  api 'com.github.johnrengelman:shadow:8.1.1'
+  api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.8'
@@ -171,8 +171,8 @@ if (project != rootProject) {
 
   allprojects {
     java {
-        targetCompatibility = JavaVersion.VERSION_11
-        sourceCompatibility = JavaVersion.VERSION_11
+        targetCompatibility = JavaVersion.VERSION_21
+        sourceCompatibility = JavaVersion.VERSION_21
     }
   }
 

buildSrc/reaper/build.gradle

@@ -12,8 +12,8 @@
 apply plugin: 'java'
 
 java {
-  targetCompatibility = JavaVersion.VERSION_11
-  sourceCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
+  sourceCompatibility = JavaVersion.VERSION_21
 }
 
 jar {

buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/.ci/java-versions.properties

@@ -27,6 +27,6 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-OPENSEARCH_BUILD_JAVA=openjdk11
-OPENSEARCH_RUNTIME_JAVA=java11
+OPENSEARCH_BUILD_JAVA=openjdk21
+OPENSEARCH_RUNTIME_JAVA=java21
 GRADLE_TASK=build

buildSrc/src/main/resources/forbidden/opensearch-server-signatures.txt

@@ -61,10 +61,6 @@ java.nio.channels.FileChannel#read(java.nio.ByteBuffer, long)
 @defaultMessage Use Lucene.parseLenient instead it strips off minor version
 org.apache.lucene.util.Version#parseLeniently(java.lang.String)
 
-@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#relativeTimeInMillis instead
-org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread()
-org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter()
-
 @defaultMessage Don't interrupt threads use FutureUtils#cancel(Future<T>) instead
 java.util.concurrent.Future#cancel(boolean)
 

buildSrc/src/main/resources/minimumCompilerVersion

@@ -1 +1 @@
-17
+21

buildSrc/src/main/resources/minimumRuntimeVersion

@@ -1 +1 @@
-11
+21

buildSrc/src/testKit/thirdPartyAudit/build.gradle

@@ -46,11 +46,11 @@ dependencies {
 }
 
 tasks.register("empty", ThirdPartyAuditTask) {
-  targetCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
   signatureFile = file('third-party-audit-empty.txt')
 }
 
 tasks.register("absurd", ThirdPartyAuditTask) {
-  targetCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
   signatureFile = file('third-party-audit-absurd.txt')
 }

client/rest-high-level/src/test/java/org/opensearch/client/HighLevelRestClientCompressionIT.java

@@ -67,7 +67,7 @@ public void testCompressesResponseIfRequested() throws IOException {
         SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, requestOptions);
 
         assertThat(searchResponse.status().getStatus(), equalTo(200));
-        assertEquals(1L, searchResponse.getHits().getTotalHits().value);
+        assertEquals(1L, searchResponse.getHits().getTotalHits().value());
         assertEquals(SAMPLE_DOCUMENT, searchResponse.getHits().getHits()[0].getSourceAsString());
     }
 

client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java

@@ -236,7 +236,7 @@ public void testDeleteByQuery() throws Exception {
             assertEquals(0, bulkResponse.getSearchFailures().size());
             assertEquals(
                 2,
-                highLevelClient().search(new SearchRequest(sourceIndex), RequestOptions.DEFAULT).getHits().getTotalHits().value
+                highLevelClient().search(new SearchRequest(sourceIndex), RequestOptions.DEFAULT).getHits().getTotalHits().value()
             );
         }
         {

client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java

@@ -237,7 +237,7 @@ public void testSearchScroll() throws IOException {
             RequestOptions.DEFAULT
         );
         assertEquals(mockSearchResponse.getScrollId(), searchResponse.getScrollId());
-        assertEquals(0, searchResponse.getHits().getTotalHits().value);
+        assertEquals(0, searchResponse.getHits().getTotalHits().value());
         assertEquals(5, searchResponse.getTotalShards());
         assertEquals(5, searchResponse.getSuccessfulShards());
         assertEquals(100, searchResponse.getTook().getMillis());