Merge branch 'lucene_snapshot' into madv_random_feature_flag

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -529,8 +529,8 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
-      - label: "{{matrix.image}} / 8.13.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.2
+      - label: "{{matrix.image}} / 8.13.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -543,7 +543,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.2
+          BWC_VERSION: 8.13.3
 
       - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0

.buildkite/pipelines/periodic.yml

@@ -312,16 +312,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.2
-      - label: 8.13.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.2#bwcTest
+      - label: 8.13.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.2
+          BWC_VERSION: 8.13.3
       - label: 8.14.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.0#bwcTest
         timeout_in_minutes: 300
@@ -396,7 +396,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -438,7 +438,7 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -30,5 +30,5 @@ BWC_VERSION:
   - "8.10.4"
   - "8.11.4"
   - "8.12.2"
-  - "8.13.2"
+  - "8.13.3"
   - "8.14.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.20"
-  - "8.13.2"
+  - "8.13.3"
   - "8.14.0"

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -130,7 +130,8 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       ':server:generateModulesList',
       ':server:generatePluginsList',
       ':generateProviderImpls',
-      ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs'].collect { elasticsearchProject.right()?.task(it) ?: it })
+      ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs',
+      ':x-pack:libs:es-opensaml-security-api:shadowJar'].collect { elasticsearchProject.right()?.task(it) ?: it })
   }
 
   // this path is produced by the extractLibs task above
@@ -239,20 +240,22 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
  * but before the XML document, e.g. a doctype or comment
  */
 void modifyXml(Object path, Action<? super Node> action, String preface = null) {
-  Node xml = parseXml(path)
-  action.execute(xml)
+  if (project.file(path).exists()) {
+    Node xml = parseXml(path)
+    action.execute(xml)
 
-  File xmlFile = project.file(path)
-  xmlFile.withPrintWriter { writer ->
-    def printer = new XmlNodePrinter(writer)
-    printer.namespaceAware = true
-    printer.preserveWhitespace = true
-    writer.write("<?xml version=\"1.0\"?>\n")
+    File xmlFile = project.file(path)
+    xmlFile.withPrintWriter { writer ->
+      def printer = new XmlNodePrinter(writer)
+      printer.namespaceAware = true
+      printer.preserveWhitespace = true
+      writer.write("<?xml version=\"1.0\"?>\n")
 
-    if (preface != null) {
-      writer.write(preface)
+      if (preface != null) {
+        writer.write(preface)
+      }
+      printer.print(xml)
     }
-    printer.print(xml)
   }
 }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -49,6 +49,7 @@
 public class MrjarPlugin implements Plugin<Project> {
 
     private static final Pattern MRJAR_SOURCESET_PATTERN = Pattern.compile("main(\\d{2})");
+    private static final String MRJAR_IDEA_ENABLED = "org.gradle.mrjar.idea.enabled";
 
     private final JavaToolchainService javaToolchains;
 
@@ -61,23 +62,30 @@ public class MrjarPlugin implements Plugin<Project> {
     public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
-
-        List<Integer> mainVersions = findSourceVersions(project);
-        List<String> mainSourceSets = new ArrayList<>();
-        mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
-        List<String> testSourceSets = new ArrayList<>(mainSourceSets);
-        testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
-        for (int javaVersion : mainVersions) {
-            String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
-            SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
-            configureSourceSetInJar(project, mainSourceSet, javaVersion);
-            mainSourceSets.add(mainSourceSetName);
-            testSourceSets.add(mainSourceSetName);
-
-            String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion;
-            SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion);
-            testSourceSets.add(testSourceSetName);
-            createTestTask(project, testSourceSet, javaVersion, mainSourceSets);
+        var isIdeaSync = System.getProperty("idea.sync.active", "false").equals("true");
+        var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true");
+
+        // Ignore version-specific source sets if we are importing into IntelliJ and have not explicitly enabled this.
+        // Avoids an IntelliJ bug:
+        // https://youtrack.jetbrains.com/issue/IDEA-285640/Compiler-Options-Settings-language-level-is-set-incorrectly-with-JDK-19ea
+        if (isIdeaSync == false || ideaSourceSetsEnabled) {
+            List<Integer> mainVersions = findSourceVersions(project);
+            List<String> mainSourceSets = new ArrayList<>();
+            mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
+            List<String> testSourceSets = new ArrayList<>(mainSourceSets);
+            testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
+            for (int javaVersion : mainVersions) {
+                String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
+                SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
+                configureSourceSetInJar(project, mainSourceSet, javaVersion);
+                mainSourceSets.add(mainSourceSetName);
+                testSourceSets.add(mainSourceSetName);
+
+                String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion;
+                SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion);
+                testSourceSets.add(testSourceSetName);
+                createTestTask(project, testSourceSet, javaVersion, mainSourceSets);
+            }
         }
 
         configureMrjar(project);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java

@@ -11,6 +11,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import org.apache.commons.compress.utils.Lists;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainDownload;
 import org.gradle.jvm.toolchain.JavaToolchainRequest;
@@ -20,25 +21,25 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
+import java.util.Comparator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.StreamSupport;
 
 import static org.gradle.jvm.toolchain.JavaToolchainDownload.fromUri;
 
 public abstract class AdoptiumJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
     // package protected for better testing
-    final Map<AdoptiumVersionRequest, Optional<String>> CACHED_RELEASES = new ConcurrentHashMap<>();
+    final Map<AdoptiumVersionRequest, Optional<AdoptiumVersionInfo>> CACHED_SEMVERS = new ConcurrentHashMap<>();
 
     @Override
     public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
         if (requestIsSupported(request) == false) {
             return Optional.empty();
         }
         AdoptiumVersionRequest versionRequestKey = toVersionRequest(request);
-        Optional<String> versionInfo = CACHED_RELEASES.computeIfAbsent(
+        Optional<AdoptiumVersionInfo> versionInfo = CACHED_SEMVERS.computeIfAbsent(
             versionRequestKey,
             (r) -> resolveAvailableVersion(versionRequestKey)
         );
@@ -53,12 +54,12 @@ private AdoptiumVersionRequest toVersionRequest(JavaToolchainRequest request) {
         return new AdoptiumVersionRequest(platform, arch, javaLanguageVersion);
     }
 
-    private Optional<String> resolveAvailableVersion(AdoptiumVersionRequest requestKey) {
+    private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionRequest requestKey) {
         ObjectMapper mapper = new ObjectMapper();
         try {
             int languageVersion = requestKey.languageVersion.asInt();
             URL source = new URL(
-                "https://api.adoptium.net/v3/info/release_names?architecture="
+                "https://api.adoptium.net/v3/info/release_versions?architecture="
                     + requestKey.arch
                     + "&image_type=jdk&os="
                     + requestKey.platform
@@ -70,8 +71,14 @@ private Optional<String> resolveAvailableVersion(AdoptiumVersionRequest requestK
                     + ")"
             );
             JsonNode jsonNode = mapper.readTree(source);
-            JsonNode versionsNode = jsonNode.get("releases");
-            return StreamSupport.stream(versionsNode.spliterator(), false).map(JsonNode::textValue).findFirst();
+            JsonNode versionsNode = jsonNode.get("versions");
+            return Optional.of(
+                Lists.newArrayList(versionsNode.iterator())
+                    .stream()
+                    .map(this::toVersionInfo)
+                    .max(Comparator.comparing(AdoptiumVersionInfo::semver))
+                    .get()
+            );
         } catch (FileNotFoundException e) {
             // request combo not supported (e.g. aarch64 + windows
             return Optional.empty();
@@ -80,10 +87,21 @@ private Optional<String> resolveAvailableVersion(AdoptiumVersionRequest requestK
         }
     }
 
-    private URI resolveDownloadURI(AdoptiumVersionRequest request, String version) {
+    private AdoptiumVersionInfo toVersionInfo(JsonNode node) {
+        return new AdoptiumVersionInfo(
+            node.get("build").asInt(),
+            node.get("major").asInt(),
+            node.get("minor").asInt(),
+            node.get("openjdk_version").asText(),
+            node.get("security").asInt(),
+            node.get("semver").asText()
+        );
+    }
+
+    private URI resolveDownloadURI(AdoptiumVersionRequest request, AdoptiumVersionInfo versionInfo) {
         return URI.create(
-            "https://api.adoptium.net/v3/binary/version/"
-                + version
+            "https://api.adoptium.net/v3/binary/version/jdk-"
+                + versionInfo.semver
                 + "/"
                 + request.platform
                 + "/"
@@ -100,5 +118,7 @@ private boolean requestIsSupported(JavaToolchainRequest request) {
         return anyVendorOr(request.getJavaToolchainSpec().getVendor().get(), JvmVendorSpec.ADOPTIUM);
     }
 
+    record AdoptiumVersionInfo(int build, int major, int minor, String openjdkVersion, int security, String semver) {}
+
     record AdoptiumVersionRequest(String platform, String arch, JavaLanguageVersion languageVersion) {}
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java

@@ -39,7 +39,11 @@ record JdkBuild(JavaLanguageVersion languageVersion, String version, String buil
     );
 
     // package private so it can be replaced by tests
-    List<JdkBuild> builds = List.of(getBundledJdkBuild());
+    List<JdkBuild> builds = List.of(
+        getBundledJdkBuild(),
+        // 22 release candidate
+        new JdkBuild(JavaLanguageVersion.of(22), "22", "36", "830ec9fcccef480bb3e73fb7ecafe059")
+    );
 
     private JdkBuild getBundledJdkBuild() {
         String bundledJdkVersion = VersionProperties.getBundledJdkVersion();

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy

@@ -11,6 +11,7 @@ package org.elasticsearch.gradle.internal.toolchain
 import org.gradle.api.services.BuildServiceParameters
 import org.gradle.jvm.toolchain.JavaLanguageVersion
 import org.gradle.jvm.toolchain.JavaToolchainResolver
+import org.gradle.platform.OperatingSystem
 
 import static org.elasticsearch.gradle.internal.toolchain.AbstractCustomJavaToolchainResolver.toArchString
 import static org.elasticsearch.gradle.internal.toolchain.AbstractCustomJavaToolchainResolver.toOsString
@@ -37,7 +38,12 @@ class AdoptiumJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
                     toOsString(it[2], it[1]),
                     toArchString(it[3]),
                     languageVersion);
-            resolver.CACHED_RELEASES.put(request, Optional.of('jdk-' + languageVersion.asInt() + '.1.1.1+37.1'))
+            resolver.CACHED_SEMVERS.put(request, Optional.of(new AdoptiumJdkToolchainResolver.AdoptiumVersionInfo(languageVersion.asInt(),
+                    1,
+                    1,
+                    "" + languageVersion.asInt() + ".1.1.1+37",
+                    0, "" + languageVersion.asInt() + ".1.1.1+37.1"
+            )))
 
         }
         return resolver

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 8.14.0
 lucene            = 9.11.0-snapshot-b9844481e3a
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 22+36@830ec9fcccef480bb3e73fb7ecafe059
+bundled_jdk = 21.0.2+13@f2283984656d49d69e91c558476027ac
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java

@@ -457,6 +457,26 @@ public void testLegacyV3() throws GeneralSecurityException, IOException {
         assertThat(toByteArray(wrapper.getFile("file_setting")), equalTo("file_value".getBytes(StandardCharsets.UTF_8)));
     }
 
+    public void testLegacyV5() throws GeneralSecurityException, IOException {
+        final Path configDir = createTempDir();
+        final Path keystore = configDir.resolve("elasticsearch.keystore");
+        try (
+            InputStream is = KeyStoreWrapperTests.class.getResourceAsStream("/format-v5-with-password-elasticsearch.keystore");
+            OutputStream os = Files.newOutputStream(keystore)
+        ) {
+            final byte[] buffer = new byte[4096];
+            int readBytes;
+            while ((readBytes = is.read(buffer)) > 0) {
+                os.write(buffer, 0, readBytes);
+            }
+        }
+        final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir);
+        assertNotNull(wrapper);
+        wrapper.decrypt("keystorepassword".toCharArray());
+        assertThat(wrapper.getFormatVersion(), equalTo(5));
+        assertThat(wrapper.getSettingNames(), equalTo(Set.of("keystore.seed")));
+    }
+
     public void testSerializationNewlyCreated() throws Exception {
         final KeyStoreWrapper wrapper = KeyStoreWrapper.create();
         wrapper.setString("string_setting", "string_value".toCharArray());