Merge branch '8.18' into deprication/add-deprication-warning-to-hands…

…hake

.buildkite/pipelines/periodic-fwc.template.yml

@@ -0,0 +1,15 @@
+steps:
+  - label: $FWC_VERSION / fwc
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
+      preemptible: true
+    matrix:
+      setup:
+        FWC_VERSION: $FWC_LIST
+    env:
+      FWC_VERSION: $FWC_VERSION

.buildkite/pipelines/periodic-fwc.yml

@@ -0,0 +1,16 @@
+# This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml
+steps:
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
+      preemptible: true
+    matrix:
+      setup:
+        FWC_VERSION: []
+    env:
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/scripts/branches.sh

@@ -2,3 +2,7 @@
 
 # This determines which branches will have pipelines triggered periodically, for dra workflows.
 BRANCHES=( $(cat branches.json | jq -r '.branches[].branch') )
+
+# Sort them to make ordering predictable
+IFS=$'\n' BRANCHES=($(sort <<<"${BRANCHES[*]}"))
+unset IFS

.buildkite/scripts/periodic.trigger.sh

@@ -46,4 +46,15 @@ EOF
       branch: "$BRANCH"
       commit: "$LAST_GOOD_COMMIT"
 EOF
+# Include forward compatibility tests only for the bugfix branch
+if [[ "${BRANCH}" == "${BRANCHES[2]}" ]]; then
+  cat <<EOF
+  - trigger: elasticsearch-periodic-fwc
+    label: Trigger periodic-fwc pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
+EOF
+fi
 done

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -24,7 +24,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:d74b1fda6b7fee2c90b410df258e005c049e0672fe16d79d00e58f14fb69f90b",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:6387bd4c462007eaecaf13a423aea99c8a8452da09244c129703324aa97769c6",
         "-wolfi",
         "apk"
     ),

build.gradle

@@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.JsonNode
 import com.fasterxml.jackson.databind.ObjectMapper
 
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin
 import org.elasticsearch.gradle.internal.ResolveAllDependencies
 import org.elasticsearch.gradle.util.GradleUtils
@@ -118,10 +119,10 @@ tasks.register("updateCIBwcVersions") {
     outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipeline
   }
 
-  // Writes a Buildkite pipelime from a template, and replaces $BWC_LIST with an array of versions
+  // Writes a Buildkite pipelime from a template, and replaces a variable with an array of versions
   // Useful for writing a list of versions in a matrix configuration
-  def expandBwcList = { String outputFilePath, String pipelineTemplatePath, List<Version> versions ->
-    writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: "BWC_LIST")])
+  def expandList = { String outputFilePath, String pipelineTemplatePath, String variable, List<Version> versions ->
+    writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: variable)])
   }
 
   // Writes a Buildkite pipeline from a template, and replaces $BWC_STEPS with a list of steps, one for each version
@@ -133,11 +134,18 @@ tasks.register("updateCIBwcVersions") {
   doLast {
     writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible))
     writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible))
-    expandBwcList(
+    expandList(
       ".buildkite/pipelines/intake.yml",
       ".buildkite/pipelines/intake.template.yml",
+      "BWC_LIST",
       filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible)
     )
+    expandList(
+      ".buildkite/pipelines/periodic-fwc.yml",
+      ".buildkite/pipelines/periodic-fwc.template.yml",
+      "FWC_LIST",
+      buildParams.bwcVersions.released.findAll { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor }
+    )
     writeBuildkitePipeline(
       ".buildkite/pipelines/periodic.yml",
       ".buildkite/pipelines/periodic.template.yml",

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement;
 
 import java.lang.instrument.Instrumentation;
 import java.lang.reflect.Constructor;
@@ -242,7 +243,14 @@ private static PolicyManager createPolicyManager() {
         if (trustStorePath != null) {
             Collections.addAll(
                 serverScopes,
-                new Scope("org.bouncycastle.fips.tls", List.of(new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))))),
+                new Scope(
+                    "org.bouncycastle.fips.tls",
+                    List.of(
+                        new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))),
+                        new OutboundNetworkEntitlement(),
+                        new ManageThreadsEntitlement()
+                    )
+                ),
                 new Scope(
                     "org.bouncycastle.fips.core",
                     // read to lib dir is required for checksum validation
@@ -258,6 +266,8 @@ private static PolicyManager createPolicyManager() {
         List<Entitlement> agentEntitlements = List.of(
             new CreateClassLoaderEntitlement(),
             new ManageThreadsEntitlement(),
+            new SetHttpsConnectionPropertiesEntitlement(),
+            new OutboundNetworkEntitlement(),
             new FilesEntitlement(
                 List.of(
                     FileData.ofPath(Path.of("/co/elastic/apm/agent/"), READ),

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -30,13 +30,74 @@
 
 public final class FileAccessTree {
 
+    /**
+     * An intermediary structure to help build exclusive paths for files entitlements.
+     */
+    record ExclusiveFileEntitlement(String componentName, String moduleName, FilesEntitlement filesEntitlement) {}
+
+    /**
+     * An intermediary structure to help globally validate exclusive paths, and then build exclusive paths for individual modules.
+     */
+    record ExclusivePath(String componentName, String moduleName, String path) {
+
+        @Override
+        public String toString() {
+            return "[[" + componentName + "] [" + moduleName + "] [" + path + "]]";
+        }
+    }
+
+    static List<ExclusivePath> buildExclusivePathList(List<ExclusiveFileEntitlement> exclusiveFileEntitlements, PathLookup pathLookup) {
+        List<ExclusivePath> exclusivePaths = new ArrayList<>();
+        for (ExclusiveFileEntitlement efe : exclusiveFileEntitlements) {
+            for (FilesEntitlement.FileData fd : efe.filesEntitlement().filesData()) {
+                if (fd.exclusive()) {
+                    List<Path> paths = fd.resolvePaths(pathLookup).toList();
+                    for (Path path : paths) {
+                        exclusivePaths.add(new ExclusivePath(efe.componentName(), efe.moduleName(), normalizePath(path)));
+                    }
+                }
+            }
+        }
+        exclusivePaths.sort((ep1, ep2) -> PATH_ORDER.compare(ep1.path(), ep2.path()));
+        return exclusivePaths;
+    }
+
+    static void validateExclusivePaths(List<ExclusivePath> exclusivePaths) {
+        if (exclusivePaths.isEmpty() == false) {
+            ExclusivePath currentExclusivePath = exclusivePaths.get(0);
+            for (int i = 1; i < exclusivePaths.size(); ++i) {
+                ExclusivePath nextPath = exclusivePaths.get(i);
+                if (currentExclusivePath.path().equals(nextPath.path) || isParent(currentExclusivePath.path(), nextPath.path())) {
+                    throw new IllegalArgumentException(
+                        "duplicate/overlapping exclusive paths found in files entitlements: " + currentExclusivePath + " and " + nextPath
+                    );
+                }
+                currentExclusivePath = nextPath;
+            }
+        }
+    }
+
     private static final Logger logger = LogManager.getLogger(FileAccessTree.class);
     private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator();
 
+    private final String[] exclusivePaths;
     private final String[] readPaths;
     private final String[] writePaths;
 
-    private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup) {
+    private FileAccessTree(
+        String componentName,
+        String moduleName,
+        FilesEntitlement filesEntitlement,
+        PathLookup pathLookup,
+        List<ExclusivePath> exclusivePaths
+    ) {
+        List<String> updatedExclusivePaths = new ArrayList<>();
+        for (ExclusivePath exclusivePath : exclusivePaths) {
+            if (exclusivePath.componentName().equals(componentName) == false || exclusivePath.moduleName().equals(moduleName) == false) {
+                updatedExclusivePaths.add(exclusivePath.path());
+            }
+        }
+
         List<String> readPaths = new ArrayList<>();
         List<String> writePaths = new ArrayList<>();
         BiConsumer<Path, Mode> addPath = (path, mode) -> {
@@ -83,9 +144,11 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         Path jdk = Paths.get(System.getProperty("java.home"));
         addPathAndMaybeLink.accept(jdk.resolve("conf"), Mode.READ);
 
+        updatedExclusivePaths.sort(PATH_ORDER);
         readPaths.sort(PATH_ORDER);
         writePaths.sort(PATH_ORDER);
 
+        this.exclusivePaths = updatedExclusivePaths.toArray(new String[0]);
         this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]);
         this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]);
     }
@@ -106,8 +169,14 @@ private static List<String> pruneSortedPaths(List<String> paths) {
         return prunedReadPaths;
     }
 
-    public static FileAccessTree of(FilesEntitlement filesEntitlement, PathLookup pathLookup) {
-        return new FileAccessTree(filesEntitlement, pathLookup);
+    public static FileAccessTree of(
+        String componentName,
+        String moduleName,
+        FilesEntitlement filesEntitlement,
+        PathLookup pathLookup,
+        List<ExclusivePath> exclusivePaths
+    ) {
+        return new FileAccessTree(componentName, moduleName, filesEntitlement, pathLookup, exclusivePaths);
     }
 
     boolean canRead(Path path) {
@@ -132,10 +201,16 @@ static String normalizePath(Path path) {
         return result;
     }
 
-    private static boolean checkPath(String path, String[] paths) {
+    private boolean checkPath(String path, String[] paths) {
         if (paths.length == 0) {
             return false;
         }
+
+        int endx = Arrays.binarySearch(exclusivePaths, path, PATH_ORDER);
+        if (endx < -1 && isParent(exclusivePaths[-endx - 2], path) || endx >= 0) {
+            return false;
+        }
+
         int ndx = Arrays.binarySearch(paths, path, PATH_ORDER);
         if (ndx < -1) {
             return isParent(paths[-ndx - 2], path);

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -13,6 +13,8 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.instrumentation.InstrumentationService;
 import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
+import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusiveFileEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusivePath;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement;
@@ -32,6 +34,7 @@
 import java.lang.module.ModuleFinder;
 import java.lang.module.ModuleReference;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -91,7 +94,7 @@ ModuleEntitlements defaultEntitlements(String componentName) {
     }
 
     // pkg private for testing
-    ModuleEntitlements policyEntitlements(String componentName, List<Entitlement> entitlements) {
+    ModuleEntitlements policyEntitlements(String componentName, String moduleName, List<Entitlement> entitlements) {
         FilesEntitlement filesEntitlement = FilesEntitlement.EMPTY;
         for (Entitlement entitlement : entitlements) {
             if (entitlement instanceof FilesEntitlement) {
@@ -101,7 +104,7 @@ ModuleEntitlements policyEntitlements(String componentName, List<Entitlement> en
         return new ModuleEntitlements(
             componentName,
             entitlements.stream().collect(groupingBy(Entitlement::getClass)),
-            FileAccessTree.of(filesEntitlement, pathLookup)
+            FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, exclusivePaths)
         );
     }
 
@@ -143,6 +146,13 @@ private static Set<Module> findSystemModules() {
      */
     private final Module entitlementsModule;
 
+    /**
+     * Paths that are only allowed for a single module. Used to generate
+     * structures to indicate other modules aren't allowed to use these
+     * files in {@link FileAccessTree}s.
+     */
+    private final List<ExclusivePath> exclusivePaths;
+
     public PolicyManager(
         Policy serverPolicy,
         List<Entitlement> apmAgentEntitlements,
@@ -162,25 +172,40 @@ public PolicyManager(
         this.apmAgentPackageName = apmAgentPackageName;
         this.entitlementsModule = entitlementsModule;
         this.pathLookup = requireNonNull(pathLookup);
-        this.defaultFileAccess = FileAccessTree.of(FilesEntitlement.EMPTY, pathLookup);
+        this.defaultFileAccess = FileAccessTree.of(
+            UNKNOWN_COMPONENT_NAME,
+            UNKNOWN_COMPONENT_NAME,
+            FilesEntitlement.EMPTY,
+            pathLookup,
+            List.of()
+        );
         this.mutedClasses = suppressFailureLogClasses;
 
+        List<ExclusiveFileEntitlement> exclusiveFileEntitlements = new ArrayList<>();
         for (var e : serverEntitlements.entrySet()) {
-            validateEntitlementsPerModule(SERVER_COMPONENT_NAME, e.getKey(), e.getValue());
+            validateEntitlementsPerModule(SERVER_COMPONENT_NAME, e.getKey(), e.getValue(), exclusiveFileEntitlements);
         }
-        validateEntitlementsPerModule(APM_AGENT_COMPONENT_NAME, "unnamed", apmAgentEntitlements);
+        validateEntitlementsPerModule(APM_AGENT_COMPONENT_NAME, ALL_UNNAMED, apmAgentEntitlements, exclusiveFileEntitlements);
         for (var p : pluginsEntitlements.entrySet()) {
             for (var m : p.getValue().entrySet()) {
-                validateEntitlementsPerModule(p.getKey(), m.getKey(), m.getValue());
+                validateEntitlementsPerModule(p.getKey(), m.getKey(), m.getValue(), exclusiveFileEntitlements);
             }
         }
+        List<ExclusivePath> exclusivePaths = FileAccessTree.buildExclusivePathList(exclusiveFileEntitlements, pathLookup);
+        FileAccessTree.validateExclusivePaths(exclusivePaths);
+        this.exclusivePaths = exclusivePaths;
     }
 
     private static Map<String, List<Entitlement>> buildScopeEntitlementsMap(Policy policy) {
         return policy.scopes().stream().collect(toUnmodifiableMap(Scope::moduleName, Scope::entitlements));
     }
 
-    private static void validateEntitlementsPerModule(String componentName, String moduleName, List<Entitlement> entitlements) {
+    private static void validateEntitlementsPerModule(
+        String componentName,
+        String moduleName,
+        List<Entitlement> entitlements,
+        List<ExclusiveFileEntitlement> exclusiveFileEntitlements
+    ) {
         Set<Class<? extends Entitlement>> found = new HashSet<>();
         for (var e : entitlements) {
             if (found.contains(e.getClass())) {
@@ -189,6 +214,9 @@ private static void validateEntitlementsPerModule(String componentName, String m
                 );
             }
             found.add(e.getClass());
+            if (e instanceof FilesEntitlement fe) {
+                exclusiveFileEntitlements.add(new ExclusiveFileEntitlement(componentName, moduleName, fe));
+            }
         }
     }
 
@@ -498,7 +526,7 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
 
         if (requestingModule.isNamed() == false && requestingClass.getPackageName().startsWith(apmAgentPackageName)) {
             // The APM agent is the only thing running non-modular in the system classloader
-            return policyEntitlements(APM_AGENT_COMPONENT_NAME, apmAgentEntitlements);
+            return policyEntitlements(APM_AGENT_COMPONENT_NAME, ALL_UNNAMED, apmAgentEntitlements);
         }
 
         return defaultEntitlements(UNKNOWN_COMPONENT_NAME);
@@ -513,7 +541,7 @@ private ModuleEntitlements getModuleScopeEntitlements(
         if (entitlements == null) {
             return defaultEntitlements(componentName);
         }
-        return policyEntitlements(componentName, entitlements);
+        return policyEntitlements(componentName, moduleName, entitlements);
     }
 
     private static boolean isServerModule(Module requestingModule) {