From 100b931543bca41c9e984932a18fe064106f6a7b Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 26 Feb 2025 13:52:17 -0800 Subject: [PATCH 01/11] Add forwards compatibility testing CI pipeline --- .buildkite/pipelines/periodic-fwc.template.yml | 15 +++++++++++++++ .buildkite/pipelines/periodic-fwc.yml | 16 ++++++++++++++++ .buildkite/scripts/branches.sh | 4 ++++ .buildkite/scripts/periodic.trigger.sh | 11 +++++++++++ build.gradle | 16 ++++++++++++---- 5 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 .buildkite/pipelines/periodic-fwc.template.yml create mode 100644 .buildkite/pipelines/periodic-fwc.yml diff --git a/.buildkite/pipelines/periodic-fwc.template.yml b/.buildkite/pipelines/periodic-fwc.template.yml new file mode 100644 index 0000000000000..b9f00a649a14b --- /dev/null +++ b/.buildkite/pipelines/periodic-fwc.template.yml @@ -0,0 +1,15 @@ +steps: + - label: $FWC_VERSION / fwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + preemptible: true + matrix: + setup: + FWC_VERSION: $FWC_LIST + env: + FWC_VERSION: $FWC_VERSION diff --git a/.buildkite/pipelines/periodic-fwc.yml b/.buildkite/pipelines/periodic-fwc.yml new file mode 100644 index 0000000000000..e94f3600d56cd --- /dev/null +++ b/.buildkite/pipelines/periodic-fwc.yml @@ -0,0 +1,16 @@ +# This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml +steps: + - label: $FWC_VERSION / fwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + preemptible: true + matrix: + setup: + FWC_VERSION: [] + env: + FWC_VERSION: $FWC_VERSION diff --git a/.buildkite/scripts/branches.sh b/.buildkite/scripts/branches.sh index 886fa59e4d02c..8e909b362ef2e 100755 --- a/.buildkite/scripts/branches.sh +++ b/.buildkite/scripts/branches.sh @@ -2,3 +2,7 @@ # This determines which branches will have pipelines triggered periodically, for dra workflows. BRANCHES=( $(cat branches.json | jq -r '.branches[].branch') ) + +# Sort them to make ordering predictable +IFS=$'\n' BRANCHES=($(sort <<<"${BRANCHES[*]}")) +unset IFS diff --git a/.buildkite/scripts/periodic.trigger.sh b/.buildkite/scripts/periodic.trigger.sh index cc10a5ae41861..30e13386f3088 100755 --- a/.buildkite/scripts/periodic.trigger.sh +++ b/.buildkite/scripts/periodic.trigger.sh @@ -46,4 +46,15 @@ EOF branch: "$BRANCH" commit: "$LAST_GOOD_COMMIT" EOF +# Include forward compatibility tests only for the bugfix branch +if [[ "${BRANCH}" == "${BRANCHES[2]}" ]]; then + cat < versions -> - writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: "BWC_LIST")]) + def expandList = { String outputFilePath, String pipelineTemplatePath, String variable, List versions -> + writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: variable)]) } // Writes a Buildkite pipeline from a template, and replaces $BWC_STEPS with a list of steps, one for each version @@ -133,11 +134,18 @@ tasks.register("updateCIBwcVersions") { doLast { writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible)) writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible)) - expandBwcList( + expandList( ".buildkite/pipelines/intake.yml", ".buildkite/pipelines/intake.template.yml", + "BWC_LIST", filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible) ) + expandList( + ".buildkite/pipelines/periodic-fwc.yml", + ".buildkite/pipelines/periodic-fwc.template.yml", + "FWC_LIST", + buildParams.bwcVersions.released.findAll { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor } + ) writeBuildkitePipeline( ".buildkite/pipelines/periodic.yml", ".buildkite/pipelines/periodic.template.yml", From b9aea477ee9f4952b9ae55056539f23ef88a7985 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 26 Feb 2025 13:53:44 -0800 Subject: [PATCH 02/11] Add ignore_url for setting based files entitlement (#123541) (#123545) File entitlements which read a setting may actually contain urls. This commit adds an optional `ignore_url` property for the entitlement to skip any values which are urls. --- .../policy/entitlements/FilesEntitlement.java | 49 +++++++++++++------ .../entitlements/FilesEntitlementTests.java | 39 +++++++++++++-- 2 files changed, 67 insertions(+), 21 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java index 609e0ed66cfa0..c47b0b93c5471 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java @@ -9,6 +9,7 @@ package org.elasticsearch.entitlement.runtime.policy.entitlements; +import org.elasticsearch.core.Booleans; import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; import org.elasticsearch.entitlement.runtime.policy.PathLookup; import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException; @@ -17,6 +18,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.stream.Stream; @@ -85,12 +87,12 @@ static FileData ofRelativePath(Path relativePath, BaseDir baseDir, Mode mode) { return new RelativePathFileData(relativePath, baseDir, mode, null); } - static FileData ofPathSetting(String setting, Mode mode) { - return new PathSettingFileData(setting, mode, null); + static FileData ofPathSetting(String setting, Mode mode, boolean ignoreUrl) { + return new PathSettingFileData(setting, mode, ignoreUrl, null); } - static FileData ofRelativePathSetting(String setting, BaseDir baseDir, Mode mode) { - return new RelativePathSettingFileData(setting, baseDir, mode, null); + static FileData ofRelativePathSetting(String setting, BaseDir baseDir, Mode mode, boolean ignoreUrl) { + return new RelativePathSettingFileData(setting, baseDir, mode, ignoreUrl, null); } /** @@ -207,10 +209,10 @@ public FileData withPlatform(Platform platform) { } } - private record PathSettingFileData(String setting, Mode mode, Platform platform) implements FileData { + private record PathSettingFileData(String setting, Mode mode, boolean ignoreUrl, Platform platform) implements FileData { @Override public Stream resolvePaths(PathLookup pathLookup) { - return resolvePathSettings(pathLookup, setting); + return resolvePathSettings(pathLookup, setting, ignoreUrl); } @Override @@ -218,17 +220,17 @@ public FileData withPlatform(Platform platform) { if (platform == platform()) { return this; } - return new PathSettingFileData(setting, mode, platform); + return new PathSettingFileData(setting, mode, ignoreUrl, platform); } } - private record RelativePathSettingFileData(String setting, BaseDir baseDir, Mode mode, Platform platform) + private record RelativePathSettingFileData(String setting, BaseDir baseDir, Mode mode, boolean ignoreUrl, Platform platform) implements FileData, RelativeFileData { @Override public Stream resolveRelativePaths(PathLookup pathLookup) { - return resolvePathSettings(pathLookup, setting); + return resolvePathSettings(pathLookup, setting, ignoreUrl); } @Override @@ -236,16 +238,22 @@ public FileData withPlatform(Platform platform) { if (platform == platform()) { return this; } - return new RelativePathSettingFileData(setting, baseDir, mode, platform); + return new RelativePathSettingFileData(setting, baseDir, mode, ignoreUrl, platform); } } - private static Stream resolvePathSettings(PathLookup pathLookup, String setting) { + private static Stream resolvePathSettings(PathLookup pathLookup, String setting, boolean ignoreUrl) { + Stream result; if (setting.contains("*")) { - return pathLookup.settingGlobResolver().apply(setting).map(Path::of); + result = pathLookup.settingGlobResolver().apply(setting); + } else { + String path = pathLookup.settingResolver().apply(setting); + result = path == null ? Stream.of() : Stream.of(path); + } + if (ignoreUrl) { + result = result.filter(s -> s.toLowerCase(Locale.ROOT).startsWith("https://") == false); } - String path = pathLookup.settingResolver().apply(setting); - return path == null ? Stream.of() : Stream.of(Path.of(path)); + return result.map(Path::of); } private static Mode parseMode(String mode) { @@ -298,6 +306,7 @@ public static FilesEntitlement build(List paths) { String relativePathSetting = file.remove("relative_path_setting"); String modeAsString = file.remove("mode"); String platformAsString = file.remove("platform"); + String ignoreUrlAsString = file.remove("ignore_url"); if (file.isEmpty() == false) { throw new PolicyValidationException("unknown key(s) [" + file + "] in a listed file for files entitlement"); @@ -324,6 +333,14 @@ public static FilesEntitlement build(List paths) { baseDir = parseBaseDir(relativeTo); } + boolean ignoreUrl = false; + if (ignoreUrlAsString != null) { + if (relativePathAsString != null || pathAsString != null) { + throw new PolicyValidationException("'ignore_url' may only be used with `path_setting` or `relative_path_setting`"); + } + ignoreUrl = Booleans.parseBoolean(ignoreUrlAsString); + } + final FileData fileData; if (relativePathAsString != null) { if (baseDir == null) { @@ -342,12 +359,12 @@ public static FilesEntitlement build(List paths) { } fileData = FileData.ofPath(path, mode); } else if (pathSetting != null) { - fileData = FileData.ofPathSetting(pathSetting, mode); + fileData = FileData.ofPathSetting(pathSetting, mode, ignoreUrl); } else if (relativePathSetting != null) { if (baseDir == null) { throw new PolicyValidationException("files entitlement with a 'relative_path_setting' must specify 'relative_to'"); } - fileData = FileData.ofRelativePathSetting(relativePathSetting, baseDir, mode); + fileData = FileData.ofRelativePathSetting(relativePathSetting, baseDir, mode, ignoreUrl); } else { throw new AssertionError("File entry validation error"); } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java index e0c08d18b8c15..a453d6cf54992 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.CONFIG; import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ; import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE; import static org.hamcrest.Matchers.contains; @@ -94,22 +95,50 @@ public void testFileDataRelativeWithEmptyDirectory() { public void testPathSettingResolve() { var entitlement = FilesEntitlement.build(List.of(Map.of("path_setting", "foo.bar", "mode", "read"))); var filesData = entitlement.filesData(); - assertThat(filesData, contains(FileData.ofPathSetting("foo.bar", READ))); + assertThat(filesData, contains(FileData.ofPathSetting("foo.bar", READ, false))); - var fileData = FileData.ofPathSetting("foo.bar", READ); + var fileData = FileData.ofPathSetting("foo.bar", READ, false); // empty settings assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), empty()); - fileData = FileData.ofPathSetting("foo.bar", READ); + fileData = FileData.ofPathSetting("foo.bar", READ, false); settings = Settings.builder().put("foo.bar", "/setting/path").build(); assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/setting/path"))); - fileData = FileData.ofPathSetting("foo.*.bar", READ); + fileData = FileData.ofPathSetting("foo.*.bar", READ, false); settings = Settings.builder().put("foo.baz.bar", "/setting/path").build(); assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/setting/path"))); - fileData = FileData.ofPathSetting("foo.*.bar", READ); + fileData = FileData.ofPathSetting("foo.*.bar", READ, false); settings = Settings.builder().put("foo.baz.bar", "/setting/path").put("foo.baz2.bar", "/other/path").build(); assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), containsInAnyOrder(Path.of("/setting/path"), Path.of("/other/path"))); } + + public void testPathSettingIgnoreUrl() { + var fileData = FileData.ofPathSetting("foo.*.bar", READ, true); + settings = Settings.builder().put("foo.nonurl.bar", "/setting/path").put("foo.url.bar", "https://mysite").build(); + assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/setting/path"))); + } + + public void testRelativePathSettingIgnoreUrl() { + var fileData = FileData.ofRelativePathSetting("foo.*.bar", CONFIG, READ, true); + settings = Settings.builder().put("foo.nonurl.bar", "path").put("foo.url.bar", "https://mysite").build(); + assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/config/path"))); + } + + public void testIgnoreUrlValidation() { + var e = expectThrows( + PolicyValidationException.class, + () -> FilesEntitlement.build(List.of(Map.of("path", "/foo", "mode", "read", "ignore_url", "true"))) + ); + assertThat(e.getMessage(), is("'ignore_url' may only be used with `path_setting` or `relative_path_setting`")); + + e = expectThrows( + PolicyValidationException.class, + () -> FilesEntitlement.build( + List.of(Map.of("relative_path", "foo", "relative_to", "config", "mode", "read", "ignore_url", "true")) + ) + ); + assertThat(e.getMessage(), is("'ignore_url' may only be used with `path_setting` or `relative_path_setting`")); + } } From fa69e79d9e7fb265204ff1b6c1a7f7c1ea755054 Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:07:29 -0800 Subject: [PATCH 03/11] Add fips attribute to .fleet-agents docs (#123406) (#123550) Co-authored-by: Julien Lind --- .../template-resources/src/main/resources/fleet-agents.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index 8b1c13f3152e8..ac2f88746eef2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -102,6 +102,9 @@ "unprivileged": { "type": "boolean" }, + "fips": { + "type": "boolean" + }, "version": { "type": "text", "fields": { From 810a969d5dbe3439f0b2d27fd6652f3b0e27f0f8 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 26 Feb 2025 14:13:13 -0800 Subject: [PATCH 04/11] Quote matrix variables --- .buildkite/pipelines/periodic-fwc.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipelines/periodic-fwc.yml b/.buildkite/pipelines/periodic-fwc.yml index e94f3600d56cd..434a091aa1dfe 100644 --- a/.buildkite/pipelines/periodic-fwc.yml +++ b/.buildkite/pipelines/periodic-fwc.yml @@ -1,7 +1,7 @@ # This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml steps: - - label: $FWC_VERSION / fwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false + - label: "{{matrix.FWC_VERSION}} / fwc" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false timeout_in_minutes: 300 agents: provider: gcp @@ -13,4 +13,4 @@ steps: setup: FWC_VERSION: [] env: - FWC_VERSION: $FWC_VERSION + FWC_VERSION: "{{matrix.FWC_VERSION}}" From c5a35a735552ff19a96d9e3a53bbd147202c63a4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 27 Feb 2025 09:33:38 +1100 Subject: [PATCH 05/11] Mute org.elasticsearch.smoketest.MlWithSecurityIT test {yaml=ml/3rd_party_deployment/Test start deployment fails while model download in progress} #120814 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index fab4577574bd6..2e19403767f44 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -453,3 +453,6 @@ tests: - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT method: test {yaml=analysis-common/40_token_filters/stemmer_override file access} issue: https://github.com/elastic/elasticsearch/issues/121625 +- class: org.elasticsearch.smoketest.MlWithSecurityIT + method: test {yaml=ml/3rd_party_deployment/Test start deployment fails while model download in progress} + issue: https://github.com/elastic/elasticsearch/issues/120814 From a0ef3cb4334fbca233019a6c684a0a2604aa420f Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 26 Feb 2025 18:44:27 -0800 Subject: [PATCH 06/11] Add an exclusive parameter for files entitlements (#123087) (#123562) This adds an exclusive parameter for FilesEntitlement where a path can be made exclusive for a certain module. Should two modules attempt to both specify the same path as exclusive an exception is thrown. --- .../runtime/policy/FileAccessTree.java | 83 ++++++++++- .../runtime/policy/PolicyManager.java | 46 ++++-- .../policy/entitlements/FilesEntitlement.java | 137 +++++++++++++----- .../runtime/policy/FileAccessTreeTests.java | 101 ++++++++++--- .../runtime/policy/PolicyManagerTests.java | 111 +++++++++++++- .../entitlements/FilesEntitlementTests.java | 29 +++- 6 files changed, 436 insertions(+), 71 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java index 336a00643e979..d46a1aeb7eade 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java @@ -30,13 +30,74 @@ public final class FileAccessTree { + /** + * An intermediary structure to help build exclusive paths for files entitlements. + */ + record ExclusiveFileEntitlement(String componentName, String moduleName, FilesEntitlement filesEntitlement) {} + + /** + * An intermediary structure to help globally validate exclusive paths, and then build exclusive paths for individual modules. + */ + record ExclusivePath(String componentName, String moduleName, String path) { + + @Override + public String toString() { + return "[[" + componentName + "] [" + moduleName + "] [" + path + "]]"; + } + } + + static List buildExclusivePathList(List exclusiveFileEntitlements, PathLookup pathLookup) { + List exclusivePaths = new ArrayList<>(); + for (ExclusiveFileEntitlement efe : exclusiveFileEntitlements) { + for (FilesEntitlement.FileData fd : efe.filesEntitlement().filesData()) { + if (fd.exclusive()) { + List paths = fd.resolvePaths(pathLookup).toList(); + for (Path path : paths) { + exclusivePaths.add(new ExclusivePath(efe.componentName(), efe.moduleName(), normalizePath(path))); + } + } + } + } + exclusivePaths.sort((ep1, ep2) -> PATH_ORDER.compare(ep1.path(), ep2.path())); + return exclusivePaths; + } + + static void validateExclusivePaths(List exclusivePaths) { + if (exclusivePaths.isEmpty() == false) { + ExclusivePath currentExclusivePath = exclusivePaths.get(0); + for (int i = 1; i < exclusivePaths.size(); ++i) { + ExclusivePath nextPath = exclusivePaths.get(i); + if (currentExclusivePath.path().equals(nextPath.path) || isParent(currentExclusivePath.path(), nextPath.path())) { + throw new IllegalArgumentException( + "duplicate/overlapping exclusive paths found in files entitlements: " + currentExclusivePath + " and " + nextPath + ); + } + currentExclusivePath = nextPath; + } + } + } + private static final Logger logger = LogManager.getLogger(FileAccessTree.class); private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator(); + private final String[] exclusivePaths; private final String[] readPaths; private final String[] writePaths; - private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup) { + private FileAccessTree( + String componentName, + String moduleName, + FilesEntitlement filesEntitlement, + PathLookup pathLookup, + List exclusivePaths + ) { + List updatedExclusivePaths = new ArrayList<>(); + for (ExclusivePath exclusivePath : exclusivePaths) { + if (exclusivePath.componentName().equals(componentName) == false || exclusivePath.moduleName().equals(moduleName) == false) { + updatedExclusivePaths.add(exclusivePath.path()); + } + } + List readPaths = new ArrayList<>(); List writePaths = new ArrayList<>(); BiConsumer addPath = (path, mode) -> { @@ -83,9 +144,11 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup) Path jdk = Paths.get(System.getProperty("java.home")); addPathAndMaybeLink.accept(jdk.resolve("conf"), Mode.READ); + updatedExclusivePaths.sort(PATH_ORDER); readPaths.sort(PATH_ORDER); writePaths.sort(PATH_ORDER); + this.exclusivePaths = updatedExclusivePaths.toArray(new String[0]); this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]); this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]); } @@ -106,8 +169,14 @@ private static List pruneSortedPaths(List paths) { return prunedReadPaths; } - public static FileAccessTree of(FilesEntitlement filesEntitlement, PathLookup pathLookup) { - return new FileAccessTree(filesEntitlement, pathLookup); + public static FileAccessTree of( + String componentName, + String moduleName, + FilesEntitlement filesEntitlement, + PathLookup pathLookup, + List exclusivePaths + ) { + return new FileAccessTree(componentName, moduleName, filesEntitlement, pathLookup, exclusivePaths); } boolean canRead(Path path) { @@ -132,10 +201,16 @@ static String normalizePath(Path path) { return result; } - private static boolean checkPath(String path, String[] paths) { + private boolean checkPath(String path, String[] paths) { if (paths.length == 0) { return false; } + + int endx = Arrays.binarySearch(exclusivePaths, path, PATH_ORDER); + if (endx < -1 && isParent(exclusivePaths[-endx - 2], path) || endx >= 0) { + return false; + } + int ndx = Arrays.binarySearch(paths, path, PATH_ORDER); if (ndx < -1) { return isParent(paths[-ndx - 2], path); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index cf3775474b79a..ddceb9f9ff1f0 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -13,6 +13,8 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.instrumentation.InstrumentationService; import org.elasticsearch.entitlement.runtime.api.NotEntitledException; +import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusiveFileEntitlement; +import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusivePath; import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement; @@ -32,6 +34,7 @@ import java.lang.module.ModuleFinder; import java.lang.module.ModuleReference; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -91,7 +94,7 @@ ModuleEntitlements defaultEntitlements(String componentName) { } // pkg private for testing - ModuleEntitlements policyEntitlements(String componentName, List entitlements) { + ModuleEntitlements policyEntitlements(String componentName, String moduleName, List entitlements) { FilesEntitlement filesEntitlement = FilesEntitlement.EMPTY; for (Entitlement entitlement : entitlements) { if (entitlement instanceof FilesEntitlement) { @@ -101,7 +104,7 @@ ModuleEntitlements policyEntitlements(String componentName, List en return new ModuleEntitlements( componentName, entitlements.stream().collect(groupingBy(Entitlement::getClass)), - FileAccessTree.of(filesEntitlement, pathLookup) + FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, exclusivePaths) ); } @@ -143,6 +146,13 @@ private static Set findSystemModules() { */ private final Module entitlementsModule; + /** + * Paths that are only allowed for a single module. Used to generate + * structures to indicate other modules aren't allowed to use these + * files in {@link FileAccessTree}s. + */ + private final List exclusivePaths; + public PolicyManager( Policy serverPolicy, List apmAgentEntitlements, @@ -162,25 +172,40 @@ public PolicyManager( this.apmAgentPackageName = apmAgentPackageName; this.entitlementsModule = entitlementsModule; this.pathLookup = requireNonNull(pathLookup); - this.defaultFileAccess = FileAccessTree.of(FilesEntitlement.EMPTY, pathLookup); + this.defaultFileAccess = FileAccessTree.of( + UNKNOWN_COMPONENT_NAME, + UNKNOWN_COMPONENT_NAME, + FilesEntitlement.EMPTY, + pathLookup, + List.of() + ); this.mutedClasses = suppressFailureLogClasses; + List exclusiveFileEntitlements = new ArrayList<>(); for (var e : serverEntitlements.entrySet()) { - validateEntitlementsPerModule(SERVER_COMPONENT_NAME, e.getKey(), e.getValue()); + validateEntitlementsPerModule(SERVER_COMPONENT_NAME, e.getKey(), e.getValue(), exclusiveFileEntitlements); } - validateEntitlementsPerModule(APM_AGENT_COMPONENT_NAME, "unnamed", apmAgentEntitlements); + validateEntitlementsPerModule(APM_AGENT_COMPONENT_NAME, ALL_UNNAMED, apmAgentEntitlements, exclusiveFileEntitlements); for (var p : pluginsEntitlements.entrySet()) { for (var m : p.getValue().entrySet()) { - validateEntitlementsPerModule(p.getKey(), m.getKey(), m.getValue()); + validateEntitlementsPerModule(p.getKey(), m.getKey(), m.getValue(), exclusiveFileEntitlements); } } + List exclusivePaths = FileAccessTree.buildExclusivePathList(exclusiveFileEntitlements, pathLookup); + FileAccessTree.validateExclusivePaths(exclusivePaths); + this.exclusivePaths = exclusivePaths; } private static Map> buildScopeEntitlementsMap(Policy policy) { return policy.scopes().stream().collect(toUnmodifiableMap(Scope::moduleName, Scope::entitlements)); } - private static void validateEntitlementsPerModule(String componentName, String moduleName, List entitlements) { + private static void validateEntitlementsPerModule( + String componentName, + String moduleName, + List entitlements, + List exclusiveFileEntitlements + ) { Set> found = new HashSet<>(); for (var e : entitlements) { if (found.contains(e.getClass())) { @@ -189,6 +214,9 @@ private static void validateEntitlementsPerModule(String componentName, String m ); } found.add(e.getClass()); + if (e instanceof FilesEntitlement fe) { + exclusiveFileEntitlements.add(new ExclusiveFileEntitlement(componentName, moduleName, fe)); + } } } @@ -498,7 +526,7 @@ private ModuleEntitlements computeEntitlements(Class requestingClass) { if (requestingModule.isNamed() == false && requestingClass.getPackageName().startsWith(apmAgentPackageName)) { // The APM agent is the only thing running non-modular in the system classloader - return policyEntitlements(APM_AGENT_COMPONENT_NAME, apmAgentEntitlements); + return policyEntitlements(APM_AGENT_COMPONENT_NAME, ALL_UNNAMED, apmAgentEntitlements); } return defaultEntitlements(UNKNOWN_COMPONENT_NAME); @@ -513,7 +541,7 @@ private ModuleEntitlements getModuleScopeEntitlements( if (entitlements == null) { return defaultEntitlements(componentName); } - return policyEntitlements(componentName, entitlements); + return policyEntitlements(componentName, moduleName, entitlements); } private static boolean isServerModule(Module requestingModule) { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java index c47b0b93c5471..37526c98868da 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java @@ -9,7 +9,6 @@ package org.elasticsearch.entitlement.runtime.policy.entitlements; -import org.elasticsearch.core.Booleans; import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; import org.elasticsearch.entitlement.runtime.policy.PathLookup; import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException; @@ -21,6 +20,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.function.BiFunction; import java.util.stream.Stream; import static java.lang.Character.isLetter; @@ -75,24 +75,28 @@ public sealed interface FileData { Mode mode(); + boolean exclusive(); + + FileData withExclusive(boolean exclusive); + Platform platform(); FileData withPlatform(Platform platform); static FileData ofPath(Path path, Mode mode) { - return new AbsolutePathFileData(path, mode, null); + return new AbsolutePathFileData(path, mode, null, false); } static FileData ofRelativePath(Path relativePath, BaseDir baseDir, Mode mode) { - return new RelativePathFileData(relativePath, baseDir, mode, null); + return new RelativePathFileData(relativePath, baseDir, mode, null, false); } static FileData ofPathSetting(String setting, Mode mode, boolean ignoreUrl) { - return new PathSettingFileData(setting, mode, ignoreUrl, null); + return new PathSettingFileData(setting, mode, ignoreUrl, null, false); } static FileData ofRelativePathSetting(String setting, BaseDir baseDir, Mode mode, boolean ignoreUrl) { - return new RelativePathSettingFileData(setting, baseDir, mode, ignoreUrl, null); + return new RelativePathSettingFileData(setting, baseDir, mode, ignoreUrl, null, false); } /** @@ -176,7 +180,13 @@ private static Stream relativePathsCombination(Path[] baseDirs, Stream resolvePaths(PathLookup pathLookup) { return Stream.of(path); @@ -187,14 +197,20 @@ public FileData withPlatform(Platform platform) { if (platform == platform()) { return this; } - return new AbsolutePathFileData(path, mode, platform); + return new AbsolutePathFileData(path, mode, platform, exclusive); } } - private record RelativePathFileData(Path relativePath, BaseDir baseDir, Mode mode, Platform platform) + private record RelativePathFileData(Path relativePath, BaseDir baseDir, Mode mode, Platform platform, boolean exclusive) implements FileData, RelativeFileData { + + @Override + public RelativePathFileData withExclusive(boolean exclusive) { + return new RelativePathFileData(relativePath, baseDir, mode, platform, exclusive); + } + @Override public Stream resolveRelativePaths(PathLookup pathLookup) { return Stream.of(relativePath); @@ -205,11 +221,19 @@ public FileData withPlatform(Platform platform) { if (platform == platform()) { return this; } - return new RelativePathFileData(relativePath, baseDir, mode, platform); + return new RelativePathFileData(relativePath, baseDir, mode, platform, exclusive); } } - private record PathSettingFileData(String setting, Mode mode, boolean ignoreUrl, Platform platform) implements FileData { + private record PathSettingFileData(String setting, Mode mode, boolean ignoreUrl, Platform platform, boolean exclusive) + implements + FileData { + + @Override + public PathSettingFileData withExclusive(boolean exclusive) { + return new PathSettingFileData(setting, mode, ignoreUrl, platform, exclusive); + } + @Override public Stream resolvePaths(PathLookup pathLookup) { return resolvePathSettings(pathLookup, setting, ignoreUrl); @@ -220,14 +244,24 @@ public FileData withPlatform(Platform platform) { if (platform == platform()) { return this; } - return new PathSettingFileData(setting, mode, ignoreUrl, platform); + return new PathSettingFileData(setting, mode, ignoreUrl, platform, exclusive); } } - private record RelativePathSettingFileData(String setting, BaseDir baseDir, Mode mode, boolean ignoreUrl, Platform platform) - implements - FileData, - RelativeFileData { + private record RelativePathSettingFileData( + String setting, + BaseDir baseDir, + Mode mode, + boolean ignoreUrl, + Platform platform, + boolean exclusive + ) implements FileData, RelativeFileData { + + @Override + public RelativePathSettingFileData withExclusive(boolean exclusive) { + return new RelativePathSettingFileData(setting, baseDir, mode, ignoreUrl, platform, exclusive); + } + @Override public Stream resolveRelativePaths(PathLookup pathLookup) { return resolvePathSettings(pathLookup, setting, ignoreUrl); @@ -238,7 +272,7 @@ public FileData withPlatform(Platform platform) { if (platform == platform()) { return this; } - return new RelativePathSettingFileData(setting, baseDir, mode, ignoreUrl, platform); + return new RelativePathSettingFileData(setting, baseDir, mode, ignoreUrl, platform, exclusive); } } @@ -296,17 +330,54 @@ public static FilesEntitlement build(List paths) { if (paths == null || paths.isEmpty()) { throw new PolicyValidationException("must specify at least one path"); } + BiFunction, String, String> checkString = (values, key) -> { + Object value = values.remove(key); + if (value == null) { + return null; + } else if (value instanceof String str) { + return str; + } + throw new PolicyValidationException( + "expected [" + + key + + "] to be type [" + + String.class.getSimpleName() + + "] but found type [" + + value.getClass().getSimpleName() + + "]" + ); + }; + BiFunction, String, Boolean> checkBoolean = (values, key) -> { + Object value = values.remove(key); + if (value == null) { + return null; + } else if (value instanceof Boolean bool) { + return bool; + } + throw new PolicyValidationException( + "expected [" + + key + + "] to be type [" + + boolean.class.getSimpleName() + + "] but found type [" + + value.getClass().getSimpleName() + + "]" + ); + }; List filesData = new ArrayList<>(); for (Object object : paths) { - Map file = new HashMap<>((Map) object); - String pathAsString = file.remove("path"); - String relativePathAsString = file.remove("relative_path"); - String relativeTo = file.remove("relative_to"); - String pathSetting = file.remove("path_setting"); - String relativePathSetting = file.remove("relative_path_setting"); - String modeAsString = file.remove("mode"); - String platformAsString = file.remove("platform"); - String ignoreUrlAsString = file.remove("ignore_url"); + Map file = new HashMap<>((Map) object); + String pathAsString = checkString.apply(file, "path"); + String relativePathAsString = checkString.apply(file, "relative_path"); + String relativeTo = checkString.apply(file, "relative_to"); + String pathSetting = checkString.apply(file, "path_setting"); + String relativePathSetting = checkString.apply(file, "relative_path_setting"); + String modeAsString = checkString.apply(file, "mode"); + String platformAsString = checkString.apply(file, "platform"); + Boolean ignoreUrlAsStringBoolean = checkBoolean.apply(file, "ignore_url"); + boolean ignoreUrlAsString = ignoreUrlAsStringBoolean != null && ignoreUrlAsStringBoolean; + Boolean exclusiveBoolean = checkBoolean.apply(file, "exclusive"); + boolean exclusive = exclusiveBoolean != null && exclusiveBoolean; if (file.isEmpty() == false) { throw new PolicyValidationException("unknown key(s) [" + file + "] in a listed file for files entitlement"); @@ -333,12 +404,8 @@ public static FilesEntitlement build(List paths) { baseDir = parseBaseDir(relativeTo); } - boolean ignoreUrl = false; - if (ignoreUrlAsString != null) { - if (relativePathAsString != null || pathAsString != null) { - throw new PolicyValidationException("'ignore_url' may only be used with `path_setting` or `relative_path_setting`"); - } - ignoreUrl = Booleans.parseBoolean(ignoreUrlAsString); + if (ignoreUrlAsStringBoolean != null && (relativePathAsString != null || pathAsString != null)) { + throw new PolicyValidationException("'ignore_url' may only be used with `path_setting` or `relative_path_setting`"); } final FileData fileData; @@ -346,7 +413,6 @@ public static FilesEntitlement build(List paths) { if (baseDir == null) { throw new PolicyValidationException("files entitlement with a 'relative_path' must specify 'relative_to'"); } - Path relativePath = Path.of(relativePathAsString); if (FileData.isAbsolutePath(relativePathAsString)) { throw new PolicyValidationException("'relative_path' [" + relativePathAsString + "] must be relative"); @@ -359,17 +425,16 @@ public static FilesEntitlement build(List paths) { } fileData = FileData.ofPath(path, mode); } else if (pathSetting != null) { - fileData = FileData.ofPathSetting(pathSetting, mode, ignoreUrl); + fileData = FileData.ofPathSetting(pathSetting, mode, ignoreUrlAsString); } else if (relativePathSetting != null) { if (baseDir == null) { throw new PolicyValidationException("files entitlement with a 'relative_path_setting' must specify 'relative_to'"); } - fileData = FileData.ofRelativePathSetting(relativePathSetting, baseDir, mode, ignoreUrl); + fileData = FileData.ofRelativePathSetting(relativePathSetting, baseDir, mode, ignoreUrlAsString); } else { throw new AssertionError("File entry validation error"); } - - filesData.add(fileData.withPlatform(platform)); + filesData.add(fileData.withPlatform(platform).withExclusive(exclusive)); } return new FilesEntitlement(filesData); } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java index 98fd98b75719e..106a7db84e087 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusivePath; import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -54,13 +55,13 @@ private static Path path(String s) { ); public void testEmpty() { - var tree = accessTree(FilesEntitlement.EMPTY); + var tree = accessTree(FilesEntitlement.EMPTY, List.of()); assertThat(tree.canRead(path("path")), is(false)); assertThat(tree.canWrite(path("path")), is(false)); } public void testRead() { - var tree = accessTree(entitlement("foo", "read")); + var tree = accessTree(entitlement("foo", "read"), List.of()); assertThat(tree.canRead(path("foo")), is(true)); assertThat(tree.canRead(path("foo/subdir")), is(true)); assertThat(tree.canRead(path("food")), is(false)); @@ -72,7 +73,7 @@ public void testRead() { } public void testWrite() { - var tree = accessTree(entitlement("foo", "read_write")); + var tree = accessTree(entitlement("foo", "read_write"), List.of()); assertThat(tree.canWrite(path("foo")), is(true)); assertThat(tree.canWrite(path("foo/subdir")), is(true)); assertThat(tree.canWrite(path("food")), is(false)); @@ -84,7 +85,7 @@ public void testWrite() { } public void testTwoPaths() { - var tree = accessTree(entitlement("foo", "read", "bar", "read")); + var tree = accessTree(entitlement("foo", "read", "bar", "read"), List.of()); assertThat(tree.canRead(path("a")), is(false)); assertThat(tree.canRead(path("bar")), is(true)); assertThat(tree.canRead(path("bar/subdir")), is(true)); @@ -95,15 +96,17 @@ public void testTwoPaths() { } public void testReadWriteUnderRead() { - var tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write")); + var tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write"), List.of()); assertThat(tree.canRead(path("foo")), is(true)); assertThat(tree.canWrite(path("foo")), is(false)); assertThat(tree.canRead(path("foo/bar")), is(true)); assertThat(tree.canWrite(path("foo/bar")), is(true)); + assertThat(tree.canRead(path("foo/baz")), is(true)); + assertThat(tree.canWrite(path("foo/baz")), is(false)); } public void testPrunedPaths() { - var tree = accessTree(entitlement("foo", "read", "foo/baz", "read", "foo/bar", "read")); + var tree = accessTree(entitlement("foo", "read", "foo/baz", "read", "foo/bar", "read"), List.of()); assertThat(tree.canRead(path("foo")), is(true)); assertThat(tree.canWrite(path("foo")), is(false)); assertThat(tree.canRead(path("foo/bar")), is(true)); @@ -114,7 +117,7 @@ public void testPrunedPaths() { assertThat(tree.canRead(path("foo/barf")), is(true)); assertThat(tree.canWrite(path("foo/barf")), is(false)); - tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write")); + tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write"), List.of()); assertThat(tree.canRead(path("foo")), is(true)); assertThat(tree.canWrite(path("foo")), is(false)); assertThat(tree.canRead(path("foo/bar")), is(true)); @@ -124,7 +127,7 @@ public void testPrunedPaths() { } public void testPathAndFileWithSamePrefix() { - var tree = accessTree(entitlement("foo/bar/", "read", "foo/bar.xml", "read")); + var tree = accessTree(entitlement("foo/bar/", "read", "foo/bar.xml", "read"), List.of()); assertThat(tree.canRead(path("foo")), is(false)); assertThat(tree.canRead(path("foo/bar")), is(true)); assertThat(tree.canRead(path("foo/bar/baz")), is(true)); @@ -134,7 +137,7 @@ public void testPathAndFileWithSamePrefix() { public void testReadWithRelativePath() { for (var dir : List.of("config", "home")) { - var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read", "relative_to", dir))); + var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read", "relative_to", dir)), List.of()); assertThat(tree.canRead(path("foo")), is(false)); assertThat(tree.canRead(path("/" + dir + "/foo")), is(true)); @@ -151,7 +154,7 @@ public void testReadWithRelativePath() { public void testWriteWithRelativePath() { for (var dir : List.of("config", "home")) { - var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", dir))); + var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", dir)), List.of()); assertThat(tree.canWrite(path("/" + dir + "/foo")), is(true)); assertThat(tree.canWrite(path("/" + dir + "/foo/subdir")), is(true)); assertThat(tree.canWrite(path("/" + dir)), is(false)); @@ -166,7 +169,7 @@ public void testWriteWithRelativePath() { } public void testMultipleDataDirs() { - var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", "data"))); + var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", "data")), List.of()); assertThat(tree.canWrite(path("/data1/foo")), is(true)); assertThat(tree.canWrite(path("/data2/foo")), is(true)); assertThat(tree.canWrite(path("/data3/foo")), is(false)); @@ -184,7 +187,7 @@ public void testMultipleDataDirs() { } public void testNormalizePath() { - var tree = accessTree(entitlement("foo/../bar", "read")); + var tree = accessTree(entitlement("foo/../bar", "read"), List.of()); assertThat(tree.canRead(path("foo/../bar")), is(true)); assertThat(tree.canRead(path("foo/../bar/")), is(true)); assertThat(tree.canRead(path("foo")), is(false)); @@ -192,7 +195,7 @@ public void testNormalizePath() { } public void testNormalizeTrailingSlashes() { - var tree = accessTree(entitlement("/trailing/slash/", "read", "/no/trailing/slash", "read")); + var tree = accessTree(entitlement("/trailing/slash/", "read", "/no/trailing/slash", "read"), List.of()); assertThat(tree.canRead(path("/trailing/slash")), is(true)); assertThat(tree.canRead(path("/trailing/slash/")), is(true)); assertThat(tree.canRead(path("/trailing/slash.xml")), is(false)); @@ -205,7 +208,7 @@ public void testNormalizeTrailingSlashes() { public void testForwardSlashes() { String sep = getDefaultFileSystem().getSeparator(); - var tree = accessTree(entitlement("a/b", "read", "m" + sep + "n", "read")); + var tree = accessTree(entitlement("a/b", "read", "m" + sep + "n", "read"), List.of()); // Native separators work assertThat(tree.canRead(path("a" + sep + "b")), is(true)); @@ -219,7 +222,7 @@ public void testForwardSlashes() { public void testJdkAccess() { Path jdkDir = Paths.get(System.getProperty("java.home")); var confDir = jdkDir.resolve("conf"); - var tree = accessTree(FilesEntitlement.EMPTY); + var tree = accessTree(FilesEntitlement.EMPTY, List.of()); assertThat(tree.canRead(confDir), is(true)); assertThat(tree.canWrite(confDir), is(false)); @@ -239,7 +242,7 @@ public void testFollowLinks() throws IOException { Path writeTarget = baseTargetDir.resolve("write_link"); Files.createSymbolicLink(readTarget, source1Dir); Files.createSymbolicLink(writeTarget, source2Dir); - var tree = accessTree(entitlement(readTarget.toString(), "read", writeTarget.toString(), "read_write")); + var tree = accessTree(entitlement(readTarget.toString(), "read", writeTarget.toString(), "read_write"), List.of()); assertThat(tree.canRead(baseSourceDir), is(false)); assertThat(tree.canRead(baseTargetDir), is(false)); @@ -256,13 +259,65 @@ public void testFollowLinks() throws IOException { } public void testTempDirAccess() { - var tree = FileAccessTree.of(FilesEntitlement.EMPTY, TEST_PATH_LOOKUP); + var tree = FileAccessTree.of("test-component", "test-module", FilesEntitlement.EMPTY, TEST_PATH_LOOKUP, List.of()); assertThat(tree.canRead(TEST_PATH_LOOKUP.tempDir()), is(true)); assertThat(tree.canWrite(TEST_PATH_LOOKUP.tempDir()), is(true)); } - FileAccessTree accessTree(FilesEntitlement entitlement) { - return FileAccessTree.of(entitlement, TEST_PATH_LOOKUP); + public void testBasicExclusiveAccess() { + var tree = accessTree(entitlement("foo", "read"), exclusivePaths("test-component", "test-module", "foo")); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + tree = accessTree(entitlement("foo", "read_write"), exclusivePaths("test-component", "test-module", "foo")); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(true)); + tree = accessTree(entitlement("foo", "read"), exclusivePaths("test-component", "diff-module", "foo/bar")); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + assertThat(tree.canRead(path("foo/baz")), is(true)); + assertThat(tree.canWrite(path("foo/baz")), is(false)); + assertThat(tree.canRead(path("foo/bar")), is(false)); + assertThat(tree.canWrite(path("foo/bar")), is(false)); + tree = accessTree( + entitlement("foo", "read", "foo.xml", "read", "foo/bar.xml", "read_write"), + exclusivePaths("test-component", "diff-module", "foo/bar", "foo/baz", "other") + ); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + assertThat(tree.canRead(path("foo.xml")), is(true)); + assertThat(tree.canWrite(path("foo.xml")), is(false)); + assertThat(tree.canRead(path("foo/baz")), is(false)); + assertThat(tree.canWrite(path("foo/baz")), is(false)); + assertThat(tree.canRead(path("foo/bar")), is(false)); + assertThat(tree.canWrite(path("foo/bar")), is(false)); + assertThat(tree.canRead(path("foo/bar.xml")), is(true)); + assertThat(tree.canWrite(path("foo/bar.xml")), is(true)); + assertThat(tree.canRead(path("foo/bar.baz")), is(true)); + assertThat(tree.canWrite(path("foo/bar.baz")), is(false)); + assertThat(tree.canRead(path("foo/biz/bar.xml")), is(true)); + assertThat(tree.canWrite(path("foo/biz/bar.xml")), is(false)); + } + + public void testInvalidExclusiveAccess() { + var tree = accessTree(entitlement("a", "read"), exclusivePaths("diff-component", "diff-module", "a/b")); + assertThat(tree.canRead(path("a")), is(true)); + assertThat(tree.canWrite(path("a")), is(false)); + assertThat(tree.canRead(path("a/b")), is(false)); + assertThat(tree.canWrite(path("a/b")), is(false)); + assertThat(tree.canRead(path("a/b/c")), is(false)); + assertThat(tree.canWrite(path("a/b/c")), is(false)); + tree = accessTree(entitlement("a/b", "read"), exclusivePaths("diff-component", "diff-module", "a")); + assertThat(tree.canRead(path("a")), is(false)); + assertThat(tree.canWrite(path("a")), is(false)); + assertThat(tree.canRead(path("a/b")), is(false)); + assertThat(tree.canWrite(path("a/b")), is(false)); + tree = accessTree(entitlement("a", "read"), exclusivePaths("diff-component", "diff-module", "a")); + assertThat(tree.canRead(path("a")), is(false)); + assertThat(tree.canWrite(path("a")), is(false)); + } + + FileAccessTree accessTree(FilesEntitlement entitlement, List exclusivePaths) { + return FileAccessTree.of("test-component", "test-module", entitlement, TEST_PATH_LOOKUP, exclusivePaths); } static FilesEntitlement entitlement(String... values) { @@ -279,4 +334,12 @@ static FilesEntitlement entitlement(String... values) { static FilesEntitlement entitlement(Map value) { return FilesEntitlement.build(List.of(value)); } + + static List exclusivePaths(String componentName, String moduleName, String... paths) { + List exclusivePaths = new ArrayList<>(); + for (String path : paths) { + exclusivePaths.add(new ExclusivePath(componentName, moduleName, path(path).toString())); + } + return exclusivePaths; + } } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java index 5a65ea81d0a0e..7f37168a3a7f8 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java @@ -368,7 +368,10 @@ public void testDuplicateEntitlements() { ) ); assertEquals( - "[(APM agent)] using module [unnamed] found duplicate entitlement " + "[" + CreateClassLoaderEntitlement.class.getName() + "]", + "[(APM agent)] using module [ALL-UNNAMED] found duplicate entitlement " + + "[" + + CreateClassLoaderEntitlement.class.getName() + + "]", iae.getMessage() ); @@ -408,6 +411,112 @@ public void testDuplicateEntitlements() { ); } + public void testFilesEntitlementsWithExclusive() { + var iae = expectThrows( + IllegalArgumentException.class, + () -> new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.of( + "plugin1", + new Policy( + "test", + List.of( + new Scope( + "test", + List.of( + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(Path.of("/tmp/test"), FilesEntitlement.Mode.READ) + .withExclusive(true) + ) + ) + ) + ) + ) + ), + "plugin2", + new Policy( + "test", + List.of( + new Scope( + "test", + List.of( + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(Path.of("/tmp/test"), FilesEntitlement.Mode.READ) + .withExclusive(true) + ) + ) + ) + ) + ) + ) + ), + c -> "", + TEST_AGENTS_PACKAGE_NAME, + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() + ) + ); + assertTrue(iae.getMessage().contains("duplicate/overlapping exclusive paths found in files entitlements:")); + assertTrue(iae.getMessage().contains("[test] [/tmp/test]]")); + + iae = expectThrows( + IllegalArgumentException.class, + () -> new PolicyManager( + new Policy( + "test", + List.of( + new Scope( + "test", + List.of( + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(Path.of("/tmp/test/foo"), FilesEntitlement.Mode.READ) + .withExclusive(true), + FilesEntitlement.FileData.ofPath(Path.of("/tmp/"), FilesEntitlement.Mode.READ) + ) + ) + ) + ) + ) + ), + List.of(), + Map.of( + "plugin1", + new Policy( + "test", + List.of( + new Scope( + "test", + List.of( + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(Path.of("/tmp/test"), FilesEntitlement.Mode.READ) + .withExclusive(true) + ) + ) + ) + ) + ) + ) + ), + c -> "", + TEST_AGENTS_PACKAGE_NAME, + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() + ) + ); + assertEquals( + "duplicate/overlapping exclusive paths found in files entitlements: " + + "[[plugin1] [test] [/tmp/test]] and [[(server)] [test] [/tmp/test/foo]]", + iae.getMessage() + ); + } + /** * If the plugin resolver tells us a class is in a plugin, don't conclude that it's in an agent. */ diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java index a453d6cf54992..60c80b41a5087 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java @@ -11,11 +11,16 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.entitlement.runtime.policy.PathLookup; +import org.elasticsearch.entitlement.runtime.policy.Policy; +import org.elasticsearch.entitlement.runtime.policy.PolicyParser; import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException; +import org.elasticsearch.entitlement.runtime.policy.Scope; import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.FileData; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -114,6 +119,26 @@ public void testPathSettingResolve() { assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), containsInAnyOrder(Path.of("/setting/path"), Path.of("/other/path"))); } + public void testExclusiveParsing() throws Exception { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - files: + - path: /test + mode: read + exclusive: true + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true).parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of( + new Scope( + "entitlement-module-name", + List.of(FilesEntitlement.build(List.of(Map.of("path", "/test", "mode", "read", "exclusive", true)))) + ) + ) + ); + assertEquals(expected, parsedPolicy); + } + public void testPathSettingIgnoreUrl() { var fileData = FileData.ofPathSetting("foo.*.bar", READ, true); settings = Settings.builder().put("foo.nonurl.bar", "/setting/path").put("foo.url.bar", "https://mysite").build(); @@ -129,14 +154,14 @@ public void testRelativePathSettingIgnoreUrl() { public void testIgnoreUrlValidation() { var e = expectThrows( PolicyValidationException.class, - () -> FilesEntitlement.build(List.of(Map.of("path", "/foo", "mode", "read", "ignore_url", "true"))) + () -> FilesEntitlement.build(List.of(Map.of("path", "/foo", "mode", "read", "ignore_url", true))) ); assertThat(e.getMessage(), is("'ignore_url' may only be used with `path_setting` or `relative_path_setting`")); e = expectThrows( PolicyValidationException.class, () -> FilesEntitlement.build( - List.of(Map.of("relative_path", "foo", "relative_to", "config", "mode", "read", "ignore_url", "true")) + List.of(Map.of("relative_path", "foo", "relative_to", "config", "mode", "read", "ignore_url", true)) ) ); assertThat(e.getMessage(), is("'ignore_url' may only be used with `path_setting` or `relative_path_setting`")); From 6cc68f5d15eb9f07b3b94e2357918a69ca57e039 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Thu, 27 Feb 2025 20:24:41 +1100 Subject: [PATCH 07/11] Update docker.elastic.co/wolfi/chainguard-base:latest Docker digest to 6387bd4 (#123395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | docker.elastic.co/wolfi/chainguard-base | digest | `d74b1fd` -> `6387bd4` | --- ### Configuration 📅 **Schedule**: Branch creation - "after 1pm on tuesday" (UTC), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://redirect.github.com/renovatebot/renovate). --- .../main/java/org/elasticsearch/gradle/internal/DockerBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 7fc08ae332fbb..22cbdef3291ec 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -24,7 +24,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:d74b1fda6b7fee2c90b410df258e005c049e0672fe16d79d00e58f14fb69f90b", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:6387bd4c462007eaecaf13a423aea99c8a8452da09244c129703324aa97769c6", "-wolfi", "apk" ), From 5e619ee2b46b1d9f3b55c7188f6b26cff4ed8fc8 Mon Sep 17 00:00:00 2001 From: Patrick Doyle <810052+prdoyle@users.noreply.github.com> Date: Thu, 27 Feb 2025 04:27:04 -0500 Subject: [PATCH 08/11] More entitlements for bouncycastle (#123548) (#123572) --- .../initialization/EntitlementInitialization.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index f39038c0cf8de..1eb78c3fced84 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -242,7 +242,14 @@ private static PolicyManager createPolicyManager() { if (trustStorePath != null) { Collections.addAll( serverScopes, - new Scope("org.bouncycastle.fips.tls", List.of(new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))))), + new Scope( + "org.bouncycastle.fips.tls", + List.of( + new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))), + new OutboundNetworkEntitlement(), + new ManageThreadsEntitlement() + ) + ), new Scope( "org.bouncycastle.fips.core", // read to lib dir is required for checksum validation From 04e8a8ea1349466ba31f74e2c6b9dc0ec89e44e5 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Thu, 27 Feb 2025 11:42:14 +0100 Subject: [PATCH 09/11] Skip selected tests in SSLErrorMessageFileTests that rely on SecurityManager to block access to resources. (#123484) (#123511) (cherry picked from commit 7e554fa4435d73be1c28577100f00276fd4380cf) --- .../elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java index 2ac2d4ebf0c32..4f64b780e1f97 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.PathUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.jdk.RuntimeVersionFeature; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ssl.SSLService; import org.junit.Before; @@ -362,6 +363,11 @@ private void checkBlockedResource( String configKey, BiConsumer configure ) throws Exception { + assumeTrue( + "Requires Security Manager to block access, entitlements are not checked for unit tests", + RuntimeVersionFeature.isSecurityManagerAvailable() + ); + final String prefix = randomSslPrefix(); final Settings.Builder settings = Settings.builder(); configure.accept(prefix, settings); From 0a47363f6d435ccabbfa124ccfe641be58b58abe Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Thu, 27 Feb 2025 13:36:44 +0100 Subject: [PATCH 10/11] Add missing APM agent entitlements (#123575) (#123579) Add missing APM agent entitlements --- .../entitlement/initialization/EntitlementInitialization.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 1eb78c3fced84..dc839a5259821 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -33,6 +33,7 @@ import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; import java.lang.instrument.Instrumentation; import java.lang.reflect.Constructor; @@ -265,6 +266,8 @@ private static PolicyManager createPolicyManager() { List agentEntitlements = List.of( new CreateClassLoaderEntitlement(), new ManageThreadsEntitlement(), + new SetHttpsConnectionPropertiesEntitlement(), + new OutboundNetworkEntitlement(), new FilesEntitlement( List.of( FileData.ofPath(Path.of("/co/elastic/apm/agent/"), READ), From b585bb1196e599bc5df6db691f228f5763470197 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Thu, 27 Feb 2025 13:48:40 +0100 Subject: [PATCH 11/11] fix testReadBlobWithPrematureConnectionClose jdk24 (#122655) (#123596) (cherry picked from commit 8b4f159aa2cd8d7862f35db9a29e40f0cf447606) Co-authored-by: Mikhail Berezovskiy --- .../blobstore/AbstractBlobContainerRetriesTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index 12094b31a049d..9e955fa727d76 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -356,6 +356,7 @@ public void testReadBlobWithPrematureConnectionClose() { containsString("premature end of chunk coded message body: closing chunk expected"), containsString("premature end of content-length delimited message body"), containsString("connection closed prematurely"), + containsString("premature eof"), // if we didn't call exchange.getResponseBody().flush() then we might not even have sent the response headers: alwaysFlushBody ? never() : containsString("the target server failed to respond") )