Merge branch 'main' into intervals_max_gaps_docs

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.jdk.RuntimeVersionFeature;
 
@@ -26,7 +27,9 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-        boolean useEntitlements = Boolean.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
+        boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled;
         return Stream.of(
             Stream.of(
                 /*

docs/reference/rest-api/security/get-service-accounts.asciidoc

@@ -276,6 +276,21 @@ GET /_security/service/elastic/fleet-server
             "view_index_metadata"
           ],
           "allow_restricted_indices": false
+        },
+        {
+          "names": [
+            "agentless-*",
+          ],
+          "privileges": [
+            "read",
+            "write",
+            "monitor",
+            "create_index",
+            "auto_configure",
+            "maintenance",
+            "view_index_metadata"
+          ],
+          "allow_restricted_indices": false
         }
       ],
       "applications": [

docs/reference/scripting/using.asciidoc

@@ -201,8 +201,13 @@ when you're creating <<runtime-mapping-fields,runtime fields>>.
 [[script-stored-scripts]]
 === Store and retrieve scripts
 You can store and retrieve scripts from the cluster state using the
-<<stored-script-apis,stored script APIs>>. Stored scripts reduce compilation
-time and make searches faster.
+<<stored-script-apis,stored script APIs>>. Stored scripts allow you to reference 
+shared scripts for operations like scoring, aggregating, filtering, and 
+reindexing. Instead of embedding scripts inline in each query, you can reference 
+these shared operations.
+
+Stored scripts can also reduce request payload size. Depending on script size 
+and request frequency, this can help lower latency and data transfer costs.
 
 NOTE: Unlike regular scripts, stored scripts require that you specify a script
 language using the `lang` parameter.

muted-tests.yml

@@ -248,6 +248,29 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceCrudIT
   method: testGetServicesWithCompletionTaskType
   issue: https://github.com/elastic/elasticsearch/issues/119959
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testSearchableSnapshotUpgrade {p0=[9.0.0, 8.18.0, 8.18.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/119978
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 8.18.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/119979
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testMountSearchableSnapshot {p0=[9.0.0, 8.18.0, 8.18.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/119550
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 8.18.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/119980
+- class: org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormatTests
+  method: testRandomExceptions
+  issue: https://github.com/elastic/elasticsearch/issues/119981
+- class: org.elasticsearch.multi_cluster.MultiClusterYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/119983
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 9.0.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/119989
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 9.0.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/119990
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.core.AbstractRefCounted;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
@@ -109,7 +110,9 @@ private static Bootstrap initPhase1() {
         final PrintStream out = getStdout();
         final PrintStream err = getStderr();
         final ServerArgs args;
-        final boolean useEntitlements = Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"));
+        final boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(System.getProperty("es.entitlements.enabled", "false"));
+        // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
+        final boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled;
         try {
             initSecurityProperties();
 

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -781,6 +781,7 @@ protected Set<String> preserveILMPolicyIds() {
             "profiling-60-days",
             "profiling-60-days@lifecycle",
             "synthetics",
+            "agentless",
             "synthetics@lifecycle",
             "traces@lifecycle",
             "7-days-default",
@@ -2215,6 +2216,7 @@ protected static boolean isXPackTemplate(String name) {
             case "metrics-tsdb-settings":
             case "metrics-mappings":
             case "synthetics":
+            case "agentless":
             case "synthetics-settings":
             case "synthetics-mappings":
             case ".snapshot-blob-cache":

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java

@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction;
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
+import org.elasticsearch.action.admin.indices.mapping.put.TransportAutoPutMappingAction;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
 import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
 import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction;
@@ -428,7 +429,6 @@ static RoleDescriptor kibanaSystem(String name) {
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(
                         "logs-cloud_security_posture.findings_latest-default*",
-                        "logs-cloud_security_posture.scores-default*",
                         "logs-cloud_security_posture.vulnerabilities_latest-default*"
                     )
                     .privileges(
@@ -440,6 +440,20 @@ static RoleDescriptor kibanaSystem(String name) {
                         TransportUpdateSettingsAction.TYPE.name()
                     )
                     .build(),
+                // For destination indices of the Cloud Security Posture packages that ships a
+                // transform (specific for scores indexes, as of 9.0.0 score indices will need to have auto_put priviliges)
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("logs-cloud_security_posture.scores-default*")
+                    .privileges(
+                        "create_index",
+                        "read",
+                        "index",
+                        "delete",
+                        TransportIndicesAliasesAction.NAME,
+                        TransportUpdateSettingsAction.TYPE.name(),
+                        TransportAutoPutMappingAction.TYPE.name()
+                    )
+                    .build(),
                 // For source indices of the Cloud Detection & Response (CDR) packages that ships a
                 // transform
                 RoleDescriptor.IndicesPrivileges.builder()

x-pack/plugin/core/template-resources/src/main/resources/agentless@mappings.json

@@ -0,0 +1,22 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "v": {
+          "type": "object",
+          "enabled": false
+        },
+        "updated_at": {
+          "type": "date",
+          "format": "strict_date_optional_time||epoch_millis"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default mappings for the agentless index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}

x-pack/plugin/core/template-resources/src/main/resources/agentless@settings.json

@@ -0,0 +1,15 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "hidden": true
+      }
+    }
+  },
+  "_meta": {
+    "description": "default settings for the agentless index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}

x-pack/plugin/core/template-resources/src/main/resources/agentless@template.json

@@ -0,0 +1,15 @@
+{
+  "index_patterns": ["agentless-*-*"],
+  "priority": 100,
+  "composed_of": [
+    "agentless@mappings",
+    "agentless@settings"
+  ],
+  "allow_auto_create": true,
+  "_meta": {
+    "description": "default agentless template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java

@@ -8,6 +8,7 @@
 
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -85,10 +86,22 @@ public ZoneId zoneId() {
         return zoneId;
     }
 
+    /**
+     * In case that the range is empty due to foldable, invalid bounds, but the bounds themselves are not yet folded, the optimizer will
+     * need two passes to fold this.
+     * That's because we shouldn't perform folding when trying to determine foldability.
+     */
     @Override
     public boolean foldable() {
         if (lower.foldable() && upper.foldable()) {
-            return areBoundariesInvalid() || value.foldable();
+            if (value().foldable()) {
+                return true;
+            }
+
+            // We cannot fold the bounds here; but if they're already literals, we can check if the range is always empty.
+            if (lower() instanceof Literal && upper() instanceof Literal) {
+                return areBoundariesInvalid();
+            }
         }
 
         return false;

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
+import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
@@ -27,11 +28,16 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
+import org.elasticsearch.xpack.esql.plan.logical.Filter;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.FIVE;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptySource;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.fieldAttribute;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOrEqualOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf;
@@ -111,6 +117,53 @@ public void testArithmeticFolding() {
         assertEquals(1, foldOperator(new Mod(EMPTY, new Literal(EMPTY, 7, DataType.INTEGER), THREE)));
     }
 
+    public void testFoldRange() {
+        // 1 + 9 < value AND value < 20-1
+        // with value = 12 and randomly replacing the `<` by `<=`
+        Expression lowerBound = new Add(EMPTY, new Literal(EMPTY, 1, DataType.INTEGER), new Literal(EMPTY, 9, DataType.INTEGER));
+        Expression upperBound = new Sub(EMPTY, new Literal(EMPTY, 20, DataType.INTEGER), new Literal(EMPTY, 1, DataType.INTEGER));
+        Expression value = new Literal(EMPTY, 12, DataType.INTEGER);
+        Range range = new Range(EMPTY, value, lowerBound, randomBoolean(), upperBound, randomBoolean(), randomZone());
+
+        Expression folded = new ConstantFolding().rule(range);
+        assertTrue((Boolean) as(folded, Literal.class).value());
+    }
+
+    public void testFoldRangeWithInvalidBoundaries() {
+        // 1 + 9 < value AND value <= 11 - 1
+        // This is always false. We also randomly test versions with `<=`.
+        Expression lowerBound;
+        boolean includeLowerBound = randomBoolean();
+        if (includeLowerBound) {
+            // 1 + 10 <= value
+            lowerBound = new Add(EMPTY, new Literal(EMPTY, 1, DataType.INTEGER), new Literal(EMPTY, 10, DataType.INTEGER));
+        } else {
+            // 1 + 9 < value
+            lowerBound = new Add(EMPTY, new Literal(EMPTY, 1, DataType.INTEGER), new Literal(EMPTY, 9, DataType.INTEGER));
+        }
+
+        boolean includeUpperBound = randomBoolean();
+        // value < 11 - 1
+        // or
+        // value <= 11 - 1
+        Expression upperBound = new Sub(EMPTY, new Literal(EMPTY, 11, DataType.INTEGER), new Literal(EMPTY, 1, DataType.INTEGER));
+
+        Expression value = fieldAttribute();
+
+        Range range = new Range(EMPTY, value, lowerBound, includeLowerBound, upperBound, includeUpperBound, randomZone());
+
+        // We need to test this as part of a logical plan, to correctly simulate how we traverse down the expression tree.
+        // Just applying this to the range directly won't perform a transformDown.
+        LogicalPlan filter = new Filter(EMPTY, emptySource(), range);
+
+        Filter foldedOnce = as(new ConstantFolding().apply(filter), Filter.class);
+        // We need to run the rule twice, because during the first run only the boundaries can be folded - the range doesn't know it's
+        // foldable, yet.
+        Filter foldedTwice = as(new ConstantFolding().apply(foldedOnce), Filter.class);
+
+        assertFalse((Boolean) as(foldedTwice.condition(), Literal.class).value());
+    }
+
     private static Object foldOperator(BinaryOperator<?, ?, ?, ?> b) {
         return ((Literal) new ConstantFolding().rule(b)).value();
     }

x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java

@@ -316,6 +316,21 @@ public class ServiceAccountIT extends ESRestTestCase {
                     "view_index_metadata"
                   ],
                   "allow_restricted_indices": false
+                },
+                {
+                  "names": [
+                    "agentless-*"
+                  ],
+                  "privileges": [
+                    "read",
+                    "write",
+                    "monitor",
+                    "create_index",
+                    "auto_configure",
+                    "maintenance",
+                    "view_index_metadata"
+                  ],
+                  "allow_restricted_indices": false
                 }
               ],
               "applications": [        {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java

@@ -166,6 +166,12 @@ final class ElasticServiceAccounts {
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices("content-*", ".search-acl-filter-*")
                     .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance", "view_index_metadata")
+                    .build(),
+                // Custom permissions required for stateful agentless integrations
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("agentless-*")
+                    .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance", "view_index_metadata")
+                    .allowRestrictedIndices(false)
                     .build(), },
             new RoleDescriptor.ApplicationResourcePrivileges[] {
                 RoleDescriptor.ApplicationResourcePrivileges.builder()