Merge branch 'main' into esql_full_sentences

.buildkite/pipelines/intake.yml

@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -577,8 +577,8 @@ steps:
         env:
           BWC_VERSION: 8.13.4
 
-      - label: "{{matrix.image}} / 8.14.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.3
+      - label: "{{matrix.image}} / 8.14.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -592,7 +592,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.14.3
+          BWC_VERSION: 8.14.4
 
       - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0

.buildkite/pipelines/periodic.yml

@@ -642,8 +642,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.14.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.3#bwcTest
+      - label: 8.14.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -653,7 +653,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.14.3
+          BWC_VERSION: 8.14.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -821,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -31,6 +31,6 @@ BWC_VERSION:
   - "8.11.4"
   - "8.12.2"
   - "8.13.4"
-  - "8.14.3"
+  - "8.14.4"
   - "8.15.0"
   - "8.16.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.23"
-  - "8.14.3"
+  - "8.14.4"
   - "8.15.0"
   - "8.16.0"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.plugin",
         "org.elasticsearch.plugin.analysis",
         "org.elasticsearch.pluginclassloader",
-        "org.elasticsearch.preallocate",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
         "org.elasticsearch.simdvec",

distribution/build.gradle

@@ -280,8 +280,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 
   dependencies {
     libs project(':server')
-    // this is a special circumstance of a jar that is not a dependency of server, but needs to be in the module path
-    libs project(':libs:elasticsearch-preallocate')
 
     libsVersionChecker project(':distribution:tools:java-version-checker')
     libsCliLauncher project(':distribution:tools:cli-launcher')

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -69,11 +69,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                  * parsing will break in an incompatible way for some date patterns and locales.
                  */
                 "-Djava.locale.providers=SPI,COMPAT",
-                /*
-                 * Temporarily suppress illegal reflective access in searchable snapshots shared cache preallocation; this is temporary
-                 * while we explore alternatives. See org.elasticsearch.xpack.searchablesnapshots.preallocate.Preallocate.
-                 */
-                "--add-opens=java.base/java.io=org.elasticsearch.preallocate",
                 maybeEnableNativeAccess(),
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),

docs/changelog/110352.yaml

@@ -0,0 +1,5 @@
+pr: 110352
+summary: Search coordinator uses `event.ingested` in cluster state to do rewrites
+area: Search
+type: enhancement
+issues: []

docs/changelog/110427.yaml

@@ -0,0 +1,6 @@
+pr: 110427
+summary: "[Inference API] Remove unused Cohere rerank service settings fields in a\
+  \ BWC way"
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/110603.yaml

@@ -0,0 +1,6 @@
+pr: 110603
+summary: Stop iterating over all fields to extract @timestamp value
+area: TSDB
+type: enhancement
+issues:
+ - 92297

docs/changelog/110665.yaml

@@ -0,0 +1,6 @@
+pr: 110665
+summary: "[ESQL] Fix parsing of large magnitude negative numbers"
+area: ES|QL
+type: bug
+issues:
+ - 104323

docs/changelog/110707.yaml

@@ -0,0 +1,5 @@
+pr: 110707
+summary: Fix issue with returning incomplete fragment for plain highlighter
+area: Highlighting
+type: bug
+issues: []

docs/reference/docs/index_.asciidoc

@@ -211,7 +211,7 @@ creates a dynamic mapping. By default, new fields and objects are
 automatically added to the mapping if needed. For more information about field
 mapping, see <<mapping,mapping>> and the <<indices-put-mapping,update mapping>> API.
 
-Automatic index creation is controlled by the `action.auto_create_index`
+Automatic index creation is controlled by the <<index-management-settings,`action.auto_create_index`>>
 setting. This setting defaults to `true`, which allows any index to be created
 automatically. You can modify this setting to explicitly allow or block
 automatic creation of indices that match specified patterns, or set it to

docs/reference/mapping/types/rank-features.asciidoc

@@ -70,13 +70,23 @@ GET my-index-000001/_search
     }
   }
 }
+
+GET my-index-000001/_search
+{
+  "query": { <6>
+    "term": {
+      "topics": "economics"
+    }
+  }
+}
 --------------------------------------------------
 
 <1> Rank features fields must use the `rank_features` field type
 <2> Rank features that correlate negatively with the score need to declare it
 <3> Rank features fields must be a hash with string keys and strictly positive numeric values
 <4> This query ranks documents by how much they are about the "politics" topic.
 <5> This query ranks documents inversely to the number of "1star" reviews they received.
+<6> This query returns documents that store the "economics" feature in the "topics" field.
 
 
 NOTE: `rank_features` fields only support single-valued features and strictly

docs/reference/modules/cluster/misc.asciidoc

@@ -11,12 +11,12 @@ An entire cluster may be set to read-only with the following setting:
       (<<dynamic-cluster-setting,Dynamic>>)
       Make the whole cluster read only (indices do not accept write
       operations), metadata is not allowed to be modified (create or delete
-      indices).
+      indices). Defaults to `false`. 
 
 `cluster.blocks.read_only_allow_delete`::
       (<<dynamic-cluster-setting,Dynamic>>)
       Identical to `cluster.blocks.read_only` but allows to delete indices
-      to free up resources.
+      to free up resources. Defaults to `false`. 
 
 WARNING: Don't rely on this setting to prevent changes to your cluster. Any
 user with access to the <<cluster-update-settings,cluster-update-settings>>

docs/reference/query-dsl.asciidoc

@@ -72,14 +72,12 @@ include::query-dsl/match-all-query.asciidoc[]
 
 include::query-dsl/span-queries.asciidoc[]
 
+include::query-dsl/vector-queries.asciidoc[]
+
 include::query-dsl/special-queries.asciidoc[]
 
 include::query-dsl/term-level-queries.asciidoc[]
 
-include::query-dsl/text-expansion-query.asciidoc[]
-
-include::query-dsl/sparse-vector-query.asciidoc[]
-
 include::query-dsl/minimum-should-match.asciidoc[]
 
 include::query-dsl/multi-term-rewrite.asciidoc[]

docs/reference/query-dsl/sparse-vector-query.asciidoc

@@ -1,5 +1,5 @@
 [[query-dsl-sparse-vector-query]]
-== Sparse vector query
+=== Sparse vector query
 
 ++++
 <titleabbrev>Sparse vector</titleabbrev>
@@ -19,7 +19,7 @@ For example, a stored vector `{"feature_0": 0.12, "feature_1": 1.2, "feature_2":
 
 [discrete]
 [[sparse-vector-query-ex-request]]
-=== Example request using an {nlp} model
+==== Example request using an {nlp} model
 
 [source,console]
 ----
@@ -37,7 +37,7 @@ GET _search
 // TEST[skip: Requires inference]
 
 [discrete]
-=== Example request using precomputed vectors
+==== Example request using precomputed vectors
 
 [source,console]
 ----
@@ -55,7 +55,7 @@ GET _search
 
 [discrete]
 [[sparse-vector-field-params]]
-=== Top level parameters for `sparse_vector`
+==== Top level parameters for `sparse_vector`
 
 `field`::
 (Required, string) The name of the field that contains the token-weight pairs to be searched against.
@@ -120,7 +120,7 @@ NOTE: The default values for `tokens_freq_ratio_threshold` and `tokens_weight_th
 
 [discrete]
 [[sparse-vector-query-example]]
-=== Example ELSER query
+==== Example ELSER query
 
 The following is an example of the `sparse_vector` query that references the ELSER model to perform semantic search.
 For a more detailed description of how to perform semantic search by using ELSER and the `sparse_vector` query, refer to <<semantic-search-elser,this tutorial>>.
@@ -241,7 +241,7 @@ GET my-index/_search
 
 [discrete]
 [[sparse-vector-query-with-pruning-config-and-rescore-example]]
-=== Example ELSER query with pruning configuration and rescore
+==== Example ELSER query with pruning configuration and rescore
 
 The following is an extension to the above example that adds a preview:[] pruning configuration to the `sparse_vector` query.
 The pruning configuration identifies non-significant tokens to prune from the query in order to improve query performance.

docs/reference/query-dsl/special-queries.asciidoc

@@ -17,10 +17,6 @@ or collection of documents.
 This query finds queries that are stored as documents that match with
 the specified document.
 
-<<query-dsl-knn-query,`knn` query>>::
-A query that finds the _k_ nearest vectors to a query
-vector, as measured by a similarity metric.
-
 <<query-dsl-rank-feature-query,`rank_feature` query>>::
 A query that computes scores based on the values of numeric features and is
 able to efficiently skip non-competitive hits.
@@ -32,9 +28,6 @@ This query allows a script to act as a filter. Also see the
 <<query-dsl-script-score-query,`script_score` query>>::
 A query that allows to modify the score of a sub-query with a script.
 
-<<query-dsl-semantic-query,`semantic` query>>::
-A query that allows you to perform semantic search.
-
 <<query-dsl-wrapper-query,`wrapper` query>>::
 A query that accepts other queries as json or yaml string.
 
@@ -50,20 +43,14 @@ include::mlt-query.asciidoc[]
 
 include::percolate-query.asciidoc[]
 
-include::knn-query.asciidoc[]
-
 include::rank-feature-query.asciidoc[]
 
 include::script-query.asciidoc[]
 
 include::script-score-query.asciidoc[]
 
-include::semantic-query.asciidoc[]
-
 include::wrapper-query.asciidoc[]
 
 include::pinned-query.asciidoc[]
 
 include::rule-query.asciidoc[]
-
-include::weighted-tokens-query.asciidoc[]

docs/reference/query-dsl/text-expansion-query.asciidoc

@@ -1,5 +1,5 @@
 [[query-dsl-text-expansion-query]]
-== Text expansion query
+=== Text expansion query
 
 ++++
 <titleabbrev>Text expansion</titleabbrev>
@@ -12,7 +12,7 @@ The text expansion query uses a {nlp} model to convert the query text into a lis
 
 [discrete]
 [[text-expansion-query-ex-request]]
-=== Example request
+==== Example request
 
 [source,console]
 ----
@@ -32,14 +32,14 @@ GET _search
 
 [discrete]
 [[text-expansion-query-params]]
-=== Top level parameters for `text_expansion`
+==== Top level parameters for `text_expansion`
 
 `<sparse_vector_field>`:::
 (Required, object) The name of the field that contains the token-weight pairs the NLP model created based on the input text.
 
 [discrete]
 [[text-expansion-rank-feature-field-params]]
-=== Top level parameters for `<sparse_vector_field>`
+==== Top level parameters for `<sparse_vector_field>`
 
 `model_id`::::
 (Required, string) The ID of the model to use to convert the query text into token-weight pairs.
@@ -84,7 +84,7 @@ NOTE: The default values for `tokens_freq_ratio_threshold` and `tokens_weight_th
 
 [discrete]
 [[text-expansion-query-example]]
-=== Example ELSER query
+==== Example ELSER query
 
 The following is an example of the `text_expansion` query that references the ELSER model to perform semantic search.
 For a more detailed description of how to perform semantic search by using ELSER and the `text_expansion` query, refer to <<semantic-search-elser,this tutorial>>.
@@ -208,7 +208,7 @@ GET my-index/_search
 
 [discrete]
 [[text-expansion-query-with-pruning-config-and-rescore-example]]
-=== Example ELSER query with pruning configuration and rescore
+==== Example ELSER query with pruning configuration and rescore
 
 The following is an extension to the above example that adds a preview:[] pruning configuration to the `text_expansion` query.
 The pruning configuration identifies non-significant tokens to prune from the query in order to improve query performance.