diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df084d8f2572..a05aa09b0f13d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Query Insights] Add X-Opaque-Id to search request metadata for top n queries ([#13374](https://github.com/opensearch-project/OpenSearch/pull/13374)) - Add support for query level resource usage tracking ([#13172](https://github.com/opensearch-project/OpenSearch/pull/13172)) - Move Remote Store Migration from DocRep to GA and modify remote migration settings name ([#14100](https://github.com/opensearch-project/OpenSearch/pull/14100)) +- Derived field object type support ([#13720](https://github.com/opensearch-project/OpenSearch/pull/13720)) - [Query Insights] Add cpu and memory metrics to top n queries ([#13739](https://github.com/opensearch-project/OpenSearch/pull/13739)) ### Dependencies diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/10_derived_field_index_mapping_definition.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/10_derived_field_index_mapping_definition.yml new file mode 100644 index 0000000000000..4f700c3b83e8f --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/10_derived_field_index_mapping_definition.yml @@ -0,0 +1,421 @@ +"Test derived_field supported type using index mapping definition": + - skip: + version: " - 2.14.99" + reason: "derived_field feature was added in 2.15" + + - do: + indices.create: + index: test + body: + mappings: + properties: + text: + type: text + keyword: + type: keyword + long: + type: long + float: + type: float + double: + type: double + date: + type: date + geo: + type: geo_point + ip: + type: ip + boolean: + type: boolean + array_of_long: + type: long + json_field: + type: text + derived: + derived_text: + type: text + script: "emit(params._source[\"text\"])" + derived_text_prefilter_field: + type: text + script: "emit(params._source[\"text\"])" + prefilter_field: "text" + derived_keyword: + type: keyword + script: "emit(params._source[\"keyword\"])" + derived_long: + type: long + script: "emit(params._source[\"long\"])" + derived_float: + type: float + script: "emit(params._source[\"float\"])" + derived_double: + type: double + script: "emit(params._source[\"double\"])" + derived_date: + type: date + script: "emit(ZonedDateTime.parse(params._source[\"date\"]).toInstant().toEpochMilli())" + derived_geo: + type: geo_point + script: "emit(params._source[\"geo\"][0], params._source[\"geo\"][1])" + derived_ip: + type: ip + script: "emit(params._source[\"ip\"])" + derived_boolean: + type: boolean + script: "emit(params._source[\"boolean\"])" + derived_array_of_long: + type: long + script: "emit(params._source[\"array_of_long\"][0]);emit(params._source[\"array_of_long\"][1]);" + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + + - do: + index: + index: test + id: 1 + body: { + text: "peter piper", + keyword: "foo", + long: 1, + float: 1.0, + double: 1.0, + date: "2017-01-01T00:00:00Z", + geo: [0.0, 20.0], + ip: "192.168.0.1", + boolean: true, + array_of_long: [1, 2], + json_field: "{\"keyword\":\"json_keyword1\",\"long\":10,\"float\":10.0,\"double\":10.0,\"date\":\"2021-01-01T00:00:00Z\",\"ip\":\"10.0.0.1\",\"boolean\":true, \"array_of_long\": [1, 2]}}" + } + + - do: + index: + index: test + id: 2 + body: { + text: "piper picked a peck", + keyword: "bar", + long: 2, + float: 2.0, + double: 2.0, + date: "2017-01-02T00:00:00Z", + geo: [10.0, 30.0], + ip: "192.168.0.2", + boolean: false, + array_of_long: [2, 3], + json_field: "{\"keyword\":\"json_keyword2\",\"long\":20,\"float\":20.0,\"double\":20.0,\"date\":\"2021-02-01T00:00:00Z\",\"ip\":\"10.0.0.2\",\"boolean\":false, \"array_of_long\": [2, 3]}}" + } + + - do: + index: + index: test + id: 3 + body: { + text: "peck of pickled peppers", + keyword: "baz", + long: -3, + float: -3.0, + double: -3.0, + date: "2017-01-03T00:00:00Z", + geo: [20.0, 40.0], + ip: "192.168.0.3", + boolean: true, + array_of_long: [3, 4], + json_field: "{\"keyword\":\"json_keyword3\",\"long\":30,\"float\":30.0,\"double\":30.0,\"date\":\"2021-03-01T00:00:00Z\",\"ip\":\"10.0.0.3\",\"boolean\":true, \"array_of_long\": [3, 4]}" + } + + - do: + index: + index: test + id: 4 + body: { + text: "pickled peppers", + keyword: "qux", + long: 4, + float: 4.0, + double: 4.0, + date: "2017-01-04T00:00:00Z", + geo: [30.0, 50.0], + ip: "192.168.0.4", + boolean: false, + array_of_long: [4, 5], + json_field: "{\"keyword\":\"json_keyword4\",\"long\":40,\"float\":40.0,\"double\":40.0,\"date\":\"2021-04-01T00:00:00Z\",\"ip\":\"10.0.0.4\",\"boolean\":false, \"array_of_long\": [4, 5]}" + } + + - do: + index: + index: test + id: 5 + body: { + text: "peppers", + keyword: "quux", + long: 5, + float: 5.0, + double: 5.0, + date: "2017-01-05T00:00:00Z", + geo: [40.0, 60.0], + ip: "192.168.0.5", + boolean: true, + array_of_long: [5, 6], + json_field: "{\"keyword\":\"json_keyword5\",\"long\":50,\"float\":50.0,\"double\":50.0,\"date\":\"2021-05-01T00:00:00Z\",\"ip\":\"10.0.0.5\",\"boolean\":true, \"array_of_long\": [5, 6]}" + } + + - do: + indices.refresh: + index: [test] + + # Tests for derived_text + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + match_phrase: + derived_text: + query: "peter piper" + + - match: { hits.total: 1 } + + # Tests for derived_keyword + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + term: + derived_keyword: + value: "foo" + + - match: { hits.total: 1 } + + # Tests for derived_long + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_long: + gte: 1 + + - match: { hits.total: 4 } + + # Tests for derived_float + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_float: + gte: 1.0 + + - match: { hits.total: 4 } + + # Tests for derived_double + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_double: + gte: 1.0 + + - match: { hits.total: 4 } + + # Tests for derived_date + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_date: + gte: "2017-01-02" + + - match: { hits.total: 4 } + + # Tests for derived_geo + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + geo_distance: + distance: "20km" + derived_geo: + lat: 0.0 + lon: 20.0 + + - match: { hits.total: 1 } + + # Tests for derived_ip + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + term: + derived_ip: + value: "192.168.0.1" + + - match: { hits.total: 1 } + + # Tests for derived_boolean + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + term: + derived_boolean: + value: true + + - match: { hits.total: 3 } + + # Tests for derived_array_of_long + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_array_of_long: + gte: 3 + + - match: { hits.total: 4 } + + # Tests for derived_object.keyword + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + term: + derived_object.keyword: + value: "json_keyword1" + + - match: { hits.total: 1 } + + # Tests for derived_object.long + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_object.long: + gte: 11 + + - match: { hits.total: 4 } + + # Tests for derived_object.float + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_object.float: + gte: 10.1 + + - match: { hits.total: 4 } + + # Tests for derived_object.double + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_object.double: + gte: 10.1 + + - match: { hits.total: 4 } + + # Tests for derived_object.date + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_object.date: + gte: "2021-03-01" + + - match: { hits.total: 3 } + + # Tests for derived_object.ip + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + term: + derived_object.ip: + value: "10.0.0.1" + + - match: { hits.total: 1 } + + # Tests for derived_object.boolean + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + term: + derived_object.boolean: + value: true + + - match: { hits.total: 3 } + + # Tests for derived_object.array_of_long + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: + derived_object.array_of_long: + gte: 3 + + - match: { hits.total: 4 } + + # Tests for query string + - do: + search: + rest_total_hits_as_int: true + index: test + q: "derived_keyword:foo" + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test + q: derived_object.keyword:json_keyword1 + + - match: { hits.total: 1 } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/20_derived_field_put_mapping.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/20_derived_field_put_mapping.yml new file mode 100644 index 0000000000000..0370fd94e8548 --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/20_derived_field_put_mapping.yml @@ -0,0 +1,123 @@ +--- +"Test create and update mapping for derived fields": + - skip: + version: " - 2.14.99" + reason: "derived_field feature was added in 2.15" + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + text: + type: text + json_field: + type: text + derived: + derived_text: + type: text + script: "emit(params._source[\"text\"])" + derived_text_prefilter_field: + type: keyword + script: "emit(params._source[\"text\"])" + prefilter_field: "text" + derived_date: + type: date + script: "emit(params._source[\"keyword\"])" + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + + - do: + indices.get_mapping: + index: test_index + + - match: {test_index.mappings.derived.derived_text.type: text} + - match: {test_index.mappings.derived.derived_text_prefilter_field.type: keyword} + - match: {test_index.mappings.derived.derived_text_prefilter_field.prefilter_field: text} + - match: {test_index.mappings.derived.derived_date.type: date} + - match: {test_index.mappings.derived.derived_object.type: object} + - match: {test_index.mappings.derived.derived_object.properties.keyword: keyword} + - match: {test_index.mappings.derived.derived_object.prefilter_field: json_field} + + + - do: + indices.put_mapping: + index: test_index + body: + properties: + text: + type: text + json_field: + type: text + derived: + derived_text: + type: keyword + script: "emit(params._source[\"text\"])" + derived_text_prefilter_field: + type: text + script: "emit(params._source[\"text\"])" + prefilter_field: "text" + derived_date: + type: keyword + script: "emit(params._source[\"keyword\"])" + derived_object: + type: object + properties: + keyword: text + script: "emit(params._source[\"text\"])" + prefilter_field: "text" + format: "dd-MM-yyyy" + ignore_malformed: true + + - do: + indices.get_mapping: + index: test_index + + - match: {test_index.mappings.derived.derived_text.type: keyword} + - match: {test_index.mappings.derived.derived_text_prefilter_field.type: text} + - match: {test_index.mappings.derived.derived_text_prefilter_field.prefilter_field: text} + - match: {test_index.mappings.derived.derived_date.type: keyword} + - match: {test_index.mappings.derived.derived_object.type: object} + - match: {test_index.mappings.derived.derived_object.properties.keyword: text} + - match: {test_index.mappings.derived.derived_object.prefilter_field: text} + - match: {test_index.mappings.derived.derived_object.format: "dd-MM-yyyy"} + - match: {test_index.mappings.derived.derived_object.ignore_malformed: true} + + + - do: + indices.put_mapping: + index: test_index + body: + properties: + text: + type: text + json_field: + type: text + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + ignore_malformed: false + + - do: + indices.get_mapping: + index: test_index + + - match: {test_index.mappings.derived.derived_text.type: keyword} + - match: {test_index.mappings.derived.derived_text_prefilter_field.type: text} + - match: {test_index.mappings.derived.derived_text_prefilter_field.prefilter_field: text} + - match: {test_index.mappings.derived.derived_date.type: keyword} + - match: {test_index.mappings.derived.derived_object.type: object} + - match: {test_index.mappings.derived.derived_object.properties.keyword: keyword} + - match: {test_index.mappings.derived.derived_object.prefilter_field: json_field} + - is_false: test_index.mappings.derived.derived_object.ignore_malformed diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/30_derived_field_search_definition.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/30_derived_field_search_definition.yml new file mode 100644 index 0000000000000..bb619dce63010 --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/30_derived_field_search_definition.yml @@ -0,0 +1,489 @@ +"Test derived_field supported type using search definition": + - skip: + version: " - 2.14.99" + reason: "derived_field feature was added in 2.15" + + - do: + indices.create: + index: test + body: + mappings: + properties: + text: + type: text + keyword: + type: keyword + long: + type: long + float: + type: float + double: + type: double + date: + type: date + geo: + type: geo_point + ip: + type: ip + boolean: + type: boolean + array_of_long: + type: long + json_field: + type: text + + - do: + index: + index: test + id: 1 + body: { + text: "peter piper", + keyword: "foo", + long: 1, + float: 1.0, + double: 1.0, + date: "2017-01-01T00:00:00Z", + geo: [0.0, 20.0], + ip: "192.168.0.1", + boolean: true, + array_of_long: [1, 2], + json_field: "{\"keyword\":\"json_keyword1\",\"long\":10,\"float\":10.0,\"double\":10.0,\"date\":\"2021-01-01T00:00:00Z\",\"ip\":\"10.0.0.1\",\"boolean\":true, \"array_of_long\": [1, 2]}}" + } + + - do: + index: + index: test + id: 2 + body: { + text: "piper picked a peck", + keyword: "bar", + long: 2, + float: 2.0, + double: 2.0, + date: "2017-01-02T00:00:00Z", + geo: [10.0, 30.0], + ip: "192.168.0.2", + boolean: false, + array_of_long: [2, 3], + json_field: "{\"keyword\":\"json_keyword2\",\"long\":20,\"float\":20.0,\"double\":20.0,\"date\":\"2021-02-01T00:00:00Z\",\"ip\":\"10.0.0.2\",\"boolean\":false, \"array_of_long\": [2, 3]}}" + } + + - do: + index: + index: test + id: 3 + body: { + text: "peck of pickled peppers", + keyword: "baz", + long: -3, + float: -3.0, + double: -3.0, + date: "2017-01-03T00:00:00Z", + geo: [20.0, 40.0], + ip: "192.168.0.3", + boolean: true, + array_of_long: [3, 4], + json_field: "{\"keyword\":\"json_keyword3\",\"long\":30,\"float\":30.0,\"double\":30.0,\"date\":\"2021-03-01T00:00:00Z\",\"ip\":\"10.0.0.3\",\"boolean\":true, \"array_of_long\": [3, 4]}" + } + + - do: + index: + index: test + id: 4 + body: { + text: "pickled peppers", + keyword: "qux", + long: 4, + float: 4.0, + double: 4.0, + date: "2017-01-04T00:00:00Z", + geo: [30.0, 50.0], + ip: "192.168.0.4", + boolean: false, + array_of_long: [4, 5], + json_field: "{\"keyword\":\"json_keyword4\",\"long\":40,\"float\":40.0,\"double\":40.0,\"date\":\"2021-04-01T00:00:00Z\",\"ip\":\"10.0.0.4\",\"boolean\":false, \"array_of_long\": [4, 5]}" + } + + - do: + index: + index: test + id: 5 + body: { + text: "peppers", + keyword: "quux", + long: 5, + float: 5.0, + double: 5.0, + date: "2017-01-05T00:00:00Z", + geo: [40.0, 60.0], + ip: "192.168.0.5", + boolean: true, + array_of_long: [5, 6], + json_field: "{\"keyword\":\"json_keyword5\",\"long\":50,\"float\":50.0,\"double\":50.0,\"date\":\"2021-05-01T00:00:00Z\",\"ip\":\"10.0.0.5\",\"boolean\":true, \"array_of_long\": [5, 6]}" + } + + - do: + indices.refresh: + index: [test] + + # Tests for derived_text + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_text: + type: text + script: "emit(params._source[\"text\"])" + query: + match_phrase: + derived_text: + query: "peter piper" + + - match: { hits.total: 1 } + + # Tests for derived_keyword + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_keyword: + type: keyword + script: "emit(params._source[\"keyword\"])" + query: + term: + derived_keyword: + value: "foo" + + - match: { hits.total: 1 } + + # Tests for derived_long + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_long: + type: long + script: "emit(params._source[\"long\"])" + query: + range: + derived_long: + gte: 1 + + - match: { hits.total: 4 } + + # Tests for derived_float + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_float: + type: float + script: "emit(params._source[\"float\"])" + query: + range: + derived_float: + gte: 1.0 + + - match: { hits.total: 4 } + + # Tests for derived_double + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_double: + type: double + script: "emit(params._source[\"double\"])" + query: + range: + derived_double: + gte: 1.0 + + - match: { hits.total: 4 } + + # Tests for derived_date + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_date: + type: date + script: "emit(ZonedDateTime.parse(params._source[\"date\"]).toInstant().toEpochMilli())" + query: + range: + derived_date: + gte: "2017-01-02" + + - match: { hits.total: 4 } + + # Tests for derived_geo + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_geo: + type: geo_point + script: "emit(params._source[\"geo\"][0], params._source[\"geo\"][1])" + query: + geo_distance: + distance: "20km" + derived_geo: + lat: 0.0 + lon: 20.0 + + - match: { hits.total: 1 } + + # Tests for derived_ip + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_ip: + type: ip + script: "emit(params._source[\"ip\"])" + query: + term: + derived_ip: + value: "192.168.0.1" + + - match: { hits.total: 1 } + + # Tests for derived_boolean + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_boolean: + type: boolean + script: "emit(params._source[\"boolean\"])" + query: + term: + derived_boolean: + value: true + + - match: { hits.total: 3 } + + # Tests for derived_array_of_long + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_array_of_long: + type: long + script: "emit(params._source[\"array_of_long\"][0]);emit(params._source[\"array_of_long\"][1]);" + query: + range: + derived_array_of_long: + gte: 3 + + - match: { hits.total: 4 } + + # Tests for derived_object.keyword + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + query: + term: + derived_object.keyword: + value: "json_keyword1" + + - match: { hits.total: 1 } + + # Tests for derived_object.long + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + query: + range: + derived_object.long: + gte: 11 + + - match: { hits.total: 4 } + + # Tests for derived_object.float + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + query: + range: + derived_object.float: + gte: 10.1 + + - match: { hits.total: 4 } + + # Tests for derived_object.double + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + query: + range: + derived_object.double: + gte: 10.1 + + - match: { hits.total: 4 } + + # Tests for derived_object.date + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + query: + range: + derived_object.date: + gte: "2021-03-01" + + - match: { hits.total: 3 } + + # Tests for derived_object.ip + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + query: + term: + derived_object.ip: + value: "10.0.0.1" + + - match: { hits.total: 1 } + + # Tests for derived_object.boolean + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + query: + term: + derived_object.boolean: + value: true + + - match: { hits.total: 3 } + + # Tests for derived_object.array_of_long + - do: + search: + rest_total_hits_as_int: true + index: test + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + query: + range: + derived_object.array_of_long: + gte: 3 + + - match: { hits.total: 4 } + + # Tests for query string + - do: + search: + body: + derived: + derived_keyword: + type: keyword + script: "emit(params._source[\"keyword\"])" + rest_total_hits_as_int: true + index: test + q: "derived_keyword:foo" + + - match: { hits.total: 1 } + + - do: + search: + body: + derived: + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + rest_total_hits_as_int: true + index: test + q: derived_object.keyword:json_keyword1 + + - match: { hits.total: 1 } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/40_derived_field_fetch_and_highlight.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/40_derived_field_fetch_and_highlight.yml new file mode 100644 index 0000000000000..52a897c341419 --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/40_derived_field_fetch_and_highlight.yml @@ -0,0 +1,279 @@ +setup: + - skip: + version: " - 2.14.99" + reason: "derived_field feature was added in 2.15" + +--- +"Test basic field retrieval": + - do: + indices.create: + index: test + body: + mappings: + properties: + text: + type: text + keyword: + type: keyword + long: + type: long + float: + type: float + double: + type: double + date: + type: date + geo: + type: geo_point + ip: + type: ip + boolean: + type: boolean + array_of_long: + type: long + json_field: + type: text + derived: + derived_text: + type: text + script: "emit(params._source[\"text\"])" + derived_text_prefilter_field: + type: text + script: "emit(params._source[\"text\"])" + prefilter_field: "text" + derived_keyword: + type: keyword + script: "emit(params._source[\"keyword\"])" + derived_long: + type: long + script: "emit(params._source[\"long\"])" + derived_float: + type: float + script: "emit(params._source[\"float\"])" + derived_double: + type: double + script: "emit(params._source[\"double\"])" + derived_date: + type: date + script: "emit(ZonedDateTime.parse(params._source[\"date\"]).toInstant().toEpochMilli())" + derived_geo: + type: geo_point + script: "emit(params._source[\"geo\"][0], params._source[\"geo\"][1])" + derived_ip: + type: ip + script: "emit(params._source[\"ip\"])" + derived_boolean: + type: boolean + script: "emit(params._source[\"boolean\"])" + derived_array_of_long: + type: long + script: "emit(params._source[\"array_of_long\"][0]);emit(params._source[\"array_of_long\"][1]);" + derived_object: + type: object + properties: + keyword: keyword + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + format: "yyyy-MM-dd" + + - do: + index: + index: test + id: 1 + body: { + text: "peter piper", + keyword: "foo", + long: 1, + float: 1.0, + double: 1.0, + date: "2017-01-01T00:00:00Z", + geo: [0.0, 20.0], + ip: "192.168.0.1", + boolean: true, + array_of_long: [1, 2], + json_field: "{\"keyword\":\"json_keyword1\",\"long\":10,\"float\":10.0,\"double\":10.0,\"date\":\"2021-01-01T00:00:00Z\",\"ip\":\"10.0.0.1\",\"boolean\":true, \"array_of_long\": [1, 2]}}" + } + + - do: + index: + index: test + id: 2 + body: { + text: "piper picked a peck", + keyword: "bar", + long: 2, + float: 2.0, + double: 2.0, + date: "2017-01-02T00:00:00Z", + geo: [10.0, 30.0], + ip: "192.168.0.2", + boolean: false, + array_of_long: [2, 3], + json_field: "{\"keyword\":\"json_keyword2\",\"long\":20,\"float\":20.0,\"double\":20.0,\"date\":\"2021-02-01T00:00:00Z\",\"ip\":\"10.0.0.2\",\"boolean\":false, \"array_of_long\": [2, 3]}}" + } + + - do: + indices.refresh: + index: [test] + + - do: + search: + index: test + body: + fields: [derived_text, derived_keyword, derived_long, derived_float, derived_double, derived_date, derived_geo, derived_ip, derived_boolean, derived_array_of_long, + derived_object, derived_object.keyword, derived_object.long, derived_object.float, derived_object.double, derived_object.date, derived_object.ip, derived_object.boolean, derived_object.array_of_long] + + - is_true: hits.hits.0._id + - is_true: hits.hits.0._source + + - match: { hits.hits.0.fields.derived_text.0: "peter piper" } + - match: { hits.hits.0.fields.derived_keyword.0: foo } + - match: { hits.hits.0.fields.derived_long.0: 1 } + - match: { hits.hits.0.fields.derived_float.0: 1.0 } + - match: { hits.hits.0.fields.derived_double.0: 1 } + - match: { hits.hits.0.fields.derived_date.0: 2017-01-01T00:00:00.000Z } + - match: { hits.hits.0.fields.derived_geo.0.lat: 0.0 } + - match: { hits.hits.0.fields.derived_geo.0.lon: 20.0 } + - match: { hits.hits.0.fields.derived_ip.0: 192.168.0.1 } + - match: { hits.hits.0.fields.derived_array_of_long.0: 1 } + - match: { hits.hits.0.fields.derived_array_of_long.1: 2 } + - match: { hits.hits.0.fields.derived_object.0: "{\"keyword\":\"json_keyword1\",\"long\":10,\"float\":10.0,\"double\":10.0,\"date\":\"2021-01-01T00:00:00Z\",\"ip\":\"10.0.0.1\",\"boolean\":true, \"array_of_long\": [1, 2]}}" } + - match: { hits.hits.0.fields.derived_object\.keyword.0: json_keyword1 } + - match: { hits.hits.0.fields.derived_object\.long.0: 10 } + - match: { hits.hits.0.fields.derived_object\.float.0: 10.0 } + - match: { hits.hits.0.fields.derived_object\.double.0: 10.0 } + - match: { hits.hits.0.fields.derived_object\.date.0: 2021-01-01 } + - match: { hits.hits.0.fields.derived_object\.ip.0: 10.0.0.1 } + - match: { hits.hits.0.fields.derived_object\.boolean.0: true } + - match: { hits.hits.0.fields.derived_object\.array_of_long.0: 1 } + - match: { hits.hits.0.fields.derived_object\.array_of_long.1: 2 } + + - match: { hits.hits.1.fields.derived_text.0: "piper picked a peck" } + - match: { hits.hits.1.fields.derived_keyword.0: bar } + - match: { hits.hits.1.fields.derived_long.0: 2 } + - match: { hits.hits.1.fields.derived_float.0: 2.0 } + - match: { hits.hits.1.fields.derived_double.0: 2 } + - match: { hits.hits.1.fields.derived_date.0: 2017-01-02T00:00:00.000Z } + - match: { hits.hits.1.fields.derived_geo.0.lat: 10.0 } + - match: { hits.hits.1.fields.derived_geo.0.lon: 30.0 } + - match: { hits.hits.1.fields.derived_ip.0: 192.168.0.2 } + - match: { hits.hits.1.fields.derived_array_of_long.0: 2 } + - match: { hits.hits.1.fields.derived_array_of_long.1: 3 } + - match: { hits.hits.1.fields.derived_object.0: "{\"keyword\":\"json_keyword2\",\"long\":20,\"float\":20.0,\"double\":20.0,\"date\":\"2021-02-01T00:00:00Z\",\"ip\":\"10.0.0.2\",\"boolean\":false, \"array_of_long\": [2, 3]}}" } + - match: { hits.hits.1.fields.derived_object\.keyword.0: json_keyword2 } + - match: { hits.hits.1.fields.derived_object\.long.0: 20 } + - match: { hits.hits.1.fields.derived_object\.float.0: 20.0 } + - match: { hits.hits.1.fields.derived_object\.double.0: 20.0 } + - match: { hits.hits.1.fields.derived_object\.date.0: 2021-02-01 } + - match: { hits.hits.1.fields.derived_object\.ip.0: 10.0.0.2 } + - match: { hits.hits.1.fields.derived_object\.boolean.0: false } + - match: { hits.hits.1.fields.derived_object\.array_of_long.0: 2 } + - match: { hits.hits.1.fields.derived_object\.array_of_long.1: 3 } + + +--- +"Test highlight": + - do: + indices.create: + index: test + body: + mappings: + properties: + text: + type: text + array_of_text: + type: text + json_field: + type: text + derived: + derived_text: + type: text + script: "emit(params._source[\"text\"])" + derived_keyword: + type: keyword + script: "emit(params._source[\"keyword\"])" + derived_array_of_text: + type: text + script: "emit(params._source[\"array_of_text\"][0]);emit(params._source[\"array_of_text\"][1]);" + derived_object: + type: object + properties: + array_of_text: text + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + + - do: + index: + index: test + id: 1 + body: { + text: "peter piper", + keyword: "foo", + long: 1, + float: 1.0, + double: 1.0, + date: "2017-01-01T00:00:00Z", + geo: [0.0, 20.0], + ip: "192.168.0.1", + boolean: true, + array_of_text: ["The quick brown fox is brown", "The quick brown fox is black"], + json_field: "{\"keyword\":\"json_keyword1\",\"long\":10,\"float\":10.0,\"double\":10.0,\"date\":\"2021-01-01T00:00:00Z\",\"ip\":\"10.0.0.1\",\"boolean\":true, \"array_of_text\": [\"The quick brown fox is brown\", \"The quick brown fox is black\"]}}" + } + + - do: + index: + index: test + id: 2 + body: { + text: "piper picked a peck", + keyword: "bar", + long: 2, + float: 2.0, + double: 2.0, + date: "2017-01-02T00:00:00Z", + geo: [10.0, 30.0], + ip: "192.168.0.2", + boolean: false, + array_of_text: ["The quick brown fox is brown", "The quick brown fox is black"], + json_field: "{\"keyword\":\"json_keyword2\",\"long\":20,\"float\":20.0,\"double\":20.0,\"date\":\"2021-02-01T00:00:00Z\",\"ip\":\"10.0.0.2\",\"boolean\":false, \"array_of_text\": [\"The quick brown fox is brown\", \"The quick brown fox is black\"]}}" + } + + - do: + indices.refresh: + index: [test] + - do: + search: + rest_total_hits_as_int: true + body: { "query" : {"multi_match" : { "query" : "piper", "fields" : [ "derived_text"] } }, + "fields": [derived_text], + "highlight" : { "type" : "unified", "fields" : { "derived_text" : {} } } + } + + - match: {hits.hits.0.highlight.derived_text.0: "peter piper"} + + + - do: + search: + rest_total_hits_as_int: true + body: { "query" : {"multi_match" : { "query" : "quick brown", "fields" : [ "derived_array_of_text"] } }, + "fields": [derived_array_of_text], + "highlight" : { "type" : "unified", "fields" : { "derived_array_of_text" : {} } } + } + + - match: {hits.hits.0.highlight.derived_array_of_text.0: "The quick brown fox is brown"} + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + match_phrase: + derived_object.array_of_text: + query: "quick brown" + highlight: + type: unified + fields: + derived_object.array_of_text: {} + + - match: {hits.hits.0.highlight.derived_object\.array_of_text.0: "The quick brown fox is brown"} diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/50_derived_field_default_analyzer.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/50_derived_field_default_analyzer.yml new file mode 100644 index 0000000000000..e10c9cb3c133f --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/derived_fields/50_derived_field_default_analyzer.yml @@ -0,0 +1,105 @@ +--- +"Test default index analyzer simple is applied on derived fields": + - do: + indices.create: + index: test + body: + settings: + index.analysis.analyzer.default.type: simple + mappings: + properties: + text: + type: text + json_field: + type: text + derived: + derived_text: + type: text + script: "emit(params._source[\"text\"])" + derived_object: + type: object + properties: + array_of_text: text + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + + - do: + index: + index: test + id: 1 + body: { + text: "Email: example@example.com, Visit https://example.com for more info.", + json_field: "{\"array_of_text\": [\"Email: example@example.com, Visit https://example.com for more info.\", \"Email: example@example.com, Visit https://example.com for more info.\"]}}" + } + + - do: + indices.refresh: + index: [test] + - do: + search: + index: test + q: "derived_text:example.com" + analyzer: standard + + - match: { hits.total.value: 0 } + + - do: + search: + index: test + q: "derived_text:example.com" + analyzer: simple + + - match: { hits.total.value: 1 } + +--- +"Test default index analyzer standard is applied on derived fields": + - do: + indices.create: + index: test + body: + settings: + index.analysis.analyzer.default.type: standard + mappings: + properties: + text: + type: text + json_field: + type: text + derived: + derived_text: + type: text + script: "emit(params._source[\"text\"])" + derived_object: + type: object + properties: + array_of_text: text + script: "emit(params._source[\"json_field\"])" + prefilter_field: "json_field" + + - do: + index: + index: test + id: 1 + body: { + text: "Email: example@example.com, Visit https://example.com for more info.", + json_field: "{\"array_of_text\": [\"Email: example@example.com, Visit https://example.com for more info.\", \"Email: example@example.com, Visit https://example.com for more info.\"]}}" + } + + - do: + indices.refresh: + index: [test] + - do: + search: + index: test + q: "derived_object.array_of_text:example.com" + analyzer: standard + + - match: { hits.total.value: 1 } + + - do: + search: + index: test + q: "derived_object.array_of_text:example.com" + analyzer: simple + + - match: { hits.total.value: 1 }