From ea34a836def79a2eaaf62a3ae1d30ff4a1cb4343 Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana Date: Fri, 19 Jul 2024 10:40:13 -0700 Subject: [PATCH] Add support for custom date format and openSearch date format for date fields as part of Lucene query (#2762) Github Issue - https://github.com/opensearch-project/sql/issues/2700 Signed-off-by: Manasvini B S --- docs/user/general/datatypes.rst | 42 +++ .../data/type/OpenSearchDataType.java | 19 +- .../data/type/OpenSearchDateType.java | 63 ++++ .../value/OpenSearchExprValueFactory.java | 7 +- .../dsl/BucketAggregationBuilder.java | 6 +- .../script/filter/lucene/LuceneQuery.java | 307 +++++++++++------- .../script/filter/lucene/RangeQuery.java | 11 +- .../script/filter/lucene/TermQuery.java | 11 +- .../client/OpenSearchNodeClientTest.java | 2 +- .../client/OpenSearchRestClientTest.java | 2 +- .../data/type/OpenSearchDataTypeTest.java | 22 +- .../data/type/OpenSearchDateTypeTest.java | 229 +++++++++++-- .../storage/OpenSearchIndexTest.java | 2 +- .../dsl/BucketAggregationBuilderTest.java | 34 ++ .../script/filter/FilterQueryBuilderTest.java | 6 +- .../script/filter/lucene/LuceneQueryTest.java | 76 ++++- .../script/filter/lucene/RangeQueryTest.java | 67 +++- .../script/filter/lucene/TermQueryTest.java | 82 +++++ 18 files changed, 791 insertions(+), 197 deletions(-) create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index c423bd7b10..042e97396e 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -400,6 +400,48 @@ Querying such index will provide a response with ``schema`` block as shown below "status": 200 } +If the sql query contains an `IndexDateField` and a literal value with an operator (such as a term query or a range query), then the literal value can be in the `IndexDateField` format. + +.. code-block:: json + + { + "mappings" : { + "properties" : { + "release_date" : { + "type" : "date", + "format": "dd-MMM-yy" + } + } + } + } + +Querying such an `IndexDateField` (``release_date``) will provide a response with ``schema`` and ``datarows`` blocks as shown below. + +.. code-block:: json + + { + "query" : "SELECT release_date FROM test_index WHERE release_date = \"03-Jan-21\"" + } + +.. code-block:: json + + { + "schema": [ + { + "name": "release_date", + "type": "date" + } + ], + "datarows": [ + [ + "2021-01-03" + ] + ], + "total": 1, + "size": 1, + "status": 200 + } + String Data Types ================= diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index ddbba61260..c35eacfc72 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -62,19 +62,23 @@ public String toString() { @EqualsAndHashCode.Exclude @Getter protected MappingType mappingType; // resolved ExprCoreType - protected ExprCoreType exprCoreType; + @Getter protected ExprCoreType exprCoreType; /** * Get a simplified type {@link ExprCoreType} if possible. To avoid returning `UNKNOWN` for - * `OpenSearch*Type`s, e.g. for IP, returns itself. + * `OpenSearch*Type`s, e.g. for IP, returns itself. If the `exprCoreType` is {@link + * ExprCoreType#DATE}, {@link ExprCoreType#TIMESTAMP}, {@link ExprCoreType#TIME}, or {@link + * ExprCoreType#UNKNOWN}, it returns the current instance; otherwise, it returns `exprCoreType`. * * @return An {@link ExprType}. */ public ExprType getExprType() { - if (exprCoreType != ExprCoreType.UNKNOWN) { - return exprCoreType; - } - return this; + return (exprCoreType == ExprCoreType.DATE + || exprCoreType == ExprCoreType.TIMESTAMP + || exprCoreType == ExprCoreType.TIME + || exprCoreType == ExprCoreType.UNKNOWN) + ? this + : exprCoreType; } /** @@ -230,6 +234,9 @@ public String legacyTypeName() { if (mappingType == null) { return exprCoreType.typeName(); } + if (mappingType.toString().equalsIgnoreCase("DATE")) { + return exprCoreType.typeName(); + } return mappingType.toString().toUpperCase(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 7e6bee77c2..5ffce655d0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -11,11 +11,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAccessor; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -137,6 +142,11 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final String CUSTOM_FORMAT_DATE_SYMBOLS = "FecEWwYqQgdMLDyuG"; + private static final List OPENSEARCH_DEFAULT_FORMATTERS = + Stream.of("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis") + .map(DateFormatter::forPattern) + .toList(); + @EqualsAndHashCode.Exclude private final List formats; private OpenSearchDateType() { @@ -235,6 +245,59 @@ public List getAllCustomFormatters() { .collect(Collectors.toList()); } + /** + * Retrieves a list of custom formatters and OpenSearch named formatters defined by the user, and + * attempts to parse the given date/time string using these formatters. + * + * @param dateTime The date/time string to parse. + * @return A ZonedDateTime representing the parsed date/time in UTC, or null if parsing fails. + */ + public ZonedDateTime getParsedDateTime(String dateTime) { + List dateFormatters = + Stream.concat(this.getAllNamedFormatters().stream(), this.getAllCustomFormatters().stream()) + .collect(Collectors.toList()); + ZonedDateTime zonedDateTime = null; + + // check if dateFormatters are empty, then set default ones + if (dateFormatters.isEmpty()) { + dateFormatters = OPENSEARCH_DEFAULT_FORMATTERS; + } + // parse using OpenSearch DateFormatters + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(dateTime); + zonedDateTime = DateFormatters.from(accessor).withZoneSameLocal(ZoneOffset.UTC); + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + return zonedDateTime; + } + + /** + * Returns a formatted date string using the internal formatter, if available. + * + * @param accessor The TemporalAccessor object containing the date/time information. + * @return A formatted date string if a formatter is available, otherwise null. + */ + public String getFormattedDate(TemporalAccessor accessor) { + if (hasNoFormatter()) { + return OPENSEARCH_DEFAULT_FORMATTERS.get(0).format(accessor); + } + // Use the first available format string to create the formatter + return DateFormatter.forPattern(this.formats.get(0)).format(accessor); + } + + /** + * Checks if the formatter is not initialized. + * + * @return True if the formatter is not set, otherwise false. + */ + public boolean hasNoFormatter() { + return this.formats.isEmpty(); + } + /** * Retrieves a list of named formatters that format for dates. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 3341e01ab2..3cb182de5b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -230,7 +230,7 @@ private Optional type(String field) { private static ExprValue parseDateTimeString(String value, OpenSearchDateType dataType) { List formatters = dataType.getAllNamedFormatters(); formatters.addAll(dataType.getAllCustomFormatters()); - ExprCoreType returnFormat = (ExprCoreType) dataType.getExprType(); + ExprCoreType returnFormat = dataType.getExprCoreType(); for (DateFormatter formatter : formatters) { try { @@ -273,8 +273,7 @@ private static ExprValue parseDateTimeString(String value, OpenSearchDateType da private static ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; - ExprType returnFormat = dt.getExprType(); - + ExprCoreType returnFormat = dt.getExprCoreType(); if (value.isNumber()) { // isNumber var numFormatters = dt.getNumericNamedFormatters(); if (numFormatters.size() > 0 || !dt.hasFormats()) { @@ -287,7 +286,7 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) epochMillis = value.longValue(); } Instant instant = Instant.ofEpochMilli(epochMillis); - switch ((ExprCoreType) returnFormat) { + switch (returnFormat) { case TIME: return new ExprTimeValue(LocalTime.from(instant.atZone(ZoneOffset.UTC))); case DATE: diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index ff66ec425a..4488128b97 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -23,6 +23,7 @@ import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.span.SpanExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; /** Bucket Aggregation Builder. */ @@ -65,7 +66,10 @@ private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( .missingOrder(missingOrder) .order(sortOrder); // Time types values are converted to LONG in ExpressionAggregationScript::execute - if (List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { + if ((expr.getDelegated().type() instanceof OpenSearchDateType + && List.of(TIMESTAMP, TIME, DATE) + .contains(((OpenSearchDateType) expr.getDelegated().type()).getExprCoreType())) + || List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { sourceBuilder.userValuetypeHint(ValueType.LONG); } return helper.build(expr.getDelegated(), sourceBuilder::field, sourceBuilder::script); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index 11533c754e..c9ef5bcca5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -8,8 +8,9 @@ import static org.opensearch.sql.analysis.NestedAnalyzer.isNestedFunction; import com.google.common.collect.ImmutableMap; +import java.time.ZonedDateTime; import java.util.Map; -import java.util.function.Function; +import java.util.function.BiFunction; import org.opensearch.index.query.QueryBuilder; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; @@ -32,6 +33,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; /** Lucene query abstraction that builds Lucene query from function expression. */ public abstract class LuceneQuery { @@ -105,135 +107,164 @@ public QueryBuilder build(FunctionExpression func) { ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); Expression expr = func.getArguments().get(1); ExprValue literalValue = - expr instanceof LiteralExpression ? expr.valueOf() : cast((FunctionExpression) expr); + expr instanceof LiteralExpression ? expr.valueOf() : cast((FunctionExpression) expr, ref); return doBuild(ref.getAttr(), ref.type(), literalValue); } - private ExprValue cast(FunctionExpression castFunction) { + private ExprValue cast(FunctionExpression castFunction, ReferenceExpression ref) { return castMap .get(castFunction.getFunctionName()) - .apply((LiteralExpression) castFunction.getArguments().get(0)); + .apply((LiteralExpression) castFunction.getArguments().get(0), ref); } /** Type converting map. */ - private final Map> castMap = - ImmutableMap.>builder() - .put( - BuiltinFunctionName.CAST_TO_STRING.getName(), - expr -> { - if (!expr.type().equals(ExprCoreType.STRING)) { - return new ExprStringValue(String.valueOf(expr.valueOf().value())); - } else { - return expr.valueOf(); - } - }) - .put( - BuiltinFunctionName.CAST_TO_BYTE.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprByteValue(expr.valueOf().byteValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprByteValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprByteValue(Byte.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_SHORT.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprShortValue(expr.valueOf().shortValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprShortValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprShortValue(Short.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_INT.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprIntegerValue(expr.valueOf().integerValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprIntegerValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprIntegerValue(Integer.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_LONG.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprLongValue(expr.valueOf().longValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprLongValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprLongValue(Long.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_FLOAT.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprFloatValue(expr.valueOf().floatValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprFloatValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprFloatValue(Float.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_DOUBLE.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprDoubleValue(expr.valueOf().doubleValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprDoubleValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprDoubleValue(Double.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_BOOLEAN.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return expr.valueOf().doubleValue() != 0 - ? ExprBooleanValue.of(true) - : ExprBooleanValue.of(false); - } else if (expr.type().equals(ExprCoreType.STRING)) { - return ExprBooleanValue.of(Boolean.valueOf(expr.valueOf().stringValue())); - } else { - return expr.valueOf(); - } - }) - .put( - BuiltinFunctionName.CAST_TO_DATE.getName(), - expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprDateValue(expr.valueOf().stringValue()); - } else { - return new ExprDateValue(expr.valueOf().dateValue()); - } - }) - .put( - BuiltinFunctionName.CAST_TO_TIME.getName(), - expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimeValue(expr.valueOf().stringValue()); - } else { - return new ExprTimeValue(expr.valueOf().timeValue()); - } - }) - .put( - BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), - expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimestampValue(expr.valueOf().stringValue()); - } else { - return new ExprTimestampValue(expr.valueOf().timestampValue()); - } - }) - .build(); + private final Map> + castMap = + ImmutableMap + .> + builder() + .put( + BuiltinFunctionName.CAST_TO_STRING.getName(), + (expr, ref) -> { + if (!expr.type().equals(ExprCoreType.STRING)) { + return new ExprStringValue(String.valueOf(expr.valueOf().value())); + } else { + return expr.valueOf(); + } + }) + .put( + BuiltinFunctionName.CAST_TO_BYTE.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprByteValue(expr.valueOf().byteValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprByteValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprByteValue(Byte.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_SHORT.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprShortValue(expr.valueOf().shortValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprShortValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprShortValue(Short.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_INT.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprIntegerValue(expr.valueOf().integerValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprIntegerValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprIntegerValue(Integer.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_LONG.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprLongValue(expr.valueOf().longValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprLongValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprLongValue(Long.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_FLOAT.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprFloatValue(expr.valueOf().floatValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprFloatValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprFloatValue(Float.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_DOUBLE.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprDoubleValue(expr.valueOf().doubleValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprDoubleValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprDoubleValue(Double.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_BOOLEAN.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return expr.valueOf().doubleValue() != 0 + ? ExprBooleanValue.of(true) + : ExprBooleanValue.of(false); + } else if (expr.type().equals(ExprCoreType.STRING)) { + return ExprBooleanValue.of(Boolean.valueOf(expr.valueOf().stringValue())); + } else { + return expr.valueOf(); + } + }) + .put( + BuiltinFunctionName.CAST_TO_DATE.getName(), + (expr, ref) -> { + if (expr.type().equals(ExprCoreType.STRING)) { + ZonedDateTime zonedDateTime = getParsedDateTime(expr, ref); + if (zonedDateTime != null) { + return new ExprDateValue(zonedDateTime.toLocalDate()); + } + return new ExprDateValue(expr.valueOf().stringValue()); + } else { + return new ExprDateValue(expr.valueOf().dateValue()); + } + }) + .put( + BuiltinFunctionName.CAST_TO_TIME.getName(), + (expr, ref) -> { + if (expr.type().equals(ExprCoreType.STRING)) { + ZonedDateTime zonedDateTime = getParsedDateTime(expr, ref); + if (zonedDateTime != null) { + return new ExprTimeValue(zonedDateTime.toLocalTime()); + } + return new ExprTimeValue(expr.valueOf().stringValue()); + } else { + return new ExprTimeValue(expr.valueOf().timeValue()); + } + }) + .put( + BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), + (expr, ref) -> { + if (expr.type().equals(ExprCoreType.STRING)) { + ZonedDateTime zonedDateTime = getParsedDateTime(expr, ref); + if (zonedDateTime != null) { + return new ExprTimestampValue(zonedDateTime.toInstant()); + } + return new ExprTimestampValue(expr.valueOf().stringValue()); + } else { + return new ExprTimestampValue(expr.valueOf().timestampValue()); + } + }) + .build(); + + /** + * Parses the date/time from the given expression if the reference type is an instance of + * OpenSearchDateType. + * + * @param expr The expression to parse. + * @return The parsed ZonedDateTime or null if the conditions are not met. + */ + private ZonedDateTime getParsedDateTime(LiteralExpression expr, ReferenceExpression ref) { + if (ref.type() instanceof OpenSearchDateType) { + return ((OpenSearchDateType) ref.type()).getParsedDateTime(expr.valueOf().stringValue()); + } + return null; + } /** * Build method that subclass implements by default which is to build query from reference and @@ -248,4 +279,36 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l throw new UnsupportedOperationException( "Subclass doesn't implement this and build method either"); } + + /** + * Converts a literal value to a formatted date or time value based on the specified field type. + * + *

If the field type is an instance of {@link OpenSearchDateType}, this method checks the type + * of the literal value and converts it to a formatted date or time if necessary. The formatting + * is applied if the {@link OpenSearchDateType} has a formatter. Otherwise, the raw value is + * returned. + * + * @param literal the literal value to be converted + * @param fieldType the field type to determine the conversion logic + * @return the formatted date or time value if the field type requires it, otherwise the raw value + */ + protected Object value(ExprValue literal, ExprType fieldType) { + if (fieldType instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) fieldType; + if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + return openSearchDateType.hasNoFormatter() + ? literal.timestampValue().toEpochMilli() + : openSearchDateType.getFormattedDate(literal.timestampValue()); + } else if (literal.type().equals(ExprCoreType.DATE)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.dateValue()); + } else if (literal.type().equals(ExprCoreType.TIME)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.timeValue()); + } + } + return literal.value(); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java index 2e33e3cc7c..e9a38b6ee3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java @@ -10,7 +10,6 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; /** Lucene query that builds range query for non-quality comparison. */ @@ -30,7 +29,7 @@ public enum Comparison { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { - Object value = value(literal); + Object value = this.value(literal, fieldType); RangeQueryBuilder query = QueryBuilders.rangeQuery(fieldName); switch (comparison) { @@ -46,12 +45,4 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l throw new IllegalStateException("Comparison is supported by range query: " + comparison); } } - - private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { - return literal.timestampValue().toEpochMilli(); - } else { - return literal.value(); - } - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java index cd506898d7..f8988b3cd9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java @@ -8,7 +8,6 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; @@ -18,14 +17,6 @@ public class TermQuery extends LuceneQuery { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { fieldName = OpenSearchTextType.convertTextToKeyword(fieldName, fieldType); - return QueryBuilders.termQuery(fieldName, value(literal)); - } - - private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { - return literal.timestampValue().toEpochMilli(); - } else { - return literal.value(); - } + return QueryBuilders.termQuery(fieldName, this.value(literal, fieldType)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 040b7d2759..9da6e05e92 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -169,7 +169,7 @@ void get_index_mappings() throws IOException { () -> assertEquals(OpenSearchTextType.of(MappingType.Double), parsedTypes.get("balance")), () -> assertEquals("KEYWORD", mapping.get("city").legacyTypeName()), () -> assertEquals(OpenSearchTextType.of(MappingType.Keyword), parsedTypes.get("city")), - () -> assertEquals("DATE", mapping.get("birthday").legacyTypeName()), + () -> assertEquals("TIMESTAMP", mapping.get("birthday").legacyTypeName()), () -> assertEquals(OpenSearchTextType.of(MappingType.Date), parsedTypes.get("birthday")), () -> assertEquals("GEO_POINT", mapping.get("location").legacyTypeName()), () -> diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 99201aae4f..b83313de07 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -169,7 +169,7 @@ void get_index_mappings() throws IOException { () -> assertEquals(OpenSearchTextType.of(MappingType.Double), parsedTypes.get("balance")), () -> assertEquals("KEYWORD", mapping.get("city").legacyTypeName()), () -> assertEquals(OpenSearchTextType.of(MappingType.Keyword), parsedTypes.get("city")), - () -> assertEquals("DATE", mapping.get("birthday").legacyTypeName()), + () -> assertEquals("TIMESTAMP", mapping.get("birthday").legacyTypeName()), () -> assertEquals(OpenSearchTextType.of(MappingType.Date), parsedTypes.get("birthday")), () -> assertEquals("GEO_POINT", mapping.get("location").legacyTypeName()), () -> diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 82e6222dc4..76fbbd6e65 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -70,7 +70,7 @@ public void typeName() { assertEquals("STRING", textType.typeName()); assertEquals("STRING", textKeywordType.typeName()); assertEquals("OBJECT", OpenSearchDataType.of(MappingType.Object).typeName()); - assertEquals("DATE", OpenSearchDataType.of(MappingType.Date).typeName()); + assertEquals("TIMESTAMP", OpenSearchDataType.of(MappingType.Date).typeName()); assertEquals("DOUBLE", OpenSearchDataType.of(MappingType.Double).typeName()); assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).typeName()); } @@ -80,7 +80,7 @@ public void legacyTypeName() { assertEquals("TEXT", textType.legacyTypeName()); assertEquals("TEXT", textKeywordType.legacyTypeName()); assertEquals("OBJECT", OpenSearchDataType.of(MappingType.Object).legacyTypeName()); - assertEquals("DATE", OpenSearchDataType.of(MappingType.Date).legacyTypeName()); + assertEquals("TIMESTAMP", OpenSearchDataType.of(MappingType.Date).legacyTypeName()); assertEquals("DOUBLE", OpenSearchDataType.of(MappingType.Double).legacyTypeName()); assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).legacyTypeName()); } @@ -104,8 +104,8 @@ private static Stream getTestDataWithType() { Arguments.of(MappingType.ScaledFloat, "scaled_float", DOUBLE), Arguments.of(MappingType.Double, "double", DOUBLE), Arguments.of(MappingType.Boolean, "boolean", BOOLEAN), - Arguments.of(MappingType.Date, "date", TIMESTAMP), - Arguments.of(MappingType.DateNanos, "date", TIMESTAMP), + Arguments.of(MappingType.Date, "timestamp", TIMESTAMP), + Arguments.of(MappingType.DateNanos, "timestamp", TIMESTAMP), Arguments.of(MappingType.Object, "object", STRUCT), Arguments.of(MappingType.Nested, "nested", ARRAY), Arguments.of(MappingType.GeoPoint, "geo_point", OpenSearchGeoPointType.of()), @@ -124,7 +124,15 @@ public void of_MappingType(MappingType mappingType, String name, ExprType dataTy assertAll( () -> assertEquals(nameForPPL, type.typeName()), () -> assertEquals(nameForSQL, type.legacyTypeName()), - () -> assertEquals(dataType, type.getExprType())); + () -> { + if (dataType == ExprCoreType.TIMESTAMP + || dataType == ExprCoreType.DATE + || dataType == ExprCoreType.TIME) { + assertEquals(dataType, type.getExprCoreType()); + } else { + assertEquals(dataType, type.getExprType()); + } + }); } @ParameterizedTest(name = "{0}") @@ -133,7 +141,7 @@ public void of_ExprCoreType(ExprCoreType coreType) { assumeFalse(coreType == UNKNOWN); var type = OpenSearchDataType.of(coreType); if (type instanceof OpenSearchDateType) { - assertEquals(coreType, type.getExprType()); + assertEquals(coreType, type.getExprCoreType()); } else { assertEquals(coreType.toString(), type.typeName()); assertEquals(coreType.toString(), type.legacyTypeName()); @@ -416,7 +424,7 @@ public void test_getExprType() { assertEquals(FLOAT, OpenSearchDataType.of(MappingType.HalfFloat).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.Double).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.ScaledFloat).getExprType()); - assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprType()); + assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprCoreType()); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index c6885c8ffe..3c1cf1bf0f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -5,12 +5,7 @@ package org.opensearch.sql.opensearch.data.type; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; @@ -22,6 +17,9 @@ import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.isDateTypeCompatible; import com.google.common.collect.Lists; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.stream.Stream; @@ -48,8 +46,6 @@ class OpenSearchDateTypeTest { OpenSearchDateType.of(defaultFormatString); private static final OpenSearchDateType dateDateType = OpenSearchDateType.of(dateFormatString); private static final OpenSearchDateType timeDateType = OpenSearchDateType.of(timeFormatString); - private static final OpenSearchDateType datetimeDateType = - OpenSearchDateType.of(timestampFormatString); @Test public void isCompatible() { @@ -76,8 +72,8 @@ public void isCompatible() { public void check_typeName() { assertAll( // always use the MappingType of "DATE" - () -> assertEquals("DATE", defaultDateType.typeName()), - () -> assertEquals("DATE", timeDateType.typeName()), + () -> assertEquals("TIMESTAMP", defaultDateType.typeName()), + () -> assertEquals("TIME", timeDateType.typeName()), () -> assertEquals("DATE", dateDateType.typeName())); } @@ -85,8 +81,8 @@ public void check_typeName() { public void check_legacyTypeName() { assertAll( // always use the legacy "DATE" type - () -> assertEquals("DATE", defaultDateType.legacyTypeName()), - () -> assertEquals("DATE", timeDateType.legacyTypeName()), + () -> assertEquals("TIMESTAMP", defaultDateType.legacyTypeName()), + () -> assertEquals("TIME", timeDateType.legacyTypeName()), () -> assertEquals("DATE", dateDateType.legacyTypeName())); } @@ -94,9 +90,9 @@ public void check_legacyTypeName() { public void check_exprTypeName() { assertAll( // exprType changes based on type (no datetime): - () -> assertEquals(TIMESTAMP, defaultDateType.getExprType()), - () -> assertEquals(TIME, timeDateType.getExprType()), - () -> assertEquals(DATE, dateDateType.getExprType())); + () -> assertEquals(TIMESTAMP, defaultDateType.getExprCoreType()), + () -> assertEquals(TIME, timeDateType.getExprCoreType()), + () -> assertEquals(DATE, dateDateType.getExprCoreType())); } private static Stream getAllSupportedFormats() { @@ -129,22 +125,22 @@ public void check_datetime_format_names(FormatNames datetimeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, camelCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } String snakeCaseName = datetimeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, snakeCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } else { fail(); } @@ -161,18 +157,22 @@ public void check_date_format_names(FormatNames dateFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - camelCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = dateFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - snakeCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -189,18 +189,22 @@ public void check_time_format_names(FormatNames timeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - camelCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = timeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - snakeCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -244,9 +248,9 @@ private static Stream get_format_combinations_for_test() { @MethodSource("get_format_combinations_for_test") public void check_ExprCoreType_of_combinations_of_custom_and_predefined_formats( ExprCoreType expected, List formats, String testName) { - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); formats = Lists.reverse(formats); - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); } @Test @@ -259,4 +263,171 @@ public void check_if_date_type_compatible() { assertTrue(isDateTypeCompatible(DATE)); assertFalse(isDateTypeCompatible(OpenSearchDataType.of(OpenSearchDataType.MappingType.Text))); } + + @Test + void test_valid_timestamp_with_custom_format() { + String timestamp = "2021-11-08T17:00:00Z"; + String format = "strict_date_time_no_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_valid_timestamp_with_multiple_formats() { + String timestamp = "2021-11-08T17:00:00Z"; + String timestamp2 = "2021/11/08T17:00:00Z"; + + List formats = Arrays.asList("strict_date_time_no_millis", "yyyy/MM/dd'T'HH:mm:ssX"); + OpenSearchDateType dateType = OpenSearchDateType.of(String.join(" || ", formats)); + + // Testing with the first timestamp + ZonedDateTime zonedDateTime1 = dateType.getParsedDateTime(timestamp); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime1.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime1.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + + // Testing with the second timestamp + ZonedDateTime zonedDateTime2 = dateType.getParsedDateTime(timestamp2); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime2.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime2.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_openSearch_datetime_named_formatter() { + String timestamp = "2019-03-23T21:34:46"; + String format = "strict_date_hour_minute_second"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + + assertEquals("2019-03-23T21:34:46", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2019-03-23"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("21:34:46"), zonedDateTime.toLocalTime()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_openSearch_datetime_with_default_formatter() { + String timestamp = "2019-03-23T21:34:46"; + OpenSearchDateType dateType = OpenSearchDateType.of(TIMESTAMP); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + // formatted using OpenSearch default formatter + assertEquals("2019-03-23T21:34:46Z", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2019-03-23"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("21:34:46"), zonedDateTime.toLocalTime()); + assertTrue(dateType.hasNoFormatter()); + } + + @Test + void test_invalid_date_with_named_formatter() { + // Incorrect date + String timestamp = "2019-23-23"; + String format = "strict_date_hour_minute_second"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + assertNull(zonedDateTime); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_invalid_time_with_custom_formatter() { + String timestamp = "invalid-timestamp"; + List formats = Arrays.asList("yyyy/MM/dd'T'HH:mm:ssX", "yyyy-MM-dd'T'HH:mm:ssX"); + OpenSearchDateType dateType = OpenSearchDateType.of(String.join(" || ", formats)); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + assertNull(zonedDateTime); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_epoch_datetime_formatter() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + String format = "epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(String.valueOf(epochTimestamp)); + + assertEquals(Long.toString(epochTimestamp), dateType.getFormattedDate(zonedDateTime)); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("17:00:00"), zonedDateTime.toLocalTime()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_timeStamp_format_with_default_formatters() { + String timestamp = "2021-11-08 17:00:00"; + String format = "strict_date_time_no_millis || epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertNull(dateType.getParsedDateTime(timestamp)); + } + + @Test + void test_valid_date_with_custom_formatter() { + String dateString = "2021-11-08"; + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(expectedDate, parsedDate); + assertEquals("2021-11-08", dateType.getFormattedDate(parsedDate)); + } + + @Test + void test_valid_date_string_with_custom_formatter() { + String dateString = "03-Jan-21"; + String format = "dd-MMM-yy"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(LocalDate.parse("2021-01-03"), parsedDate); + assertEquals("03-Jan-21", dateType.getFormattedDate(parsedDate)); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_valid_date_with_multiple_formatters() { + String dateString = "2021-11-08"; + String format = "yyyy/MM/dd || yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyy-MM-dd")); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(expectedDate, parsedDate); + assertEquals("2021/11/08", dateType.getFormattedDate(parsedDate)); + } + + @Test + void test_valid_time_with_custom_formatter() { + String timeString = "12:10:30.000"; + String format = "HH:mm:ss.SSS"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern(format)); + LocalTime parsedTime = dateType.getParsedDateTime(timeString).toLocalTime(); + + assertEquals(expectedTime, parsedTime); + assertEquals("12:10:30.000", dateType.getFormattedDate(parsedTime)); + } + + @Test + void test_valid_time_with_multiple_formatters() { + String timeString = "12:10:30"; + String format = "HH:mm:ss.SSS || HH:mm:ss"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss")); + LocalTime parsedTime = dateType.getParsedDateTime(timeString).toLocalTime(); + + assertEquals(expectedTime, parsedTime); + assertEquals("12:10:30.000", dateType.getFormattedDate(parsedTime)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 3ddb07d86a..3ca566fac6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -148,7 +148,7 @@ void getFieldTypes() { hasEntry("gender", ExprCoreType.BOOLEAN), hasEntry("family", ExprCoreType.ARRAY), hasEntry("employer", ExprCoreType.STRUCT), - hasEntry("birthday", ExprCoreType.TIMESTAMP), + hasEntry("birthday", (ExprType) OpenSearchDataType.of(MappingType.Date)), hasEntry("id1", ExprCoreType.BYTE), hasEntry("id2", ExprCoreType.SHORT), hasEntry("blob", (ExprType) OpenSearchDataType.of(MappingType.Binary)))); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java index 4250b3297f..08c4017f1d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java @@ -40,6 +40,7 @@ import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.parse.ParseExpression; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @@ -134,6 +135,39 @@ void should_build_bucket_with_parse_expression() { buildQuery(Arrays.asList(asc(named("name", parseExpression))))); } + @Test + void terms_bucket_for_opensearchdate_type_uses_long() { + OpenSearchDateType dataType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); + + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"field\" : \"date\",\n" + + " \"missing_bucket\" : true,\n" + + " \"value_type\" : \"long\",\n" + + " \"missing_order\" : \"first\",\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery(Arrays.asList(asc(named("date", ref("date", dataType)))))); + } + + @Test + void terms_bucket_for_opensearchdate_type_uses_long_false() { + OpenSearchDateType dataType = OpenSearchDateType.of(STRING); + + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"field\" : \"date\",\n" + + " \"missing_bucket\" : true,\n" + + " \"missing_order\" : \"first\",\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery(Arrays.asList(asc(named("date", ref("date", dataType)))))); + } + @ParameterizedTest(name = "{0}") @EnumSource( value = ExprCoreType.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 90b982e017..bd2a9901ed 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -1772,9 +1772,9 @@ void cast_to_date_in_filter() { + " }\n" + " }\n" + "}"; - assertJsonEquals( json, buildQuery(DSL.equal(ref("date_value", DATE), DSL.castDate(literal("2021-11-08"))))); + assertJsonEquals( json, buildQuery( @@ -1821,7 +1821,7 @@ void cast_to_timestamp_in_filter() { "{\n" + " \"term\" : {\n" + " \"timestamp_value\" : {\n" - + " \"value\" : 1636390800000,\n" + + " \"value\" : \"2021-11-08 17:00:00\",\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" @@ -1847,7 +1847,7 @@ void cast_in_range_query() { "{\n" + " \"range\" : {\n" + " \"timestamp_value\" : {\n" - + " \"from\" : 1636390800000,\n" + + " \"from\" : \"2021-11-08 17:00:00\",\n" + " \"to\" : null," + " \"include_lower\" : false," + " \"include_upper\" : true," diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQueryTest.java index df3a730bad..1713d1dd1b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQueryTest.java @@ -8,18 +8,22 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.expression.DSL.literal; +import static org.opensearch.sql.expression.DSL.ref; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class LuceneQueryTest { @Test void should_not_support_single_argument_by_default() { - assertFalse(new LuceneQuery() {}.canSupport(DSL.abs(DSL.ref("age", INTEGER)))); + assertFalse(new LuceneQuery() {}.canSupport(DSL.abs(ref("age", INTEGER)))); } @Test @@ -27,4 +31,74 @@ void should_throw_exception_if_not_implemented() { assertThrows( UnsupportedOperationException.class, () -> new LuceneQuery() {}.doBuild(null, null, null)); } + + @Test + void should_cast_to_time_with_format() { + String format = "HH:mm:ss.SSS || HH:mm:ss"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal(ref("time_value", dateType), DSL.castTime(literal("17:00:00"))))); + } + + @Test + void should_cast_to_time_with_no_format() { + String format = "HH:mm"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal(ref("time_value", dateType), DSL.castTime(literal("17:00:00"))))); + } + + @Test + void should_cast_to_date_with_format() { + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal(ref("date_value", dateType), DSL.castDate(literal("2017-01-02"))))); + } + + @Test + void should_cast_to_date_with_no_format() { + String format = "yyyy/MM/dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal(ref("date_value", dateType), DSL.castDate(literal("2017-01-02"))))); + } + + @Test + void should_cast_to_timestamp_with_format() { + String format = "yyyy-MM-dd HH:mm:ss"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal( + ref("timestamp_value", dateType), + DSL.castTimestamp(literal("2021-11-08 17:00:00"))))); + } + + @Test + void should_cast_to_timestamp_with_no_format() { + String format = "2021/11/08T17:00:00Z"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + SemanticCheckException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal( + ref("timestamp_value", dateType), + DSL.castTimestamp(literal("2021-11-08 17:00:00 "))))); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java index ca87f42900..2f5482171d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java @@ -5,13 +5,17 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene; +import static org.junit.Assert.*; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import java.time.*; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; -import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.model.*; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.RangeQuery.Comparison; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -26,4 +30,65 @@ void should_throw_exception_for_unsupported_comparison() { new RangeQuery(Comparison.BETWEEN) .doBuild("name", STRING, ExprValueUtils.stringValue("John"))); } + + @Test + void test_timestamp_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); + assertNotNull( + new RangeQuery(Comparison.LT) + .doBuild("time", openSearchDateType, new ExprTimestampValue("2021-11-08 17:00:00"))); + } + + @Test + void test_timestamp_has_format() { + String timestamp = "2019-03-23 21:34:46"; + OpenSearchDateType dateType = OpenSearchDateType.of("yyyy-MM-dd HH:mm:ss"); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + ExprValue literal = ExprValueUtils.timestampValue(zonedDateTime.toInstant()); + assertNotNull(new RangeQuery(Comparison.LT).doBuild("time_stamp", dateType, literal)); + } + + @Test + void test_time_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.TIME); + assertNotNull( + new RangeQuery(Comparison.LT) + .doBuild("time", openSearchDateType, new ExprTimeValue("17:00:00"))); + } + + @Test + void test_time_has_format() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + String format = "epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(String.valueOf(epochTimestamp)); + ExprValue literal = ExprValueUtils.timeValue(zonedDateTime.toLocalTime()); + assertNotNull(new RangeQuery(Comparison.LT).doBuild("time", dateType, literal)); + } + + @Test + void test_date_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.DATE); + assertNotNull( + new RangeQuery(Comparison.LT) + .doBuild("date", openSearchDateType, new ExprDateValue("2021-11-08"))); + } + + @Test + void test_date_has_format() { + String dateString = "2021-11-08"; + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + ExprValue literal = ExprValueUtils.dateValue(parsedDate); + assertNotNull(new RangeQuery(Comparison.LT).doBuild("date", dateType, literal)); + } + + @Test + void test_non_date_field_type() { + String dateString = "2021-11-08"; + OpenSearchDateType dateType = OpenSearchDateType.of(STRING); + ExprValue literal = ExprValueUtils.stringValue(dateString); + assertNotNull(new RangeQuery(Comparison.LT).doBuild("string_value", dateType, literal)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java new file mode 100644 index 0000000000..def9fafba3 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java @@ -0,0 +1,82 @@ +package org.opensearch.sql.opensearch.storage.script.filter.lucene; + +import static org.junit.Assert.*; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.time.*; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.*; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class TermQueryTest { + + @Test + void test_timestamp_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); + assertNotNull( + new TermQuery() + .doBuild("time", openSearchDateType, new ExprTimestampValue("2021-11-08 17:00:00"))); + } + + @Test + void test_timestamp_has_format() { + String timestamp = "2019-03-23 21:34:46"; + OpenSearchDateType dateType = OpenSearchDateType.of("yyyy-MM-dd HH:mm:ss"); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + ExprValue literal = ExprValueUtils.timestampValue(zonedDateTime.toInstant()); + assertNotNull(new TermQuery().doBuild("time_stamp", dateType, literal)); + } + + @Test + void test_time_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.TIME); + assertNotNull( + new TermQuery().doBuild("time", openSearchDateType, new ExprTimeValue("17:00:00"))); + } + + @Test + void test_time_has_format() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + String format = "epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(String.valueOf(epochTimestamp)); + ExprValue literal = ExprValueUtils.timeValue(zonedDateTime.toLocalTime()); + assertNotNull(new TermQuery().doBuild("time", dateType, literal)); + } + + @Test + void test_date_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.DATE); + assertNotNull( + new TermQuery().doBuild("date", openSearchDateType, new ExprDateValue("2021-11-08"))); + } + + @Test + void test_date_has_format() { + String dateString = "2021-11-08"; + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + ExprValue literal = ExprValueUtils.dateValue(parsedDate); + assertNotNull(new TermQuery().doBuild("date", dateType, literal)); + } + + @Test + void test_invalid_date_field_type() { + String dateString = "2021-11-08"; + OpenSearchDateType dateType = OpenSearchDateType.of(STRING); + ExprValue literal = ExprValueUtils.stringValue(dateString); + assertNotNull(new TermQuery().doBuild("string_value", dateType, literal)); + } + + @Test + void test_string_field_type() { + String dateString = "2021-11-08"; + ExprValue literal = ExprValueUtils.stringValue(dateString); + assertNotNull(new TermQuery().doBuild("string_value", STRING, literal)); + } +}