diff --git a/core/build.gradle b/core/build.gradle index 0c44b0780d..655e7d92c2 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -56,7 +56,6 @@ dependencies { api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' - api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" api project(':common') testImplementation('org.junit.jupiter:junit-jupiter:5.9.3') diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java index 92883e604a..394535750b 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java @@ -15,15 +15,10 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.time.temporal.TemporalAccessor; -import java.util.List; import lombok.RequiredArgsConstructor; -import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.utils.DateTimeFormatters; /** Expression Date Value. */ @RequiredArgsConstructor @@ -31,41 +26,10 @@ public class ExprDateValue extends AbstractExprValue { private final LocalDate date; - /** Formatted date using user defined/default format */ - private String formattedDate; - /** Constructor of ExprDateValue. */ public ExprDateValue(String date) { - this(date, List.of()); - } - - /** Constructor of ExprDateValue to support custom/OpenSearch date formats in mappings. */ - public ExprDateValue(String date, List dateFormatters) { try { - LocalDate localDate = null; - // check if dateFormatters are empty, then set default ones - if (dateFormatters == null || dateFormatters.isEmpty()) { - dateFormatters = DateTimeFormatters.initializeDateFormatters(); - } - // parse using OpenSearch DateFormatters - for (DateFormatter formatter : dateFormatters) { - try { - TemporalAccessor accessor = formatter.parse(date); - ZonedDateTime zonedDateTime = DateFormatters.from(accessor); - localDate = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toLocalDate(); - - this.formattedDate = formatter.format(accessor); - break; - } catch (IllegalArgumentException ignored) { - // nothing to do, try another format - } - } - if (localDate == null) { - // Default constructor behavior, parse using java DateTimeFormatter - localDate = LocalDate.parse(date, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); - } - this.date = localDate; - + this.date = LocalDate.parse(date, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); } catch (DateTimeParseException e) { throw new SemanticCheckException( String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", date)); @@ -74,10 +38,7 @@ public ExprDateValue(String date, List dateFormatters) { @Override public String value() { - if (this.formattedDate == null) { - return DateTimeFormatter.ISO_LOCAL_DATE.format(date); - } - return this.formattedDate; + return DateTimeFormatter.ISO_LOCAL_DATE.format(date); } @Override diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java index ec32e64a7d..ad278160c6 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java @@ -14,17 +14,12 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; -import java.time.temporal.TemporalAccessor; -import java.util.List; import java.util.Objects; import lombok.RequiredArgsConstructor; -import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionProperties; -import org.opensearch.sql.utils.DateTimeFormatters; /** Expression Time Value. */ @RequiredArgsConstructor @@ -32,39 +27,10 @@ public class ExprTimeValue extends AbstractExprValue { private final LocalTime time; - /** Formatted time using user defined/default format */ - private String formattedTime; - /** Constructor of ExprTimeValue. */ public ExprTimeValue(String time) { - this(time, List.of()); - } - - /** Constructor of ExprTimeValue to support custom/OpenSearch date formats in mappings. */ - public ExprTimeValue(String time, List dateFormatters) { try { - LocalTime localTime = null; - // check if dateFormatters are empty, then set default ones - if (dateFormatters == null || dateFormatters.isEmpty()) { - dateFormatters = DateTimeFormatters.initializeDateFormatters(); - } - // parse using OpenSearch DateFormatters - for (DateFormatter formatter : dateFormatters) { - try { - TemporalAccessor accessor = formatter.parse(time); - ZonedDateTime zonedDateTime = DateFormatters.from(accessor); - localTime = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toLocalTime(); - this.formattedTime = formatter.format(accessor); - break; - } catch (IllegalArgumentException ignored) { - // nothing to do, try another format - } - } - if (localTime == null) { - // Default constructor behavior, parse using java DateTimeFormatter - localTime = LocalTime.parse(time, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); - } - this.time = localTime; + this.time = LocalTime.parse(time, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); } catch (DateTimeParseException e) { throw new SemanticCheckException( String.format("time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", time)); @@ -73,10 +39,7 @@ public ExprTimeValue(String time, List dateFormatters) { @Override public String value() { - if (this.formattedTime == null) { - return ISO_LOCAL_TIME.format(time); - } - return this.formattedTime; + return ISO_LOCAL_TIME.format(time); } @Override diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java index 3176a96f5f..666550377a 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java @@ -12,19 +12,13 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalAccessor; -import java.util.List; import java.util.Objects; import lombok.RequiredArgsConstructor; -import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.utils.DateTimeFormatters; /** Expression Timestamp Value. */ @RequiredArgsConstructor @@ -32,44 +26,13 @@ public class ExprTimestampValue extends AbstractExprValue { private final Instant timestamp; - /** Formatted dateTime using user defined/default format */ - private String formattedDateTime; - /** Constructor. */ public ExprTimestampValue(String timestamp) { - this(timestamp, List.of()); - } - - /** - * Constructor of ExprTimestampValue to support custom/OpenSearch dateTime formats in mappings. - */ - public ExprTimestampValue(String timestamp, List dateFormatters) { try { - Instant localDateTime = null; - // check if dateFormatters are empty, then set default ones - if (dateFormatters == null || dateFormatters.isEmpty()) { - dateFormatters = DateTimeFormatters.initializeDateFormatters(); - } - // parse using OpenSearch DateFormatters - for (DateFormatter formatter : dateFormatters) { - try { - TemporalAccessor accessor = formatter.parse(timestamp); - ZonedDateTime zonedDateTime = DateFormatters.from(accessor); - localDateTime = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toInstant(); - this.formattedDateTime = formatter.format(accessor); - break; - } catch (IllegalArgumentException ignored) { - // nothing to do, try another format - } - } - if (localDateTime == null) { - // Default constructor behavior, parse using java DateTimeFormatter - localDateTime = - LocalDateTime.parse(timestamp, DATE_TIME_FORMATTER_VARIABLE_NANOS) - .atZone(ZoneOffset.UTC) - .toInstant(); - } - this.timestamp = localDateTime; + this.timestamp = + LocalDateTime.parse(timestamp, DATE_TIME_FORMATTER_VARIABLE_NANOS) + .atZone(ZoneOffset.UTC) + .toInstant(); } catch (DateTimeParseException e) { throw new SemanticCheckException( String.format( @@ -81,23 +44,15 @@ public ExprTimestampValue(String timestamp, List dateFormatters) /** localDateTime Constructor. */ public ExprTimestampValue(LocalDateTime localDateTime) { this.timestamp = localDateTime.atZone(ZoneOffset.UTC).toInstant(); - this.formattedDateTime = null; - } - - public boolean hasNoFormatter() { - return this.formattedDateTime == null; } @Override public String value() { - if (this.formattedDateTime == null) { - return timestamp.getNano() == 0 - ? DATE_TIME_FORMATTER_WITHOUT_NANO - .withZone(ZoneOffset.UTC) - .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) - : DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp); - } - return this.formattedDateTime; + return timestamp.getNano() == 0 + ? DATE_TIME_FORMATTER_WITHOUT_NANO + .withZone(ZoneOffset.UTC) + .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) + : DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp); } @Override diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 24a3e20349..18e6541514 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -18,12 +18,8 @@ import java.time.format.ResolverStyle; import java.time.format.SignStyle; import java.time.temporal.ChronoField; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.Locale; import lombok.experimental.UtilityClass; -import org.opensearch.common.time.DateFormatter; /** DateTimeFormatter. Reference org.opensearch.common.time.DateFormatters. */ @UtilityClass @@ -47,9 +43,6 @@ public class DateTimeFormatters { private static final int MIN_FRACTION_SECONDS = 0; private static final int MAX_FRACTION_SECONDS = 9; - public static final List OPENSEARCH_DEFAULT_FORMATS = - Arrays.asList("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis"); - public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder() .appendOffset("+HHmm", "Z") @@ -116,9 +109,6 @@ public class DateTimeFormatters { public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - public static final DateTimeFormatter SQL_LITERAL_DEFAULT_DATE_TIME_FORMAT = - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ"); - public static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) @@ -216,12 +206,4 @@ public class DateTimeFormatters { .appendFraction( ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) .toFormatter(); - - public static List initializeDateFormatters() { - List dateFormatters = new ArrayList<>(); - for (String pattern : OPENSEARCH_DEFAULT_FORMATS) { - dateFormatters.add(DateFormatter.forPattern(pattern)); - } - return dateFormatters; - } } diff --git a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java index 88336a58c1..aef3ad368e 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java @@ -7,23 +7,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; import org.junit.jupiter.api.Test; -import org.opensearch.common.time.DateFormatter; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionProperties; @@ -111,379 +104,16 @@ public void dateInUnsupportedFormat() { } @Test - void testValidTimestampWithCustomFormatter() { - String timestamp = "2021-11-08T17:00:00Z"; - DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); - ExprTimestampValue timestampValue = - new ExprTimestampValue(timestamp, Collections.singletonList(formatter)); - - assertEquals("2021-11-08T17:00:00Z", timestampValue.value()); - assertEquals(LocalDate.parse("2021-11-08"), timestampValue.dateValue()); - assertEquals(LocalTime.parse("17:00:00"), timestampValue.timeValue()); - assertEquals(TIMESTAMP, timestampValue.type()); - assertEquals( - ZonedDateTime.of(LocalDateTime.parse("2021-11-08T17:00:00"), ZoneOffset.UTC).toInstant(), - timestampValue.timestampValue()); - assertEquals("TIMESTAMP '2021-11-08T17:00:00Z'", timestampValue.toString()); - assertEquals( - LocalDateTime.parse("2021-11-08T17:00:00"), - LocalDateTime.ofInstant(timestampValue.timestampValue(), ZoneOffset.UTC)); - assertThrows( - ExpressionEvaluationException.class, - () -> integerValue(1).timestampValue(), - "invalid to get timestampValue from value of type INTEGER"); - } - - @Test - void testValidTimestampWithMultipleFormatters() { - String timestamp = "2021-11-08T17:00:00Z"; - DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd'T'HH:mm:ssX"); - DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); - - ExprTimestampValue timestampValue = - new ExprTimestampValue(timestamp, Arrays.asList(formatter1, formatter2)); - - assertEquals("2021-11-08T17:00:00Z", timestampValue.value()); - assertEquals(LocalDate.parse("2021-11-08"), timestampValue.dateValue()); - assertEquals(LocalTime.parse("17:00:00"), timestampValue.timeValue()); - assertEquals(TIMESTAMP, timestampValue.type()); - - String timestamp2 = "2021/11/08T17:00:00Z"; - - ExprTimestampValue timestampValue2 = - new ExprTimestampValue(timestamp2, Arrays.asList(formatter1, formatter2)); - - assertEquals("2021/11/08T17:00:00Z", timestampValue2.value()); - assertEquals(LocalDate.parse("2021-11-08"), timestampValue2.dateValue()); - assertEquals(LocalTime.parse("17:00:00"), timestampValue2.timeValue()); - assertEquals(TIMESTAMP, timestampValue2.type()); - } - - @Test - void testEmptyFormatterListForTimeStamp() { - String timestamp = "2021-11-08T17:00:00"; - Instant expectedDateTime = - ZonedDateTime.of(2021, 11, 8, 17, 0, 0, 0, ZoneOffset.UTC).toInstant(); - - // Test with null formatter list - ExprTimestampValue valueWithNullFormatter = new ExprTimestampValue(timestamp, null); - assertEquals(expectedDateTime, valueWithNullFormatter.timestampValue()); - assertEquals("2021-11-08T17:00:00.000Z", valueWithNullFormatter.value()); - - // Test with empty formatter list - ExprTimestampValue valueWithEmptyFormatter = - new ExprTimestampValue(timestamp, Collections.emptyList()); - assertEquals(expectedDateTime, valueWithEmptyFormatter.timestampValue()); - assertEquals("2021-11-08T17:00:00.000Z", valueWithEmptyFormatter.value()); - } - - @Test - void testOpenSearchDateTimeNamedFormatter() { - String timestamp = "2019-03-23T21:34:46"; - DateFormatter formatter = DateFormatter.forPattern("strict_date_hour_minute_second"); - ExprTimestampValue value = - new ExprTimestampValue(timestamp, Collections.singletonList(formatter)); - - assertEquals("2019-03-23T21:34:46", value.value()); - assertEquals(LocalDate.parse("2019-03-23"), value.dateValue()); - assertEquals(LocalTime.parse("21:34:46"), value.timeValue()); - assertEquals(TIMESTAMP, value.type()); - } - - @Test - void testInvalidTimestamp() { - String timestamp = "invalid-timestamp"; - DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd'T'HH:mm:ssX"); - DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); - - List formatters = Arrays.asList(formatter1, formatter2); - - try { - new ExprTimestampValue(timestamp, formatters); - } catch (SemanticCheckException e) { - assertEquals( - String.format( - "timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", - timestamp), - e.getMessage()); - } - } - - @Test - void testEpochDateTimeFormatter() { - long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" - DateFormatter formatter = DateFormatter.forPattern("epoch_millis"); - - ExprTimestampValue value = - new ExprTimestampValue(Long.toString(epochTimestamp), Collections.singletonList(formatter)); - assertEquals("1636390800000", value.value()); - assertEquals(LocalDate.parse("2021-11-08"), value.dateValue()); - assertEquals(LocalTime.parse("17:00:00"), value.timeValue()); - assertEquals(TIMESTAMP, value.type()); - } - - @Test - void testValidTimeStampWithDefaultFormatters() { - String timestamp = "2021-11-08 17:00:00"; - Instant expectedDateTime = - ZonedDateTime.of(2021, 11, 8, 17, 0, 0, 0, ZoneOffset.UTC).toInstant(); - - // Test with null formatter list - ExprTimestampValue valueWithNullFormatter = new ExprTimestampValue(timestamp, null); - assertEquals(expectedDateTime, valueWithNullFormatter.timestampValue()); - assertEquals("2021-11-08 17:00:00", valueWithNullFormatter.value()); - - // Test with empty formatter list - ExprTimestampValue valueWithEmptyFormatter = - new ExprTimestampValue(timestamp, Collections.emptyList()); - assertEquals(expectedDateTime, valueWithEmptyFormatter.timestampValue()); - assertEquals("2021-11-08 17:00:00", valueWithEmptyFormatter.value()); - } - - @Test - void testValidTimeStampHasNoFormatters() { - String timestamp = "2021-11-08 17:00:00"; - Instant expectedDateTime = - ZonedDateTime.of(2021, 11, 8, 17, 0, 0, 0, ZoneOffset.UTC).toInstant(); - - // Test with null formatter list - ExprTimestampValue valueWithNullFormatter = new ExprTimestampValue(timestamp, null); - assertEquals(expectedDateTime, valueWithNullFormatter.timestampValue()); - assertEquals("2021-11-08 17:00:00", valueWithNullFormatter.value()); - assertTrue(valueWithNullFormatter.hasNoFormatter()); - } - - @Test - void testValidDateWithCustomFormatter() { - String dateString = "2021-11-08"; - LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); - DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd"); - ExprDateValue value = new ExprDateValue(dateString, Collections.singletonList(formatter)); - assertEquals(expectedDate, value.dateValue()); - assertEquals("2021-11-08", value.value()); - } - - @Test - void testValidDateWithMultipleFormatters() { - String dateString = "2021-11-08"; - LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyy-MM-dd")); - DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd"); - DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd"); - - ExprDateValue value = new ExprDateValue(dateString, List.of(formatter1, formatter2)); - - assertEquals(expectedDate, value.dateValue()); - assertEquals("2021-11-08", value.value()); - } - - @Test - void testInvalidDate() { - String dateString = "invalid-date"; - DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd"); - DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd"); - - try { - new ExprDateValue(dateString, List.of(formatter1, formatter2)); - } catch (SemanticCheckException e) { - assertEquals( - String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", dateString), - e.getMessage()); - } - } - - @Test - void testEmptyFormatterListForDate() { - String dateString = "2021-11-08"; - LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_LOCAL_DATE); - - // Test with null formatter list - ExprDateValue valueWithNullFormatter = new ExprDateValue(dateString, null); - assertEquals(expectedDate, valueWithNullFormatter.dateValue()); - // Formatted to default strict_date_hour_minute_second format - assertEquals("2021-11-08T00:00:00.000Z", valueWithNullFormatter.value()); - - // Test with empty formatter list - ExprDateValue valueWithEmptyFormatter = new ExprDateValue(dateString, Collections.emptyList()); - assertEquals(expectedDate, valueWithEmptyFormatter.dateValue()); - // Formatted to default strict_date_hour_minute_second format - assertEquals("2021-11-08T00:00:00.000Z", valueWithEmptyFormatter.value()); - } - - @Test - void testEpochMillisFormat() { - String dateString = "1636358400000"; // Epoch millis for 2021-11-08 - LocalDate expectedDate = LocalDate.of(2021, 11, 8); - - ExprDateValue value = new ExprDateValue(dateString, null); - assertEquals(expectedDate, value.dateValue()); - assertEquals("1636358400000", value.value()); - } - - @Test - void testInvalidDateWithFallbackDefaultFormatter() { - String dateString = "invalid-date"; - DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd"); - DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd"); - - // Test with invalid date and fallback formatter also fails - Exception exception = - assertThrows( - SemanticCheckException.class, - () -> { - new ExprDateValue(dateString, List.of(formatter1, formatter2)); - }); - assertEquals( - String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", dateString), - exception.getMessage()); - } - - @Test - void testInvalidDateThrowsException() { - String dateString = "invalid-date"; - - Exception exception = - assertThrows( - SemanticCheckException.class, - () -> { - new ExprDateValue(dateString, null); - }); - assertEquals( - String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", dateString), - exception.getMessage()); - } - - @Test - void testValidDateWithDefaultFormatters() { - String dateString = "2021-11-08 00:00:00"; - LocalDate expectedDate = LocalDate.of(2021, 11, 8); - - // Test with null formatter list - ExprDateValue valueWithNullFormatter = new ExprDateValue(dateString, null); - assertEquals(expectedDate, valueWithNullFormatter.dateValue()); - // Formatted to default strict_date_optional_time format - assertEquals("2021-11-08", valueWithNullFormatter.value()); - - // Test with empty formatter list - ExprDateValue valueWithEmptyFormatter = new ExprDateValue(dateString, Collections.emptyList()); - assertEquals(expectedDate, valueWithEmptyFormatter.dateValue()); - // Formatted to default strict_date_optional_time format - assertEquals("2021-11-08", valueWithEmptyFormatter.value()); - } - - @Test - void testValidTimeWithCustomFormatter() { - String timeString = "12:10:30.000"; - LocalTime expectedTime = - LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss.SSS")); - DateFormatter formatter = DateFormatter.forPattern("HH:mm:ss.SSS"); - - ExprTimeValue value = new ExprTimeValue(timeString, Collections.singletonList(formatter)); - - assertEquals(expectedTime, value.timeValue()); - assertEquals("12:10:30.000", value.value()); - assertEquals("TIME '12:10:30'", value.toString()); - } - - @Test - void testValidTimeWithMultipleFormatters() { - String timeString = "12:10:30"; - LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss")); - DateFormatter formatter1 = DateFormatter.forPattern("HH:mm:ss.SSS"); - DateFormatter formatter2 = DateFormatter.forPattern("HH:mm:ss"); - - ExprTimeValue value = new ExprTimeValue(timeString, List.of(formatter1, formatter2)); - - assertEquals(expectedTime, value.timeValue()); - assertEquals("12:10:30", value.value()); - } - - @Test - void testInvalidTime() { - String timeString = "invalid-time"; - DateFormatter formatter1 = DateFormatter.forPattern("HH:mm:ss.SSS"); - DateFormatter formatter2 = DateFormatter.forPattern("HH:mm:ss"); - - try { - new ExprTimeValue(timeString, List.of(formatter1, formatter2)); - } catch (SemanticCheckException e) { - assertEquals( - String.format( - "time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", timeString), - e.getMessage()); - } - } - - @Test - void testEmptyFormatterListForTime() { - String timeString = "12:10:30"; - LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ISO_LOCAL_TIME); - - // Test with null formatter list - ExprTimeValue valueWithNullFormatter = new ExprTimeValue(timeString, null); - assertEquals(expectedTime, valueWithNullFormatter.timeValue()); - assertEquals("12:10:30", valueWithNullFormatter.value()); - - // Test with empty formatter list - ExprTimeValue valueWithEmptyFormatter = new ExprTimeValue(timeString, Collections.emptyList()); - assertEquals(expectedTime, valueWithEmptyFormatter.timeValue()); - assertEquals("12:10:30", valueWithEmptyFormatter.value()); - } - - @Test - void testEpochTimeFormatter() { - long epochMilli = 1420070400000L; // epoch time in milliseconds - LocalTime expectedTime = - ZonedDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC).toLocalTime(); - DateFormatter epochFormatter = DateFormatter.forPattern("epoch_millis"); - - ExprTimeValue value = - new ExprTimeValue(String.valueOf(epochMilli), Collections.singletonList(epochFormatter)); - - assertEquals(expectedTime, value.timeValue()); - assertEquals(String.valueOf(epochMilli), value.value()); - assertEquals(TIME, value.type()); - assertTrue(value.isDateTime()); - assertEquals("TIME '00:00:00'", value.toString()); - - var exception = assertThrows(ExpressionEvaluationException.class, value::dateValue); - assertEquals("invalid to get dateValue from value of type TIME", exception.getMessage()); - exception = assertThrows(ExpressionEvaluationException.class, value::timestampValue); - assertEquals("invalid to get timestampValue from value of type TIME", exception.getMessage()); - exception = - assertThrows(ExpressionEvaluationException.class, () -> integerValue(1).timeValue()); - assertEquals("invalid to get timeValue from value of type INTEGER", exception.getMessage()); - - var functionProperties = new FunctionProperties(); - var today = LocalDate.now(functionProperties.getQueryStartClock()); - assertEquals(today, value.dateValue(functionProperties)); - assertEquals( - today.atTime(0, 0, 0), - LocalDateTime.ofInstant(value.timestampValue(functionProperties), ZoneOffset.UTC)); - assertEquals( - ZonedDateTime.of(LocalTime.parse("00:00:00").atDate(today), ZoneOffset.UTC).toInstant(), - value.timestampValue(functionProperties)); - } - - @Test - public void timeInUnsupportedFormat() { + public void timestampInUnsupportedFormat() { SemanticCheckException exception = - assertThrows(SemanticCheckException.class, () -> new ExprTimeValue("01:01:0")); + assertThrows( + SemanticCheckException.class, () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); assertEquals( - "time:01:01:0 in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", + "timestamp:2020-07-07T01:01:01Z in unsupported format, " + + "please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", exception.getMessage()); } - @Test - public void timestampSupportedFormat() { - String timestamp = "2020-07-07T01:01:01Z"; - ExprTimestampValue timestampValue = new ExprTimestampValue(timestamp); - - assertEquals("2020-07-07T01:01:01Z", timestampValue.value()); - assertEquals(LocalDate.parse("2020-07-07"), timestampValue.dateValue()); - assertEquals(LocalTime.parse("01:01:01"), timestampValue.timeValue()); - assertEquals(TIMESTAMP, timestampValue.type()); - } - @Test public void stringTimestampValue() { ExprValue stringValue = new ExprStringValue("2020-08-17 19:44:00"); @@ -495,9 +125,13 @@ public void stringTimestampValue() { assertEquals(LocalTime.parse("19:44:00"), stringValue.timeValue()); assertEquals("\"2020-08-17 19:44:00\"", stringValue.toString()); - Instant timestampValue = new ExprStringValue("2020-07-07T01:01:01Z").timestampValue(); + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> new ExprStringValue("2020-07-07T01:01:01Z").timestampValue()); assertEquals( - ZonedDateTime.of(2020, 07, 07, 1, 1, 1, 0, ZoneOffset.UTC).toInstant(), timestampValue); + "date:2020-07-07T01:01:01Z in unsupported format, " + "please use 'yyyy-MM-dd'", + exception.getMessage()); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 27e5796d17..0ec77f9f31 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -1434,11 +1434,11 @@ public void testTimeBracket() throws IOException { public void testDateBracket() throws IOException { JSONObject result = executeQuery("select {date '2020-09-16'}"); verifySchema(result, schema("{date '2020-09-16'}", null, "date")); - verifyDataRows(result, rows("2020-09-16T00:00:00.000Z")); + verifyDataRows(result, rows("2020-09-16")); result = executeQuery("select {d '2020-09-16'}"); verifySchema(result, schema("{d '2020-09-16'}", null, "date")); - verifyDataRows(result, rows("2020-09-16T00:00:00.000Z")); + verifyDataRows(result, rows("2020-09-16")); } private void compareBrackets(String query1, String query2, String timestamp) throws IOException { @@ -1455,25 +1455,20 @@ public void testBracketedEquivalent() throws IOException { compareBrackets("timestamp", "ts", "2020-09-16 17:30:00"); compareBrackets("timestamp", "timestamp", "2020-09-16 17:30:00.123"); compareBrackets("timestamp", "ts", "2020-09-16 17:30:00.123"); - compareBrackets("date", "date", "2020-09-16T00:00:00.000Z"); - compareBrackets("date", "d", "2020-09-16T00:00:00.000Z"); + compareBrackets("date", "date", "2020-09-16"); + compareBrackets("date", "d", "2020-09-16"); compareBrackets("time", "time", "17:30:00"); compareBrackets("time", "t", "17:30:00"); } @Test - public void testBrackets() { - try { - executeQuery("select {time '2020-09-16'}"); - executeQuery("select {t '2020-09-16'}"); - executeQuery("select {timestamp '2020-09-16'}"); - executeQuery("select {ts '2020-09-16'}"); - } catch (Exception e) { - // exception should not be thrown - assertTrue(false); - } + public void testBracketFails() { + assertThrows(ResponseException.class, () -> executeQuery("select {time '2020-09-16'}")); + assertThrows(ResponseException.class, () -> executeQuery("select {t '2020-09-16'}")); assertThrows(ResponseException.class, () -> executeQuery("select {date '17:30:00'}")); assertThrows(ResponseException.class, () -> executeQuery("select {d '17:30:00'}")); + assertThrows(ResponseException.class, () -> executeQuery("select {timestamp '2020-09-16'}")); + assertThrows(ResponseException.class, () -> executeQuery("select {ts '2020-09-16'}")); assertThrows(ResponseException.class, () -> executeQuery("select {timestamp '17:30:00'}")); assertThrows(ResponseException.class, () -> executeQuery("select {ts '17:30:00'}")); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 3d7468b95a..215c0b0690 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -11,11 +11,17 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAccessor; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -26,6 +32,9 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final OpenSearchDateType instance = new OpenSearchDateType(); + /** Could be user defined custom or OpenSearch named formatter * */ + private DateFormatter formatter; + /** Numeric formats which support full datetime. */ public static final List SUPPORTED_NAMED_NUMERIC_FORMATS = List.of(FormatNames.EPOCH_MILLIS, FormatNames.EPOCH_SECOND); @@ -137,6 +146,9 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final String CUSTOM_FORMAT_DATE_SYMBOLS = "FecEWwYqQgdMLDyuG"; + private static final List OPENSEARCH_DEFAULT_FORMATS = + Arrays.asList("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis"); + @EqualsAndHashCode.Exclude private final List formats; private OpenSearchDateType() { @@ -236,13 +248,67 @@ public List getAllCustomFormatters() { } /** - * Retrieves a list of custom formatters and OpenSearch named formatters defined by the user. + * Retrieves a list of custom formatters and OpenSearch named formatters defined by the user, and + * attempts to parse the given date/time string using these formatters. * - * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. + * @param dateTime The date/time string to parse. + * @return A ZonedDateTime representing the parsed date/time in UTC, or null if parsing fails. */ - public List getAllFormatters() { + public ZonedDateTime getParsedDateTime(String dateTime) { List dateFormatters = this.getAllNamedFormatters(); dateFormatters.addAll(this.getAllCustomFormatters()); + ZonedDateTime zonedDateTime = null; + + // check if dateFormatters are empty, then set default ones + if (dateFormatters.isEmpty()) { + dateFormatters = initializeDateFormatters(); + } + // parse using OpenSearch DateFormatters + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(dateTime); + zonedDateTime = DateFormatters.from(accessor).withZoneSameLocal(ZoneOffset.UTC); + this.formatter = formatter; + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + return zonedDateTime; + } + + /** + * Returns a formatted date string using the internal formatter, if available. + * + * @param accessor The TemporalAccessor object containing the date/time information. + * @return A formatted date string if a formatter is available, otherwise null. + */ + public String getFormattedDate(TemporalAccessor accessor) { + if (!hasNoFormatter()) { + return this.formatter.format(accessor); + } + return null; + } + + /** + * Checks if the formatter is not initialized. + * + * @return True if the formatter is not set, otherwise false. + */ + public boolean hasNoFormatter() { + return this.formatter == null; + } + + /** + * Initializes and returns a list of default OpenSearch date formatters. + * + * @return A list of DateFormatter objects initialized with default patterns. + */ + private static List initializeDateFormatters() { + List dateFormatters = new ArrayList<>(); + for (String pattern : OPENSEARCH_DEFAULT_FORMATS) { + dateFormatters.add(DateFormatter.forPattern(pattern)); + } return dateFormatters; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index 9ef2b77fac..4488128b97 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -66,11 +66,10 @@ private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( .missingOrder(missingOrder) .order(sortOrder); // Time types values are converted to LONG in ExpressionAggregationScript::execute - if (expr.getDelegated().type() instanceof OpenSearchDateType - && List.of(TIMESTAMP, TIME, DATE) - .contains(((OpenSearchDateType) expr.getDelegated().type()).getExprCoreType())) { - sourceBuilder.userValuetypeHint(ValueType.LONG); - } else if (List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { + if ((expr.getDelegated().type() instanceof OpenSearchDateType + && List.of(TIMESTAMP, TIME, DATE) + .contains(((OpenSearchDateType) expr.getDelegated().type()).getExprCoreType())) + || List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { sourceBuilder.userValuetypeHint(ValueType.LONG); } return helper.build(expr.getDelegated(), sourceBuilder::field, sourceBuilder::script); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index 55a1e57c16..6aebee561c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.analysis.NestedAnalyzer.isNestedFunction; import com.google.common.collect.ImmutableMap; +import java.time.ZonedDateTime; import java.util.Map; import java.util.function.Function; import org.opensearch.index.query.QueryBuilder; @@ -215,9 +216,13 @@ private ExprValue cast(FunctionExpression castFunction) { expr -> { if (expr.type().equals(ExprCoreType.STRING) && this.ref.type() instanceof OpenSearchDateType) { - return new ExprDateValue( - expr.valueOf().stringValue(), - ((OpenSearchDateType) this.ref.type()).getAllFormatters()); + ZonedDateTime zonedDateTime = + ((OpenSearchDateType) this.ref.type()) + .getParsedDateTime(expr.valueOf().stringValue()); + if (zonedDateTime != null) { + return new ExprDateValue(zonedDateTime.toLocalDate()); + } + return new ExprDateValue(expr.valueOf().stringValue()); } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprDateValue(expr.valueOf().stringValue()); } else { @@ -229,9 +234,13 @@ private ExprValue cast(FunctionExpression castFunction) { expr -> { if (expr.type().equals(ExprCoreType.STRING) && this.ref.type() instanceof OpenSearchDateType) { - return new ExprTimeValue( - expr.valueOf().stringValue(), - ((OpenSearchDateType) this.ref.type()).getAllFormatters()); + ZonedDateTime zonedDateTime = + ((OpenSearchDateType) this.ref.type()) + .getParsedDateTime(expr.valueOf().stringValue()); + if (zonedDateTime != null) { + return new ExprTimeValue(zonedDateTime.toLocalTime()); + } + return new ExprTimeValue(expr.valueOf().stringValue()); } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprTimeValue(expr.valueOf().stringValue()); } else { @@ -243,9 +252,13 @@ private ExprValue cast(FunctionExpression castFunction) { expr -> { if (expr.type().equals(ExprCoreType.STRING) && this.ref.type() instanceof OpenSearchDateType) { - return new ExprTimestampValue( - expr.valueOf().stringValue(), - ((OpenSearchDateType) this.ref.type()).getAllFormatters()); + ZonedDateTime zonedDateTime = + ((OpenSearchDateType) this.ref.type()) + .getParsedDateTime(expr.valueOf().stringValue()); + if (zonedDateTime != null) { + return new ExprTimestampValue(zonedDateTime.toInstant()); + } + return new ExprTimestampValue(expr.valueOf().stringValue()); } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprTimestampValue(expr.valueOf().stringValue()); } else { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java index 5ccc57f0ca..aba9bc6fd2 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java @@ -9,10 +9,10 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RangeQueryBuilder; -import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; /** Lucene query that builds range query for non-quality comparison. */ @RequiredArgsConstructor @@ -32,7 +32,7 @@ public enum Comparison { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { - Object value = value(literal); + Object value = value(literal, fieldType); RangeQueryBuilder query = QueryBuilders.rangeQuery(fieldName); switch (comparison) { @@ -49,13 +49,23 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l } } - private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP) - && ((ExprTimestampValue) literal).hasNoFormatter()) { - // Formatting to default Epoch when no custom formatter - return literal.timestampValue().toEpochMilli(); - } else { - return literal.value(); + private Object value(ExprValue literal, ExprType fieldType) { + if (fieldType instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) fieldType; + if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + return openSearchDateType.hasNoFormatter() + ? literal.timestampValue().toEpochMilli() + : openSearchDateType.getFormattedDate(literal.timestampValue()); + } else if (literal.type().equals(ExprCoreType.DATE)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.dateValue()); + } else if (literal.type().equals(ExprCoreType.TIME)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.timeValue()); + } } + return literal.value(); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java index 0b06658ea3..fddeb13a0b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java @@ -7,10 +7,10 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; /** Lucene query that build term query for equality comparison. */ @@ -19,16 +19,26 @@ public class TermQuery extends LuceneQuery { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { fieldName = OpenSearchTextType.convertTextToKeyword(fieldName, fieldType); - return QueryBuilders.termQuery(fieldName, value(literal)); + return QueryBuilders.termQuery(fieldName, value(literal, fieldType)); } - private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP) - && ((ExprTimestampValue) literal).hasNoFormatter()) { - // Formatting to default Epoch when no custom formatter - return literal.timestampValue().toEpochMilli(); - } else { - return literal.value(); + private Object value(ExprValue literal, ExprType fieldType) { + if (fieldType instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) fieldType; + if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + return openSearchDateType.hasNoFormatter() + ? literal.timestampValue().toEpochMilli() + : openSearchDateType.getFormattedDate(literal.timestampValue()); + } else if (literal.type().equals(ExprCoreType.DATE)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.dateValue()); + } else if (literal.type().equals(ExprCoreType.TIME)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.timeValue()); + } } + return literal.value(); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index bd3e426077..4728b454f9 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -5,12 +5,7 @@ package org.opensearch.sql.opensearch.data.type; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; @@ -22,6 +17,9 @@ import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.isDateTypeCompatible; import com.google.common.collect.Lists; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.stream.Stream; @@ -31,7 +29,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.type.ExprCoreType; @@ -268,14 +265,136 @@ public void check_if_date_type_compatible() { } @Test - public void testGetAllFormatters() { - List namedFormatters = timeDateType.getAllNamedFormatters(); - List customFormatters = timeDateType.getAllCustomFormatters(); + void testValidTimestampWithCustomFormat() { + String timestamp = "2021-11-08T17:00:00Z"; + String format = "strict_date_time_no_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void testValidTimestampWithMultipleFormats() { + String timestamp = "2021-11-08T17:00:00Z"; + String timestamp2 = "2021/11/08T17:00:00Z"; + + List formats = Arrays.asList("strict_date_time_no_millis", "yyyy/MM/dd'T'HH:mm:ssX"); + OpenSearchDateType dateType = OpenSearchDateType.of(String.join(" || ", formats)); + + // Testing with the first timestamp + ZonedDateTime zonedDateTime1 = dateType.getParsedDateTime(timestamp); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime1.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime1.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + + // Testing with the second timestamp + ZonedDateTime zonedDateTime2 = dateType.getParsedDateTime(timestamp2); + + assertEquals("2021/11/08T17:00:00Z", dateType.getFormattedDate(zonedDateTime2.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime2.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void testOpenSearchDateTimeNamedFormatter() { + String timestamp = "2019-03-23T21:34:46"; + String format = "strict_date_hour_minute_second"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + + assertEquals("2019-03-23T21:34:46", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2019-03-23"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("21:34:46"), zonedDateTime.toLocalTime()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void testInvalidTimestamp() { + String timestamp = "invalid-timestamp"; + List formats = Arrays.asList("yyyy/MM/dd'T'HH:mm:ssX", "yyyy-MM-dd'T'HH:mm:ssX"); + OpenSearchDateType dateType = OpenSearchDateType.of(String.join(" || ", formats)); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + assertNull(dateType.getFormattedDate(zonedDateTime)); + assertNull(zonedDateTime); + assertTrue(dateType.hasNoFormatter()); + } + + @Test + void testEpochDateTimeFormatter() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + String format = "epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(String.valueOf(epochTimestamp)); + + assertEquals(Long.toString(epochTimestamp), dateType.getFormattedDate(zonedDateTime)); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("17:00:00"), zonedDateTime.toLocalTime()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void testCustomTimeStampFormatWithDefaultFormatters() { + String timestamp = "2021-11-08 17:00:00"; + String format = "strict_date_optional_time || epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + assertNull(dateType.getParsedDateTime(timestamp)); + assertNull(dateType.getFormattedDate(zonedDateTime)); + } + + @Test + void testValidDateWithCustomFormatter() { + String dateString = "2021-11-08"; + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(expectedDate, parsedDate); + assertEquals("2021-11-08", dateType.getFormattedDate(parsedDate)); + } + + @Test + void testValidDateWithMultipleFormatters() { + String dateString = "2021-11-08"; + String format = "yyyy/MM/dd || yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyy-MM-dd")); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(expectedDate, parsedDate); + assertEquals("2021-11-08", dateType.getFormattedDate(parsedDate)); + } + + @Test + void testValidTimeWithCustomFormatter() { + String timeString = "12:10:30.000"; + String format = "HH:mm:ss.SSS"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern(format)); + LocalTime parsedTime = dateType.getParsedDateTime(timeString).toLocalTime(); + + assertEquals(expectedTime, parsedTime); + assertEquals("12:10:30.000", dateType.getFormattedDate(parsedTime)); + } + + @Test + void testValidTimeWithMultipleFormatters() { + String timeString = "12:10:30"; + String format = "HH:mm:ss.SSS || HH:mm:ss"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); - List allFormatters = timeDateType.getAllFormatters(); + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss")); + LocalTime parsedTime = dateType.getParsedDateTime(timeString).toLocalTime(); - assertEquals(namedFormatters.size() + customFormatters.size(), allFormatters.size()); - assertTrue(allFormatters.containsAll(namedFormatters)); - assertTrue(allFormatters.containsAll(customFormatters)); + assertEquals(expectedTime, parsedTime); + assertEquals("12:10:30", dateType.getFormattedDate(parsedTime)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java index 4250b3297f..08c4017f1d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java @@ -40,6 +40,7 @@ import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.parse.ParseExpression; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @@ -134,6 +135,39 @@ void should_build_bucket_with_parse_expression() { buildQuery(Arrays.asList(asc(named("name", parseExpression))))); } + @Test + void terms_bucket_for_opensearchdate_type_uses_long() { + OpenSearchDateType dataType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); + + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"field\" : \"date\",\n" + + " \"missing_bucket\" : true,\n" + + " \"value_type\" : \"long\",\n" + + " \"missing_order\" : \"first\",\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery(Arrays.asList(asc(named("date", ref("date", dataType)))))); + } + + @Test + void terms_bucket_for_opensearchdate_type_uses_long_false() { + OpenSearchDateType dataType = OpenSearchDateType.of(STRING); + + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"field\" : \"date\",\n" + + " \"missing_bucket\" : true,\n" + + " \"missing_order\" : \"first\",\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery(Arrays.asList(asc(named("date", ref("date", dataType)))))); + } + @ParameterizedTest(name = "{0}") @EnumSource( value = ExprCoreType.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 53b228622c..bd2a9901ed 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -1767,23 +1767,14 @@ void cast_to_date_in_filter() { "{\n" + " \"term\" : {\n" + " \"date_value\" : {\n" - + " \"value\" : \"2021-11-08T00:00:00.000Z\",\n" + + " \"value\" : \"2021-11-08\",\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" + "}"; - assertJsonEquals( json, buildQuery(DSL.equal(ref("date_value", DATE), DSL.castDate(literal("2021-11-08"))))); - json = - "{\n" - + " \"term\" : {\n" - + " \"date_value\" : {\n" - + " \"value\" : \"2021-11-08\",\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + "}"; + assertJsonEquals( json, buildQuery( @@ -1830,7 +1821,7 @@ void cast_to_timestamp_in_filter() { "{\n" + " \"term\" : {\n" + " \"timestamp_value\" : {\n" - + " \"value\" : 1636390800000,\n" + + " \"value\" : \"2021-11-08 17:00:00\",\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" @@ -1856,7 +1847,7 @@ void cast_in_range_query() { "{\n" + " \"range\" : {\n" + " \"timestamp_value\" : {\n" - + " \"from\" : 1636390800000,\n" + + " \"from\" : \"2021-11-08 17:00:00\",\n" + " \"to\" : null," + " \"include_lower\" : false," + " \"include_upper\" : true,"