From 4776f623833acf63b1ad404e58d45d10d88b930c Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 18 Sep 2024 12:38:53 +0800 Subject: [PATCH] add fe ut --- .../apache/doris/common/util/TimeUtils.java | 27 ++++++++++ .../tablefunction/MetadataGenerator.java | 38 +++----------- .../doris/common/util/TimeUtilsTest.java | 49 +++++++++++++++++++ 3 files changed, 83 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java index 55b2ed1c58d972..5768ca7b8052ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -340,4 +340,31 @@ private static String formatDateStr(String dateStr) { parts.length > 3 ? String.format(" %02d:%02d:%02d", Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5])) : ""); } + + + // Refer to be/src/vec/runtime/vdatetime_value.h + public static long convertToDateTimeV2( + int year, int month, int day, int hour, int minute, int second, int microsecond) { + return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32 + | (long) day << 37 | (long) month << 42 | (long) year << 46; + } + + // Refer to be/src/vec/runtime/vdatetime_value.h + public static long convertToDateV2( + int year, int month, int day) { + return (long) day | (long) month << 5 | (long) year << 9; + } + + public static long convertStringToDateTimeV2(String dateTimeStr) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateTimeStr, formatter); + return convertToDateTimeV2(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), + dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond(), dateTime.getNano() / 1000); + } + + public static long convertStringToDateV2(String dateStr) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateStr, formatter); + return convertToDateV2(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index aad02b4ec05344..4411db584fe314 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -107,7 +107,6 @@ import java.text.SimpleDateFormat; import java.time.Instant; import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -343,7 +342,8 @@ private static TFetchSchemaTableDataResult icebergMetadataResult(TMetadataTableR TRow trow = new TRow(); LocalDateTime committedAt = LocalDateTime.ofInstant(Instant.ofEpochMilli( snapshot.timestampMillis()), TimeUtils.getTimeZone().toZoneId()); - long encodedDatetime = convertToDateTimeV2(committedAt.getYear(), committedAt.getMonthValue(), + long encodedDatetime = TimeUtils.convertToDateTimeV2(committedAt.getYear(), + committedAt.getMonthValue(), committedAt.getDayOfMonth(), committedAt.getHour(), committedAt.getMinute(), committedAt.getSecond(), committedAt.getNano() / 1000); @@ -799,32 +799,6 @@ private static void filterColumns(TFetchSchemaTableDataResult result, result.setDataBatch(filterColumnsRows); } - // Refer to be/src/vec/runtime/vdatetime_value.h - private static long convertToDateTimeV2( - int year, int month, int day, int hour, int minute, int second, int microsecond) { - return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32 - | (long) day << 37 | (long) month << 42 | (long) year << 46; - } - - // Refer to be/src/vec/runtime/vdatetime_value.h - private static long convertToDateV2( - int year, int month, int day) { - return (long) day | (long) month << 5 | (long) year << 9; - } - - private static long convertStringToDateTimeV2(String dateTimeStr) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateTimeStr, formatter); - return convertToDateTimeV2(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), - dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond(), dateTime.getNano() / 1000); - } - - private static long convertStringToDateV2(String dateStr) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateStr, formatter); - return convertToDateV2(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth()); - } - private static TFetchSchemaTableDataResult mtmvMetadataResult(TMetadataTableRequestParams params) { if (LOG.isDebugEnabled()) { LOG.debug("mtmvMetadataResult() start"); @@ -1559,7 +1533,7 @@ private static List partitionValuesMetadataResultForHmsTable(HMSExternalTa for (int i = 0; i < colIdxs.size(); ++i) { int idx = colIdxs.get(i); String partitionValue = values.get(idx); - if (partitionValue == null && partitionValue.equals(TablePartitionValues.HIVE_DEFAULT_PARTITION)) { + if (partitionValue == null || partitionValue.equals(TablePartitionValues.HIVE_DEFAULT_PARTITION)) { trow.addToColumnValue(new TCell().setIsNull(true)); } else { Type type = types.get(i); @@ -1588,10 +1562,12 @@ private static List partitionValuesMetadataResultForHmsTable(HMSExternalTa break; case DATE: case DATEV2: - trow.addToColumnValue(new TCell().setLongVal(convertStringToDateV2(partitionValue))); + trow.addToColumnValue( + new TCell().setLongVal(TimeUtils.convertStringToDateV2(partitionValue))); case DATETIME: case DATETIMEV2: - trow.addToColumnValue(new TCell().setLongVal(convertStringToDateTimeV2(partitionValue))); + trow.addToColumnValue( + new TCell().setLongVal(TimeUtils.convertStringToDateTimeV2(partitionValue))); break; default: throw new AnalysisException( diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java index 114ddada9cdc2e..a93b33c3dcc84c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; import mockit.Expectations; @@ -31,6 +32,7 @@ import org.junit.Test; import java.time.ZoneId; +import java.time.format.DateTimeParseException; import java.util.Calendar; import java.util.Date; import java.util.LinkedList; @@ -201,4 +203,51 @@ public void testGetHourAsDate() { Assert.assertNull(TimeUtils.getHourAsDate("111")); Assert.assertNull(TimeUtils.getHourAsDate("-1")); } + + @Test + public void testConvertToBEDateType() { + long result = TimeUtils.convertStringToDateV2("2021-01-01"); + Assert.assertEquals(1034785, result); + result = TimeUtils.convertStringToDateV2("1900-01-01"); + Assert.assertEquals(972833, result); + result = TimeUtils.convertStringToDateV2("1899-12-31"); + Assert.assertEquals(972703, result); + result = TimeUtils.convertStringToDateV2("9999-12-31"); + Assert.assertEquals(5119903, result); + + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("2021-1-1")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("1900-01-1")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("20210101")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("")); + ExceptionChecker.expectThrows(NullPointerException.class, () -> TimeUtils.convertStringToDateV2(null)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateV2("2024:12:31")); + } + + @Test + public void testConvertToBEDatetimeV2Type() { + long result = TimeUtils.convertStringToDateTimeV2("2021-01-01 10:10:10"); + Assert.assertEquals(142219811099770880L, result); + result = TimeUtils.convertStringToDateTimeV2("1900-01-01 00:00:00"); + Assert.assertEquals(133705149423026176L, result); + result = TimeUtils.convertStringToDateTimeV2("1899-12-31 23:59:59"); + Assert.assertEquals(133687385164611584L, result); + result = TimeUtils.convertStringToDateTimeV2("9999-12-31 23:59:59"); + Assert.assertEquals(703674213003689984L, result); + + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2021-1-1")); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("1900-01-1")); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("20210101")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateTimeV2("")); + ExceptionChecker.expectThrows(NullPointerException.class, () -> TimeUtils.convertStringToDateTimeV2(null)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024-10-10")); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024-10-10 10")); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateV2("2024:12:31")); + } }