Skip to content

Commit

Permalink
Adding timestamp rounding support in star tree
Browse files Browse the repository at this point in the history 
Signed-off-by: Bharathwaj G <[email protected]>
  • Loading branch information
@bharath-techie
bharath-techie committed Aug 14, 2024
1 parent ef1a79f commit 5fdce71
Show file tree
Hide file tree
Showing 18 changed files with 1,086 additions and 155 deletions.
16 changes: 8 additions & 8 deletions server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
.startObject("startree-1")
.field("type", "star_tree")
.startObject("config")
.startArray("ordered_dimensions")
.startObject()
.startObject("date_dimension")
.field("name", "timestamp")
.endObject()
.startArray("ordered_dimensions")
.startObject()
.field("name", getDim(invalidDim, keywordDim))
.endObject()
Expand Down Expand Up @@ -97,14 +97,14 @@ private static XContentBuilder createMaxDimTestMapping() {
.startObject("startree-1")
.field("type", "star_tree")
.startObject("config")
.startArray("ordered_dimensions")
.startObject()
.startObject("date_dimension")
.field("name", "timestamp")
.startArray("calendar_intervals")
.value("day")
.value("month")
.endArray()
.endObject()
.startArray("ordered_dimensions")
.startObject()
.field("name", "dim2")
.endObject()
Expand Down Expand Up @@ -139,7 +139,7 @@ private static XContentBuilder createMaxDimTestMapping() {
}
}

private static XContentBuilder createTestMappingWithoutStarTree(boolean invalidDim, boolean invalidMetric, boolean keywordDim) {
private static XContentBuilder createTestMappingWithoutStarTree() {
try {
return jsonBuilder().startObject()
.startObject("properties")
Expand Down Expand Up @@ -176,10 +176,10 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea
.startObject(sameStarTree ? "startree-1" : "startree-2")
.field("type", "star_tree")
.startObject("config")
.startArray("ordered_dimensions")
.startObject()
.startObject("date_dimension")
.field("name", "timestamp")
.endObject()
.startArray("ordered_dimensions")
.startObject()
.field("name", changeDim ? "numeric_new" : getDim(false, false))
.endObject()
Expand Down Expand Up @@ -295,7 +295,7 @@ public void testUpdateIndexWithAdditionOfStarTree() {
}

public void testUpdateIndexWithNewerStarTree() {
prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree(false, false, false)).get();
prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree()).get();

IllegalArgumentException ex = expectThrows(
IllegalArgumentException.class,
Expand Down
53 changes: 43 additions & 10 deletions server/src/main/java/org/opensearch/common/Rounding.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,15 @@
import java.time.temporal.TemporalQueries;
import java.time.zone.ZoneOffsetTransition;
import java.time.zone.ZoneRules;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.OptionalLong;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

/**
* A strategy for rounding milliseconds since epoch.
Expand Down Expand Up @@ -95,7 +99,7 @@ public enum DateTimeUnit {
WEEK_OF_WEEKYEAR((byte) 1, "week", IsoFields.WEEK_OF_WEEK_BASED_YEAR, true, TimeUnit.DAYS.toMillis(7)) {
private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(7);

long roundFloor(long utcMillis) {
public long roundFloor(long utcMillis) {
return DateUtils.roundWeekOfWeekYear(utcMillis);
}

Expand All @@ -107,7 +111,7 @@ long extraLocalOffsetLookup() {
YEAR_OF_CENTURY((byte) 2, "year", ChronoField.YEAR_OF_ERA, false, 12) {
private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(366);

long roundFloor(long utcMillis) {
public long roundFloor(long utcMillis) {
return DateUtils.roundYear(utcMillis);
}

Expand All @@ -118,7 +122,7 @@ long extraLocalOffsetLookup() {
QUARTER_OF_YEAR((byte) 3, "quarter", IsoFields.QUARTER_OF_YEAR, false, 3) {
private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(92);

long roundFloor(long utcMillis) {
public long roundFloor(long utcMillis) {
return DateUtils.roundQuarterOfYear(utcMillis);
}

Expand All @@ -129,7 +133,7 @@ long extraLocalOffsetLookup() {
MONTH_OF_YEAR((byte) 4, "month", ChronoField.MONTH_OF_YEAR, false, 1) {
private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(31);

long roundFloor(long utcMillis) {
public long roundFloor(long utcMillis) {
return DateUtils.roundMonthOfYear(utcMillis);
}

Expand All @@ -138,7 +142,7 @@ long extraLocalOffsetLookup() {
}
},
DAY_OF_MONTH((byte) 5, "day", ChronoField.DAY_OF_MONTH, true, ChronoField.DAY_OF_MONTH.getBaseUnit().getDuration().toMillis()) {
long roundFloor(long utcMillis) {
public long roundFloor(long utcMillis) {
return DateUtils.roundFloor(utcMillis, this.ratio);
}

Expand All @@ -147,7 +151,7 @@ long extraLocalOffsetLookup() {
}
},
HOUR_OF_DAY((byte) 6, "hour", ChronoField.HOUR_OF_DAY, true, ChronoField.HOUR_OF_DAY.getBaseUnit().getDuration().toMillis()) {
long roundFloor(long utcMillis) {
public long roundFloor(long utcMillis) {
return DateUtils.roundFloor(utcMillis, ratio);
}

Expand All @@ -162,7 +166,7 @@ long extraLocalOffsetLookup() {
true,
ChronoField.MINUTE_OF_HOUR.getBaseUnit().getDuration().toMillis()
) {
long roundFloor(long utcMillis) {
public long roundFloor(long utcMillis) {
return DateUtils.roundFloor(utcMillis, ratio);
}

Expand All @@ -177,7 +181,7 @@ long extraLocalOffsetLookup() {
true,
ChronoField.SECOND_OF_MINUTE.getBaseUnit().getDuration().toMillis()
) {
long roundFloor(long utcMillis) {
public long roundFloor(long utcMillis) {
return DateUtils.roundFloor(utcMillis, ratio);
}

Expand Down Expand Up @@ -210,7 +214,7 @@ public long extraLocalOffsetLookup() {
* @param utcMillis the milliseconds since the epoch
* @return the rounded down milliseconds since the epoch
*/
abstract long roundFloor(long utcMillis);
public abstract long roundFloor(long utcMillis);

/**
* When looking up {@link LocalTimeOffset} go this many milliseconds
Expand Down Expand Up @@ -260,6 +264,35 @@ public static DateTimeUnit resolve(byte id) {
}
}

/**
* DateTimeUnit Comparator which tracks dateTimeUnits from second unit to year unit
*/
public static class DateTimeUnitComparator implements Comparator<DateTimeUnit> {
public static final Map<DateTimeUnit, Integer> ORDERED_DATE_TIME_UNIT = new HashMap<>();

static {
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.SECOND_OF_MINUTE, 1);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MINUTES_OF_HOUR, 2);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.HOUR_OF_DAY, 3);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.DAY_OF_MONTH, 4);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.WEEK_OF_WEEKYEAR, 5);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MONTH_OF_YEAR, 6);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.QUARTER_OF_YEAR, 7);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.YEAR_OF_CENTURY, 8);
}

@Override
public int compare(DateTimeUnit unit1, DateTimeUnit unit2) {
return Integer.compare(ORDERED_DATE_TIME_UNIT.get(unit1), ORDERED_DATE_TIME_UNIT.get(unit2));
}
}

public static List<DateTimeUnit> getSortedDateTimeUnits(List<DateTimeUnit> dateTimeUnits) {
return dateTimeUnits.stream()
.sorted(Comparator.comparingInt(DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT::get))
.collect(Collectors.toList());
}

public abstract void innerWriteTo(StreamOutput out) throws IOException;

@Override
Expand Down Expand Up @@ -947,7 +980,7 @@ public final long nextRoundingValue(long utcMillis) {
*
* @opensearch.internal
*/
static class TimeIntervalRounding extends Rounding {
public static class TimeIntervalRounding extends Rounding {
static final byte ID = 2;

private final long interval;
Expand Down
63 changes: 61 additions & 2 deletions server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@

import org.opensearch.common.Rounding;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.common.time.DateUtils;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.mapper.CompositeDataCubeFieldType;
import org.opensearch.index.mapper.DateFieldMapper;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

Expand All @@ -28,19 +31,70 @@ public class DateDimension implements Dimension {
public static final String CALENDAR_INTERVALS = "calendar_intervals";
public static final String DATE = "date";
private final String field;
private final List<Rounding.DateTimeUnit> sortedCalendarIntervals;
private final DateFieldMapper.Resolution resolution;

public DateDimension(String field, List<Rounding.DateTimeUnit> calendarIntervals) {
public DateDimension(String field, List<Rounding.DateTimeUnit> calendarIntervals, DateFieldMapper.Resolution resolution) {
this.field = field;
this.calendarIntervals = calendarIntervals;
// Sort from the lowest unit to the highest unit
this.sortedCalendarIntervals = Rounding.getSortedDateTimeUnits(calendarIntervals);
if (resolution == null) {
this.resolution = DateFieldMapper.Resolution.MILLISECONDS;
} else {
this.resolution = resolution;
}
}

public List<Rounding.DateTimeUnit> getIntervals() {
return calendarIntervals;
}

public List<Rounding.DateTimeUnit> getSortedCalendarIntervals() {
return sortedCalendarIntervals;
}

/**
* Sets the dimension values in sorted order in the provided array starting from the given index.
*
* @param val The value to be set
* @param dims The dimensions array to set the values in
* @param index The starting index in the array
* @return The next available index in the array
*/
@Override
public int setDimensionValues(final Long val, final Long[] dims, int index) {
for (Rounding.DateTimeUnit dateTimeUnit : sortedCalendarIntervals) {
if (val == null) {
dims[index++] = null;
continue;
}
dims[index++] = dateTimeUnit.roundFloor(convertNanosToMillis(val));
}
return index;
}

private long convertNanosToMillis(long nanoSecondsSinceEpoch) {
if (resolution.equals(DateFieldMapper.Resolution.NANOSECONDS)) return DateUtils.toMilliSeconds(nanoSecondsSinceEpoch);
return nanoSecondsSinceEpoch;
}

/**
* Returns the list of fields that represent the dimension
*/
@Override
public List<String> getDimensionFieldsNames() {
List<String> fields = new ArrayList<>(calendarIntervals.size());
for (Rounding.DateTimeUnit interval : sortedCalendarIntervals) {
// TODO : revisit this post file format changes
fields.add(field + "_" + interval.shortName());
}
return fields;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startObject("date_dimension");
builder.field(CompositeDataCubeFieldType.NAME, this.getField());
builder.field(CompositeDataCubeFieldType.TYPE, DATE);
builder.startArray(CALENDAR_INTERVALS);
Expand Down Expand Up @@ -69,4 +123,9 @@ public int hashCode() {
public String getField() {
return field;
}

@Override
public int getNumDimensions() {
return calendarIntervals.size();
}
}
24 changes: 24 additions & 0 deletions server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,36 @@
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.core.xcontent.ToXContent;

import java.util.List;

/**
* Base interface for data-cube dimensions
*
* @opensearch.experimental
*/
@ExperimentalApi
public interface Dimension extends ToXContent {

String getField();

/**
* Returns the number of dimension values that gets added to star tree document
* as part of this dimension
*/
int getNumDimensions();

/**
* Sets the dimension values in the provided array starting from the given index.
*
* @param value The value to be set
* @param dims The dimensions array to set the values in
* @param index The starting index in the array
* @return The next available index in the array
*/
int setDimensionValues(Long value, Long[] dims, int index);

/**
* Returns the list of dimension fields that represent the dimension
*/
List<String> getDimensionFieldsNames();
}
8 changes: 7 additions & 1 deletion server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ private static DateDimension parseAndCreateDateDimension(
.stream()
.map(Object::toString)
.collect(Collectors.toList());
System.out.println(intervalStrings);
if (intervalStrings == null || intervalStrings.isEmpty()) {
calendarIntervals = StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings());
} else {
Expand All @@ -94,6 +95,11 @@ private static DateDimension parseAndCreateDateDimension(
calendarIntervals = new ArrayList<>(calendarIntervals);
}
dimensionMap.remove(CALENDAR_INTERVALS);
return new DateDimension(name, calendarIntervals);
DateFieldMapper.Resolution resolution = null;
if (c != null && c.mapperService() != null && c.mapperService().fieldType(name) != null) {
resolution = ((DateFieldMapper.DateFieldType) c.mapperService().fieldType(name)).resolution();
}

return new DateDimension(name, calendarIntervals, resolution);
}
}
18 changes: 18 additions & 0 deletions server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.opensearch.index.mapper.CompositeDataCubeFieldType;

import java.io.IOException;
import java.util.List;
import java.util.Objects;

/**
Expand All @@ -33,6 +34,23 @@ public String getField() {
return field;
}

@Override
public int getNumDimensions() {
return 1;
}

@Override
public int setDimensionValues(final Long val, final Long[] dims, int index) {
dims[index++] = val;
return index;
}

@Override
public List<String> getDimensionFieldsNames() {
// TODO : revisit this post file format changes
return List.of(field);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
Expand Down
Loading

0 comments on commit 5fdce71

Please sign in to comment.