Skip to content

Commit

Permalink
[IOTDB-4356] De-duplication of PathPatternTree (apache#7289)
Browse files Browse the repository at this point in the history 
[IOTDB-4356] De-duplication of PathPatternTree apache#7289
  • Loading branch information
@Cpaulyz
Cpaulyz authored Sep 13, 2022
1 parent d33f7dc commit 641e2b9
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 5 deletions.
58 changes: 58 additions & 0 deletions node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,64 @@ public boolean prefixMatchFullPath(PartialPath rPath) {
return true;
}

/**
* Test if this path pattern includes input path pattern. e.g. "root.**" includes "root.sg.**",
* "root.*.d.s" includes "root.sg.d.s", "root.sg.**" does not include "root.**.s", "root.*.d.s"
* does not include "root.sg.d1.*"
*
* @param rPath a pattern path of a timeseries
* @return true if this path pattern includes input path pattern, otherwise return false
*/
public boolean include(PartialPath rPath) {
String[] rNodes = rPath.getNodes();
String[] lNodes = nodes.clone();
// Replace * with ** if they are adjacent to each other
for (int i = 1; i < lNodes.length; i++) {
if (MULTI_LEVEL_PATH_WILDCARD.equals(lNodes[i - 1])
&& ONE_LEVEL_PATH_WILDCARD.equals(lNodes[i])) {
lNodes[i] = MULTI_LEVEL_PATH_WILDCARD;
}
if (MULTI_LEVEL_PATH_WILDCARD.equals(lNodes[lNodes.length - i])
&& ONE_LEVEL_PATH_WILDCARD.equals(lNodes[lNodes.length - 1 - i])) {
lNodes[lNodes.length - 1 - i] = MULTI_LEVEL_PATH_WILDCARD;
}
}
// dp[i][j] means if nodes1[0:i) includes nodes[0:j)
// for example: "root.sg.**" includes "root.sg.d1.*"
// 1 0 0 0 0 |→| 1 0 0 0 0 |→| 1 0 0 0 0 |→| 1 0 0 0 0
// 0 0 0 0 0 |↓| 0 1 0 0 0 |→| 0 1 0 0 0 |→| 0 1 0 0 0
// 0 0 0 0 0 |↓| 0 0 0 0 0 |↓| 0 0 1 0 0 |→| 0 0 1 0 0
// 0 0 0 0 0 |↓| 0 0 0 0 0 |↓| 0 0 0 0 0 |↓| 0 0 0 1 1
// Since the derivation of the next row depends only on the previous row, the calculation can
// be performed using a one-dimensional array
boolean[] dp = new boolean[rNodes.length + 1];
dp[0] = true;
for (int i = 1; i <= lNodes.length; i++) {
boolean[] newDp = new boolean[rNodes.length + 1];
for (int j = i; j <= rNodes.length; j++) {
if (lNodes[i - 1].equals(MULTI_LEVEL_PATH_WILDCARD)) {
// if encounter MULTI_LEVEL_PATH_WILDCARD
if (dp[j - 1]) {
for (int k = j; k <= rNodes.length; k++) {
newDp[k] = true;
}
break;
}
} else {
// if without MULTI_LEVEL_PATH_WILDCARD, scan and check
if (!rNodes[j - 1].equals(MULTI_LEVEL_PATH_WILDCARD)
&& (lNodes[i - 1].equals(ONE_LEVEL_PATH_WILDCARD)
|| lNodes[i - 1].equals(rNodes[j - 1]))) {
// if nodes1[i-1] includes rNodes[j-1], dp[i][j] = dp[i-1][j-1]
newDp[j] |= dp[j - 1];
}
}
}
dp = newDp;
}
return dp[rNodes.length];
}

/**
* Test if this path pattern overlaps with input path pattern. Overlap means the result sets
* generated by two path pattern share some common elements. e.g. "root.sg.**" overlaps with
Expand Down
43 changes: 43 additions & 0 deletions node-commons/src/test/java/org/apache/iotdb/commons/path/PartialPathTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,49 @@ public void testOverlapWith() throws IllegalPathException {
}
}

@Test
public void testInclude() throws IllegalPathException {
PartialPath[][] pathPairs =
new PartialPath[][] {
new PartialPath[] {new PartialPath("root.**"), new PartialPath("root.sg.**")},
new PartialPath[] {new PartialPath("root.**.*"), new PartialPath("root.**")},
new PartialPath[] {new PartialPath("root.**.*"), new PartialPath("root.sg.**")},
new PartialPath[] {new PartialPath("root.**.s"), new PartialPath("root.sg.**")},
new PartialPath[] {new PartialPath("root.*.**"), new PartialPath("root.sg.**")},
new PartialPath[] {new PartialPath("root.*.d.s"), new PartialPath("root.sg1.d.s")},
new PartialPath[] {new PartialPath("root.**.s"), new PartialPath("root.*.d.s")},
new PartialPath[] {new PartialPath("root.*.d.s"), new PartialPath("root.**.s")},
new PartialPath[] {new PartialPath("root.*.d.s"), new PartialPath("root.sg.*.s")},
new PartialPath[] {new PartialPath("root.*.d.s"), new PartialPath("root.sg.d2.s")},
new PartialPath[] {new PartialPath("root.*.d.s.*"), new PartialPath("root.sg.d.s")},
new PartialPath[] {new PartialPath("root.**.d.s"), new PartialPath("root.**.d2.s")},
new PartialPath[] {new PartialPath("root.**.*.s"), new PartialPath("root.**.d2.s")},
new PartialPath[] {new PartialPath("root.**.d1.*"), new PartialPath("root.*")},
new PartialPath[] {new PartialPath("root.**.d1.*"), new PartialPath("root.d2.*.s")},
new PartialPath[] {new PartialPath("root.**.d1.**"), new PartialPath("root.d2.**")},
new PartialPath[] {
new PartialPath("root.**.*.**.**"), new PartialPath("root.d2.*.s1.**")
},
new PartialPath[] {new PartialPath("root.**.s1.d1"), new PartialPath("root.s1.d1.**")},
new PartialPath[] {new PartialPath("root.**.s1"), new PartialPath("root.**.s2.s1")},
new PartialPath[] {
new PartialPath("root.**.s1.s2.**"), new PartialPath("root.d1.s1.s2.*")
},
new PartialPath[] {new PartialPath("root.**.s1"), new PartialPath("root.**.s2")},
new PartialPath[] {new PartialPath("root.*.*.**"), new PartialPath("root.**.*")},
new PartialPath[] {new PartialPath("root.**.**"), new PartialPath("root.*.**.**.*")},
};
boolean[] results =
new boolean[] {
true, false, true, false, true, true, true, false, false, false, false, false, true,
false, false, false, true, false, true, true, false, false, true
};
Assert.assertEquals(pathPairs.length, results.length);
for (int i = 0; i < pathPairs.length; i++) {
Assert.assertEquals(results[i], pathPairs[i][0].include(pathPairs[i][1]));
}
}

private void checkNodes(String[] expected, String[] actual) {
Assert.assertEquals(expected.length, actual.length);
for (int i = 0; i < expected.length; i++) {
Expand Down
12 changes: 7 additions & 5 deletions server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/PathPatternTree.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,10 @@
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

public class PathPatternTree {
Expand Down Expand Up @@ -79,7 +81,7 @@ public void appendFullPath(PartialPath devicePath, String measurement) {
public void appendPathPattern(PartialPath pathPattern) {
boolean isExist = false;
for (PartialPath path : pathPatternList) {
if (path.matchFullPath(pathPattern)) {
if (path.include(pathPattern)) {
// path already exists in pathPatternList
isExist = true;
break;
Expand All @@ -88,7 +90,7 @@ public void appendPathPattern(PartialPath pathPattern) {
if (!isExist) {
// remove duplicate path in pathPatternList
pathPatternList.removeAll(
pathPatternList.stream().filter(pathPattern::matchFullPath).collect(Collectors.toList()));
pathPatternList.stream().filter(pathPattern::include).collect(Collectors.toList()));
pathPatternList.add(pathPattern);
}
}
Expand Down Expand Up @@ -134,13 +136,13 @@ public boolean isEmpty() {

public List<String> getAllDevicePatterns() {
List<String> nodes = new ArrayList<>();
List<String> results = new ArrayList<>();
Set<String> results = new HashSet<>();
searchDevicePattern(root, nodes, results);
return results;
return new ArrayList<>(results);
}

private void searchDevicePattern(
PathPatternNode curNode, List<String> nodes, List<String> results) {
PathPatternNode curNode, List<String> nodes, Set<String> results) {
nodes.add(curNode.getName());
if (curNode.isLeaf()) {
if (!curNode.getName().equals(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD)) {
Expand Down
33 changes: 33 additions & 0 deletions server/src/test/java/org/apache/iotdb/db/mpp/common/schematree/PathPatternTreeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,35 @@ public void pathPatternTreeTest6() throws IllegalPathException, IOException {
new PartialPath("root.sg1.d3.t1")));
}

/** This use case is used to test the de-duplication of getAllPathPatterns results */
@Test
public void pathPatternTreeTest7() throws IllegalPathException, IOException {
checkPathPatternTree(
Arrays.asList(
new PartialPath("root.sg1.d1.s1"),
new PartialPath("root.sg1.*.s2"),
new PartialPath("root.sg1.d1.t1.s1"),
new PartialPath("root.sg1.*.s1"),
new PartialPath("root.sg1.**.s1")),
Arrays.asList(new PartialPath("root.sg1.*.s2"), new PartialPath("root.sg1.**.s1")),
Arrays.asList(new PartialPath("root.sg1.*"), new PartialPath("root.sg1.**")));
}

/** This use case is used to test the de-duplication of getAllDevicePatterns results */
@Test
public void pathPatternTreeTest8() throws IllegalPathException, IOException {
checkPathPatternTree(
Arrays.asList(new PartialPath("root.sg1.d1.s1"), new PartialPath("root.sg1.d1.s2")),
Arrays.asList(new PartialPath("root.sg1.d1.s1"), new PartialPath("root.sg1.d1.s2")),
Collections.singletonList(new PartialPath("root.sg1.d1")));
}

/**
* @param paths PartialPath list to create PathPatternTree
* @param compressedPaths Expected PartialPath list of getAllPathPatterns
* @param compressedDevicePaths Expected PartialPath list of getAllDevicePatterns
* @throws IOException
*/
private void checkPathPatternTree(
List<PartialPath> paths,
List<PartialPath> compressedPaths,
Expand All @@ -147,6 +176,10 @@ private void checkPathPatternTree(
}
patternTree.constructTree();

Assert.assertEquals(
compressedPaths.stream().sorted().collect(Collectors.toList()),
patternTree.getAllPathPatterns().stream().sorted().collect(Collectors.toList()));

PathPatternTree resultPatternTree = new PathPatternTree();
for (PartialPath path : compressedPaths) {
resultPatternTree.appendPathPattern(path);
Expand Down

0 comments on commit 641e2b9

Please sign in to comment.