diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 97a38b98645205..1db50c93fbee10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -51,7 +51,6 @@ import org.apache.doris.planner.ListPartitionPrunerV2; import org.apache.doris.planner.PartitionPrunerV2Base.UniqueId; import org.apache.doris.planner.external.FileSplit; -import org.apache.doris.spi.Split; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -66,6 +65,7 @@ import com.google.common.collect.Streams; import com.google.common.collect.TreeRangeMap; import lombok.Data; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; @@ -957,7 +957,7 @@ public static class FileCacheValue { // File Cache for self splitter. private final List files = Lists.newArrayList(); // File split cache for old splitter. This is a temp variable. - private final List splits = Lists.newArrayList(); + private final List splits = Lists.newArrayList(); private boolean isSplittable; // The values of partitions. // e.g for file : hdfs://path/to/table/part1=a/part2=b/datafile @@ -967,17 +967,21 @@ public static class FileCacheValue { private AcidInfo acidInfo; public void addFile(RemoteFile file) { - HiveFileStatus status = new HiveFileStatus(); - status.setBlockLocations(file.getBlockLocations()); - status.setPath(file.getPath()); - status.length = file.getSize(); - status.blockSize = file.getBlockSize(); - status.modificationTime = file.getModificationTime(); - files.add(status); + if (isFileVisible(file.getName())) { + HiveFileStatus status = new HiveFileStatus(); + status.setBlockLocations(file.getBlockLocations()); + status.setPath(file.getPath()); + status.length = file.getSize(); + status.blockSize = file.getBlockSize(); + status.modificationTime = file.getModificationTime(); + files.add(status); + } } - public void addSplit(Split split) { - splits.add(split); + public void addSplit(FileSplit split) { + if (isFileVisible(split.getPath().getName())) { + splits.add(split); + } } public int getValuesSize() { @@ -992,6 +996,12 @@ public AcidInfo getAcidInfo() { public void setAcidInfo(AcidInfo acidInfo) { this.acidInfo = acidInfo; } + + private boolean isFileVisible(String filename) { + return StringUtils.isNotEmpty(filename) + && !filename.startsWith(".") + && !filename.startsWith("_"); + } } @Data