From 2b4ff1fa9a415ea1fa3a5c3118f97d71d3a8fa9e Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 14 Sep 2024 12:20:03 +0800 Subject: [PATCH] 4 --- .../apache/doris/analysis/DescribeStmt.java | 27 +++++++++-- .../apache/doris/datasource/CatalogIf.java | 7 ++- .../datasource/hive/HMSExternalCatalog.java | 47 +++++++++++++++---- .../nereids/rules/analysis/BindRelation.java | 2 +- 4 files changed, 69 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java index 7e503d525868c7c..91bf7713db02365 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -32,6 +32,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.proc.IndexSchemaProcNode; import org.apache.doris.common.proc.ProcNodeInterface; @@ -46,6 +47,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import jdk.internal.joptsimple.internal.Strings; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -55,6 +57,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; public class DescribeStmt extends ShowStmt implements NotFallbackInParser { @@ -123,6 +126,25 @@ public boolean isAllTables() { @Override public void analyze(Analyzer analyzer) throws UserException { + // First handle meta table. + // It will convert this to corresponding table valued functions + // eg: DESC table$partitions -> partition_values(...) + if (dbTableName != null) { + dbTableName.analyze(analyzer); + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(dbTableName.getCtl()); + Pair sourceTableNameWithMetaName = catalog.getSourceTableNameWithMetaTableName( + dbTableName.getTbl()); + if (!Strings.isNullOrEmpty(sourceTableNameWithMetaName.second)) { + isTableValuedFunction = true; + Optional optTvfRef = catalog.getMetaTableFunctionRef( + dbTableName.getDb(), dbTableName.getTbl()); + if (!optTvfRef.isPresent()) { + throw new AnalysisException("meta table not found: " + sourceTableNameWithMetaName.second); + } + tableValuedFunctionRef = optTvfRef.get(); + } + } + if (!isAllTables && isTableValuedFunction) { tableValuedFunctionRef.analyze(analyzer); List columns = tableValuedFunctionRef.getTable().getBaseSchema(); @@ -148,8 +170,6 @@ public void analyze(Analyzer analyzer) throws UserException { } } - dbTableName.analyze(analyzer); - if (!Env.getCurrentEnv().getAccessManager() .checkTblPriv(ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "DESCRIBE", @@ -159,8 +179,7 @@ public void analyze(Analyzer analyzer) throws UserException { CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(dbTableName.getCtl()); DatabaseIf db = catalog.getDbOrAnalysisException(dbTableName.getDb()); - TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl()); - + TableIf table = db.getTableOrDdlException(dbTableName.getTbl()); table.readLock(); try { if (!isAllTables) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 18605efc5512b33..41cb44ef0b5fa3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.DropDbStmt; import org.apache.doris.analysis.DropTableStmt; import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.TableValuedFunctionRef; import org.apache.doris.analysis.TruncateTableStmt; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; @@ -212,7 +213,11 @@ default Pair getSourceTableNameWithMetaTableName(String tableNam return Pair.of(tableName, ""); } - default Optional getMetaTableFunction(TableIf table, String sourceNameWithMetaName) { + default Optional getMetaTableFunction(String dbName, String sourceNameWithMetaName) { + return Optional.empty(); + } + + default Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { return Optional.empty(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 438fde26e1b1a84..02c0e814b07e193 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -17,9 +17,9 @@ package org.apache.doris.datasource.hive; +import org.apache.doris.analysis.TableValuedFunctionRef; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.HdfsResource; -import org.apache.doris.catalog.TableIf; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -49,6 +49,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import lombok.Getter; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -295,10 +296,20 @@ public Pair getSourceTableNameWithMetaTableName(String tableName } @Override - public Optional getMetaTableFunction(TableIf table, String sourceNameWithMetaName) { + public Optional getMetaTableFunction(String dbName, String sourceNameWithMetaName) { for (MetaTableFunction metaFunction : MetaTableFunction.values()) { if (metaFunction.containsMetaTable(sourceNameWithMetaName)) { - return Optional.of(metaFunction.createFunction(table)); + return Optional.of(metaFunction.createFunction(name, dbName, sourceNameWithMetaName)); + } + } + return Optional.empty(); + } + + @Override + public Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { + for (MetaTableFunction metaFunction : MetaTableFunction.values()) { + if (metaFunction.containsMetaTable(sourceNameWithMetaName)) { + return Optional.of(metaFunction.createFunctionRef(name, dbName, sourceNameWithMetaName)); } } return Optional.empty(); @@ -325,12 +336,14 @@ public boolean isEnableHmsEventsIncrementalSync() { * eg: tbl$partitions */ private enum MetaTableFunction { - PARTITIONS; + PARTITIONS("partition_values"); private final String suffix; + private final String tvfName; - MetaTableFunction() { + MetaTableFunction(String tvfName) { this.suffix = "$" + name().toLowerCase(); + this.tvfName = tvfName; } boolean containsMetaTable(String tableName) { @@ -341,15 +354,33 @@ String getSourceTableName(String tableName) { return tableName.substring(0, tableName.length() - suffix.length()); } - public TableValuedFunction createFunction(TableIf table) { + public TableValuedFunction createFunction(String ctlName, String dbName, String sourceNameWithMetaName) { switch (this) { case PARTITIONS: - List nameParts = Lists.newArrayList(table.getDatabase().getCatalog().getName(), - table.getDatabase().getFullName(), table.getName()); + List nameParts = Lists.newArrayList(ctlName, dbName, + getSourceTableName(sourceNameWithMetaName)); return PartitionValues.create(nameParts); default: throw new AnalysisException("Unsupported meta function type: " + this); } } + + public TableValuedFunctionRef createFunctionRef(String ctlName, String dbName, String sourceNameWithMetaName) { + switch (this) { + case PARTITIONS: + Map params = Maps.newHashMap(); + params.put("catalog", ctlName); + params.put("database", dbName); + params.put("tblName", getSourceTableName(sourceNameWithMetaName)); + try { + return new TableValuedFunctionRef(tvfName, null, params); + } catch (org.apache.doris.common.AnalysisException e) { + LOG.warn("should not happen. {}.{}.{}", ctlName, dbName, tblName); + return null; + } + default: + throw new AnalysisException("Unsupported meta function type: " + this); + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 74411dfef1e0bdb..fee31b68fe46eb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -371,7 +371,7 @@ public static LogicalPlan checkAndAddDeleteSignFilter(LogicalOlapScan scan, Conn private Optional handleMetaTable(TableIf table, UnboundRelation unboundRelation, List qualifiedTableName) { Optional tvf = table.getDatabase().getCatalog().getMetaTableFunction( - table, qualifiedTableName.get(2)); + qualifiedTableName.get(1), qualifiedTableName.get(2)); if (tvf.isPresent()) { return Optional.of(new LogicalTVFRelation(unboundRelation.getRelationId(), tvf.get())); }