Skip to content

Commit

Permalink
Spark3.4,3.5: In describe extended view command: fix wrong view catal…
Browse files Browse the repository at this point in the history
…og and namespace.
  • Loading branch information
wangpeipei3 committed Dec 17, 2024
1 parent ff81344 commit 606eaee
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 3 deletions.
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ systemProp.defaultFlinkVersions=1.20
systemProp.knownFlinkVersions=1.18,1.19,1.20
systemProp.defaultHiveVersions=2
systemProp.knownHiveVersions=2,3
systemProp.defaultSparkVersions=3.5
systemProp.defaultSparkVersions=3.5,3.4
systemProp.knownSparkVersions=3.3,3.4,3.5
systemProp.defaultKafkaVersions=3
systemProp.knownKafkaVersions=3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,14 @@ case class DescribeV2ViewExec(
private def describeExtended: Seq[InternalRow] = {
val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
val properties: Map[String, String] = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
val viewCatalogAndNamespace: Seq[String] = view.currentCatalog +: view.currentNamespace.toSeq
val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2)
val viewProperties = properties.toSeq.sortBy(_._1).map {
case (key, value) =>
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
}.mkString("[", ", ", "]")


//omiited view text, because it can show in SHOW CREATE.
toCatalystRow("# Detailed View Information", "", "") ::
toCatalystRow("Comment", view.properties.getOrDefault(ViewCatalog.PROP_COMMENT, ""), "") ::
toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.iceberg.view.View;
import org.apache.iceberg.view.ViewHistoryEntry;
import org.apache.iceberg.view.ViewProperties;
import org.apache.iceberg.view.ViewUtil;
import org.apache.iceberg.view.ViewVersion;
import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.Dataset;
Expand Down Expand Up @@ -1388,6 +1389,68 @@ public void describeExtendedView() {
""));
}

@Test
public void createAndDescribeViewInDefaultNamespace() {
String viewName = viewName("createViewInDefaultNamespace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName);

sql("CREATE VIEW %s (id, data) AS %s", viewName, sql);
TableIdentifier identifier = TableIdentifier.of(NAMESPACE, viewName);
View view = viewCatalog().loadView(identifier);
assertThat(view.currentVersion().defaultCatalog()).isNull();
assertThat(view.name()).isEqualTo(ViewUtil.fullViewName(catalogName, identifier));
assertThat(view.currentVersion().defaultNamespace()).isEqualTo(NAMESPACE);

String location = viewCatalog().loadView(identifier).location();
assertThat(sql("DESCRIBE EXTENDED %s.%s", NAMESPACE, viewName))
.contains(
row("id", "int", ""),
row("data", "string", ""),
row("", "", ""),
row("# Detailed View Information", "", ""),
row("Comment", "", ""),
row("View Catalog and Namespace", String.format("%s.%s", catalogName, NAMESPACE), ""),
row("View Query Output Columns", "[id, data]", ""),
row(
"View Properties",
String.format(
"['format-version' = '1', 'location' = '%s', 'provider' = 'iceberg']",
location),
""));
}

@Test
public void createAndDescribeViewWithoutCurrentNamespace() {
String viewName = viewName("createViewWithoutCurrentNamespace");
Namespace namespace = Namespace.of("test_namespace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName);

sql("CREATE NAMESPACE IF NOT EXISTS %s", namespace);
sql("CREATE VIEW %s.%s (id, data) AS %s", namespace, viewName, sql);
TableIdentifier identifier = TableIdentifier.of(namespace, viewName);
View view = viewCatalog().loadView(identifier);
assertThat(view.currentVersion().defaultCatalog()).isNull();
assertThat(view.name()).isEqualTo(ViewUtil.fullViewName(catalogName, identifier));
assertThat(view.currentVersion().defaultNamespace()).isEqualTo(NAMESPACE);

String location = viewCatalog().loadView(identifier).location();
assertThat(sql("DESCRIBE EXTENDED %s.%s", namespace, viewName))
.contains(
row("id", "int", ""),
row("data", "string", ""),
row("", "", ""),
row("# Detailed View Information", "", ""),
row("Comment", "", ""),
row("View Catalog and Namespace", String.format("%s.%s", catalogName, namespace), ""),
row("View Query Output Columns", "[id, data]", ""),
row(
"View Properties",
String.format(
"['format-version' = '1', 'location' = '%s', 'provider' = 'iceberg']",
location),
""));
}

@Test
public void showViewProperties() {
String viewName = viewName("showViewProps");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,14 @@ case class DescribeV2ViewExec(
private def describeExtended: Seq[InternalRow] = {
val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
val properties: Map[String, String] = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
val viewCatalogAndNamespace: Seq[String] = view.currentCatalog +: view.currentNamespace.toSeq
val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2)
val viewProperties = properties.toSeq.sortBy(_._1).map {
case (key, value) =>
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
}.mkString("[", ", ", "]")


//omiited view text, because it can show in SHOW CREATE.
toCatalystRow("# Detailed View Information", "", "") ::
toCatalystRow("Comment", view.properties.getOrDefault(ViewCatalog.PROP_COMMENT, ""), "") ::
toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.iceberg.view.View;
import org.apache.iceberg.view.ViewHistoryEntry;
import org.apache.iceberg.view.ViewProperties;
import org.apache.iceberg.view.ViewUtil;
import org.apache.iceberg.view.ViewVersion;
import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.Dataset;
Expand Down Expand Up @@ -1417,6 +1418,68 @@ public void describeExtendedView() {
""));
}

@TestTemplate
public void createAndDescribeViewInDefaultNamespace() {
String viewName = viewName("createViewInDefaultNamespace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName);

sql("CREATE VIEW %s (id, data) AS %s", viewName, sql);
TableIdentifier identifier = TableIdentifier.of(NAMESPACE, viewName);
View view = viewCatalog().loadView(identifier);
assertThat(view.currentVersion().defaultCatalog()).isNull();
assertThat(view.name()).isEqualTo(ViewUtil.fullViewName(catalogName, identifier));
assertThat(view.currentVersion().defaultNamespace()).isEqualTo(NAMESPACE);

String location = viewCatalog().loadView(identifier).location();
assertThat(sql("DESCRIBE EXTENDED %s.%s", NAMESPACE, viewName))
.contains(
row("id", "int", ""),
row("data", "string", ""),
row("", "", ""),
row("# Detailed View Information", "", ""),
row("Comment", "", ""),
row("View Catalog and Namespace", String.format("%s.%s", catalogName, NAMESPACE), ""),
row("View Query Output Columns", "[id, data]", ""),
row(
"View Properties",
String.format(
"['format-version' = '1', 'location' = '%s', 'provider' = 'iceberg']",
location),
""));
}

@TestTemplate
public void createAndDescribeViewWithoutCurrentNamespace() {
String viewName = viewName("createViewWithoutCurrentNamespace");
Namespace namespace = Namespace.of("test_namespace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName);

sql("CREATE NAMESPACE IF NOT EXISTS %s", namespace);
sql("CREATE VIEW %s.%s (id, data) AS %s", namespace, viewName, sql);
TableIdentifier identifier = TableIdentifier.of(namespace, viewName);
View view = viewCatalog().loadView(identifier);
assertThat(view.currentVersion().defaultCatalog()).isNull();
assertThat(view.name()).isEqualTo(ViewUtil.fullViewName(catalogName, identifier));
assertThat(view.currentVersion().defaultNamespace()).isEqualTo(NAMESPACE);

String location = viewCatalog().loadView(identifier).location();
assertThat(sql("DESCRIBE EXTENDED %s.%s", namespace, viewName))
.contains(
row("id", "int", ""),
row("data", "string", ""),
row("", "", ""),
row("# Detailed View Information", "", ""),
row("Comment", "", ""),
row("View Catalog and Namespace", String.format("%s.%s", catalogName, namespace), ""),
row("View Query Output Columns", "[id, data]", ""),
row(
"View Properties",
String.format(
"['format-version' = '1', 'location' = '%s', 'provider' = 'iceberg']",
location),
""));
}

@TestTemplate
public void showViewProperties() {
String viewName = viewName("showViewProps");
Expand Down

0 comments on commit 606eaee

Please sign in to comment.