Skip to content

Commit

Permalink
Support Spark SHOW TABLES command (unitycatalog#293)
Browse files Browse the repository at this point in the history
**Description of changes**

<!-- Please state what you've changed and how it might affect the users.
-->
Support Spark SHOW TABLES command
  • Loading branch information
amaliujia authored Jul 29, 2024
1 parent 0de24a0 commit 3feba9d
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package io.unitycatalog.connectors.spark

import io.unitycatalog.client.{ApiClient, ApiException}
import io.unitycatalog.client.api.{TablesApi, TemporaryTableCredentialsApi}
import io.unitycatalog.client.model.{AwsCredentials, GenerateTemporaryTableCredential, TableOperation, TableType}
import io.unitycatalog.client.model.{AwsCredentials, GenerateTemporaryTableCredential, ListTablesResponse, TableOperation, TableType}

import java.net.URI
import java.util
Expand All @@ -14,6 +14,7 @@ import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.types.{DataType, StructField, StructType}
import org.apache.spark.sql.util.CaseInsensitiveStringMap

import scala.collection.convert.ImplicitConversions._
import scala.jdk.CollectionConverters._

/**
Expand All @@ -33,7 +34,7 @@ class UCSingleCatalog extends TableCatalog {

override def name(): String = deltaCatalog.name()

override def listTables(namespace: Array[String]): Array[Identifier] = ???
override def listTables(namespace: Array[String]): Array[Identifier] = deltaCatalog.listTables(namespace)

override def loadTable(ident: Identifier): Table = deltaCatalog.loadTable(ident)

Expand Down Expand Up @@ -91,7 +92,18 @@ private class UCProxy extends TableCatalog {
this.name
}

override def listTables(namespace: Array[String]): Array[Identifier] = ???
override def listTables(namespace: Array[String]): Array[Identifier] = {
if (namespace.length > 1) {
throw new ApiException("Nested namespaces are not supported: " + namespace.mkString("."))
}
val catalogName = this.name
val schemaName = namespace.head
val maxResults = 0
val pageToken = null
val response: ListTablesResponse = tablesApi.listTables(catalogName, schemaName, maxResults, pageToken)
response.getTables.toSeq.map(table => Identifier.of(namespace, table.getName)).toArray
}


override def loadTable(ident: Identifier): Table = {
val t = try {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package io.unitycatalog.connectors.spark;

import static io.unitycatalog.server.utils.TestUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.*;

import io.unitycatalog.client.ApiException;
import io.unitycatalog.client.model.*;
Expand All @@ -19,6 +18,7 @@
import java.io.IOException;
import java.util.*;
import org.apache.spark.network.util.JavaUtils;
import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.BeforeEach;
Expand Down Expand Up @@ -143,6 +143,24 @@ public void testCredentialDelta() throws ApiException, IOException {
session.stop();
}

@Test
public void testShowTables() throws ApiException, IOException {
createCommonResources();
SparkSession session = createSparkSessionWithCatalogs(SPARK_CATALOG);
setupExternalParquetTable(PARQUET_TABLE, new ArrayList<>(0));

Row[] tables = (Row[]) session.sql("SHOW TABLES in " + SCHEMA_NAME).collect();
assertEquals(tables.length, 1);
assertEquals(tables[0].getString(0), SCHEMA_NAME);
assertEquals(tables[0].getString(1), PARQUET_TABLE);

AnalysisException exception =
assertThrows(AnalysisException.class, () -> session.sql("SHOW TABLES in a.b.c").collect());
assertTrue(exception.getMessage().contains("a.b.c"));

session.stop();
}

private String generateTableLocation(String catalogName, String tableName) throws IOException {
return new File(new File(dataDir, catalogName), tableName).getCanonicalPath();
}
Expand Down

0 comments on commit 3feba9d

Please sign in to comment.