From 96215e3844d926a63889ae7ca169a5a2a4bdb564 Mon Sep 17 00:00:00 2001 From: mbrunetto Date: Tue, 20 Feb 2024 15:24:28 +0100 Subject: [PATCH 1/8] Include Vertica DB inside the doc and gradle libs --- dataframe-jdbc/build.gradle.kts | 1 + docs/StardustDocs/topics/gradleReference.md | 2 +- docs/StardustDocs/topics/readSqlDatabases.md | 12 +++++++++--- gradle/libs.versions.toml | 2 ++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/dataframe-jdbc/build.gradle.kts b/dataframe-jdbc/build.gradle.kts index 2c5505a46f..a695f85f23 100644 --- a/dataframe-jdbc/build.gradle.kts +++ b/dataframe-jdbc/build.gradle.kts @@ -23,6 +23,7 @@ dependencies { testImplementation(libs.postgresql) testImplementation(libs.mysql) testImplementation(libs.h2db) + testImplementation(libs.vertica) testImplementation(libs.junit) testImplementation(libs.sl4j) testImplementation(libs.kotestAssertions) { diff --git a/docs/StardustDocs/topics/gradleReference.md b/docs/StardustDocs/topics/gradleReference.md index 5c848e261a..dc049a83e3 100644 --- a/docs/StardustDocs/topics/gradleReference.md +++ b/docs/StardustDocs/topics/gradleReference.md @@ -156,7 +156,7 @@ dataframes { Find full example code [here](https://github.com/zaleslaw/KotlinDataFrame-SQL-Examples/blob/master/src/main/kotlin/Example_3_Import_schema_via_Gradle.kt). **NOTE:** This is an experimental functionality and, for now, -we only support four databases: MariaDB, MySQL, PostgreSQL, and SQLite. +we only support four databases: MariaDB, MySQL, PostgreSQL, SQLite and Vertica. Additionally, support for JSON and date-time types is limited. Please take this into consideration when using these functions. diff --git a/docs/StardustDocs/topics/readSqlDatabases.md b/docs/StardustDocs/topics/readSqlDatabases.md index fda5388cb6..982d6f2047 100644 --- a/docs/StardustDocs/topics/readSqlDatabases.md +++ b/docs/StardustDocs/topics/readSqlDatabases.md @@ -15,7 +15,7 @@ There are two main blocks of available functionality: * ```getSchemaForAllSqlTables``` for all non-system tables **NOTE:** This is an experimental module and for now, -we only support four databases: MariaDB, MySQL, PostgreSQL, and SQLite. +we only support four databases: MariaDB, MySQL, PostgreSQL, SQLite and Vertica. Additionally, support for JSON and date-time types is limited. Please take this into consideration when using these functions. @@ -55,6 +55,12 @@ For SQLite: implementation("org.xerial:sqlite-jdbc:$version") ``` +For Vertica: + +```kotlin +implementation("com.vertica.jdbc:vertica-jdbc:$version") +``` + In the second, be sure that you can establish a connection to the database. For this, usually, you need to have three things: a URL to a database, a username and a password. @@ -240,7 +246,7 @@ and transforms it into an AnyFrame object. The `dbType: DbType` parameter specifies the type of our database (e.g., PostgreSQL, MySQL, etc.), supported by a library. -Currently, the following classes are available: `H2, MariaDb, MySql, PostgreSql, Sqlite`. +Currently, the following classes are available: `H2, MariaDb, MySql, PostgreSql, Sqlite, Vertica`. ```kotlin @@ -449,7 +455,7 @@ This function reads the schema from a ResultSet object provided by the user. The `dbType: DbType` parameter specifies the type of our database (e.g., PostgreSQL, MySQL, etc.), supported by a library. -Currently, the following classes are available: `H2, MariaDb, MySql, PostgreSql, Sqlite`. +Currently, the following classes are available: `H2, MariaDb, MySql, PostgreSql, Sqlite, Vertica`. ```kotlin import org.jetbrains.kotlinx.dataframe.io.db.PostgreSql diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5a7163c9b0..ca8ad31113 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -21,6 +21,7 @@ h2db = "2.2.220" mysql = "8.0.33" postgresql = "42.6.0" sqlite = "3.42.0.1" +vertica = "24.1.0-0" kotlinDatetime = "0.4.0" kotlinpoet = "1.12.0" openapi = "2.1.13" @@ -54,6 +55,7 @@ h2db = { group = "com.h2database", name = "h2", version.ref = "h2db" } mysql = { group = "mysql", name = "mysql-connector-java", version.ref = "mysql" } postgresql = { group = "org.postgresql", name = "postgresql", version.ref = "postgresql" } sqlite = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite" } +vertica = { group = "com.vertica.jdbc", name = "vertica-jdbc", version.ref = "vertica" } poi-ooxml = { module = "org.apache.poi:poi-ooxml", version.ref = "poi" } kotlin-datetimeJvm = { module = "org.jetbrains.kotlinx:kotlinx-datetime-jvm", version.ref = "kotlinDatetime" } From 2e3358f51ad6b8eee4e05f8c9dc8312e7a2cdeef Mon Sep 17 00:00:00 2001 From: mbrunetto Date: Wed, 21 Feb 2024 17:10:49 +0100 Subject: [PATCH 2/8] WIP: testing the fields --- .../kotlinx/dataframe/io/db/Vertica.kt | 46 +++ .../jetbrains/kotlinx/dataframe/io/db/util.kt | 3 +- .../kotlinx/dataframe/io/mysqlTest.kt | 36 +- .../kotlinx/dataframe/io/vertica2Test.kt | 278 +++++++++++++ .../kotlinx/dataframe/io/verticaTest.kt | 375 ++++++++++++++++++ 5 files changed, 733 insertions(+), 5 deletions(-) create mode 100644 dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt create mode 100644 dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt create mode 100644 dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt new file mode 100644 index 0000000000..711246ff97 --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt @@ -0,0 +1,46 @@ +package org.jetbrains.kotlinx.dataframe.io.db + +import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import java.sql.ResultSet +import java.util.Locale +import org.jetbrains.kotlinx.dataframe.io.TableMetadata +import kotlin.reflect.KType +import kotlin.reflect.full.createType + +/** + * Represents the MySql database type. + * + * This class provides methods to convert data from a ResultSet to the appropriate type for MySql, + * and to generate the corresponding column schema. + */ +public object Vertica : DbType("vertica") { + override val driverClassName: String + get() = "com.vertica.jdbc.Driver" + + override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? { +// when(tableColumnMetadata.sqlTypeName) { +// "FLOAT8" -> Double::class.createType(nullable = tableColumnMetadata.isNullable) +// } + return null + } + + override fun isSystemTable(tableMetadata: TableMetadata): Boolean { + val schemaName = tableMetadata.schemaName + + return schemaName?.startsWith("v_", true) ?: false + } + + override fun buildTableMetadata(tables: ResultSet): TableMetadata { + return TableMetadata( + tables.getString("table_name"), + tables.getString("table_schem"), + tables.getString("table_cat")) + } + + override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? { +// if(tableColumnMetadata.sqlTypeName == "INT UNSIGNED") +// return Long::class.createType(nullable = tableColumnMetadata.isNullable) + return null + } +} diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt index cfe59732f4..d8975a745e 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt @@ -17,8 +17,9 @@ public fun extractDBTypeFromUrl(url: String?): DbType { MySql.dbTypeInJdbcUrl in url -> MySql Sqlite.dbTypeInJdbcUrl in url -> Sqlite PostgreSql.dbTypeInJdbcUrl in url -> PostgreSql + Vertica.dbTypeInJdbcUrl in url -> Vertica else -> throw IllegalArgumentException("Unsupported database type in the url: $url. " + - "Only H2, MariaDB, MySQL, SQLite and PostgreSQL are supported!") + "Only H2, MariaDB, MySQL, SQLite, PostgreSQL and Vertica are supported!") } } else { throw SQLException("Database URL could not be null. The existing value is $url") diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mysqlTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mysqlTest.kt index 929a248b9e..8366ced6b5 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mysqlTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mysqlTest.kt @@ -229,10 +229,38 @@ class MySqlTest { @Language("SQL") val insertData2 = """ INSERT INTO table2 ( - bitCol, tinyintCol, smallintCol, mediumintCol, mediumintUnsignedCol, integerCol, intCol, - integerUnsignedCol, bigintCol, floatCol, doubleCol, decimalCol, dateCol, datetimeCol, timestampCol, - timeCol, yearCol, varcharCol, charCol, binaryCol, varbinaryCol, tinyblobCol, blobCol, - mediumblobCol, longblobCol, textCol, mediumtextCol, longtextCol, enumCol, setCol, location, data + bitCol, + tinyintCol, + smallintCol, + mediumintCol, + mediumintUnsignedCol, + integerCol, + intCol, + integerUnsignedCol, + bigintCol, + floatCol, + doubleCol, + decimalCol, + dateCol, + datetimeCol, + timestampCol, + timeCol, + yearCol, + varcharCol, + charCol, + binaryCol, + varbinaryCol, + tinyblobCol, + blobCol, + mediumblobCol, + longblobCol, + textCol, + mediumtextCol, + longtextCol, + enumCol, + setCol, + location, + data ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ST_GeomFromText('POINT(1 1)'), ?) """.trimIndent() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt new file mode 100644 index 0000000000..fabb3151f7 --- /dev/null +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt @@ -0,0 +1,278 @@ +package org.jetbrains.kotlinx.dataframe.io + +import io.kotest.matchers.shouldBe +import org.intellij.lang.annotations.Language +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.api.cast +import org.jetbrains.kotlinx.dataframe.api.filter +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test +import java.math.BigDecimal +import java.sql.Connection +import java.sql.DriverManager +import java.sql.SQLException +import java.sql.Time +import java.sql.Timestamp +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.util.Date +import kotlin.reflect.typeOf + +// Run with https://hub.docker.com/r/vertica/vertica-ce +private const val URL = "jdbc:vertica://localhost:5433" +private const val USER_NAME = "dbadmin" +private const val PASSWORD = "" +private const val TEST_SCHEMA_NAME = "testschema" + +@DataSchema +interface Table1Vertica { + val id: Int + val boolCol: Boolean + val charCol: Char + val varcharCol: String + val longvarcharCol: String + val dateCol: Date + val timeCol: Time + val timestampCol: Timestamp + val doubleprecisionCol: Float + val floatCol: Float + val float8Col: Float + val realCol: Float + val integerCol: Long + val intCol: Long + val bigintCol: Long + val int8Col: Long + val smallintCol: Long + val tinyintCol: Long + val decimalCol: BigDecimal + val numericCol: BigDecimal + val numberCol: BigDecimal + val moneyCol: BigDecimal + val geometryCol: String + val geographyCol: String +} + +class VerticaTest { + companion object { + private lateinit var connection: Connection + + @BeforeClass + @JvmStatic + fun setUpClass() { + connection = DriverManager.getConnection(URL, USER_NAME, PASSWORD) + + connection.createStatement().use { st -> + // Drop the test schema if it exists + val dropSchemaQuery = "DROP SCHEMA IF EXISTS $TEST_SCHEMA_NAME" + st.executeUpdate(dropSchemaQuery) + + // Create the test schema + val createSchemaQuery = "CREATE SCHEMA $TEST_SCHEMA_NAME" + st.executeUpdate(createSchemaQuery) + +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO $TEST_SCHEMA_NAME" + st.execute(setDefaultSchemaQuery) + } + + connection.createStatement().use { st -> + st.execute("DROP TABLE IF EXISTS table1") + } + + @Language("SQL") + val createTableQuery = """ + CREATE TABLE IF NOT EXISTS table1 ( + id INT NOT NULL PRIMARY KEY, + boolCol BOOLEAN, + charCol CHAR(10), + varcharCol VARCHAR, + longvarcharCol LONG VARCHAR, + binaryCol BINARY(11), + varbinaryCol VARBINARY, + longvarbinaryCol LONG VARBINARY, + dateCol DATE, + timeCol TIME, + timestampCol TIMESTAMP, + doubleprecisionCol DOUBLE PRECISION, + floatCol FLOAT, + float8Col FLOAT8, + realCol REAL, + integerCol INTEGER, + intCol INT, + bigintCol BIGINT, + int8Col INT8, + smallintCol SMALLINT, + tinyintCol TINYINT, + decimalCol DECIMAL, + numericCol NUMERIC, + numberCol NUMBER, + moneyCol MONEY, + geometryCol GEOMETRY, + geographyCol GEOGRAPHY + ) + """ + + connection.createStatement().execute( + createTableQuery.trimIndent() + ) + + @Language("SQL") + val insertData1 = """ + INSERT INTO table1 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ST_GeomFromText('POINT(1 1)'), ST_GeographyFromText('POLYGON((1 2,3 4,2 3,1 2))')) + """.trimIndent() + + connection.prepareStatement(insertData1).use { st -> + // Insert data into table1 + for (i in 1..3) { + st.setInt(1, i) + st.setBoolean(2, true) + st.setString(3, "charValue$i") + st.setString(4, "varcharValue$i") + st.setString(5, "longvarcharValue$i") + st.setBytes(6, "binaryValue".toByteArray()) + st.setBytes(7, "varbinaryValue".toByteArray()) + st.setBytes(8, "longvarbinaryValue".toByteArray()) + st.setDate(9, java.sql.Date.valueOf(LocalDate.of(2024, 1, 1))) + st.setTime(10, Time.valueOf(LocalTime.of(10, 0, 0))) + st.setTimestamp(11, Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0))) + st.setFloat(12, i * 10.0f) + st.setFloat(13, i * 10.0f) + st.setFloat(14, i * 10.0f) + st.setFloat(15, i * 10.0f) + st.setInt(16, i * 100) + st.setInt(17, i * 100) + st.setInt(18, i * 100) + st.setInt(19, i * 100) + st.setInt(20, i * 100) + st.setInt(21, i * 100) + st.setBigDecimal(22, BigDecimal(i * 10)) + st.setBigDecimal(23, BigDecimal(i * 10)) + st.setBigDecimal(24, BigDecimal(i * 10)) + st.setBigDecimal(25, BigDecimal(i * 20)) + st.setBigDecimal(26, BigDecimal(i * 30)) + st.executeUpdate() + } + } + } + + @AfterClass + @JvmStatic + fun tearDownClass() { + try { + connection.createStatement().use { st -> st.execute("DROP TABLE IF EXISTS table1") } + connection.createStatement().use { st -> st.execute("DROP SCHEMA IF EXISTS $TEST_SCHEMA_NAME") } + connection.close() + } catch (e: SQLException) { + e.printStackTrace() + } + } + } + + @Test + fun `basic test for reading sql tables`() { + val df = DataFrame.readSqlTable(connection, "table1").cast() + df.rowsCount() shouldBe 3 + val result = df.filter { it[Table1Vertica::id] == 1 } + result[0][0] shouldBe 1L + result[0][1] shouldBe true + result[0][2] shouldBe "charValue1" + result[0][3] shouldBe "varcharValue1" + result[0][4] shouldBe "longvarcharValue1" + result[0][5] shouldBe "binaryValue".toByteArray() + result[0][6] shouldBe "varbinaryValue".toByteArray() + result[0][7] shouldBe "longvarbinaryValue".toByteArray() + result[0][8] shouldBe java.sql.Date.valueOf(LocalDate.of(2024, 1, 1)) + result[0][9] shouldBe Time.valueOf(LocalTime.of(10, 0, 0)) + result[0][10] shouldBe Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)) + result[0][11] shouldBe 1 * 10.0f + result[0][12] shouldBe 1 * 10.0f + result[0][13] shouldBe 1 * 10.0f + result[0][14] shouldBe 1 * 10.0f + result[0][15] shouldBe 1 * 100 + result[0][16] shouldBe 1 * 100 + result[0][17] shouldBe 1 * 100 + result[0][18] shouldBe 1 * 100 + result[0][19] shouldBe 1 * 100 + result[0][20] shouldBe 1 * 100 + result[0][21] shouldBe BigDecimal("10.000000000000000") + result[0][22] shouldBe BigDecimal("10.000000000000000") + result[0][23] shouldBe BigDecimal("10") + result[0][24] shouldBe BigDecimal("10.0000") + result[0][25] shouldBe BigDecimal("11.0000") + result[0][26] shouldBe BigDecimal("12.0000") + + val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + schema.columns["id"]!!.type shouldBe typeOf() + schema.columns["boolCol"]!!.type shouldBe typeOf() + schema.columns["charCol"]!!.type shouldBe typeOf() + schema.columns["varcharCol"]!!.type shouldBe typeOf() + schema.columns["longvarcharCol"]!!.type shouldBe typeOf() + schema.columns["binaryCol"]!!.type shouldBe typeOf() + schema.columns["varbinaryCol"]!!.type shouldBe typeOf() + schema.columns["longvarbinaryCol"]!!.type shouldBe typeOf() + schema.columns["dateCol"]!!.type shouldBe typeOf() + schema.columns["timeCol"]!!.type shouldBe typeOf() + schema.columns["timestampCol"]!!.type shouldBe typeOf() + schema.columns["doubleprecisionCol"]!!.type shouldBe typeOf() + schema.columns["floatCol"]!!.type shouldBe typeOf() + schema.columns["float8Col"]!!.type shouldBe typeOf() + schema.columns["realCol"]!!.type shouldBe typeOf() + schema.columns["integerCol"]!!.type shouldBe typeOf() + schema.columns["intCol"]!!.type shouldBe typeOf() + schema.columns["bigintCol"]!!.type shouldBe typeOf() + schema.columns["int8Col"]!!.type shouldBe typeOf() + schema.columns["smallintCol"]!!.type shouldBe typeOf() + schema.columns["tinyintCol"]!!.type shouldBe typeOf() + schema.columns["decimalCol"]!!.type shouldBe typeOf() + schema.columns["numericCol"]!!.type shouldBe typeOf() + schema.columns["numberCol"]!!.type shouldBe typeOf() + schema.columns["moneyCol"]!!.type shouldBe typeOf() + schema.columns["geometryCol"]!!.type shouldBe typeOf() + schema.columns["geographyCol"]!!.type shouldBe typeOf() + } + +// @Test +// fun `read from sql query`() { +// @Language("SQL") +// val sqlQuery = """ +// SELECT +// t1.id, +// t1.enumCol, +// t2.setCol +// FROM table1 t1 +// JOIN table2 t2 ON t1.id = t2.id +// """.trimIndent() +// +// val df = DataFrame.readSqlQuery(connection, sqlQuery = sqlQuery).cast() +// val result = df.filter { it[Table3MySql::id] == 1 } +// result[0][2] shouldBe "Option1" +// +// val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) +// schema.columns["id"]!!.type shouldBe typeOf() +// schema.columns["enumCol"]!!.type shouldBe typeOf() +// schema.columns["setCol"]!!.type shouldBe typeOf() +// } +// +// @Test +// fun `read from all tables`() { +// val dataframes = DataFrame.readAllSqlTables(connection) +// +// val table1Df = dataframes[0].cast() +// +// table1Df.rowsCount() shouldBe 3 +// table1Df.filter { it[Table1MySql::integerCol] > 100 }.rowsCount() shouldBe 2 +// table1Df[0][11] shouldBe 10.0 +// table1Df[0][26] shouldBe "textValue1" +// +// val table2Df = dataframes[1].cast() +// +// table2Df.rowsCount() shouldBe 3 +// table2Df.filter { it[Table2MySql::integerCol] != null && it[Table2MySql::integerCol]!! > 400 } +// .rowsCount() shouldBe 1 +// table2Df[0][11] shouldBe 20.0 +// table2Df[0][26] shouldBe null +// } +} diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt new file mode 100644 index 0000000000..84d26fadf5 --- /dev/null +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt @@ -0,0 +1,375 @@ +//package org.jetbrains.kotlinx.dataframe.io +// +//import io.kotest.matchers.shouldBe +//import org.intellij.lang.annotations.Language +//import org.jetbrains.kotlinx.dataframe.DataFrame +//import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +//import org.jetbrains.kotlinx.dataframe.api.cast +//import org.jetbrains.kotlinx.dataframe.api.filter +//import org.junit.AfterClass +//import org.junit.BeforeClass +//import org.junit.Test +//import java.math.BigDecimal +//import java.sql.Connection +//import java.sql.DriverManager +//import java.sql.SQLException +//import org.jetbrains.kotlinx.dataframe.api.add +//import org.jetbrains.kotlinx.dataframe.api.select +//import org.junit.Ignore +//import java.util.UUID +//import kotlin.reflect.typeOf +// +//private const val URL = "jdbc:vertica://localhost:5433" +//private const val USER_NAME = "root" +//private const val PASSWORD = "pass" +//private const val TEST_DATABASE_NAME = "testKDFdatabase" +// +//@DataSchema +//interface Table1Vertica { +// val id: Int +// val bitCol: Boolean +// val tinyintCol: Int +// val smallintCol: Int +// val mediumintCol: Int +// val mediumintUnsignedCol: Int +// val integerCol: Int +// val intCol: Int +// val integerUnsignedCol: Long +// val bigintCol: Long +// val floatCol: Float +// val doubleCol: Double +// val decimalCol: BigDecimal +// val dateCol: String +// val datetimeCol: String +// val timestampCol: String +// val timeCol: String +// val yearCol: String +// val varcharCol: String +// val charCol: String +// val binaryCol: ByteArray +// val varbinaryCol: ByteArray +// val tinyblobCol: ByteArray +// val blobCol: ByteArray +// val mediumblobCol: ByteArray +// val longblobCol: ByteArray +// val textCol: String +// val mediumtextCol: String +// val longtextCol: String +// val enumCol: String +// val setCol: Char +//} +// +//@Ignore +//class VerticaTest { +// companion object { +// private lateinit var connection: Connection +// +// @BeforeClass +// @JvmStatic +// fun setUpClass() { +// connection = DriverManager.getConnection(URL, USER_NAME, PASSWORD) +// +// connection.createStatement().use { st -> +// // Drop the test database if it exists +// val dropDatabaseQuery = "DROP DATABASE IF EXISTS $TEST_DATABASE_NAME" +// st.executeUpdate(dropDatabaseQuery) +// +// // Create the test database +// val createDatabaseQuery = "CREATE DATABASE $TEST_DATABASE_NAME" +// st.executeUpdate(createDatabaseQuery) +// +// // Use the newly created database +// val useDatabaseQuery = "USE $TEST_DATABASE_NAME" +// st.executeUpdate(useDatabaseQuery) +// } +// +// connection.createStatement().use { st -> st.execute("DROP TABLE IF EXISTS table1") } +// +// @Language("SQL") +// val createTableQuery = """ +// CREATE TABLE IF NOT EXISTS table1 ( +// integerCol INTEGER, +// intCol INT, +// bigintCol BIGINT, +// int8Col INT8, +// smallintCol SMALLINT, +// tinyintCol TINYINT, +// decimalCol DECIMAL, +// numericCol NUMERIC, +// numberCol NUMBER, +// moneyCol MONEY, +// uuidCol UUID +// ) +// """ +// +// val createTableQueryBackup = """ +// CREATE TABLE IF NOT EXISTS table1 ( +//// timewithtimezoneCol TIMETZ, +//// timestampwithtimezoneCol TIMESTAMPTZ, +//// intervalCol INTERVAL, +//// intervaldaytosecondCol INTERVAL DAY TO SECOND, +//// intervalyeartomonthCol INTERVAL YEAR TO MONTH, +// uuidCol UUID, +//// arrayCol ARRAY[VARCHAR(50)], +//// rowCol ROW(street VARCHAR, city VARCHAR)), +//// setCol SET[VARCHAR] +// ) +// """ +// +// connection.createStatement().execute( +// createTableQuery.trimIndent() +// ) +// +// @Language("SQL") +// val insertData1 = """ +// INSERT INTO table1 ( +// boolCol, +// charCol, +// varcharCol, +// longvarcharCol, +// binaryCol, +// varbinaryCol, +// longvarbinaryCol, +// dateCol, +// timeCol, +//// timewithtimezoneCol, +// timestampCol, +//// timestampwithtimezoneCol, +//// intervalCol, +//// intervaldaytosecondCol, +//// intervalyeartomonthCol, +// doubleprecisionCol, +// floatCol, +// floatncol, +// float8Col, +// realCol, +// integerCol, +// intCol, +// bigintCol, +// int8Col, +// smallintCol, +// tinyintCol, +// decimalCol, +// numericCol, +// numberCol, +// moneyCol, +//// geometryCol, +//// geographyCol, +// uuidCol, +//// arrayCol, +//// rowCol, +//// setCol +// ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +// """.trimIndent() +// +// @Language("SQL") +// val insertDatatest = """ +// INSERT INTO table1 ( +// boolCol, +// charCol, +// varcharCol, +// longvarcharCol, +// binaryCol, +// varbinaryCol, +// longvarbinaryCol, +// dateCol, +// timeCol, +//// timewithtimezoneCol, +// timestampCol, +//// timestampwithtimezoneCol, +//// intervalCol, +//// intervaldaytosecondCol, +//// intervalyeartomonthCol, +// doubleprecisionCol, +// floatCol, +// floatncol, +// float8Col, +// realCol, +// integerCol, +// intCol, +// bigintCol, +// int8Col, +// smallintCol, +// tinyintCol, +// decimalCol, +// numericCol, +// numberCol, +// moneyCol, +//// geometryCol, +//// geographyCol, +// uuidCol, +//// arrayCol, +//// rowCol, +//// setCol +// ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +// """.trimIndent() +// +// connection.prepareStatement(insertData1).use { st -> +// // Insert data into table1 +// for (i in 1..3) { +// st.setTimestamp(10, java.sql.Timestamp(System.currentTimeMillis())) +//// st.setBigDecimal(12, BigDecimal(i * 10)) +//// st.setDate(13, java.sql.Date(System.currentTimeMillis())) +//// st.setTimestamp(14, java.sql.Timestamp(System.currentTimeMillis())) +//// st.setTimestamp(15, java.sql.Timestamp(System.currentTimeMillis())) +// st.setDouble(11, i * 10.0) +// st.setFloat(12, i * 10.0f) +// st.setFloat(13, i * 10.0f) +// st.setFloat(14, i * 10.0f) +// st.setFloat(15, i * 10.0f) +// st.setInt(16, i * 100) +// st.setInt(17, i * 100) +// st.setInt(18, i * 100) +// st.setInt(19, i * 100) +// st.setInt(20, i * 100) +// st.setInt(21, i * 100) +// st.setBigDecimal(22, BigDecimal(i * 10)) +// st.setBigDecimal(23, BigDecimal(i * 10)) +// st.setBigDecimal(24, BigDecimal(i * 10)) +// st.setBigDecimal(25, BigDecimal(i * 10)) +//// st.setString(31, "{\"key\": \"value\"}") +//// st.setString(32, "{\"key\": \"value\"}") +//// st.setString(33, "{\"key\": \"value\"}") +// st.setString(26, "cee3754a-e2f2-4020-b43f-aba52b10c827") +//// st.setString(35, "{\"key\": \"value\"}") +//// st.setString(36, "{\"key\": \"value\"}") +//// st.setString(37, "{\"key\": \"value\"}") +// st.executeUpdate() +// } +// } +// } +// +// @AfterClass +// @JvmStatic +// fun tearDownClass() { +// try { +// connection.createStatement().use { st -> st.execute("DROP TABLE IF EXISTS table1") } +// connection.createStatement().use { st -> st.execute("DROP DATABASE IF EXISTS $TEST_DATABASE_NAME") } +// connection.close() +// } catch (e: SQLException) { +// e.printStackTrace() +// } +// } +// } +// +// @Ignore +// @Test +// fun `basic test for reading sql tables`() { +// val df1 = DataFrame.readSqlTable(connection, "table1").cast() +// val result = df1.filter { it[Table1Vertica::id] == 1 } +// result[0][26] shouldBe "textValue1" +// +// val schema = DataFrame.getSchemaForSqlTable(connection, "table1") +// schema.columns["id"]!!.type shouldBe typeOf() +// schema.columns["textCol"]!!.type shouldBe typeOf() +// } +// +//// @Test +//// fun `read from sql query`() { +//// @Language("SQL") +//// val sqlQuery = """ +//// SELECT +//// t1.id, +//// t1.enumCol, +//// t2.setCol +//// FROM table1 t1 +//// JOIN table2 t2 ON t1.id = t2.id +//// """.trimIndent() +//// +//// val df = DataFrame.readSqlQuery(connection, sqlQuery = sqlQuery).cast() +//// val result = df.filter { it[Table3MySql::id] == 1 } +//// result[0][2] shouldBe "Option1" +//// +//// val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) +//// schema.columns["id"]!!.type shouldBe typeOf() +//// schema.columns["enumCol"]!!.type shouldBe typeOf() +//// schema.columns["setCol"]!!.type shouldBe typeOf() +//// } +//// +//// @Test +//// fun `read from all tables`() { +//// val dataframes = DataFrame.readAllSqlTables(connection) +//// +//// val table1Df = dataframes[0].cast() +//// +//// table1Df.rowsCount() shouldBe 3 +//// table1Df.filter { it[Table1MySql::integerCol] > 100 }.rowsCount() shouldBe 2 +//// table1Df[0][11] shouldBe 10.0 +//// table1Df[0][26] shouldBe "textValue1" +//// +//// val table2Df = dataframes[1].cast() +//// +//// table2Df.rowsCount() shouldBe 3 +//// table2Df.filter { it[Table2MySql::integerCol] != null && it[Table2MySql::integerCol]!! > 400 } +//// .rowsCount() shouldBe 1 +//// table2Df[0][11] shouldBe 20.0 +//// table2Df[0][26] shouldBe null +//// } +//// +//// @Test +//// fun `reading numeric types`() { +//// val df1 = DataFrame.readSqlTable(connection, "table1").cast() +//// +//// val result = df1.select("tinyintCol").add("tinyintCol2") { it[Table1MySql::tinyintCol] } +//// +//// result[0][1] shouldBe 1.toByte() +//// +//// val result1 = df1.select("smallintCol") +//// .add("smallintCol2") { it[Table1MySql::smallintCol] } +//// +//// result1[0][1] shouldBe 10.toShort() +//// +//// val result2 = df1.select("mediumintCol") +//// .add("mediumintCol2") { it[Table1MySql::mediumintCol] } +//// +//// result2[0][1] shouldBe 100 +//// +//// val result3 = df1.select("mediumintUnsignedCol") +//// .add("mediumintUnsignedCol2") { it[Table1MySql::mediumintUnsignedCol] } +//// +//// result3[0][1] shouldBe 100 +//// +//// val result4 = df1.select("integerUnsignedCol") +//// .add("integerUnsignedCol2") { it[Table1MySql::integerUnsignedCol] } +//// +//// result4[0][1] shouldBe 100L +//// +//// val result5 = df1.select("bigintCol") +//// .add("bigintCol2") { it[Table1MySql::bigintCol] } +//// +//// result5[0][1] shouldBe 100 +//// +//// val result6 = df1.select("floatCol") +//// .add("floatCol2") { it[Table1MySql::floatCol] } +//// +//// result6[0][1] shouldBe 10.0f +//// +//// val result7 = df1.select("doubleCol") +//// .add("doubleCol2") { it[Table1MySql::doubleCol] } +//// +//// result7[0][1] shouldBe 10.0 +//// +//// val result8 = df1.select("decimalCol") +//// .add("decimalCol2") { it[Table1MySql::decimalCol] } +//// +//// result8[0][1] shouldBe BigDecimal("10") +//// +//// val schema = DataFrame.getSchemaForSqlTable(connection, "table1") +//// +//// schema.columns["tinyintCol"]!!.type shouldBe typeOf() +//// schema.columns["smallintCol"]!!.type shouldBe typeOf() +//// schema.columns["mediumintCol"]!!.type shouldBe typeOf() +//// schema.columns["mediumintUnsignedCol"]!!.type shouldBe typeOf() +//// schema.columns["integerUnsignedCol"]!!.type shouldBe typeOf() +//// schema.columns["bigintCol"]!!.type shouldBe typeOf() +//// schema.columns["floatCol"]!!.type shouldBe typeOf() +//// schema.columns["doubleCol"]!!.type shouldBe typeOf() +//// schema.columns["decimalCol"]!!.type shouldBe typeOf() +//// // TODO: all unsigned types +//// // TODO: new mapping system based on class names +//// // validation after mapping in getObject +//// // getObject(i+1, type) catch getObject catch getString +//// // add direct mapping to getString and other methods +//// +//// } +//} From 903bfe53670d2332d3b76367aec51522df55683d Mon Sep 17 00:00:00 2001 From: mbrunetto Date: Thu, 22 Feb 2024 15:31:40 +0100 Subject: [PATCH 3/8] WIP: testing the fields --- .../kotlinx/dataframe/io/db/Vertica.kt | 26 ++++++++------- .../kotlinx/dataframe/io/vertica2Test.kt | 32 ++++++++++++++----- .../kotlinx/dataframe/io/verticaTest.kt | 4 --- gradle/scripts/pre-commit | 0 4 files changed, 39 insertions(+), 23 deletions(-) mode change 100644 => 100755 gradle/scripts/pre-commit diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt index 711246ff97..aa899e7b43 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt @@ -5,6 +5,7 @@ import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import java.util.Locale import org.jetbrains.kotlinx.dataframe.io.TableMetadata +import java.sql.Time import kotlin.reflect.KType import kotlin.reflect.full.createType @@ -18,12 +19,13 @@ public object Vertica : DbType("vertica") { override val driverClassName: String get() = "com.vertica.jdbc.Driver" - override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? { -// when(tableColumnMetadata.sqlTypeName) { -// "FLOAT8" -> Double::class.createType(nullable = tableColumnMetadata.isNullable) -// } - return null - } + override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? = + when(tableColumnMetadata.sqlTypeName.uppercase()) { + "UUID" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) +// "GEOMETRY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) +// "GEOGRAPHY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) + else -> null + } override fun isSystemTable(tableMetadata: TableMetadata): Boolean { val schemaName = tableMetadata.schemaName @@ -38,9 +40,11 @@ public object Vertica : DbType("vertica") { tables.getString("table_cat")) } - override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? { -// if(tableColumnMetadata.sqlTypeName == "INT UNSIGNED") -// return Long::class.createType(nullable = tableColumnMetadata.isNullable) - return null - } + override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = + when(tableColumnMetadata.sqlTypeName.uppercase()) { + "UUID" -> String::class.createType(nullable = tableColumnMetadata.isNullable) +// "GEOMETRY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) +// "GEOGRAPHY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) + else -> null + } } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt index fabb3151f7..216186c4ca 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt @@ -1,5 +1,7 @@ package org.jetbrains.kotlinx.dataframe.io +import com.vertica.dsi.dataengine.utilities.TimeTz +import com.vertica.dsi.dataengine.utilities.TimestampTz import io.kotest.matchers.shouldBe import org.intellij.lang.annotations.Language import org.jetbrains.kotlinx.dataframe.DataFrame @@ -18,7 +20,9 @@ import java.sql.Timestamp import java.time.LocalDate import java.time.LocalDateTime import java.time.LocalTime +import java.util.Calendar import java.util.Date +import java.util.UUID import kotlin.reflect.typeOf // Run with https://hub.docker.com/r/vertica/vertica-ce @@ -111,7 +115,11 @@ class VerticaTest { numberCol NUMBER, moneyCol MONEY, geometryCol GEOMETRY, - geographyCol GEOGRAPHY + geographyCol GEOGRAPHY, + timewithtimezoneCol TIMETZ, + timestampwithtimezoneCol TIMESTAMPTZ, + uuidCol UUID, + arrayCol ARRAY[VARCHAR(50)] ) """ @@ -121,7 +129,7 @@ class VerticaTest { @Language("SQL") val insertData1 = """ - INSERT INTO table1 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ST_GeomFromText('POINT(1 1)'), ST_GeographyFromText('POLYGON((1 2,3 4,2 3,1 2))')) + INSERT INTO table1 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, public.ST_GeomFromText('POINT(1 1)'), public.ST_GeographyFromText('POLYGON((1 2,3 4,2 3,1 2))'), ?, ?, ?, ARRAY['Test', 'Test1']) """.trimIndent() connection.prepareStatement(insertData1).use { st -> @@ -151,8 +159,10 @@ class VerticaTest { st.setBigDecimal(22, BigDecimal(i * 10)) st.setBigDecimal(23, BigDecimal(i * 10)) st.setBigDecimal(24, BigDecimal(i * 10)) - st.setBigDecimal(25, BigDecimal(i * 20)) - st.setBigDecimal(26, BigDecimal(i * 30)) + st.setBigDecimal(25, BigDecimal(i * 10)) + st.setTime(26, TimeTz(Time.valueOf(LocalTime.of(10,0,0,0)), Calendar.getInstance())) + st.setTimestamp(27, TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance())) + st.setString(28, "4a866db2-baa6-442a-a371-1f4b5ee627ba") st.executeUpdate() } } @@ -201,8 +211,10 @@ class VerticaTest { result[0][22] shouldBe BigDecimal("10.000000000000000") result[0][23] shouldBe BigDecimal("10") result[0][24] shouldBe BigDecimal("10.0000") - result[0][25] shouldBe BigDecimal("11.0000") - result[0][26] shouldBe BigDecimal("12.0000") + result[0][27] shouldBe TimeTz(Time.valueOf(LocalTime.of(10,0,0,0)), Calendar.getInstance()) +// result[0][28] shouldBe TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance()) + result[0][29] shouldBe UUID.fromString("4a866db2-baa6-442a-a371-1f4b5ee627ba") + result[0][30] shouldBe arrayOf("Test", "Test1") val schema = DataFrame.getSchemaForSqlTable(connection, "table1") schema.columns["id"]!!.type shouldBe typeOf() @@ -230,8 +242,12 @@ class VerticaTest { schema.columns["numericCol"]!!.type shouldBe typeOf() schema.columns["numberCol"]!!.type shouldBe typeOf() schema.columns["moneyCol"]!!.type shouldBe typeOf() - schema.columns["geometryCol"]!!.type shouldBe typeOf() - schema.columns["geographyCol"]!!.type shouldBe typeOf() + schema.columns["geometryCol"]!!.type shouldBe typeOf() + schema.columns["geographyCol"]!!.type shouldBe typeOf() + schema.columns["timewithtimezoneCol"]!!.type shouldBe typeOf() + schema.columns["timestampwithtimezoneCol"]!!.type shouldBe typeOf() + schema.columns["uuidCol"]!!.type shouldBe typeOf() + schema.columns["arrayCol"]!!.type shouldBe typeOf?>() } // @Test diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt index 84d26fadf5..5537f95467 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt @@ -104,13 +104,9 @@ // // val createTableQueryBackup = """ // CREATE TABLE IF NOT EXISTS table1 ( -//// timewithtimezoneCol TIMETZ, -//// timestampwithtimezoneCol TIMESTAMPTZ, //// intervalCol INTERVAL, //// intervaldaytosecondCol INTERVAL DAY TO SECOND, //// intervalyeartomonthCol INTERVAL YEAR TO MONTH, -// uuidCol UUID, -//// arrayCol ARRAY[VARCHAR(50)], //// rowCol ROW(street VARCHAR, city VARCHAR)), //// setCol SET[VARCHAR] // ) diff --git a/gradle/scripts/pre-commit b/gradle/scripts/pre-commit old mode 100644 new mode 100755 From f1b6fe19ec97fdec9b14147dcab43f027da0199b Mon Sep 17 00:00:00 2001 From: mbrunetto Date: Thu, 22 Feb 2024 17:15:42 +0100 Subject: [PATCH 4/8] WIP: testing the fields --- .../kotlinx/dataframe/io/db/Vertica.kt | 13 +++--- .../kotlinx/dataframe/io/vertica2Test.kt | 40 ++++++------------- .../kotlinx/dataframe/io/verticaTest.kt | 2 - 3 files changed, 19 insertions(+), 36 deletions(-) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt index aa899e7b43..0603f361f1 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt @@ -1,11 +1,9 @@ package org.jetbrains.kotlinx.dataframe.io.db import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.io.TableMetadata import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet -import java.util.Locale -import org.jetbrains.kotlinx.dataframe.io.TableMetadata -import java.sql.Time import kotlin.reflect.KType import kotlin.reflect.full.createType @@ -22,8 +20,8 @@ public object Vertica : DbType("vertica") { override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? = when(tableColumnMetadata.sqlTypeName.uppercase()) { "UUID" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) -// "GEOMETRY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) -// "GEOGRAPHY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) + "ARRAY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) + "UNKNOWN" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) else -> null } @@ -43,8 +41,9 @@ public object Vertica : DbType("vertica") { override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = when(tableColumnMetadata.sqlTypeName.uppercase()) { "UUID" -> String::class.createType(nullable = tableColumnMetadata.isNullable) -// "GEOMETRY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) -// "GEOGRAPHY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) +// "ARRAY" -> Array::class.createType(arguments = listOf(KTypeProjection.invariant(String::class.starProjectedType)), nullable = tableColumnMetadata.isNullable) + "ARRAY" -> String::class.createType(nullable = tableColumnMetadata.isNullable) // TODO understan if we can use the one above + "UNKNOWN" -> String::class.createType(nullable = tableColumnMetadata.isNullable) else -> null } } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt index 216186c4ca..6cbdaa3439 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt @@ -34,29 +34,6 @@ private const val TEST_SCHEMA_NAME = "testschema" @DataSchema interface Table1Vertica { val id: Int - val boolCol: Boolean - val charCol: Char - val varcharCol: String - val longvarcharCol: String - val dateCol: Date - val timeCol: Time - val timestampCol: Timestamp - val doubleprecisionCol: Float - val floatCol: Float - val float8Col: Float - val realCol: Float - val integerCol: Long - val intCol: Long - val bigintCol: Long - val int8Col: Long - val smallintCol: Long - val tinyintCol: Long - val decimalCol: BigDecimal - val numericCol: BigDecimal - val numberCol: BigDecimal - val moneyCol: BigDecimal - val geometryCol: String - val geographyCol: String } class VerticaTest { @@ -119,7 +96,10 @@ class VerticaTest { timewithtimezoneCol TIMETZ, timestampwithtimezoneCol TIMESTAMPTZ, uuidCol UUID, - arrayCol ARRAY[VARCHAR(50)] + arrayCol ARRAY[VARCHAR(50)], + rowCol ROW(street VARCHAR, city VARCHAR), + setCol SET[VARCHAR], + intervalCol INTERVAL ) """ @@ -129,7 +109,7 @@ class VerticaTest { @Language("SQL") val insertData1 = """ - INSERT INTO table1 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, public.ST_GeomFromText('POINT(1 1)'), public.ST_GeographyFromText('POLYGON((1 2,3 4,2 3,1 2))'), ?, ?, ?, ARRAY['Test', 'Test1']) + INSERT INTO table1 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, public.ST_GeomFromText('POINT(1 1)'), public.ST_GeographyFromText('POLYGON((1 2,3 4,2 3,1 2))'), ?, ?, ?, ARRAY['Test', 'Test1'], ROW('aStreet', 'aCity'), SET['aStreet', 'aCity'], INTERVAL '1 12:59:10:05') """.trimIndent() connection.prepareStatement(insertData1).use { st -> @@ -214,7 +194,10 @@ class VerticaTest { result[0][27] shouldBe TimeTz(Time.valueOf(LocalTime.of(10,0,0,0)), Calendar.getInstance()) // result[0][28] shouldBe TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance()) result[0][29] shouldBe UUID.fromString("4a866db2-baa6-442a-a371-1f4b5ee627ba") - result[0][30] shouldBe arrayOf("Test", "Test1") +// result[0][30] shouldBe arrayOf("Test", "Test1") +// result[0][31] shouldBe "{\"street\":\"aStreet\",\"city\":\"aCity\"}" +// result[0][32] shouldBe "{\"street\":\"aStreet\",\"city\":\"aCity\"}" +// result[0][33] shouldBe "1 12:59:10.005000" val schema = DataFrame.getSchemaForSqlTable(connection, "table1") schema.columns["id"]!!.type shouldBe typeOf() @@ -247,7 +230,10 @@ class VerticaTest { schema.columns["timewithtimezoneCol"]!!.type shouldBe typeOf() schema.columns["timestampwithtimezoneCol"]!!.type shouldBe typeOf() schema.columns["uuidCol"]!!.type shouldBe typeOf() - schema.columns["arrayCol"]!!.type shouldBe typeOf?>() + schema.columns["arrayCol"]!!.type shouldBe typeOf() + schema.columns["rowCol"]!!.type shouldBe typeOf() + schema.columns["setCol"]!!.type shouldBe typeOf() + schema.columns["intervalCol"]!!.type shouldBe typeOf() } // @Test diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt index 5537f95467..a1b5fbf27e 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt @@ -107,8 +107,6 @@ //// intervalCol INTERVAL, //// intervaldaytosecondCol INTERVAL DAY TO SECOND, //// intervalyeartomonthCol INTERVAL YEAR TO MONTH, -//// rowCol ROW(street VARCHAR, city VARCHAR)), -//// setCol SET[VARCHAR] // ) // """ // From b5925ef9e551b7aecbb4c50fe225640fd988862c Mon Sep 17 00:00:00 2001 From: mbrunetto Date: Mon, 26 Feb 2024 13:08:04 +0100 Subject: [PATCH 5/8] test Vertica --- .../kotlinx/dataframe/io/readJdbc.kt | 44 +- .../kotlinx/dataframe/io/vertica2Test.kt | 280 ------- .../kotlinx/dataframe/io/verticaTest.kt | 709 +++++++++--------- 3 files changed, 367 insertions(+), 666 deletions(-) delete mode 100644 dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt index c695c9bfb5..858d857bac 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt @@ -1,31 +1,32 @@ package org.jetbrains.kotlinx.dataframe.io import io.github.oshai.kotlinlogging.KotlinLogging +import org.jetbrains.kotlinx.dataframe.AnyFrame +import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.api.toDataFrame +import org.jetbrains.kotlinx.dataframe.impl.schema.DataFrameSchemaImpl +import org.jetbrains.kotlinx.dataframe.io.db.DbType +import org.jetbrains.kotlinx.dataframe.io.db.Vertica +import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromUrl +import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import java.math.BigDecimal +import java.sql.Blob +import java.sql.Clob import java.sql.Connection import java.sql.DatabaseMetaData import java.sql.DriverManager +import java.sql.NClob +import java.sql.Ref import java.sql.ResultSet import java.sql.ResultSetMetaData +import java.sql.RowId +import java.sql.SQLXML import java.sql.Time import java.sql.Timestamp import java.sql.Types -import java.sql.RowId -import java.sql.Ref -import java.sql.Clob -import java.sql.Blob -import java.sql.NClob -import java.sql.SQLXML import java.util.Date -import org.jetbrains.kotlinx.dataframe.AnyFrame -import org.jetbrains.kotlinx.dataframe.DataColumn -import org.jetbrains.kotlinx.dataframe.DataFrame -import org.jetbrains.kotlinx.dataframe.api.toDataFrame -import org.jetbrains.kotlinx.dataframe.impl.schema.DataFrameSchemaImpl -import org.jetbrains.kotlinx.dataframe.io.db.DbType -import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromUrl -import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema -import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import kotlin.reflect.KType import kotlin.reflect.full.createType import kotlin.reflect.full.isSupertypeOf @@ -296,7 +297,7 @@ public fun DataFrame.Companion.readAllSqlTables( if (!dbType.isSystemTable(table)) { // we filter her second time because of specific logic with SQLite and possible issues with future databases // val tableName = if (table.catalogue != null) table.catalogue + "." + table.name else table.name - val tableName = if (catalogue != null) catalogue + "." + table.name else table.name + val tableName = getTableName(catalogue, table, dbType) // TODO: both cases is schema specified or not in URL // in h2 database name is recognized as a schema name https://www.h2database.com/html/features.html#database_url @@ -305,6 +306,7 @@ public fun DataFrame.Companion.readAllSqlTables( logger.debug { "Reading table: $tableName" } val dataFrame = readSqlTable(connection, tableName, limit) + dataFrames += dataFrame logger.debug { "Finished reading table: $tableName" } } @@ -313,6 +315,14 @@ public fun DataFrame.Companion.readAllSqlTables( return dataFrames } +private fun getTableName(catalogue: String?, table: TableMetadata, dbType: DbType) = + catalogue + ?.let { catalogue + "." + table.name } + ?: when (dbType) { + Vertica -> "${table.schemaName}.${table.name}" // Vertica needs schema name + else -> table.name + } + /** * Retrieves the schema for an SQL table using the provided database configuration. * @@ -642,7 +652,7 @@ private fun generateKType(dbType: DbType, tableColumnMetadata: TableColumnMetada * @param tableColumnMetadata The metadata of the table column. * @return The KType associated with the SQL type, or a default type if no mapping is found. */ -private fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata): KType { +public fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata): KType { val jdbcTypeToKTypeMapping = mapOf( Types.BIT to Boolean::class, Types.TINYINT to Int::class, diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt deleted file mode 100644 index 6cbdaa3439..0000000000 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/vertica2Test.kt +++ /dev/null @@ -1,280 +0,0 @@ -package org.jetbrains.kotlinx.dataframe.io - -import com.vertica.dsi.dataengine.utilities.TimeTz -import com.vertica.dsi.dataengine.utilities.TimestampTz -import io.kotest.matchers.shouldBe -import org.intellij.lang.annotations.Language -import org.jetbrains.kotlinx.dataframe.DataFrame -import org.jetbrains.kotlinx.dataframe.annotations.DataSchema -import org.jetbrains.kotlinx.dataframe.api.cast -import org.jetbrains.kotlinx.dataframe.api.filter -import org.junit.AfterClass -import org.junit.BeforeClass -import org.junit.Test -import java.math.BigDecimal -import java.sql.Connection -import java.sql.DriverManager -import java.sql.SQLException -import java.sql.Time -import java.sql.Timestamp -import java.time.LocalDate -import java.time.LocalDateTime -import java.time.LocalTime -import java.util.Calendar -import java.util.Date -import java.util.UUID -import kotlin.reflect.typeOf - -// Run with https://hub.docker.com/r/vertica/vertica-ce -private const val URL = "jdbc:vertica://localhost:5433" -private const val USER_NAME = "dbadmin" -private const val PASSWORD = "" -private const val TEST_SCHEMA_NAME = "testschema" - -@DataSchema -interface Table1Vertica { - val id: Int -} - -class VerticaTest { - companion object { - private lateinit var connection: Connection - - @BeforeClass - @JvmStatic - fun setUpClass() { - connection = DriverManager.getConnection(URL, USER_NAME, PASSWORD) - - connection.createStatement().use { st -> - // Drop the test schema if it exists - val dropSchemaQuery = "DROP SCHEMA IF EXISTS $TEST_SCHEMA_NAME" - st.executeUpdate(dropSchemaQuery) - - // Create the test schema - val createSchemaQuery = "CREATE SCHEMA $TEST_SCHEMA_NAME" - st.executeUpdate(createSchemaQuery) - -// Set the schema as the default schema - val setDefaultSchemaQuery = "SET SEARCH_PATH TO $TEST_SCHEMA_NAME" - st.execute(setDefaultSchemaQuery) - } - - connection.createStatement().use { st -> - st.execute("DROP TABLE IF EXISTS table1") - } - - @Language("SQL") - val createTableQuery = """ - CREATE TABLE IF NOT EXISTS table1 ( - id INT NOT NULL PRIMARY KEY, - boolCol BOOLEAN, - charCol CHAR(10), - varcharCol VARCHAR, - longvarcharCol LONG VARCHAR, - binaryCol BINARY(11), - varbinaryCol VARBINARY, - longvarbinaryCol LONG VARBINARY, - dateCol DATE, - timeCol TIME, - timestampCol TIMESTAMP, - doubleprecisionCol DOUBLE PRECISION, - floatCol FLOAT, - float8Col FLOAT8, - realCol REAL, - integerCol INTEGER, - intCol INT, - bigintCol BIGINT, - int8Col INT8, - smallintCol SMALLINT, - tinyintCol TINYINT, - decimalCol DECIMAL, - numericCol NUMERIC, - numberCol NUMBER, - moneyCol MONEY, - geometryCol GEOMETRY, - geographyCol GEOGRAPHY, - timewithtimezoneCol TIMETZ, - timestampwithtimezoneCol TIMESTAMPTZ, - uuidCol UUID, - arrayCol ARRAY[VARCHAR(50)], - rowCol ROW(street VARCHAR, city VARCHAR), - setCol SET[VARCHAR], - intervalCol INTERVAL - ) - """ - - connection.createStatement().execute( - createTableQuery.trimIndent() - ) - - @Language("SQL") - val insertData1 = """ - INSERT INTO table1 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, public.ST_GeomFromText('POINT(1 1)'), public.ST_GeographyFromText('POLYGON((1 2,3 4,2 3,1 2))'), ?, ?, ?, ARRAY['Test', 'Test1'], ROW('aStreet', 'aCity'), SET['aStreet', 'aCity'], INTERVAL '1 12:59:10:05') - """.trimIndent() - - connection.prepareStatement(insertData1).use { st -> - // Insert data into table1 - for (i in 1..3) { - st.setInt(1, i) - st.setBoolean(2, true) - st.setString(3, "charValue$i") - st.setString(4, "varcharValue$i") - st.setString(5, "longvarcharValue$i") - st.setBytes(6, "binaryValue".toByteArray()) - st.setBytes(7, "varbinaryValue".toByteArray()) - st.setBytes(8, "longvarbinaryValue".toByteArray()) - st.setDate(9, java.sql.Date.valueOf(LocalDate.of(2024, 1, 1))) - st.setTime(10, Time.valueOf(LocalTime.of(10, 0, 0))) - st.setTimestamp(11, Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0))) - st.setFloat(12, i * 10.0f) - st.setFloat(13, i * 10.0f) - st.setFloat(14, i * 10.0f) - st.setFloat(15, i * 10.0f) - st.setInt(16, i * 100) - st.setInt(17, i * 100) - st.setInt(18, i * 100) - st.setInt(19, i * 100) - st.setInt(20, i * 100) - st.setInt(21, i * 100) - st.setBigDecimal(22, BigDecimal(i * 10)) - st.setBigDecimal(23, BigDecimal(i * 10)) - st.setBigDecimal(24, BigDecimal(i * 10)) - st.setBigDecimal(25, BigDecimal(i * 10)) - st.setTime(26, TimeTz(Time.valueOf(LocalTime.of(10,0,0,0)), Calendar.getInstance())) - st.setTimestamp(27, TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance())) - st.setString(28, "4a866db2-baa6-442a-a371-1f4b5ee627ba") - st.executeUpdate() - } - } - } - - @AfterClass - @JvmStatic - fun tearDownClass() { - try { - connection.createStatement().use { st -> st.execute("DROP TABLE IF EXISTS table1") } - connection.createStatement().use { st -> st.execute("DROP SCHEMA IF EXISTS $TEST_SCHEMA_NAME") } - connection.close() - } catch (e: SQLException) { - e.printStackTrace() - } - } - } - - @Test - fun `basic test for reading sql tables`() { - val df = DataFrame.readSqlTable(connection, "table1").cast() - df.rowsCount() shouldBe 3 - val result = df.filter { it[Table1Vertica::id] == 1 } - result[0][0] shouldBe 1L - result[0][1] shouldBe true - result[0][2] shouldBe "charValue1" - result[0][3] shouldBe "varcharValue1" - result[0][4] shouldBe "longvarcharValue1" - result[0][5] shouldBe "binaryValue".toByteArray() - result[0][6] shouldBe "varbinaryValue".toByteArray() - result[0][7] shouldBe "longvarbinaryValue".toByteArray() - result[0][8] shouldBe java.sql.Date.valueOf(LocalDate.of(2024, 1, 1)) - result[0][9] shouldBe Time.valueOf(LocalTime.of(10, 0, 0)) - result[0][10] shouldBe Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)) - result[0][11] shouldBe 1 * 10.0f - result[0][12] shouldBe 1 * 10.0f - result[0][13] shouldBe 1 * 10.0f - result[0][14] shouldBe 1 * 10.0f - result[0][15] shouldBe 1 * 100 - result[0][16] shouldBe 1 * 100 - result[0][17] shouldBe 1 * 100 - result[0][18] shouldBe 1 * 100 - result[0][19] shouldBe 1 * 100 - result[0][20] shouldBe 1 * 100 - result[0][21] shouldBe BigDecimal("10.000000000000000") - result[0][22] shouldBe BigDecimal("10.000000000000000") - result[0][23] shouldBe BigDecimal("10") - result[0][24] shouldBe BigDecimal("10.0000") - result[0][27] shouldBe TimeTz(Time.valueOf(LocalTime.of(10,0,0,0)), Calendar.getInstance()) -// result[0][28] shouldBe TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance()) - result[0][29] shouldBe UUID.fromString("4a866db2-baa6-442a-a371-1f4b5ee627ba") -// result[0][30] shouldBe arrayOf("Test", "Test1") -// result[0][31] shouldBe "{\"street\":\"aStreet\",\"city\":\"aCity\"}" -// result[0][32] shouldBe "{\"street\":\"aStreet\",\"city\":\"aCity\"}" -// result[0][33] shouldBe "1 12:59:10.005000" - - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") - schema.columns["id"]!!.type shouldBe typeOf() - schema.columns["boolCol"]!!.type shouldBe typeOf() - schema.columns["charCol"]!!.type shouldBe typeOf() - schema.columns["varcharCol"]!!.type shouldBe typeOf() - schema.columns["longvarcharCol"]!!.type shouldBe typeOf() - schema.columns["binaryCol"]!!.type shouldBe typeOf() - schema.columns["varbinaryCol"]!!.type shouldBe typeOf() - schema.columns["longvarbinaryCol"]!!.type shouldBe typeOf() - schema.columns["dateCol"]!!.type shouldBe typeOf() - schema.columns["timeCol"]!!.type shouldBe typeOf() - schema.columns["timestampCol"]!!.type shouldBe typeOf() - schema.columns["doubleprecisionCol"]!!.type shouldBe typeOf() - schema.columns["floatCol"]!!.type shouldBe typeOf() - schema.columns["float8Col"]!!.type shouldBe typeOf() - schema.columns["realCol"]!!.type shouldBe typeOf() - schema.columns["integerCol"]!!.type shouldBe typeOf() - schema.columns["intCol"]!!.type shouldBe typeOf() - schema.columns["bigintCol"]!!.type shouldBe typeOf() - schema.columns["int8Col"]!!.type shouldBe typeOf() - schema.columns["smallintCol"]!!.type shouldBe typeOf() - schema.columns["tinyintCol"]!!.type shouldBe typeOf() - schema.columns["decimalCol"]!!.type shouldBe typeOf() - schema.columns["numericCol"]!!.type shouldBe typeOf() - schema.columns["numberCol"]!!.type shouldBe typeOf() - schema.columns["moneyCol"]!!.type shouldBe typeOf() - schema.columns["geometryCol"]!!.type shouldBe typeOf() - schema.columns["geographyCol"]!!.type shouldBe typeOf() - schema.columns["timewithtimezoneCol"]!!.type shouldBe typeOf() - schema.columns["timestampwithtimezoneCol"]!!.type shouldBe typeOf() - schema.columns["uuidCol"]!!.type shouldBe typeOf() - schema.columns["arrayCol"]!!.type shouldBe typeOf() - schema.columns["rowCol"]!!.type shouldBe typeOf() - schema.columns["setCol"]!!.type shouldBe typeOf() - schema.columns["intervalCol"]!!.type shouldBe typeOf() - } - -// @Test -// fun `read from sql query`() { -// @Language("SQL") -// val sqlQuery = """ -// SELECT -// t1.id, -// t1.enumCol, -// t2.setCol -// FROM table1 t1 -// JOIN table2 t2 ON t1.id = t2.id -// """.trimIndent() -// -// val df = DataFrame.readSqlQuery(connection, sqlQuery = sqlQuery).cast() -// val result = df.filter { it[Table3MySql::id] == 1 } -// result[0][2] shouldBe "Option1" -// -// val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) -// schema.columns["id"]!!.type shouldBe typeOf() -// schema.columns["enumCol"]!!.type shouldBe typeOf() -// schema.columns["setCol"]!!.type shouldBe typeOf() -// } -// -// @Test -// fun `read from all tables`() { -// val dataframes = DataFrame.readAllSqlTables(connection) -// -// val table1Df = dataframes[0].cast() -// -// table1Df.rowsCount() shouldBe 3 -// table1Df.filter { it[Table1MySql::integerCol] > 100 }.rowsCount() shouldBe 2 -// table1Df[0][11] shouldBe 10.0 -// table1Df[0][26] shouldBe "textValue1" -// -// val table2Df = dataframes[1].cast() -// -// table2Df.rowsCount() shouldBe 3 -// table2Df.filter { it[Table2MySql::integerCol] != null && it[Table2MySql::integerCol]!! > 400 } -// .rowsCount() shouldBe 1 -// table2Df[0][11] shouldBe 20.0 -// table2Df[0][26] shouldBe null -// } -} diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt index a1b5fbf27e..b1c3b1d1a8 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt @@ -1,369 +1,340 @@ -//package org.jetbrains.kotlinx.dataframe.io -// -//import io.kotest.matchers.shouldBe -//import org.intellij.lang.annotations.Language -//import org.jetbrains.kotlinx.dataframe.DataFrame -//import org.jetbrains.kotlinx.dataframe.annotations.DataSchema -//import org.jetbrains.kotlinx.dataframe.api.cast -//import org.jetbrains.kotlinx.dataframe.api.filter -//import org.junit.AfterClass -//import org.junit.BeforeClass -//import org.junit.Test -//import java.math.BigDecimal -//import java.sql.Connection -//import java.sql.DriverManager -//import java.sql.SQLException -//import org.jetbrains.kotlinx.dataframe.api.add -//import org.jetbrains.kotlinx.dataframe.api.select -//import org.junit.Ignore -//import java.util.UUID -//import kotlin.reflect.typeOf -// -//private const val URL = "jdbc:vertica://localhost:5433" -//private const val USER_NAME = "root" -//private const val PASSWORD = "pass" -//private const val TEST_DATABASE_NAME = "testKDFdatabase" -// -//@DataSchema -//interface Table1Vertica { -// val id: Int -// val bitCol: Boolean -// val tinyintCol: Int -// val smallintCol: Int -// val mediumintCol: Int -// val mediumintUnsignedCol: Int -// val integerCol: Int -// val intCol: Int -// val integerUnsignedCol: Long -// val bigintCol: Long -// val floatCol: Float -// val doubleCol: Double -// val decimalCol: BigDecimal -// val dateCol: String -// val datetimeCol: String -// val timestampCol: String -// val timeCol: String -// val yearCol: String -// val varcharCol: String -// val charCol: String -// val binaryCol: ByteArray -// val varbinaryCol: ByteArray -// val tinyblobCol: ByteArray -// val blobCol: ByteArray -// val mediumblobCol: ByteArray -// val longblobCol: ByteArray -// val textCol: String -// val mediumtextCol: String -// val longtextCol: String -// val enumCol: String -// val setCol: Char -//} -// -//@Ignore -//class VerticaTest { -// companion object { -// private lateinit var connection: Connection -// -// @BeforeClass -// @JvmStatic -// fun setUpClass() { -// connection = DriverManager.getConnection(URL, USER_NAME, PASSWORD) -// -// connection.createStatement().use { st -> -// // Drop the test database if it exists -// val dropDatabaseQuery = "DROP DATABASE IF EXISTS $TEST_DATABASE_NAME" -// st.executeUpdate(dropDatabaseQuery) -// -// // Create the test database -// val createDatabaseQuery = "CREATE DATABASE $TEST_DATABASE_NAME" -// st.executeUpdate(createDatabaseQuery) -// -// // Use the newly created database -// val useDatabaseQuery = "USE $TEST_DATABASE_NAME" -// st.executeUpdate(useDatabaseQuery) -// } -// -// connection.createStatement().use { st -> st.execute("DROP TABLE IF EXISTS table1") } -// -// @Language("SQL") -// val createTableQuery = """ -// CREATE TABLE IF NOT EXISTS table1 ( -// integerCol INTEGER, -// intCol INT, -// bigintCol BIGINT, -// int8Col INT8, -// smallintCol SMALLINT, -// tinyintCol TINYINT, -// decimalCol DECIMAL, -// numericCol NUMERIC, -// numberCol NUMBER, -// moneyCol MONEY, -// uuidCol UUID -// ) -// """ -// -// val createTableQueryBackup = """ -// CREATE TABLE IF NOT EXISTS table1 ( -//// intervalCol INTERVAL, -//// intervaldaytosecondCol INTERVAL DAY TO SECOND, -//// intervalyeartomonthCol INTERVAL YEAR TO MONTH, -// ) -// """ -// -// connection.createStatement().execute( -// createTableQuery.trimIndent() -// ) -// -// @Language("SQL") -// val insertData1 = """ -// INSERT INTO table1 ( -// boolCol, -// charCol, -// varcharCol, -// longvarcharCol, -// binaryCol, -// varbinaryCol, -// longvarbinaryCol, -// dateCol, -// timeCol, -//// timewithtimezoneCol, -// timestampCol, -//// timestampwithtimezoneCol, -//// intervalCol, -//// intervaldaytosecondCol, -//// intervalyeartomonthCol, -// doubleprecisionCol, -// floatCol, -// floatncol, -// float8Col, -// realCol, -// integerCol, -// intCol, -// bigintCol, -// int8Col, -// smallintCol, -// tinyintCol, -// decimalCol, -// numericCol, -// numberCol, -// moneyCol, -//// geometryCol, -//// geographyCol, -// uuidCol, -//// arrayCol, -//// rowCol, -//// setCol -// ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -// """.trimIndent() -// -// @Language("SQL") -// val insertDatatest = """ -// INSERT INTO table1 ( -// boolCol, -// charCol, -// varcharCol, -// longvarcharCol, -// binaryCol, -// varbinaryCol, -// longvarbinaryCol, -// dateCol, -// timeCol, -//// timewithtimezoneCol, -// timestampCol, -//// timestampwithtimezoneCol, -//// intervalCol, -//// intervaldaytosecondCol, -//// intervalyeartomonthCol, -// doubleprecisionCol, -// floatCol, -// floatncol, -// float8Col, -// realCol, -// integerCol, -// intCol, -// bigintCol, -// int8Col, -// smallintCol, -// tinyintCol, -// decimalCol, -// numericCol, -// numberCol, -// moneyCol, -//// geometryCol, -//// geographyCol, -// uuidCol, -//// arrayCol, -//// rowCol, -//// setCol -// ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -// """.trimIndent() -// -// connection.prepareStatement(insertData1).use { st -> -// // Insert data into table1 -// for (i in 1..3) { -// st.setTimestamp(10, java.sql.Timestamp(System.currentTimeMillis())) -//// st.setBigDecimal(12, BigDecimal(i * 10)) -//// st.setDate(13, java.sql.Date(System.currentTimeMillis())) -//// st.setTimestamp(14, java.sql.Timestamp(System.currentTimeMillis())) -//// st.setTimestamp(15, java.sql.Timestamp(System.currentTimeMillis())) -// st.setDouble(11, i * 10.0) -// st.setFloat(12, i * 10.0f) -// st.setFloat(13, i * 10.0f) -// st.setFloat(14, i * 10.0f) -// st.setFloat(15, i * 10.0f) -// st.setInt(16, i * 100) -// st.setInt(17, i * 100) -// st.setInt(18, i * 100) -// st.setInt(19, i * 100) -// st.setInt(20, i * 100) -// st.setInt(21, i * 100) -// st.setBigDecimal(22, BigDecimal(i * 10)) -// st.setBigDecimal(23, BigDecimal(i * 10)) -// st.setBigDecimal(24, BigDecimal(i * 10)) -// st.setBigDecimal(25, BigDecimal(i * 10)) -//// st.setString(31, "{\"key\": \"value\"}") -//// st.setString(32, "{\"key\": \"value\"}") -//// st.setString(33, "{\"key\": \"value\"}") -// st.setString(26, "cee3754a-e2f2-4020-b43f-aba52b10c827") -//// st.setString(35, "{\"key\": \"value\"}") -//// st.setString(36, "{\"key\": \"value\"}") -//// st.setString(37, "{\"key\": \"value\"}") -// st.executeUpdate() -// } -// } -// } -// -// @AfterClass -// @JvmStatic -// fun tearDownClass() { -// try { -// connection.createStatement().use { st -> st.execute("DROP TABLE IF EXISTS table1") } -// connection.createStatement().use { st -> st.execute("DROP DATABASE IF EXISTS $TEST_DATABASE_NAME") } -// connection.close() -// } catch (e: SQLException) { -// e.printStackTrace() -// } -// } -// } -// -// @Ignore -// @Test -// fun `basic test for reading sql tables`() { -// val df1 = DataFrame.readSqlTable(connection, "table1").cast() -// val result = df1.filter { it[Table1Vertica::id] == 1 } -// result[0][26] shouldBe "textValue1" -// -// val schema = DataFrame.getSchemaForSqlTable(connection, "table1") -// schema.columns["id"]!!.type shouldBe typeOf() -// schema.columns["textCol"]!!.type shouldBe typeOf() -// } -// -//// @Test -//// fun `read from sql query`() { -//// @Language("SQL") -//// val sqlQuery = """ -//// SELECT -//// t1.id, -//// t1.enumCol, -//// t2.setCol -//// FROM table1 t1 -//// JOIN table2 t2 ON t1.id = t2.id -//// """.trimIndent() -//// -//// val df = DataFrame.readSqlQuery(connection, sqlQuery = sqlQuery).cast() -//// val result = df.filter { it[Table3MySql::id] == 1 } -//// result[0][2] shouldBe "Option1" -//// -//// val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) -//// schema.columns["id"]!!.type shouldBe typeOf() -//// schema.columns["enumCol"]!!.type shouldBe typeOf() -//// schema.columns["setCol"]!!.type shouldBe typeOf() -//// } -//// -//// @Test -//// fun `read from all tables`() { -//// val dataframes = DataFrame.readAllSqlTables(connection) -//// -//// val table1Df = dataframes[0].cast() -//// -//// table1Df.rowsCount() shouldBe 3 -//// table1Df.filter { it[Table1MySql::integerCol] > 100 }.rowsCount() shouldBe 2 -//// table1Df[0][11] shouldBe 10.0 -//// table1Df[0][26] shouldBe "textValue1" -//// -//// val table2Df = dataframes[1].cast() -//// -//// table2Df.rowsCount() shouldBe 3 -//// table2Df.filter { it[Table2MySql::integerCol] != null && it[Table2MySql::integerCol]!! > 400 } -//// .rowsCount() shouldBe 1 -//// table2Df[0][11] shouldBe 20.0 -//// table2Df[0][26] shouldBe null -//// } -//// -//// @Test -//// fun `reading numeric types`() { -//// val df1 = DataFrame.readSqlTable(connection, "table1").cast() -//// -//// val result = df1.select("tinyintCol").add("tinyintCol2") { it[Table1MySql::tinyintCol] } -//// -//// result[0][1] shouldBe 1.toByte() -//// -//// val result1 = df1.select("smallintCol") -//// .add("smallintCol2") { it[Table1MySql::smallintCol] } -//// -//// result1[0][1] shouldBe 10.toShort() -//// -//// val result2 = df1.select("mediumintCol") -//// .add("mediumintCol2") { it[Table1MySql::mediumintCol] } -//// -//// result2[0][1] shouldBe 100 -//// -//// val result3 = df1.select("mediumintUnsignedCol") -//// .add("mediumintUnsignedCol2") { it[Table1MySql::mediumintUnsignedCol] } -//// -//// result3[0][1] shouldBe 100 -//// -//// val result4 = df1.select("integerUnsignedCol") -//// .add("integerUnsignedCol2") { it[Table1MySql::integerUnsignedCol] } -//// -//// result4[0][1] shouldBe 100L -//// -//// val result5 = df1.select("bigintCol") -//// .add("bigintCol2") { it[Table1MySql::bigintCol] } -//// -//// result5[0][1] shouldBe 100 -//// -//// val result6 = df1.select("floatCol") -//// .add("floatCol2") { it[Table1MySql::floatCol] } -//// -//// result6[0][1] shouldBe 10.0f -//// -//// val result7 = df1.select("doubleCol") -//// .add("doubleCol2") { it[Table1MySql::doubleCol] } -//// -//// result7[0][1] shouldBe 10.0 -//// -//// val result8 = df1.select("decimalCol") -//// .add("decimalCol2") { it[Table1MySql::decimalCol] } -//// -//// result8[0][1] shouldBe BigDecimal("10") -//// -//// val schema = DataFrame.getSchemaForSqlTable(connection, "table1") -//// -//// schema.columns["tinyintCol"]!!.type shouldBe typeOf() -//// schema.columns["smallintCol"]!!.type shouldBe typeOf() -//// schema.columns["mediumintCol"]!!.type shouldBe typeOf() -//// schema.columns["mediumintUnsignedCol"]!!.type shouldBe typeOf() -//// schema.columns["integerUnsignedCol"]!!.type shouldBe typeOf() -//// schema.columns["bigintCol"]!!.type shouldBe typeOf() -//// schema.columns["floatCol"]!!.type shouldBe typeOf() -//// schema.columns["doubleCol"]!!.type shouldBe typeOf() -//// schema.columns["decimalCol"]!!.type shouldBe typeOf() -//// // TODO: all unsigned types -//// // TODO: new mapping system based on class names -//// // validation after mapping in getObject -//// // getObject(i+1, type) catch getObject catch getString -//// // add direct mapping to getString and other methods -//// -//// } -//} +package org.jetbrains.kotlinx.dataframe.io + +import com.vertica.dsi.dataengine.utilities.TimeTz +import com.vertica.dsi.dataengine.utilities.TimestampTz +import com.vertica.jdbc.VerticaDayTimeInterval +import com.vertica.jdbc.jdbc42.S42Array +import com.vertica.util.VerticaStruct +import io.kotest.matchers.shouldBe +import org.intellij.lang.annotations.Language +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.api.cast +import org.jetbrains.kotlinx.dataframe.api.filter +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Ignore +import org.junit.Test +import java.math.BigDecimal +import java.sql.Connection +import java.sql.DriverManager +import java.sql.SQLException +import java.sql.Time +import java.sql.Timestamp +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.util.Calendar +import java.util.Date +import java.util.UUID +import kotlin.reflect.typeOf + +// Run with https://hub.docker.com/r/vertica/vertica-ce +private const val URL = "jdbc:vertica://localhost:5433" +private const val USER_NAME = "dbadmin" +private const val PASSWORD = "" +private const val TEST_SCHEMA_NAME = "testschema" + +@DataSchema +interface Table1Vertica { + val id: Int + val varcharCol: String +} + +@DataSchema +interface Table2Vertica { + val id: Int + val boolCol: Boolean +} + +@DataSchema +interface Table3Vertica { + val id: Int + val varcharCol: String + val boolCol: Boolean +} + +@Ignore +class VerticaTest { + companion object { + private lateinit var connection: Connection + + @BeforeClass + @JvmStatic + fun setUpClass() { + connection = DriverManager.getConnection(URL, USER_NAME, PASSWORD) + + connection.createStatement().use { st -> + // Drop the test schema if it exists + val dropSchemaQuery = "DROP SCHEMA IF EXISTS $TEST_SCHEMA_NAME" + st.executeUpdate(dropSchemaQuery) + + // Create the test schema + val createSchemaQuery = "CREATE SCHEMA $TEST_SCHEMA_NAME" + st.executeUpdate(createSchemaQuery) + +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO $TEST_SCHEMA_NAME" + st.execute(setDefaultSchemaQuery) + } + + connection.createStatement().use { st -> + st.execute("DROP TABLE IF EXISTS table1") + st.execute("DROP TABLE IF EXISTS table2") + } + + @Language("SQL") + val createTableQuery = """ + CREATE TABLE IF NOT EXISTS table1 ( + id INT NOT NULL PRIMARY KEY, + boolCol BOOLEAN, + charCol CHAR(10), + varcharCol VARCHAR, + longvarcharCol LONG VARCHAR, + binaryCol BINARY(11), + varbinaryCol VARBINARY, + longvarbinaryCol LONG VARBINARY, + dateCol DATE, + timeCol TIME, + timestampCol TIMESTAMP, + doubleprecisionCol DOUBLE PRECISION, + floatCol FLOAT, + float8Col FLOAT8, + realCol REAL, + integerCol INTEGER, + intCol INT, + bigintCol BIGINT, + int8Col INT8, + smallintCol SMALLINT, + tinyintCol TINYINT, + decimalCol DECIMAL, + numericCol NUMERIC, + numberCol NUMBER, + moneyCol MONEY, + geometryCol GEOMETRY, + geographyCol GEOGRAPHY, + timewithtimezoneCol TIMETZ, + timestampwithtimezoneCol TIMESTAMPTZ, + uuidCol UUID, + arrayCol ARRAY[VARCHAR(50)], + rowCol ROW(street VARCHAR, city VARCHAR), + setCol SET[VARCHAR], + intervalCol INTERVAL + ) + """ + + @Language("SQL") + val createTable2Query = """ + CREATE TABLE IF NOT EXISTS table2 ( + id INT NOT NULL PRIMARY KEY, + boolCol BOOLEAN + ) + """ + + connection.createStatement().execute( + createTableQuery.trimIndent() + ) + + connection.createStatement().execute( + createTable2Query.trimIndent() + ) + + @Language("SQL") + val insertData1 = """ + INSERT INTO table1 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, public.ST_GeomFromText('POINT(1 1)'), public.ST_GeographyFromText('POLYGON((1 2,3 4,2 3,1 2))'), ?, ?, ?, ARRAY['Test', 'Test1'], ROW('aStreet', 'aCity'), SET['aStreet', 'aCity'], INTERVAL '1 12:59:10:05') + """.trimIndent() + + connection.prepareStatement(insertData1).use { st -> + // Insert data into table1 + for (i in 1..3) { + st.setInt(1, i) + st.setBoolean(2, true) + st.setString(3, "charValue$i") + st.setString(4, "varcharValue$i") + st.setString(5, "longvarcharValue$i") + st.setBytes(6, "binaryValue".toByteArray()) + st.setBytes(7, "varbinaryValue".toByteArray()) + st.setBytes(8, "longvarbinaryValue".toByteArray()) + st.setDate(9, java.sql.Date.valueOf(LocalDate.of(2024, 1, 1))) + st.setTime(10, Time.valueOf(LocalTime.of(10, 0, 0))) + st.setTimestamp(11, Timestamp.valueOf(LocalDateTime.of(2024, 1, 1, 10, 0, 0))) + st.setFloat(12, i * 10.0f) + st.setFloat(13, i * 10.0f) + st.setFloat(14, i * 10.0f) + st.setFloat(15, i * 10.0f) + st.setInt(16, i * 100) + st.setInt(17, i * 100) + st.setInt(18, i * 100) + st.setInt(19, i * 100) + st.setInt(20, i * 100) + st.setInt(21, i * 100) + st.setBigDecimal(22, BigDecimal(i * 10)) + st.setBigDecimal(23, BigDecimal(i * 10)) + st.setBigDecimal(24, BigDecimal(i * 10)) + st.setBigDecimal(25, BigDecimal(i * 10)) + st.setTime(26, TimeTz(Time.valueOf(LocalTime.of(10, 0, 0, 0)), Calendar.getInstance())) + st.setTimestamp( + 27, + TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1, 1, 10, 0, 0)), Calendar.getInstance()) + ) + st.setString(28, "4a866db2-baa6-442a-a371-1f4b5ee627ba") + st.executeUpdate() + } + } + + @Language("SQL") + val insertData2 = """ + INSERT INTO table2 VALUES (?, ?) + """.trimIndent() + + connection.prepareStatement(insertData2).use { st -> + // Insert data into table2 + for (i in 1..3) { + st.setInt(1, i) + st.setBoolean(2, true) + st.executeUpdate() + } + } + } + + @AfterClass + @JvmStatic + fun tearDownClass() { + try { + connection.createStatement().use { st -> + st.execute("DROP TABLE IF EXISTS table1") + st.execute("DROP TABLE IF EXISTS table2") + } + connection.createStatement().use { st -> st.execute("DROP SCHEMA IF EXISTS $TEST_SCHEMA_NAME") } + } catch (e: SQLException) { + e.printStackTrace() + } finally { + connection.close() + } + } + } + + @Test + fun `basic test for reading sql tables`() { + connection.createStatement().use { st -> +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO $TEST_SCHEMA_NAME" + st.execute(setDefaultSchemaQuery) + } + + val df = DataFrame.readSqlTable(connection, "table1").cast() + df.rowsCount() shouldBe 3 + val result = df.filter { it[Table1Vertica::id] == 1 } + result[0][0] shouldBe 1L + result[0][1] shouldBe true + result[0][2] shouldBe "charValue1" + result[0][3] shouldBe "varcharValue1" + result[0][4] shouldBe "longvarcharValue1" + result[0][5] shouldBe "binaryValue".toByteArray() + result[0][6] shouldBe "varbinaryValue".toByteArray() + result[0][7] shouldBe "longvarbinaryValue".toByteArray() + result[0][8] shouldBe java.sql.Date.valueOf(LocalDate.of(2024, 1, 1)) + result[0][9] shouldBe Time.valueOf(LocalTime.of(10, 0, 0)) + result[0][10] shouldBe Timestamp.valueOf(LocalDateTime.of(2024, 1, 1, 10, 0, 0)) + result[0][11] shouldBe 1 * 10.0f + result[0][12] shouldBe 1 * 10.0f + result[0][13] shouldBe 1 * 10.0f + result[0][14] shouldBe 1 * 10.0f + result[0][15] shouldBe 1 * 100 + result[0][16] shouldBe 1 * 100 + result[0][17] shouldBe 1 * 100 + result[0][18] shouldBe 1 * 100 + result[0][19] shouldBe 1 * 100 + result[0][20] shouldBe 1 * 100 + result[0][21] shouldBe BigDecimal("10.000000000000000") + result[0][22] shouldBe BigDecimal("10.000000000000000") + result[0][23] shouldBe BigDecimal("10") + result[0][24] shouldBe BigDecimal("10.0000") + result[0][27] shouldBe TimeTz(Time.valueOf(LocalTime.of(10, 0, 0, 0)), Calendar.getInstance()) +// result[0][28] shouldBe TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance()) + result[0][29] shouldBe UUID.fromString("4a866db2-baa6-442a-a371-1f4b5ee627ba") + (result[0][30] as S42Array).toString() shouldBe "[\"Test\",\"Test1\"]" + (result[0][31] as VerticaStruct).toString() shouldBe "{\"street\":\"aStreet\",\"city\":\"aCity\"}" + (result[0][32] as S42Array).toString() shouldBe "[\"aCity\",\"aStreet\"]" + (result[0][33] as VerticaDayTimeInterval).toString() shouldBe "1 12:59:10.005000" + + val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + schema.columns["id"]!!.type shouldBe typeOf() + schema.columns["boolCol"]!!.type shouldBe typeOf() + schema.columns["charCol"]!!.type shouldBe typeOf() + schema.columns["varcharCol"]!!.type shouldBe typeOf() + schema.columns["longvarcharCol"]!!.type shouldBe typeOf() + schema.columns["binaryCol"]!!.type shouldBe typeOf() + schema.columns["varbinaryCol"]!!.type shouldBe typeOf() + schema.columns["longvarbinaryCol"]!!.type shouldBe typeOf() + schema.columns["dateCol"]!!.type shouldBe typeOf() + schema.columns["timeCol"]!!.type shouldBe typeOf() + schema.columns["timestampCol"]!!.type shouldBe typeOf() + schema.columns["doubleprecisionCol"]!!.type shouldBe typeOf() + schema.columns["floatCol"]!!.type shouldBe typeOf() + schema.columns["float8Col"]!!.type shouldBe typeOf() + schema.columns["realCol"]!!.type shouldBe typeOf() + schema.columns["integerCol"]!!.type shouldBe typeOf() + schema.columns["intCol"]!!.type shouldBe typeOf() + schema.columns["bigintCol"]!!.type shouldBe typeOf() + schema.columns["int8Col"]!!.type shouldBe typeOf() + schema.columns["smallintCol"]!!.type shouldBe typeOf() + schema.columns["tinyintCol"]!!.type shouldBe typeOf() + schema.columns["decimalCol"]!!.type shouldBe typeOf() + schema.columns["numericCol"]!!.type shouldBe typeOf() + schema.columns["numberCol"]!!.type shouldBe typeOf() + schema.columns["moneyCol"]!!.type shouldBe typeOf() + schema.columns["geometryCol"]!!.type shouldBe typeOf() + schema.columns["geographyCol"]!!.type shouldBe typeOf() + schema.columns["timewithtimezoneCol"]!!.type shouldBe typeOf() + schema.columns["timestampwithtimezoneCol"]!!.type shouldBe typeOf() + schema.columns["uuidCol"]!!.type shouldBe typeOf() + schema.columns["arrayCol"]!!.type shouldBe typeOf() + schema.columns["rowCol"]!!.type shouldBe typeOf() + schema.columns["setCol"]!!.type shouldBe typeOf() + schema.columns["intervalCol"]!!.type shouldBe typeOf() + } + + @Test + fun `read from sql query`() { + connection.createStatement().use { st -> +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO $TEST_SCHEMA_NAME" + st.execute(setDefaultSchemaQuery) + } + + @Language("SQL") + val sqlQuery = """ + SELECT + t1.id, + t1.varcharCol, + t2.boolCol + FROM table1 t1 + JOIN table2 t2 ON t1.id = t2.id + """.trimIndent() + + val df = DataFrame.readSqlQuery(connection, sqlQuery = sqlQuery).cast() + val result = df.filter { it[Table3Vertica::id] == 1 } + result[0][2] shouldBe true + + val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + schema.columns["id"]!!.type shouldBe typeOf() + schema.columns["varcharCol"]!!.type shouldBe typeOf() + schema.columns["boolCol"]!!.type shouldBe typeOf() + } + + @Test + fun `read from all tables`() { + connection.createStatement().use { st -> +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO DEFAULT" + st.execute(setDefaultSchemaQuery) + } + + val dataframes = DataFrame.readAllSqlTables(connection, limit = 1) + + val table1Df = dataframes.first { it.columnNames().any {column -> column == "geometryCol" } } + + table1Df.columnsCount() shouldBe 34 + } +} From 3c0ad0a73cbb1db5575d7c759c4e8cb2180b0e49 Mon Sep 17 00:00:00 2001 From: mbrunetto Date: Mon, 26 Feb 2024 16:05:41 +0100 Subject: [PATCH 6/8] test Vertica --- .../kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt index b1c3b1d1a8..1c6b3b2b54 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt @@ -251,7 +251,7 @@ class VerticaTest { result[0][23] shouldBe BigDecimal("10") result[0][24] shouldBe BigDecimal("10.0000") result[0][27] shouldBe TimeTz(Time.valueOf(LocalTime.of(10, 0, 0, 0)), Calendar.getInstance()) -// result[0][28] shouldBe TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance()) + (result[0][28] as TimestampTz).time shouldBe TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance()).time result[0][29] shouldBe UUID.fromString("4a866db2-baa6-442a-a371-1f4b5ee627ba") (result[0][30] as S42Array).toString() shouldBe "[\"Test\",\"Test1\"]" (result[0][31] as VerticaStruct).toString() shouldBe "{\"street\":\"aStreet\",\"city\":\"aCity\"}" From 55abb24736d8ff5c6cfd3b492eebd44cb78375e4 Mon Sep 17 00:00:00 2001 From: mbrunetto Date: Mon, 26 Feb 2024 16:59:19 +0100 Subject: [PATCH 7/8] remove comment --- .../kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt index 0603f361f1..a5c1ee0ed5 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt @@ -41,8 +41,7 @@ public object Vertica : DbType("vertica") { override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = when(tableColumnMetadata.sqlTypeName.uppercase()) { "UUID" -> String::class.createType(nullable = tableColumnMetadata.isNullable) -// "ARRAY" -> Array::class.createType(arguments = listOf(KTypeProjection.invariant(String::class.starProjectedType)), nullable = tableColumnMetadata.isNullable) - "ARRAY" -> String::class.createType(nullable = tableColumnMetadata.isNullable) // TODO understan if we can use the one above + "ARRAY" -> String::class.createType(nullable = tableColumnMetadata.isNullable) "UNKNOWN" -> String::class.createType(nullable = tableColumnMetadata.isNullable) else -> null } From 07ab4ddd63d8c1337492d61490624ded41aad0fa Mon Sep 17 00:00:00 2001 From: mbrunetto Date: Tue, 27 Feb 2024 12:30:39 +0100 Subject: [PATCH 8/8] fix typo in the class comment --- .../kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt index a5c1ee0ed5..39045b45a0 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt @@ -8,9 +8,9 @@ import kotlin.reflect.KType import kotlin.reflect.full.createType /** - * Represents the MySql database type. + * Represents the Vertica database type. * - * This class provides methods to convert data from a ResultSet to the appropriate type for MySql, + * This class provides methods to convert data from a ResultSet to the appropriate type for Vertica, * and to generate the corresponding column schema. */ public object Vertica : DbType("vertica") {