Skip to content

Commit

Permalink
SNOW-1563484 Remove Advanced Query Pushdown Feature (#572)
Browse files Browse the repository at this point in the history 
* Upgrade JDBC to 3.17.0

* rerun test

* disable test

* add comment

* temp

* remove query push down

* fix test

* fix test
  • Loading branch information
@sfc-gh-bli
sfc-gh-bli authored Jul 30, 2024
1 parent 491d299 commit 8350590
Show file tree
Hide file tree
Showing 50 changed files with 73 additions and 7,996 deletions.
34 changes: 0 additions & 34 deletions src/it/scala/net/snowflake/spark/snowflake/ColumnNameCaseSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,14 +134,6 @@ class ColumnNameCaseSuite extends IntegrationSuiteBase {
.agg(count("*").alias("new_col"))
.count()

val result =
s"""
|SELECT ( COUNT ( 1 ) ) AS "SUBQUERY_2_COL_0" FROM ( SELECT * FROM
|( SELECT * FROM ( $table3 ) AS "SF_CONNECTOR_QUERY_ALIAS" ) AS
| "SUBQUERY_0" GROUP BY "SUBQUERY_0"."col" ) AS "SUBQUERY_1" LIMIT 1
|""".stripMargin.replaceAll("\\s", "")

assert(Utils.getLastSelect.replaceAll("\\s", "").equals(result))
}

test("Test row_number function") {
Expand All @@ -160,32 +152,6 @@ class ColumnNameCaseSuite extends IntegrationSuiteBase {
df.withColumn("rank", row_number().over(windowsSpec))
.filter("rank=1")
.collect()

// scalastyle:off println
println(Utils.getLastSelect)
// scalastyle:on println
assert(
Utils.getLastSelect
.replaceAll("\\s", "")
.equals(s"""SELECT * FROM ( SELECT ( "SUBQUERY_0"."id" ) AS "SUBQUERY_1_COL_0" ,
|( "SUBQUERY_0"."time" ) AS "SUBQUERY_1_COL_1" , ( ROW_NUMBER () OVER
|( PARTITION BY "SUBQUERY_0"."id" ORDER BY ( "SUBQUERY_0"."time" ) DESC
|) ) AS "SUBQUERY_1_COL_2" FROM ( SELECT * FROM ( $table3 ) AS
|"SF_CONNECTOR_QUERY_ALIAS" ) AS "SUBQUERY_0" ) AS "SUBQUERY_1" WHERE
|( ( "SUBQUERY_1"."SUBQUERY_1_COL_2" IS NOT NULL ) AND ( "SUBQUERY_1".
|"SUBQUERY_1_COL_2" = 1 ) )
|""".stripMargin.replaceAll("\\s", "")) ||
Utils.getLastSelect
.replaceAll("\\s", "")
.equals(s"""SELECT * FROM ( SELECT ( "SUBQUERY_0"."id" ) AS "SUBQUERY_1_COL_0" ,
| ( "SUBQUERY_0"."time" ) AS "SUBQUERY_1_COL_1" , ( ROW_NUMBER () OVER
| ( PARTITION BY "SUBQUERY_0"."id" ORDER BY ( "SUBQUERY_0"."time" ) DESC
| ) ) AS "SUBQUERY_1_COL_2" FROM ( SELECT * FROM ( $table3 ) AS
| "SF_CONNECTOR_QUERY_ALIAS" ) AS "SUBQUERY_0" ) AS "SUBQUERY_1" WHERE
| ( "SUBQUERY_1"."SUBQUERY_1_COL_2" = 1 )
|""".stripMargin.replaceAll("\\s", ""))
)

}

}
93 changes: 0 additions & 93 deletions src/it/scala/net/snowflake/spark/snowflake/DataTypesIntegrationSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,28 +191,6 @@ class DataTypesIntegrationSuite extends IntegrationSuiteBase {
.agg(count("*").alias("abc"))
.collect()

assert(
Utils.getLastSelect.equals(
s"""SELECT ( "SUBQUERY_2"."SUBQUERY_2_COL_0" ) AS "SUBQUERY_3_COL_0" , ( COUNT ( 1 ) )
| AS "SUBQUERY_3_COL_1" FROM ( SELECT ( "SUBQUERY_1"."id" ) AS "SUBQUERY_2_COL_0" FROM
| ( SELECT * FROM ( SELECT * FROM ( $test_table ) AS "SF_CONNECTOR_QUERY_ALIAS" )
| AS "SUBQUERY_0" WHERE ( ( ( "SUBQUERY_0"."time" IS NOT NULL ) AND ( "SUBQUERY_0"."time" >=
| DATEADD(day, 18140 , TO_DATE('1970-01-01')) ) ) AND ( "SUBQUERY_0"."time" <=
| DATEADD(day, 18201 , TO_DATE('1970-01-01')) ) ) ) AS "SUBQUERY_1" ) AS "SUBQUERY_2"
| GROUP BY "SUBQUERY_2"."SUBQUERY_2_COL_0"""".stripMargin.filter(_ >= ' ')
) ||
Utils.getLastSelect.equals(
s"""SELECT ( "SUBQUERY_2"."SUBQUERY_2_COL_0" ) AS "SUBQUERY_3_COL_0" , ( COUNT ( 1 ) )
| AS "SUBQUERY_3_COL_1" FROM ( SELECT ( "SUBQUERY_1"."id" ) AS "SUBQUERY_2_COL_0" FROM
| ( SELECT * FROM ( SELECT * FROM ( $test_table ) AS
| "SF_CONNECTOR_QUERY_ALIAS" ) AS "SUBQUERY_0" WHERE
| ( ( "SUBQUERY_0"."time" IS NOT NULL ) AND ( ( "SUBQUERY_0"."time" >= DATEADD(day,
| 18140 , TO_DATE('1970-01-01')) ) AND ( "SUBQUERY_0"."time" <= DATEADD(day, 18201 ,
| TO_DATE('1970-01-01')) ) ) ) ) AS "SUBQUERY_1" ) AS "SUBQUERY_2"
| GROUP BY "SUBQUERY_2"."SUBQUERY_2_COL_0"""".stripMargin.filter(_ >= ' ')
)
)

assert(result.length == 1)
assert(result(0)(1) == 2)

Expand All @@ -229,31 +207,6 @@ class DataTypesIntegrationSuite extends IntegrationSuiteBase {
.agg(count("*").alias("abc"))
.show()

assert(
Utils.getLastSelect.equals(
s"""SELECT * FROM ( SELECT ( CAST ( "SUBQUERY_2"."SUBQUERY_2_COL_0" AS VARCHAR ) )
| AS "SUBQUERY_3_COL_0" , ( CAST ( COUNT ( 1 ) AS VARCHAR ) ) AS "SUBQUERY_3_COL_1"
| FROM ( SELECT ( "SUBQUERY_1"."id" ) AS "SUBQUERY_2_COL_0" FROM ( SELECT * FROM
| ( SELECT * FROM ( $test_table ) AS "SF_CONNECTOR_QUERY_ALIAS" ) AS "SUBQUERY_0"
| WHERE ( ( ( "SUBQUERY_0"."time" IS NOT NULL ) AND ( "SUBQUERY_0"."time" <=
| DATEADD(day, 18140 , TO_DATE('1970-01-01')) ) ) AND ( "SUBQUERY_0"."time" >=
| DATEADD(day, 18201 , TO_DATE('1970-01-01')) ) ) ) AS "SUBQUERY_1" ) AS "SUBQUERY_2"
| GROUP BY "SUBQUERY_2"."SUBQUERY_2_COL_0" ) AS "SUBQUERY_3" LIMIT 21
|""".stripMargin.filter(_ >= ' ')) ||
Utils.getLastSelect.equals(
s"""SELECT * FROM ( SELECT ( CAST ( "SUBQUERY_2"."SUBQUERY_2_COL_0" AS VARCHAR ) )
| AS "SUBQUERY_3_COL_0" , ( CAST ( COUNT ( 1 ) AS VARCHAR ) ) AS "SUBQUERY_3_COL_1"
| FROM ( SELECT ( "SUBQUERY_1"."id" ) AS "SUBQUERY_2_COL_0" FROM ( SELECT * FROM
| ( SELECT * FROM ( $test_table ) AS "SF_CONNECTOR_QUERY_ALIAS" )
| AS "SUBQUERY_0" WHERE ( ( "SUBQUERY_0"."time" IS NOT NULL ) AND
| ( ( "SUBQUERY_0"."time" <= DATEADD(day, 18140 , TO_DATE('1970-01-01')) ) AND
| ( "SUBQUERY_0"."time" >= DATEADD(day, 18201 , TO_DATE('1970-01-01')) ) ) ) )
| AS "SUBQUERY_1" ) AS "SUBQUERY_2" GROUP BY "SUBQUERY_2"."SUBQUERY_2_COL_0" )
| AS "SUBQUERY_3" LIMIT 21
|""".stripMargin.filter(_ >= ' ')
)
)

jdbcUpdate(s"drop table $test_table")
}

Expand Down Expand Up @@ -287,27 +240,6 @@ class DataTypesIntegrationSuite extends IntegrationSuiteBase {
.agg(count("*").alias("abc"))
.collect()

assert(
Utils.getLastSelect.equals(
s"""SELECT ( "SUBQUERY_2"."SUBQUERY_2_COL_0" ) AS "SUBQUERY_3_COL_0" , ( COUNT ( 1 ) )
| AS "SUBQUERY_3_COL_1" FROM ( SELECT ( "SUBQUERY_1"."id" ) AS "SUBQUERY_2_COL_0" FROM
| ( SELECT * FROM ( SELECT * FROM ( $test_table ) AS "SF_CONNECTOR_QUERY_ALIAS" ) AS
| "SUBQUERY_0" WHERE ( ( ( "SUBQUERY_0"."time" IS NOT NULL ) AND ( "SUBQUERY_0"."time"
| >= to_timestamp_ntz( 1564358400000000 , 6) ) ) AND ( "SUBQUERY_0"."time" <=
| to_timestamp_ntz( 1567036800000000 , 6) ) ) ) AS "SUBQUERY_1" ) AS "SUBQUERY_2"
| GROUP BY "SUBQUERY_2"."SUBQUERY_2_COL_0"""".stripMargin.filter(_ >= ' ')
) ||
Utils.getLastSelect.equals(
s"""SELECT ( "SUBQUERY_2"."SUBQUERY_2_COL_0" ) AS "SUBQUERY_3_COL_0" , ( COUNT ( 1 ) )
| AS "SUBQUERY_3_COL_1" FROM ( SELECT ( "SUBQUERY_1"."id" ) AS "SUBQUERY_2_COL_0" FROM
| ( SELECT * FROM ( SELECT * FROM ( $test_table ) AS "SF_CONNECTOR_QUERY_ALIAS" ) AS
| "SUBQUERY_0" WHERE ( ( "SUBQUERY_0"."time" IS NOT NULL ) AND ( ( "SUBQUERY_0"."time"
| >= to_timestamp_ntz( 1564358400000000 , 6) ) AND ( "SUBQUERY_0"."time" <=
| to_timestamp_ntz( 1567036800000000 , 6) ) ) ) ) AS "SUBQUERY_1" ) AS "SUBQUERY_2"
| GROUP BY "SUBQUERY_2"."SUBQUERY_2_COL_0"""".stripMargin.filter(_ >= ' ')
)
)

assert(result.length == 1)
assert(result(0)(1) == 2)

Expand All @@ -328,31 +260,6 @@ class DataTypesIntegrationSuite extends IntegrationSuiteBase {
.agg(count("*").alias("abc"))
.show()

assert(
Utils.getLastSelect.equals(
s"""SELECT * FROM ( SELECT ( CAST ( "SUBQUERY_2"."SUBQUERY_2_COL_0" AS VARCHAR ) )
| AS "SUBQUERY_3_COL_0" , ( CAST ( COUNT ( 1 ) AS VARCHAR ) ) AS "SUBQUERY_3_COL_1"
| FROM ( SELECT ( "SUBQUERY_1"."id" ) AS "SUBQUERY_2_COL_0" FROM ( SELECT * FROM
| ( SELECT * FROM ( $test_table ) AS "SF_CONNECTOR_QUERY_ALIAS" ) AS "SUBQUERY_0"
| WHERE ( ( ( "SUBQUERY_0"."time" IS NOT NULL ) AND ( "SUBQUERY_0"."time" <=
| to_timestamp_ntz( 1564358400000000 , 6) ) ) AND ( "SUBQUERY_0"."time" >=
| to_timestamp_ntz( 1567036800000000 , 6) ) ) ) AS "SUBQUERY_1" ) AS "SUBQUERY_2"
| GROUP BY "SUBQUERY_2"."SUBQUERY_2_COL_0" ) AS "SUBQUERY_3" LIMIT 21
|""".stripMargin.filter(_ >= ' ')) ||
Utils.getLastSelect.equals(
s"""SELECT * FROM ( SELECT ( CAST ( "SUBQUERY_2"."SUBQUERY_2_COL_0" AS VARCHAR ) )
| AS "SUBQUERY_3_COL_0" , ( CAST ( COUNT ( 1 ) AS VARCHAR ) ) AS "SUBQUERY_3_COL_1"
| FROM ( SELECT ( "SUBQUERY_1"."id" ) AS "SUBQUERY_2_COL_0" FROM ( SELECT * FROM
| ( SELECT * FROM ( $test_table ) AS "SF_CONNECTOR_QUERY_ALIAS" ) AS "SUBQUERY_0"
| WHERE ( ( "SUBQUERY_0"."time" IS NOT NULL ) AND ( ( "SUBQUERY_0"."time" <=
| to_timestamp_ntz( 1564358400000000 , 6) ) AND ( "SUBQUERY_0"."time" >=
| to_timestamp_ntz( 1567036800000000 , 6) ) ) ) ) AS "SUBQUERY_1" ) AS "SUBQUERY_2"
| GROUP BY "SUBQUERY_2"."SUBQUERY_2_COL_0" ) AS "SUBQUERY_3" LIMIT 21
|""".stripMargin.filter(_ >= ' '))
)

jdbcUpdate(s"drop table $test_table")

}

}
Loading

0 comments on commit 8350590

Please sign in to comment.