Skip to content

Commit 7c6c0e1

Browse files
mcmppmcmpp
authored
Merge pull request #33 from swisscom-bigdata/Testing_Phase
Testing phase
2 parents 181a749 + 288bb80 commit 7c6c0e1

File tree

1 file changed

+82
-85
lines changed

1 file changed

+82
-85
lines changed

‎src/metabase/driver/teradata.clj‎

Lines changed: 82 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
(:require [clojure
33
[set :as set]
44
[string :as s]]
5-
[medley.core :as m]
5+
[medley.core :as m]
66
[clojure.tools.logging :as log]
77
[clojure.java.jdbc :as jdbc]
88
[java-time :as t]
@@ -14,11 +14,11 @@
1414
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
1515
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
1616
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
17-
[metabase.driver.sql-jdbc.sync.common :as sql-jdbc.sync.common]
18-
[metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync.interface]
19-
[metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]
2017
[metabase.driver.sql.query-processor :as sql.qp]
2118
[metabase.driver.sql.util.deduplicate :as deduplicateutil]
19+
[metabase.driver.sql-jdbc.sync.common :as sql-jdbc.sync.common]
20+
[metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]
21+
[metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync.interface]
2222
[metabase.util.i18n :refer [trs]]
2323
[metabase.util.honey-sql-2 :as h2x])
2424
(:import [java.sql Connection DatabaseMetaData ResultSet Types PreparedStatement]
@@ -27,6 +27,9 @@
2727

2828
(driver/register! :teradata, :parent :sql-jdbc)
2929

30+
(doseq [[feature supported?] {:metadata/key-constraints false}]
31+
(defmethod driver/database-supports? [:teradata feature] [_driver _feature _db] supported?))
32+
3033
(defmethod sql-jdbc.sync/database-type->base-type :teradata [_ column-type]
3134
({:BIGINT :type/BigInteger
3235
:BIGSERIAL :type/BigInteger
@@ -88,35 +91,36 @@
8891
[dbnames]
8992
(when dbnames
9093
(set (map #(s/trim %) (s/split (s/trim dbnames) #",")))))
94+
9195
(defn- jdbc-fields-metadata
9296
"Reducible metadata about the Fields belonging to a Table, fetching using JDBC DatabaseMetaData methods."
9397
[driver ^Connection conn db-name-or-nil schema table-name]
9498
(sql-jdbc.sync.common/reducible-results
95-
#(.getColumns (.getMetaData conn)
96-
db-name-or-nil
97-
(some->> schema (driver/escape-entity-name-for-metadata driver))
98-
(some->> table-name (driver/escape-entity-name-for-metadata driver))
99-
nil)
100-
(fn [^ResultSet rs]
101-
#(let [default (.getString rs "COLUMN_DEF")
102-
no-default? (contains? #{nil "NULL" "null"} default)
103-
nullable (.getInt rs "NULLABLE")
104-
not-nullable? (= 0 nullable)
105-
column-name (.getString rs "COLUMN_NAME")
106-
required? (and no-default? not-nullable?)]
107-
(merge
108-
{:name column-name
109-
:database-type (.getString rs "TYPE_NAME")
110-
:database-required required?}
111-
(when-let [remarks (.getString rs "REMARKS")]
112-
(when-not (s/blank? remarks)
113-
{:field-comment remarks})))))))
99+
#(.getColumns (.getMetaData conn)
100+
db-name-or-nil
101+
(some->> schema (driver/escape-entity-name-for-metadata driver))
102+
(some->> table-name (driver/escape-entity-name-for-metadata driver))
103+
nil)
104+
(fn [^ResultSet rs]
105+
#(let [default (.getString rs "COLUMN_DEF")
106+
no-default? (contains? #{nil "NULL" "null"} default)
107+
nullable (.getInt rs "NULLABLE")
108+
not-nullable? (= 0 nullable)
109+
column-name (.getString rs "COLUMN_NAME")
110+
required? (and no-default? not-nullable?)]
111+
(merge
112+
{:name column-name
113+
:database-type (.getString rs "TYPE_NAME")
114+
:database-required required?}
115+
(when-let [remarks (.getString rs "REMARKS")]
116+
(when-not (s/blank? remarks)
117+
{:field-comment remarks})))))))
114118

115119
(defn fallback-fields-metadata-from-select-query
116120
"In some rare cases `:column_name` is blank (eg. SQLite's views with group by) fallback to sniffing the type from a
117121
SELECT * query."
118-
[driver ^Connection conn table-schema table-name]
119-
(let [[sql & params] (sql-jdbc.sync.interface/fallback-metadata-query driver table-schema table-name)]
122+
[driver ^Connection conn db-name-or-nil table-schema table-name]
123+
(let [[sql & params] (sql-jdbc.sync.interface/fallback-metadata-query driver db-name-or-nil table-schema table-name)]
120124
(reify clojure.lang.IReduceInit
121125
(reduce [_ rf init]
122126
(with-open [stmt (sql-jdbc.sync.common/prepare-statement driver conn sql params)
@@ -142,59 +146,55 @@
142146
;; 3. Filter out any duplicates between the two methods using `m/distinct-by`.
143147
(let [has-fields-without-type-info? (volatile! false)
144148
jdbc-metadata (eduction
145-
(remove (fn [{:keys [database-type]}]
146-
(when (s/blank? database-type)
147-
(vreset! has-fields-without-type-info? true)
148-
true)))
149-
(jdbc-fields-metadata driver conn db-name-or-nil schema table-name))
149+
(remove (fn [{:keys [database-type]}]
150+
(when (s/blank? database-type)
151+
(vreset! has-fields-without-type-info? true)
152+
true)))
153+
(jdbc-fields-metadata driver conn db-name-or-nil schema table-name))
150154
fallback-metadata (reify clojure.lang.IReduceInit
151155
(reduce [_ rf init]
152156
(reduce
153-
rf
154-
init
155-
(when @has-fields-without-type-info?
156-
(fallback-fields-metadata-from-select-query driver conn schema table-name)))))]
157+
rf
158+
init
159+
(when @has-fields-without-type-info?
160+
(fallback-fields-metadata-from-select-query driver conn db-name-or-nil schema table-name)))))]
157161
;; VERY IMPORTANT! DO NOT REWRITE THIS TO BE LAZY! IT ONLY WORKS BECAUSE AS NORMAL-FIELDS GETS REDUCED,
158162
;; HAS-FIELDS-WITHOUT-TYPE-INFO? WILL GET SET TO TRUE IF APPLICABLE AND THEN FALLBACK-FIELDS WILL RUN WHEN
159163
;; IT'S TIME TO START EVALUATING THAT.
160164
(reduce
161-
((comp cat (m/distinct-by :name)) rf)
162-
init
163-
[jdbc-metadata fallback-metadata])))))
165+
((comp cat (m/distinct-by :name)) rf)
166+
init
167+
[jdbc-metadata fallback-metadata])))))
164168

165169
(defmethod sql-jdbc.describe-table/describe-table-fields :teradata
166170
[driver conn table db-name-or-nil]
167171
(into
168-
#{}
169-
(sql-jdbc.describe-table/describe-table-fields-xf driver table)
170-
(fields-metadata driver conn table db-name-or-nil)))
172+
#{}
173+
(sql-jdbc.describe-table/describe-table-fields-xf driver table)
174+
(fields-metadata driver conn table db-name-or-nil)))
171175

172176
(defn- teradata-spec
173-
"Create a database specification for a teradata database. Opts should include keys
174-
for :db, :user, and :password. You can also optionally set host and port.
175-
Delimiters are automatically set to \"`\"."
177+
"Create a database specification for a Teradata database."
176178
[{:keys [host user password port dbnames charset tmode encrypt-data ssl additional-options]
177179
:or {host "localhost", charset "UTF8", tmode "ANSI", encrypt-data true, ssl false}
178180
:as opts}]
179181
(merge {:classname "com.teradata.jdbc.TeraDriver"
180182
:subprotocol "teradata"
181183
:subname (str "//" host "/"
182184
(->> (merge
183-
(when dbnames
184-
{"DATABASE" (first (dbnames-set dbnames))})
185-
(when port
186-
{"DBS_PORT" port})
187-
{"CHARSET" charset
188-
"TMODE" tmode
189-
"ENCRYPTDATA" (if encrypt-data "ON" "OFF")
190-
"FINALIZE_AUTO_CLOSE" "ON"
191-
;; We don't need lob support in metabase. This also removes the limitation of 16 open statements per session which would interfere metadata crawling.
192-
"LOB_SUPPORT" "OFF"
193-
}
185+
(when dbnames
186+
{"DATABASE" (first (dbnames-set dbnames))})
187+
(when port
188+
{"DBS_PORT" port})
189+
{"CHARSET" charset
190+
"TMODE" tmode
191+
"ENCRYPTDATA" (if encrypt-data "ON" "OFF")
192+
"FINALIZE_AUTO_CLOSE" "ON"
193+
"LOB_SUPPORT" "OFF"}
194194
(if ssl
195195
{"SSLMODE" "REQUIRE"}))
196-
(map #(format "%s=%s" (first %) (second %)))
197-
(clojure.string/join ",")))}
196+
(map #(format "%s=%s" (first %) (second %)))
197+
(clojure.string/join ",")))}
198198
(dissoc opts :host :port :dbnames :tmode :charset :ssl :encrypt-data)))
199199

200200
(defmethod sql-jdbc.conn/connection-details->spec :teradata
@@ -206,8 +206,8 @@
206206
;; (more keys might be added in the future to `default-advanced-options` => see metabase-plugin.yaml
207207
;; thus we switched from using `dissoc` to `select-keys`)
208208
(select-keys details-map [:host :port :user :password :dbnames :charset :tmode :encrypt-data :ssl :additional-options])
209-
teradata-spec
210-
(sql-jdbc.common/handle-additional-options details-map, :seperator-style :comma)))
209+
teradata-spec
210+
(sql-jdbc.common/handle-additional-options details-map, :seperator-style :comma)))
211211

212212
(defn- trunc [format-template v]
213213
[:trunc v (h2x/literal format-template)])
@@ -244,7 +244,7 @@
244244
(op (if (= unit :month)
245245
(trunc :month hsql-form)
246246
(h2x/->timestamp hsql-form))
247-
(case unit
247+
(case unit
248248
:second (num-to-interval :second amount)
249249
:minute (num-to-interval :minute amount)
250250
:hour (num-to-interval :hour amount)
@@ -265,16 +265,15 @@
265265

266266
(defmethod sql.qp/apply-top-level-clause [:teradata :limit]
267267
[_ _ honeysql-form {value :limit}]
268-
(update honeysql-form :select deduplicateutil/deduplicate-identifiers)
269-
)
268+
(update honeysql-form :select deduplicateutil/deduplicate-identifiers))
270269

271270
(defmethod sql.qp/apply-top-level-clause [:teradata :page] [_ _ honeysql-form {{:keys [items page]} :page}]
272271
(assoc honeysql-form :offset (:raw (format "QUALIFY ROW_NUMBER() OVER (%s) BETWEEN %d AND %d"
273-
(first (format (select-keys honeysql-form [:order-by])
274-
:allow-dashed-names? true
275-
:quoting :ansi))
276-
(inc (* items (dec page)))
277-
(* items page)))))
272+
(first (format (select-keys honeysql-form [:order-by])
273+
:allow-dashed-names? true
274+
:quoting :ansi))
275+
(inc (* items (dec page)))
276+
(* items page)))))
278277

279278
(def excluded-schemas
280279
#{"SystemFe" "SYSLIB" "LockLogShredder" "Sys_Calendar" "SYSBAR" "SYSUIF"
@@ -292,7 +291,7 @@
292291
"Fetch a JDBC Metadata ResultSet of tables in the DB, optionally limited to ones belonging to a given schema."
293292
^ResultSet [^DatabaseMetaData metadata, ^String schema-or-nil]
294293
(jdbc/result-set-seq (.getTables metadata nil schema-or-nil "%" ; tablePattern "%" = match all tables
295-
(into-array String ["TABLE", "VIEW", "FOREIGN TABLE"]))))
294+
(into-array String ["TABLE", "VIEW", "FOREIGN TABLE"]))))
296295

297296
(defn- fast-active-tables
298297
"Teradata, fast implementation of `fast-active-tables` to support inclusion list."
@@ -351,10 +350,8 @@
351350
t (t/sql-timestamp t)]
352351
(.setTimestamp ps i t cal)))
353352

354-
;; Run the query itself without setting the timezone connection parameter as this must not be changed on a Teradata connection.
355-
;; Setting connection attributes like timezone would make subsequent queries behave unexpectedly.
356-
(defmethod sql-jdbc.execute/connection-with-timezone :teradata
357-
[driver database ^String timezone-id]
353+
(defmethod sql-jdbc.execute/do-with-connection-with-options :teradata
354+
[driver database _options f]
358355
(let [conn (.getConnection (sql-jdbc.execute/datasource database))]
359356
(try
360357
(sql-jdbc.execute/set-best-transaction-level! driver conn)
@@ -366,7 +363,7 @@
366363
(.setHoldability conn ResultSet/CLOSE_CURSORS_AT_COMMIT)
367364
(catch Throwable e
368365
(log/debug e (trs "Error setting default holdability for connection"))))
369-
conn
366+
(f conn) ;; Pass the connection to the provided function
370367
(catch Throwable e
371368
(.close conn)
372369
(throw e)))))
@@ -378,9 +375,7 @@
378375

379376
(defmethod driver/execute-reducible-query :teradata
380377
[driver query context respond]
381-
(
382-
(get-method driver/execute-reducible-query :sql-jdbc) driver (cleanup-query query) context respond)
383-
)
378+
((get-method driver/execute-reducible-query :sql-jdbc) driver (cleanup-query query) context respond))
384379

385380
(defmethod sql.qp/current-datetime-honeysql-form :teradata [_] now)
386381

@@ -390,18 +385,20 @@
390385
;; https://www.mchange.com/projects/c3p0/#acquireRetryDelay
391386
(defmethod sql-jdbc.conn/data-warehouse-connection-pool-properties :teradata
392387
[driver database]
393-
{
394-
"acquireRetryDelay" (or (config/config-int :mb-jdbc-c3po-acquire-retry-delay) 1000)
388+
{"acquireRetryDelay" (or (config/config-int :mb-jdbc-c3po-acquire-retry-delay) 1000)
395389
"acquireIncrement" 1
396-
"maxIdleTime" (* 3 60 60) ; 3 hours
390+
"maxIdleTime" (* 6 60 60) ; 6 hours
397391
"minPoolSize" 1
398392
"initialPoolSize" 1
399-
"maxPoolSize" (or (config/config-int :mb-jdbc-data-warehouse-max-connection-pool-size) 15)
393+
"maxPoolSize" (or (config/config-int :mb-jdbc-data-warehouse-max-connection-pool-size) 30)
400394
"testConnectionOnCheckout" true
401395
"maxIdleTimeExcessConnections" (* 5 60)
402-
"dataSourceName" (format "db-%d-%s-%s" (u/the-id database) (name driver) (->> database
403-
:details
404-
((some-fn :db
405-
:dbname
406-
:sid
407-
:catalog))))})
396+
"checkoutTimeout" 300000 ; 300 seconds (increase this if needed)
397+
"idleConnectionTestPeriod" 300 ; Test idle connections every 5 minutes
398+
"maxConnectionAge" (* 6 60 60) ; 6 hours
399+
" dataSourceName " (format " db-%d-%s-%s " (u/the-id database) (name driver) (->> database
400+
:details
401+
((some-fn :db
402+
:dbname
403+
:sid
404+
:catalog))))})

0 commit comments

Comments
 (0)