review

cli/README.md

@@ -18,7 +18,7 @@ Get the help simply by running the CLI (`npx azimutt@latest`) or for a specific
   - ex: `npx azimutt export "mongodb+srv://user:[email protected]"`
   - ex: `npx azimutt export mysql://user:[email protected]:3306/my_db`
   - ex: `npx azimutt export postgresql://postgres:postgres@localhost:5432/azimutt_dev`
-  - ex: `npx azimutt export https://user:[email protected]/my_db`
+  - ex: `npx azimutt export snowflake://user:[email protected]?db=my_db`
   - ex: `npx azimutt export Server=host.com,1433;Database=db;User Id=user;Password=pass`
   - `url` the database connection url, must contain everything needed (user, pass, port...)
   - `--database` is optional, restrict schema extraction to this database

frontend/src/Services/DatabaseSource.elm

@@ -71,7 +71,7 @@ databases =
     , { key = "mysql", sampleUrl = "mysql://<user>:<pass>@<host>:<port>/<db>", issue = Nothing }
     , { key = "couchbase", sampleUrl = "couchbases://<user>:<pass>@<host>", issue = Nothing }
     , { key = "mongodb", sampleUrl = "mongodb+srv://<user>:<pass>@<host>", issue = Nothing }
-    , { key = "snowflake", sampleUrl = "https://<user>:<pass>@<account>.snowflakecomputing.com/<database>", issue = Nothing }
+    , { key = "snowflake", sampleUrl = "snowflake://<user>:<pass>@<account>.snowflakecomputing.com?db=<database>", issue = Nothing }
     , { key = "oracle", sampleUrl = "oracle:thin:<user>/<pass>@<host>:<port>:<db>", issue = Just "https://github.com/azimuttapp/azimutt/issues/217" }
     , { key = "sqlite", sampleUrl = "file:<path>", issue = Just "https://github.com/azimuttapp/azimutt/issues/115" }
     ]

libs/connector-snowflake/README.md

@@ -2,7 +2,7 @@
 
 This library is able to connect to [Snowflake](https://www.snowflake.com), extract its schema and more...
 
-It lists all schemas, tables, columns, relations and types and format them in a JSON Schema.
+It lists all schemas, tables, columns and relations and format them in a JSON Schema.
 
 This library is made by [Azimutt](https://azimutt.app) to allow people to explore their Snowflake database.
 It's accessible through the [Desktop app](../../desktop) (soon), the [CLI](https://www.npmjs.com/package/azimutt) or even the website using the [gateway](../../gateway) server.

libs/connector-snowflake/src/index.ts

@@ -21,9 +21,9 @@ export const snowflake: Connector = {
         return formatSchema(schema, opts.inferRelations || false)
     },
     getTableStats: (application: string, url: DatabaseUrlParsed, id: TableId): Promise<TableStats> =>
-        connect(application, url, getTableStats(id)),
+        Promise.reject('Not implemented'), // connect(application, url, getTableStats(id)),
     getColumnStats: (application: string, url: DatabaseUrlParsed, ref: ColumnRef): Promise<ColumnStats> =>
-        connect(application, url, getColumnStats(ref)),
+        Promise.reject('Not implemented'), // connect(application, url, getColumnStats(ref)),
     query: (application: string, url: DatabaseUrlParsed, query: string, parameters: any[]): Promise<DatabaseQueryResults> =>
-        connect(application, url, execQuery(query, parameters)),
+        Promise.reject('Not implemented'), // connect(application, url, execQuery(query, parameters)),
 }

libs/connector-snowflake/src/snowflake.ts

@@ -1,16 +1,11 @@
-import {groupBy, Logger, removeUndefined, sequence, zip} from "@azimutt/utils";
-import {AzimuttSchema, ColumnName, SchemaName, TableName} from "@azimutt/database-types";
-import {schemaToColumns, ValueSchema, valuesToSchema} from "@azimutt/json-infer-schema";
+import {groupBy, Logger, removeUndefined} from "@azimutt/utils";
+import {AzimuttSchema} from "@azimutt/database-types";
 import {Conn} from "./common";
-import {buildSqlColumn, buildSqlTable} from "./helpers";
 
 export type SnowflakeSchema = { tables: SnowflakeTable[], relations: SnowflakeRelation[] }
-export type SnowflakeTable = { catalog: SnowflakeCatalogName, schema: SnowflakeSchemaName, table: SnowflakeTableName, view: boolean, columns: SnowflakeColumn[], primaryKey: SnowflakePrimaryKey | null, uniques: SnowflakeUnique[], indexes: SnowflakeIndex[], checks: SnowflakeCheck[], comment: string | null }
-export type SnowflakeColumn = { name: SnowflakeColumnName, type: SnowflakeColumnType, nullable: boolean, default: string | null, comment: string | null, values: string[] | null, schema: ValueSchema | null }
+export type SnowflakeTable = { catalog: SnowflakeCatalogName, schema: SnowflakeSchemaName, table: SnowflakeTableName, view: boolean, columns: SnowflakeColumn[], primaryKey: SnowflakePrimaryKey | null, comment: string | null }
+export type SnowflakeColumn = { name: SnowflakeColumnName, type: SnowflakeColumnType, nullable: boolean, default: string | null, comment: string | null }
 export type SnowflakePrimaryKey = { name: string, columns: SnowflakeColumnName[] }
-export type SnowflakeUnique = { name: string, columns: SnowflakeColumnName[], definition: string | null }
-export type SnowflakeIndex = { name: string, columns: SnowflakeColumnName[], definition: string | null }
-export type SnowflakeCheck = { name: string, columns: SnowflakeColumnName[], predicate: string | null }
 export type SnowflakeRelation = { name: SnowflakeRelationName, src: SnowflakeTableRef, ref: SnowflakeTableRef, columns: SnowflakeColumnLink[], comment: string | null }
 export type SnowflakeTableRef = { catalog: SnowflakeCatalogName, schema: SnowflakeSchemaName, table: SnowflakeTableName }
 export type SnowflakeColumnLink = { src: SnowflakeColumnName, ref: SnowflakeColumnName }
@@ -41,14 +36,9 @@ export const getSchema = (schema: SnowflakeSchemaName | undefined, sampleSize: n
                     type: column.type,
                     nullable: column.nullable === 'YES',
                     default: column.default,
-                    comment: column.comment,
-                    values: null, // TODO
-                    schema: null // TODO
+                    comment: column.comment
                 })),
                 primaryKey: pk ? {name: pk[0].constraint_name, columns: pk.sort((a, b) => a.key_sequence - b.key_sequence).map(c => c.column_name)} : null,
-                uniques: [], // TODO
-                indexes: [], // TODO
-                checks: [], // TODO
                 comment: table.comment
             }
         }),
@@ -78,14 +68,17 @@ export function formatSchema(schema: SnowflakeSchema, inferRelations: boolean):
                 nullable: c.nullable || undefined,
                 default: c.default || undefined,
                 comment: c.comment || undefined,
-                values: c.values && c.values.length > 0 ? c.values : undefined,
-                columns: c.schema ? schemaToColumns(c.schema, 0) : undefined
+                values: undefined, // TODO
+                columns: undefined // TODO
             })),
             view: t.view || undefined,
-            primaryKey: undefined,
-            uniques: undefined,
-            indexes: undefined,
-            checks: undefined,
+            primaryKey: t.primaryKey ? removeUndefined({
+                name: t.primaryKey.name || undefined,
+                columns: t.primaryKey.columns,
+            }) : undefined,
+            uniques: undefined, // TODO
+            indexes: undefined, // TODO
+            checks: undefined, // TODO
             comment: t.comment || undefined
         })),
         relations: schema.relations.flatMap(r => r.columns.map(c => ({

libs/connector-snowflake/tests/connect.test.ts

@@ -1,6 +1,6 @@
 import {describe, test} from "@jest/globals";
 import {Connection, ConnectionOptions, createConnection, SnowflakeError, Statement} from "snowflake-sdk";
-import {parseDatabaseUrl, DatabaseUrlParsed} from "@azimutt/database-types";
+import {parseDatabaseUrl} from "@azimutt/database-types";
 import {connect} from "../src/connect";
 import {application} from "./constants";
 import {execQuery} from "../src/common";
@@ -15,16 +15,20 @@ describe('connect', () => {
     const url = 'snowflake://<user>:<pass>@<account>.snowflakecomputing.com?db=<database>'
 
     // TODO 2: write a valid query for your database
-    const query = 'SELECT * FROM TPCDS_SF100TCL.WEB_SITE LIMIT 10;'
-    // const query = 'SHOW TABLES;'
+    const query = `
+        SELECT cc.CC_CALL_CENTER_SK, cc.CC_NAME, c.C_CUSTOMER_SK, c.C_EMAIL_ADDRESS, r.CR_REFUNDED_CASH
+        FROM TPCDS_SF10TCL.CATALOG_RETURNS r
+            JOIN TPCDS_SF10TCL.CALL_CENTER cc ON r.CR_CALL_CENTER_SK = cc.CC_CALL_CENTER_SK
+            JOIN TPCDS_SF10TCL.CUSTOMER c ON r.CR_REFUNDED_CUSTOMER_SK = c.C_CUSTOMER_SK
+        LIMIT 30;`
     const parameters: any[] = []
 
     // TODO 3: unskip the this test first and run it: `npm run test -- tests/connect.test.ts`
-    test('Azimutt should connect', async () => {
+    test.skip('Azimutt should connect', async () => {
         const parsedUrl = parseDatabaseUrl(url)
         const results = await connect(application, parsedUrl, execQuery(query, parameters))
         console.log('results', results)
-    })
+    }, 10000)
 
     // TODO 4: if previous test failed, unskip this one an find how https://www.npmjs.com/package/snowflake-sdk can connect to your database
     // tips: check lib version in package.json, ping us if you need help

libs/connector-snowflake/tests/query.test.ts

@@ -6,7 +6,7 @@ import {execQuery} from "../src/common";
 
 describe('query', () => {
     // local url, install db or replace it to test
-    const url: DatabaseUrlParsed = parseDatabaseUrl('https://<user>:<pass>@<account>.snowflakecomputing.com/<database>')
+    const url: DatabaseUrlParsed = parseDatabaseUrl('snowflake://<user>:<pass>@<account>.snowflakecomputing.com?db=<database>')
     /* FIXME
     test.skip('query', async () => {
         const query = 'SELECT u.id, e.id, o.id FROM users u JOIN events e ON u.id = e.created_by JOIN organizations o on o.id = e.organization_id LIMIT 10;'

libs/connector-snowflake/tests/snowflake.test.ts

@@ -1,13 +1,12 @@
 import {describe, expect, test} from "@jest/globals";
-import {AzimuttSchema, DatabaseUrlParsed, parseDatabaseUrl} from "@azimutt/database-types";
+import {DatabaseUrlParsed, parseDatabaseUrl} from "@azimutt/database-types";
 import {application, logger} from "./constants";
-import {execQuery} from "../src/common";
 import {connect} from "../src/connect";
 import {getColumns, getForeignKeys, getPrimaryKeys, getSchema, getTables, SnowflakeSchemaName} from "../src/snowflake";
 
 describe('snowflake', () => {
     // local url, install db or replace it to test
-    const url: DatabaseUrlParsed = parseDatabaseUrl('https://<user>:<pass>@<account>.snowflakecomputing.com/<database>')
+    const url: DatabaseUrlParsed = parseDatabaseUrl('snowflake://<user>:<pass>@<account>.snowflakecomputing.com?db=<database>')
     const schema: SnowflakeSchemaName | undefined = 'TPCDS_SF10TCL'
 
     test.skip('getSchema', async () => {

libs/connector-snowflake/tests/stats.test.ts

@@ -6,7 +6,7 @@ import {getColumnStats, getTableStats} from "../src/stats";
 
 describe('stats', () => {
     // local url, install db or replace it to test
-    const url: DatabaseUrlParsed = parseDatabaseUrl('https://<user>:<pass>@<account>.snowflakecomputing.com/<database>')
+    const url: DatabaseUrlParsed = parseDatabaseUrl('snowflake://<user>:<pass>@<account>.snowflakecomputing.com?db=<database>')
     /* FIXME
     test.skip('getTableStats', async () => {
         const stats = await connect(application, url, getTableStats('public.users'))