publish libs

cli/README.md

@@ -42,7 +42,6 @@ Start with `npm run setup` to install dependencies and set up the CLI, then you
 
 ## Publish
 
-- connect to npm account
 - update `package.json` and `src/version.ts` versions
 - update lib versions (`npm run update` + manual) & run `npm install`
 - test with `npm run dry-publish` and check `azimutt-x.y.z.tgz` content

cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "azimutt",
-  "version": "0.0.31",
+  "version": "0.0.32",
   "description": "Export database schema from relational or document databases. Import it to https://azimutt.app",
   "keywords": [
     "database",
@@ -44,7 +44,7 @@
     "dry-publish": "npm run build && npm test && npm pack"
   },
   "dependencies": {
-    "@azimutt/gateway": "^0.0.19",
+    "@azimutt/gateway": "^0.0.20",
     "chalk": "4.1.2",
     "clear": "0.1.0",
     "commander": "11.0.0",

cli/src/index.ts

@@ -36,7 +36,7 @@ program.command('explore')
     .action((url, args) => exec(launchExplore(url, args.instance || 'https://azimutt.app', logger), args))
 
 program.command('export')
-    .description('Export a database schema in a file to easily import it in Azimutt.\nWorks with Couchbase, MariaDB, MongoDB, MySQL, PostgreSQL..., issues and PR are welcome in https://github.com/azimuttapp/azimutt ;)')
+    .description('Export a database schema in a file to easily import it in Azimutt.\nWorks with Couchbase, MariaDB, MongoDB, MySQL, PostgreSQL, Snowflake..., issues and PR are welcome in https://github.com/azimuttapp/azimutt ;)')
     .argument('<url>', 'the url to connect to the source, including credentials')
     .option('-d, --database <database>', 'Limit to a specific database (ex for MongoDB)')
     .option('-s, --schema <schema>', 'Limit to a specific schema (ex for PostgreSQL)')

cli/src/version.ts

@@ -1 +1 @@
-export const version = '0.0.31'
+export const version = '0.0.32'

frontend/src/DataSources/DbMiner/DbQuery.elm

@@ -7,6 +7,7 @@ import DataSources.DbMiner.QueryMongoDB as QueryMongoDB
 import DataSources.DbMiner.QueryMySQL as QueryMySQL
 import DataSources.DbMiner.QueryPostgreSQL as QueryPostgreSQL
 import DataSources.DbMiner.QuerySQLServer as QuerySQLServer
+import DataSources.DbMiner.QuerySnowflake as QuerySnowflake
 import Dict exposing (Dict)
 import Libs.Models.DatabaseKind as DatabaseKind exposing (DatabaseKind)
 import Models.Project.ColumnPath exposing (ColumnPath)
@@ -35,6 +36,9 @@ exploreTable db table =
             DatabaseKind.PostgreSQL ->
                 QueryPostgreSQL.exploreTable table
 
+            DatabaseKind.Snowflake ->
+                QuerySnowflake.exploreTable table
+
             DatabaseKind.SQLServer ->
                 QuerySQLServer.exploreTable table
 
@@ -64,6 +68,9 @@ exploreColumn db table column =
             DatabaseKind.PostgreSQL ->
                 QueryPostgreSQL.exploreColumn table column
 
+            DatabaseKind.Snowflake ->
+                QuerySnowflake.exploreColumn table column
+
             DatabaseKind.SQLServer ->
                 QuerySQLServer.exploreColumn table column
 
@@ -143,6 +150,9 @@ addLimit db query =
         DatabaseKind.PostgreSQL ->
             { sql = QueryPostgreSQL.addLimit query.sql, origin = query.origin, db = query.db }
 
+        DatabaseKind.Snowflake ->
+            { sql = QuerySnowflake.addLimit query.sql, origin = query.origin, db = query.db }
+
         _ ->
             query
 

frontend/src/DataSources/DbMiner/QuerySnowflake.elm

@@ -0,0 +1,49 @@
+module DataSources.DbMiner.QuerySnowflake exposing (addLimit, exploreColumn, exploreTable)
+
+import Libs.Regex as Regex
+import Models.Project.ColumnPath exposing (ColumnPath)
+import Models.Project.TableId exposing (TableId)
+import Models.SqlQuery exposing (SqlQuery)
+
+
+exploreTable : TableId -> SqlQuery
+exploreTable table =
+    "SELECT *\nFROM " ++ formatTable table ++ ";\n"
+
+
+exploreColumn : TableId -> ColumnPath -> SqlQuery
+exploreColumn table column =
+    formatColumn column
+        |> (\col -> "SELECT\n  " ++ col ++ ",\n  count(*)\nFROM " ++ formatTable table ++ "\nGROUP BY " ++ col ++ "\nORDER BY count(*) DESC, " ++ col ++ ";\n")
+
+
+addLimit : SqlQuery -> SqlQuery
+addLimit query =
+    case query |> String.trim |> Regex.matches "^([\\s\\S]+?)(\\slimit \\d+)?(\\soffset \\d+)?;$" of
+        (Just q) :: Nothing :: Nothing :: [] ->
+            q ++ "\nLIMIT 100;\n"
+
+        (Just q) :: Nothing :: (Just offset) :: [] ->
+            q ++ "\nLIMIT 100" ++ offset ++ ";\n"
+
+        _ ->
+            query
+
+
+
+-- generic helpers
+
+
+formatTable : TableId -> String
+formatTable ( schema, table ) =
+    if schema == "" then
+        "\"" ++ table ++ "\""
+
+    else
+        "\"" ++ schema ++ "\"" ++ "." ++ "\"" ++ table ++ "\""
+
+
+formatColumn : ColumnPath -> String
+formatColumn column =
+    -- FIXME: don't handle https://docs.snowflake.com/en/sql-reference/data-types-semistructured
+    "\"" ++ column.head ++ "\""

frontend/src/Libs/Models/DatabaseKind.elm

@@ -15,13 +15,14 @@ type DatabaseKind
     | MongoDB
     | MySQL
     | PostgreSQL
+    | Snowflake
     | SQLServer
     | Other
 
 
 all : List DatabaseKind
 all =
-    [ Couchbase, MariaDB, MongoDB, MySQL, PostgreSQL, SQLServer, Other ]
+    [ Couchbase, MariaDB, MongoDB, MySQL, PostgreSQL, Snowflake, SQLServer, Other ]
 
 
 fromUrl : DatabaseUrl -> DatabaseKind
@@ -41,6 +42,9 @@ fromUrl url =
     else if url |> String.contains "postgre" then
         PostgreSQL
 
+    else if url |> String.contains "snowflake" then
+        Snowflake
+
     else if (url |> String.contains "sqlserver") || (url |> String.toLower |> String.contains "user id=") then
         SQLServer
 
@@ -66,6 +70,9 @@ toString kind =
         PostgreSQL ->
             "PostgreSQL"
 
+        Snowflake ->
+            "Snowflake"
+
         SQLServer ->
             "SQLServer"
 
@@ -91,6 +98,9 @@ fromString kind =
         "PostgreSQL" ->
             Just PostgreSQL
 
+        "Snowflake" ->
+            Just Snowflake
+
         "SQLServer" ->
             Just SQLServer
 

frontend/src/Libs/Models/DatabaseUrl.elm

@@ -16,7 +16,7 @@ databaseName url =
         url |> String.split ";" |> List.find (\p -> p |> String.toLower |> String.startsWith "Database") |> Maybe.mapOrElse (\p -> p |> String.split "=" |> List.get 1 |> Maybe.withDefault url) url
 
     else
-        url |> String.split "/" |> List.reverse |> List.head |> Maybe.andThen (String.split "?" >> List.head) |> Maybe.withDefault url
+        url |> String.split "/" |> List.reverse |> List.head |> Maybe.andThen (String.split "?" >> List.head) |> Maybe.andThen (String.split "@" >> List.last) |> Maybe.withDefault url
 
 
 encode : DatabaseUrl -> Value

frontend/tests/DataSources/DbMiner/QuerySnowflakeTest.elm

@@ -0,0 +1,68 @@
+module DataSources.DbMiner.QuerySnowflakeTest exposing (..)
+
+import DataSources.DbMiner.QuerySnowflake exposing (addLimit, exploreColumn, exploreTable)
+import Expect
+import Models.Project.ColumnPath as ColumnPath
+import Test exposing (Test, describe, test)
+
+
+suite : Test
+suite =
+    describe "QuerySnowflake"
+        [ describe "exploreTable" exploreTableSuite
+        , describe "exploreColumn" exploreColumnSuite
+        , describe "addLimit" addLimitSuite
+        ]
+
+
+exploreTableSuite : List Test
+exploreTableSuite =
+    [ test "with schema" (\_ -> exploreTable ( "schema", "table" ) |> Expect.equal """SELECT *
+FROM "schema"."table";
+""")
+    , test "with empty schema" (\_ -> exploreTable ( "", "table" ) |> Expect.equal """SELECT *
+FROM "table";
+""")
+    ]
+
+
+exploreColumnSuite : List Test
+exploreColumnSuite =
+    [ test "with schema" (\_ -> exploreColumn ( "schema", "table" ) (ColumnPath.fromString "column") |> Expect.equal """SELECT
+  "column",
+  count(*)
+FROM "schema"."table"
+GROUP BY "column"
+ORDER BY count(*) DESC, "column";
+""")
+    , test "with empty schema" (\_ -> exploreColumn ( "", "table" ) (ColumnPath.fromString "column") |> Expect.equal """SELECT
+  "column",
+  count(*)
+FROM "table"
+GROUP BY "column"
+ORDER BY count(*) DESC, "column";
+""")
+    , test "with json column" (\_ -> exploreColumn ( "", "table" ) (ColumnPath.fromString "data:email") |> Expect.equal """SELECT
+  "data",
+  count(*)
+FROM "table"
+GROUP BY "data"
+ORDER BY count(*) DESC, "data";
+""")
+    ]
+
+
+addLimitSuite : List Test
+addLimitSuite =
+    [ test "without limit" (\_ -> addLimit "SELECT * FROM users;" |> Expect.equal "SELECT * FROM users\nLIMIT 100;\n")
+    , test "with limit" (\_ -> addLimit "SELECT * FROM users LIMIT 10;" |> Expect.equal "SELECT * FROM users LIMIT 10;")
+    , test "with offset" (\_ -> addLimit "SELECT * FROM users OFFSET 10;" |> Expect.equal "SELECT * FROM users\nLIMIT 100 OFFSET 10;\n")
+    , test "with limit & offset" (\_ -> addLimit "SELECT * FROM users LIMIT 10 OFFSET 10;" |> Expect.equal "SELECT * FROM users LIMIT 10 OFFSET 10;")
+    , test "multiline" (\_ -> addLimit """SELECT e.id, e.name
+FROM events e
+WHERE e.name='project_loaded';  """ |> Expect.equal """SELECT e.id, e.name
+FROM events e
+WHERE e.name='project_loaded'
+LIMIT 100;
+""")
+    ]