From c4e4e4f114cf346128da53f3944c38c34fd109a2 Mon Sep 17 00:00:00 2001 From: Yazidane Date: Tue, 9 Jul 2024 07:54:47 +0800 Subject: [PATCH 1/8] feat: add sql component --- data/sql/v0/README.mdx | 140 ++++++++++++ data/sql/v0/assets/sql.svg | 7 + data/sql/v0/client.go | 98 ++++++++ data/sql/v0/component_test.go | 350 +++++++++++++++++++++++++++++ data/sql/v0/config/definition.json | 20 ++ data/sql/v0/config/setup.json | 107 +++++++++ data/sql/v0/config/tasks.json | 319 ++++++++++++++++++++++++++ data/sql/v0/main.go | 105 +++++++++ data/sql/v0/tasks.go | 304 +++++++++++++++++++++++++ store/store.go | 2 + 10 files changed, 1452 insertions(+) create mode 100644 data/sql/v0/README.mdx create mode 100644 data/sql/v0/assets/sql.svg create mode 100644 data/sql/v0/client.go create mode 100644 data/sql/v0/component_test.go create mode 100644 data/sql/v0/config/definition.json create mode 100644 data/sql/v0/config/setup.json create mode 100644 data/sql/v0/config/tasks.json create mode 100644 data/sql/v0/main.go create mode 100644 data/sql/v0/tasks.go diff --git a/data/sql/v0/README.mdx b/data/sql/v0/README.mdx new file mode 100644 index 00000000..c860e224 --- /dev/null +++ b/data/sql/v0/README.mdx @@ -0,0 +1,140 @@ +--- +title: "SQL" +lang: "en-US" +draft: false +description: "Learn about how to set up a VDP SQL component https://github.com/instill-ai/instill-core" +--- + +The SQL component is a data component that allows users to access the SQL database of your choice. +It can carry out the following tasks: + +- [Insert](#insert) +- [Update](#update) +- [Select](#select) +- [Delete](#delete) + + + +## Release Stage + +`Alpha` + + + +## Configuration + +The component configuration is defined and maintained [here](https://github.com/instill-ai/component/blob/main/data/sql/v0/config/definition.json). + + + + +## Setup + + +| Field | Field ID | Type | Note | +| :--- | :--- | :--- | :--- | +| Engine (required) | `engine` | string | Choose the engine of your database | +| Username (required) | `user` | string | Fill in your account username | +| Password | `password` | string | Fill in your account password | +| Database Name (required) | `name` | string | Fill in your database name | +| Host (required) | `host` | string | Fill in your database host | +| Port (required) | `port` | number | Fill in your database port | + + + + +## Supported Tasks + +### Insert + +Perform an insert operation based on specified criteria + + +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_INSERT` | +| Table Name (required) | `table-name` | string | The table name in the database to insert data into | +| Data (required) | `data` | semi-structured/json | The data to be inserted | + + + +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Status | `status` | string | Insert status | + + + + + + +### Update + +Perform an update operation based on specified criteria + + +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_UPDATE` | +| Table Name (required) | `table-name` | string | The table name in the database to update data into | +| Criteria (required) | `criteria` | semi-structured/json | The data criteria to be updated | +| Update (required) | `update` | semi-structured/json | The new data to be updated to | + + + +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Status | `status` | string | Update status | + + + + + + +### Select + +Perform a select operation based on specified criteria + + +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_SELECT` | +| Table Name (required) | `table-name` | string | The table name in the database to be selected | +| Criteria (required) | `criteria` | semi-structured/json | The data criteria to be selected, if JSON value of a key is null e.g \{'name':null\}, then name column will be selected without any criteria, if JSON is empty e.g \{\}, then all columns will be selected | +| From (required) | `from` | number | The starting row to be selected, if both 'from' and 'to' equals 0 then all rows will be selected | +| To (required) | `to` | number | The starting row to be selected, if both 'from' and 'to' equals 0 then all rows will be selected | + + + +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Rows | `rows` | array | The rows returned from the select operation | +| Status | `status` | string | Select status | + + + + + + +### Delete + +Perform a delete operation based on specified criteria + + +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_DELETE` | +| Table Name (required) | `table-name` | string | The table name in the database to be deleted | +| Criteria (required) | `criteria` | semi-structured/json | The data criteria to be deleted | + + + +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Status | `status` | string | Delete status | + + + + + + + diff --git a/data/sql/v0/assets/sql.svg b/data/sql/v0/assets/sql.svg new file mode 100644 index 00000000..f5be6803 --- /dev/null +++ b/data/sql/v0/assets/sql.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/data/sql/v0/client.go b/data/sql/v0/client.go new file mode 100644 index 00000000..fcbbdc7c --- /dev/null +++ b/data/sql/v0/client.go @@ -0,0 +1,98 @@ +package sql + +import ( + "fmt" + "strconv" + + "github.com/jmoiron/sqlx" + "google.golang.org/protobuf/types/known/structpb" + + // Import all the SQL drivers + _ "github.com/denisenkom/go-mssqldb" // SQL Server + _ "github.com/go-sql-driver/mysql" // MySQL and MariaDB + _ "github.com/lib/pq" // PostgreSQL + _ "github.com/nakagami/firebirdsql" // Firebird + _ "github.com/sijms/go-ora/v2" // Oracle +) + +var engines = map[string]string{ + "PostgreSQL": "postgresql://%s:%s@%s/%s", // PostgreSQL + "SQL Server": "sqlserver://%s:%s@%s?database=%s", // SQL Server + "Oracle": "oracle://%s:%s@%s/%s", // Oracle + "MySQL": "%s:%s@tcp(%s)/%s", // MySQL and MariaDB + "Firebird": "firebirdsql://%s:%s@%s/%s", // Firebird +} + +var enginesType = map[string]string{ + "PostgreSQL": "postgres", // PostgreSQL + "SQL Server": "sqlserver", // SQL Server + "Oracle": "godror", // Oracle + "MySQL": "mysql", // MySQL and MariaDB + "Firebird": "firebirdsql", // Firebird +} + +type Config struct { + DBUser string + DBPassword string + DBName string + DBHost string + DBPort string + DBEngine string +} + +func LoadConfig(setup *structpb.Struct) *Config { + return &Config{ + DBUser: getUser(setup), + DBPassword: getPassword(setup), + DBName: getName(setup), + DBHost: getHost(setup), + DBPort: getPort(setup), + DBEngine: getEngine(setup), + } +} + +func newClient(setup *structpb.Struct) SQLClient { + cfg := LoadConfig(setup) + + DBEndpoint := fmt.Sprintf("%v:%v", cfg.DBHost, cfg.DBPort) + + // Test every engines to find the correct one + var db *sqlx.DB + var err error + + // Get the correct engine + engine := engines[cfg.DBEngine] + engineType := enginesType[cfg.DBEngine] + + dsn := fmt.Sprintf(engine, + cfg.DBUser, cfg.DBPassword, DBEndpoint, cfg.DBName, + ) + + db, err = sqlx.Open(engineType, dsn) + if err != nil { + return nil + } + + return db +} + +func getUser(setup *structpb.Struct) string { + return setup.GetFields()["user"].GetStringValue() +} +func getPassword(setup *structpb.Struct) string { + return setup.GetFields()["password"].GetStringValue() +} +func getName(setup *structpb.Struct) string { + return setup.GetFields()["name"].GetStringValue() +} +func getHost(setup *structpb.Struct) string { + return setup.GetFields()["host"].GetStringValue() +} +func getPort(setup *structpb.Struct) string { + port := setup.GetFields()["port"].GetNumberValue() + portStr := strconv.FormatFloat(port, 'f', -1, 64) + return portStr +} +func getEngine(setup *structpb.Struct) string { + return setup.GetFields()["engine"].GetStringValue() +} diff --git a/data/sql/v0/component_test.go b/data/sql/v0/component_test.go new file mode 100644 index 00000000..359be1e3 --- /dev/null +++ b/data/sql/v0/component_test.go @@ -0,0 +1,350 @@ +package sql + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + qt "github.com/frankban/quicktest" + "github.com/instill-ai/component/base" + "github.com/jmoiron/sqlx" + + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/structpb" +) + +type MockSQLClient struct{} + +func (m *MockSQLClient) Queryx(query string, args ...interface{}) (*sqlx.Rows, error) { + mockDB, mock, _ := sqlmock.New() + defer mockDB.Close() + + sqlxDB := sqlx.NewDb(mockDB, "sqlmock") + mock.ExpectQuery("SELECT (.+) FROM users WHERE id = (.+) AND name = (.+) AND email = (.+) LIMIT (.+) OFFSET (.+)"). + WithArgs("1", "john", "john@example.com", 1, 0). + WillReturnRows(sqlmock.NewRows([]string{"id", "name", "email"}).AddRow("1", "john", "john@example.com")) + + return sqlxDB.Queryx("SELECT id, name, email FROM users WHERE id = ? AND name = ? AND email = ? LIMIT ? OFFSET ?", "1", "john", "john@example.com", 1, 0) +} + +func (m *MockSQLClient) NamedExec(query string, arg interface{}) (sql.Result, error) { + if strings.Contains(query, "INSERT") { + mockDB, mock, _ := sqlmock.New() + defer mockDB.Close() + + sqlxDB := sqlx.NewDb(mockDB, "sqlmock") + fmt.Print(arg) + arg = map[string]interface{}{ + "id": "1", + "name": "John Doe", + } + + mock.ExpectExec("INSERT INTO users \\(id, name\\) VALUES \\(\\?, \\?\\)"). + WithArgs("1", "John Doe").WillReturnResult(sqlmock.NewResult(1, 1)) + + return sqlxDB.NamedExec("INSERT INTO users (id, name) VALUES (:id, :name)", arg) + } else if strings.Contains(query, "DELETE") { + mockDB, mock, _ := sqlmock.New() + defer mockDB.Close() + + sqlxDB := sqlx.NewDb(mockDB, "sqlmock") + arg = map[string]interface{}{ + "id": "1", + "name": "john", + } + + mock.ExpectExec("DELETE FROM users WHERE id = \\? AND name = \\?"). + WithArgs("1", "john").WillReturnResult(sqlmock.NewResult(1, 1)) + + return sqlxDB.NamedExec("DELETE FROM users WHERE id = :id AND name = :name", arg) + + } else { + mockDB, mock, _ := sqlmock.New() + defer mockDB.Close() + + sqlxDB := sqlx.NewDb(mockDB, "sqlmock") + arg = map[string]interface{}{ + "id": "1", + "name": "John Doe Updated", + } + + mock.ExpectExec("UPDATE users SET id = \\?, name = \\? WHERE id = \\? AND name = \\?"). + WithArgs("1", "John Doe Updated", "1", "John Doe Updated").WillReturnResult(sqlmock.NewResult(1, 1)) + + return sqlxDB.NamedExec("UPDATE users SET id = :id, name = :name WHERE id = :id AND name = :name", arg) + } +} + +func TestInsertUser(t *testing.T) { + c := qt.New(t) + ctx := context.Background() + bc := base.Component{Logger: zap.NewNop()} + connector := Init(bc) + + testcases := []struct { + name string + tableName string + input InsertInput + wantResp InsertOutput + wantErr string + }{ + { + name: "insert user", + tableName: "users", + input: InsertInput{ + Data: map[string]any{ + "id": "1", + "name": "John Doe", + }, + TableName: "users", + }, + wantResp: InsertOutput{ + Status: "Successfully inserted document", + }, + }, + } + + for _, tc := range testcases { + c.Run(tc.name, func(c *qt.C) { + setup, err := structpb.NewStruct(map[string]any{ + "user": "test_user", + "password": "test_pass", + "name": "test_db", + "host": "localhost", + "port": "3306", + "region": "us-west-2", + }) + c.Assert(err, qt.IsNil) + + e := &execution{ + ComponentExecution: base.ComponentExecution{Component: connector, SystemVariables: nil, Setup: setup, Task: TaskInsert}, + client: &MockSQLClient{}, + } + e.execute = e.insert + exec := &base.ExecutionWrapper{Execution: e} + + pbIn, err := base.ConvertToStructpb(tc.input) + c.Assert(err, qt.IsNil) + + got, err := exec.Execution.Execute(ctx, []*structpb.Struct{pbIn}) + + if tc.wantErr != "" { + c.Assert(err, qt.ErrorMatches, tc.wantErr) + return + } + + wantJSON, err := json.Marshal(tc.wantResp) + c.Assert(err, qt.IsNil) + c.Check(wantJSON, qt.JSONEquals, got[0].AsMap()) + }) + } +} + +func TestUpdateUser(t *testing.T) { + c := qt.New(t) + ctx := context.Background() + bc := base.Component{Logger: zap.NewNop()} + connector := Init(bc) + + testcases := []struct { + name string + tableName string + input UpdateInput + wantResp UpdateOutput + wantErr string + }{ + { + name: "update user", + tableName: "users", + input: UpdateInput{ + Criteria: map[string]any{ + "id": "1", + "name": "John Doe", + }, + Update: map[string]any{ + "id": "1", + "name": "John Doe Updated", + }, + TableName: "users", + }, + wantResp: UpdateOutput{ + Status: "Successfully updated document", + }, + }, + } + + for _, tc := range testcases { + c.Run(tc.name, func(c *qt.C) { + setup, err := structpb.NewStruct(map[string]any{ + "user": "test_user", + "password": "test_pass", + "name": "test_db", + "host": "localhost", + "port": "3306", + "region": "us-west-2", + }) + c.Assert(err, qt.IsNil) + + e := &execution{ + ComponentExecution: base.ComponentExecution{Component: connector, SystemVariables: nil, Setup: setup, Task: TaskInsert}, + client: &MockSQLClient{}, + } + e.execute = e.update + exec := &base.ExecutionWrapper{Execution: e} + + pbIn, err := base.ConvertToStructpb(tc.input) + c.Assert(err, qt.IsNil) + + got, err := exec.Execution.Execute(ctx, []*structpb.Struct{pbIn}) + + if tc.wantErr != "" { + c.Assert(err, qt.ErrorMatches, tc.wantErr) + return + } + + wantJSON, err := json.Marshal(tc.wantResp) + c.Assert(err, qt.IsNil) + c.Check(wantJSON, qt.JSONEquals, got[0].AsMap()) + }) + } +} + +func TestSelectUser(t *testing.T) { + c := qt.New(t) + ctx := context.Background() + bc := base.Component{Logger: zap.NewNop()} + connector := Init(bc) + + testcases := []struct { + name string + tableName string + input SelectInput + wantResp SelectOutput + wantErr string + }{ + { + name: "select users", + tableName: "users", + input: SelectInput{ + Criteria: map[string]any{ + "id": "1", + "name": "john", + "email": "john@example.com", + }, + TableName: "users", + From: 0, + To: 1, + }, + wantResp: SelectOutput{ + Status: "Successfully selected document", + Rows: []map[string]any{ + {"id": "1", "name": "john", "email": "john@example.com"}, + }, + }, + }, + } + + for _, tc := range testcases { + c.Run(tc.name, func(c *qt.C) { + setup, err := structpb.NewStruct(map[string]any{ + "user": "test_user", + "password": "test_pass", + "name": "test_db", + "host": "localhost", + "port": "3306", + "region": "us-west-2", + }) + c.Assert(err, qt.IsNil) + + e := &execution{ + ComponentExecution: base.ComponentExecution{Component: connector, SystemVariables: nil, Setup: setup, Task: TaskSelect}, + client: &MockSQLClient{}, + } + e.execute = e.selects + exec := &base.ExecutionWrapper{Execution: e} + + pbIn, err := base.ConvertToStructpb(tc.input) + c.Assert(err, qt.IsNil) + + got, err := exec.Execution.Execute(ctx, []*structpb.Struct{pbIn}) + + if tc.wantErr != "" { + c.Assert(err, qt.ErrorMatches, tc.wantErr) + return + } + + wantJSON, err := json.Marshal(tc.wantResp) + c.Assert(err, qt.IsNil) + c.Check(wantJSON, qt.JSONEquals, got[0].AsMap()) + }) + } +} + +func TestDeleteUser(t *testing.T) { + c := qt.New(t) + ctx := context.Background() + bc := base.Component{Logger: zap.NewNop()} + connector := Init(bc) + + testcases := []struct { + name string + tableName string + input DeleteInput + wantResp DeleteOutput + wantErr string + }{ + { + name: "delete user", + tableName: "users", + input: DeleteInput{ + Criteria: map[string]any{ + "id": "1", + "name": "john", + }, + TableName: "users", + }, + wantResp: DeleteOutput{ + Status: "Successfully deleted document", + }, + }, + } + + for _, tc := range testcases { + c.Run(tc.name, func(c *qt.C) { + setup, err := structpb.NewStruct(map[string]any{ + "user": "test_user", + "password": "test_pass", + "name": "test_db", + "host": "localhost", + "port": "3306", + "region": "us-west-2", + }) + c.Assert(err, qt.IsNil) + + e := &execution{ + ComponentExecution: base.ComponentExecution{Component: connector, SystemVariables: nil, Setup: setup, Task: TaskDelete}, + client: &MockSQLClient{}, + } + e.execute = e.delete + exec := &base.ExecutionWrapper{Execution: e} + + pbIn, err := base.ConvertToStructpb(tc.input) + c.Assert(err, qt.IsNil) + + got, err := exec.Execution.Execute(ctx, []*structpb.Struct{pbIn}) + + if tc.wantErr != "" { + c.Assert(err, qt.ErrorMatches, tc.wantErr) + return + } + + wantJSON, err := json.Marshal(tc.wantResp) + c.Assert(err, qt.IsNil) + c.Check(wantJSON, qt.JSONEquals, got[0].AsMap()) + }) + } +} diff --git a/data/sql/v0/config/definition.json b/data/sql/v0/config/definition.json new file mode 100644 index 00000000..da16e1c2 --- /dev/null +++ b/data/sql/v0/config/definition.json @@ -0,0 +1,20 @@ +{ + "availableTasks": [ + "TASK_INSERT", + "TASK_UPDATE", + "TASK_SELECT", + "TASK_DELETE" + ], + "documentationUrl": "https://www.instill.tech/docs/component/data/sql", + "icon": "assets/sql.svg", + "id": "sql", + "public": true, + "title": "SQL", + "description": "Access the SQL database of your choice", + "tombstone": false, + "type": "COMPONENT_TYPE_DATA", + "uid": "f12d6db7-45be-4f76-96aa-497bd7c0e1d6", + "version": "0.1.0", + "sourceUrl": "https://github.com/instill-ai/component/blob/main/data/sql/v0", + "releaseStage": "RELEASE_STAGE_ALPHA" +} \ No newline at end of file diff --git a/data/sql/v0/config/setup.json b/data/sql/v0/config/setup.json new file mode 100644 index 00000000..9d2576a7 --- /dev/null +++ b/data/sql/v0/config/setup.json @@ -0,0 +1,107 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "engine": { + "description": "Choose the engine of your database", + "instillUpstreamTypes": [ + "value", + "reference", + "template" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 0, + "title": "Engine", + "enum": [ + "MySQL", + "PostgreSQL", + "SQL Server", + "Oracle", + "MariaDB", + "Firebird" + ], + "type": "string", + "x-oaiTypeLabel": "string" + }, + "user": { + "description": "Fill in your account username", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 1, + "title": "Username", + "type": "string" + }, + "password": { + "description": "Fill in your account password", + "instillUpstreamTypes": [ + "reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillSecret": true, + "instillUIOrder": 2, + "title": "Password", + "type": "string" + }, + "name": { + "description": "Fill in your database name", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 3, + "title": "Database Name", + "type": "string" + }, + "host": { + "description": "Fill in your database host", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 4, + "title": "Host", + "type": "string" + }, + "port": { + "description": "Fill in your database port", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "number" + ], + "instillUIOrder": 5, + "title": "Port", + "type": "number" + } + }, + "required": [ + "engine", + "user", + "name", + "host", + "port" + ], + "instillEditOnNodeFields": [ + "engine", + "user", + "password", + "name", + "host", + "port" + ], + "title": "SQL Connection", + "type": "object" +} \ No newline at end of file diff --git a/data/sql/v0/config/tasks.json b/data/sql/v0/config/tasks.json new file mode 100644 index 00000000..6d9b1609 --- /dev/null +++ b/data/sql/v0/config/tasks.json @@ -0,0 +1,319 @@ +{ + "TASK_INSERT": { + "instillShortDescription": "Perform an insert operation based on specified criteria", + "input": { + "instillUIOrder": 0, + "properties": { + "data": { + "description": "The data to be inserted", + "instillAcceptFormats": [ + "semi-structured/*","structured/*","object","array" + ], + "instillShortDescription": "JSON Data", + "instillUIOrder": 1, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "items": { + "title": "Object", + "instillFormat": "semi-structured/json" + }, + "title": "Data" + }, + "table-name": { + "description": "The table name in the database to insert data into", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "Database Table Name", + "instillUIOrder": 0, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Table Name", + "type":"string" + } + }, + "required": [ + "data", + "table-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "instillUIOrder": 0, + "properties": { + "status": { + "description": "Insert status", + "instillFormat": "string", + "required": [], + "instillUIOrder": 0, + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_UPDATE": { + "instillShortDescription": "Perform an update operation based on specified criteria", + "input": { + "instillUIOrder": 0, + "properties": { + "criteria": { + "description": "The data criteria to be updated", + "instillAcceptFormats": [ + "semi-structured/*","structured/*","object" + ], + "instillShortDescription": "JSON Data", + "instillUIOrder": 1, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Criteria" + }, + "update": { + "description": "The new data to be updated to", + "instillAcceptFormats": [ + "semi-structured/*","structured/*","object","array" + ], + "instillShortDescription": "JSON Data", + "instillUIOrder": 2, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "items": { + "title": "Object", + "instillFormat": "semi-structured/json" + }, + "title": "Update" + }, + "table-name": { + "description": "The table name in the database to update data into", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "Database Table Name", + "instillUIOrder": 0, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Table Name", + "type":"string" + } + }, + "required": [ + "criteria", + "update", + "table-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "instillUIOrder": 0, + "properties": { + "status": { + "description": "Update status", + "instillFormat": "string", + "required": [], + "instillUIOrder": 0, + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_SELECT":{ + "instillShortDescription": "Perform a select operation based on specified criteria", + "input": { + "instillUIOrder": 0, + "properties": { + "criteria": { + "description": "The data criteria to be selected, if JSON value of a key is null e.g {'name':null}, then name column will be selected without any criteria, if JSON is empty e.g {}, then all columns will be selected", + "instillAcceptFormats": [ + "semi-structured/*","structured/*","object" + ], + "instillShortDescription": "JSON Data", + "instillUIOrder": 1, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Criteria" + }, + "from": { + "description": "The starting row to be selected, if both 'from' and 'to' equals 0 then all rows will be selected", + "instillAcceptFormats": [ + "number" + ], + "instillShortDescription": "Starting of rows (check description for more details)", + "instillUIOrder": 2, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "From", + "type":"number" + }, + "to": { + "description": "The starting row to be selected, if both 'from' and 'to' equals 0 then all rows will be selected", + "instillAcceptFormats": [ + "number" + ], + "instillShortDescription": "Ending of rows (check description for more details)", + "instillUIOrder": 3, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "To", + "type":"number" + }, + "table-name": { + "description": "The table name in the database to be selected", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "Database Table Name", + "instillUIOrder": 0, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Table Name", + "type":"string" + } + }, + "required": [ + "table-name", + "criteria", + "from", + "to" + ], + "title": "Input", + "type": "object" + }, + "output": { + "description": "Output", + "instillEditOnNodeFields": [ + "json" + ], + "instillUIOrder": 0, + "properties": { + "rows": { + "description": "The rows returned from the select operation", + "instillEditOnNodeFields": [], + "instillUIOrder": 0, + "required": [], + "title": "Rows", + "type": "array", + "instillFormat": "array:semi-structured/json", + "items": { + "title": "Result", + "instillFormat": "semi-structured/json" + } + }, + "status": { + "description": "Select status", + "instillFormat": "string", + "required": [], + "instillUIOrder": 0, + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "rows" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_DELETE": { + "instillShortDescription": "Perform a delete operation based on specified criteria", + "input": { + "instillUIOrder": 0, + "properties": { + "criteria": { + "description": "The data criteria to be deleted", + "instillAcceptFormats": [ + "semi-structured/*","structured/*","object" + ], + "instillShortDescription": "JSON Data", + "instillUIOrder": 1, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Criteria" + }, + "table-name": { + "description": "The table name in the database to be deleted", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "Database Table Name", + "instillUIOrder": 0, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Table Name", + "type":"string" + } + }, + "required": [ + "criteria", + "table-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "instillUIOrder": 0, + "properties": { + "status": { + "description": "Delete status", + "instillFormat": "string", + "required": [], + "instillUIOrder": 0, + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "Output", + "type": "object" + } + } +} diff --git a/data/sql/v0/main.go b/data/sql/v0/main.go new file mode 100644 index 00000000..804096a1 --- /dev/null +++ b/data/sql/v0/main.go @@ -0,0 +1,105 @@ +//go:generate compogen readme ./config ./README.mdx +package sql + +import ( + "context" + "database/sql" + _ "embed" + "fmt" + "sync" + + "github.com/instill-ai/component/base" + "github.com/instill-ai/x/errmsg" + "github.com/jmoiron/sqlx" + "google.golang.org/protobuf/types/known/structpb" +) + +const ( + TaskInsert = "TASK_INSERT" + TaskUpdate = "TASK_UPDATE" + TaskSelect = "TASK_SELECT" + TaskDelete = "TASK_DELETE" +) + +//go:embed config/definition.json +var definitionJSON []byte + +//go:embed config/setup.json +var setupJSON []byte + +//go:embed config/tasks.json +var tasksJSON []byte + +var once sync.Once +var comp *component + +type SQLClient interface { + NamedExec(query string, arg interface{}) (sql.Result, error) + Queryx(query string, args ...interface{}) (*sqlx.Rows, error) +} + +type component struct { + base.Component +} + +type execution struct { + base.ComponentExecution + + execute func(*structpb.Struct) (*structpb.Struct, error) + client SQLClient +} + +func Init(bc base.Component) *component { + once.Do(func() { + comp = &component{Component: bc} + err := comp.LoadDefinition(definitionJSON, setupJSON, tasksJSON, nil) + if err != nil { + panic(err) + } + }) + return comp +} + +func (c *component) CreateExecution(sysVars map[string]any, setup *structpb.Struct, task string) (*base.ExecutionWrapper, error) { + e := &execution{ + ComponentExecution: base.ComponentExecution{Component: c, SystemVariables: sysVars, Setup: setup, Task: task}, + client: newClient(setup), + } + + switch task { + case TaskInsert: + e.execute = e.insert + case TaskUpdate: + e.execute = e.update + case TaskSelect: + e.execute = e.selects + case TaskDelete: + e.execute = e.delete + default: + return nil, errmsg.AddMessage( + fmt.Errorf("not supported task: %s", task), + fmt.Sprintf("%s task is not supported.", task), + ) + } + return &base.ExecutionWrapper{Execution: e}, nil +} + +func (e *execution) Execute(_ context.Context, inputs []*structpb.Struct) ([]*structpb.Struct, error) { + outputs := make([]*structpb.Struct, len(inputs)) + + for i, input := range inputs { + output, err := e.execute(input) + if err != nil { + return nil, err + } + + outputs[i] = output + } + + return outputs, nil +} + +func (c *component) Test(sysVars map[string]any, setup *structpb.Struct) error { + + return nil +} diff --git a/data/sql/v0/tasks.go b/data/sql/v0/tasks.go new file mode 100644 index 00000000..a1469610 --- /dev/null +++ b/data/sql/v0/tasks.go @@ -0,0 +1,304 @@ +package sql + +import ( + "fmt" + "strconv" + "strings" + + "github.com/instill-ai/component/base" + "google.golang.org/protobuf/types/known/structpb" +) + +type InsertInput struct { + Data map[string]any `json:"data"` + TableName string `json:"table-name"` +} + +type InsertOutput struct { + Status string `json:"status"` +} + +type UpdateInput struct { + Update map[string]any `json:"update"` + Criteria map[string]any `json:"criteria"` + TableName string `json:"table-name"` +} + +type UpdateOutput struct { + Status string `json:"status"` +} + +type SelectInput struct { + Criteria map[string]any `json:"criteria"` + TableName string `json:"table-name"` + From int `json:"from"` + To int `json:"to"` +} + +type SelectOutput struct { + Rows []map[string]any `json:"rows"` + Status string `json:"status"` +} + +type DeleteInput struct { + Criteria map[string]any `json:"criteria"` + TableName string `json:"table-name"` +} + +type DeleteOutput struct { + Status string `json:"status"` +} + +func buildSQLStatementInsert(tableName string, data *map[string]any) (string, map[string]any) { + sqlStatement := "INSERT INTO " + tableName + " (" + var columns []string + var placeholders []string + values := make(map[string]any) + + for dataKey, dataValue := range *data { + columns = append(columns, dataKey) + placeholders = append(placeholders, ":"+dataKey) + values[dataKey] = dataValue + } + + sqlStatement += strings.Join(columns, ", ") + ") VALUES (" + strings.Join(placeholders, ", ") + ")" + + return sqlStatement, values +} + +func buildSQLStatementUpdate(tableName string, updateData map[string]interface{}, criteria map[string]interface{}, e execution) (string, map[string]interface{}) { + // Take all columns + sqlStatementCols := "SELECT * FROM " + tableName + + // Prepare and execute the statement + rows, _ := e.client.Queryx(sqlStatementCols) + defer rows.Close() + + sqlStatement := "UPDATE " + tableName + " SET " + values := make(map[string]interface{}) + + // Get column names from the query result + columns, _ := rows.Columns() + + // Build SET clauses + var setClauses []string + for _, col := range columns { + if updateValue, found := updateData[col]; found { + setClauses = append(setClauses, fmt.Sprintf("%s = :%s", col, col)) + values[col] = updateValue + } else { + setClauses = append(setClauses, fmt.Sprintf("%s = NULL", col)) + } + } + + sqlStatement += strings.Join(setClauses, ", ") + + // Build WHERE clauses + var whereClauses []string + for col, criteriaValue := range criteria { + whereClauses = append(whereClauses, fmt.Sprintf("%s = :%s_criteria", col, col)) + values[col+"_criteria"] = criteriaValue + } + + sqlStatement += " WHERE " + strings.Join(whereClauses, " AND ") + + return sqlStatement, values +} + +func buildSQLStatementSelect(tableName string, criteria *map[string]any, to int, from int) string { + // Begin constructing SQL statement + sqlStatement := "SELECT " + var where []string + var columns []string + + for criteriaKey, criteriaValue := range *criteria { + if criteriaValue != nil { + switch criteriaValue.(type) { + case string: + // If the value is a string, quote it in the SQL statement + where = append(where, fmt.Sprintf("%s = '%v'", criteriaKey, criteriaValue)) + case map[string]any: + // If the value is json, quote it in the SQL statement + where = append(where, fmt.Sprintf("%s = '%v'", criteriaKey, criteriaValue)) + default: + // If the value is a number or bool, use it directly without quotes + where = append(where, fmt.Sprintf("%s = %v", criteriaKey, criteriaValue)) + } + } + + columns = append(columns, criteriaKey) + } + + var notAll string + if to == 0 && from == 0 { + notAll = "" + } else { + notAll = " LIMIT " + strconv.Itoa(to-from+1) + " OFFSET " + strconv.Itoa(from-1) + } + + if len(columns) > 0 { + sqlStatement += strings.Join(columns, ", ") + } else { + sqlStatement += "*" + } + + sqlStatement += " FROM " + tableName + if len(where) > 0 { + sqlStatement += " WHERE " + strings.Join(where, " AND ") + } + sqlStatement += notAll + + return sqlStatement +} + +func buildSQLStatementDelete(tableName string, criteria *map[string]any) (string, map[string]any) { + // Begin constructing SQL statement + sqlStatement := "DELETE FROM " + tableName + " WHERE " + var where []string + values := make(map[string]any) // Initialize the map + + for criteriaKey, criteriaValue := range *criteria { + where = append(where, fmt.Sprintf("%s = :%s", criteriaKey, criteriaKey)) + values[criteriaKey] = criteriaValue + } + + sqlStatement += strings.Join(where, " AND ") + + return sqlStatement, values +} + +func (e *execution) insert(in *structpb.Struct) (*structpb.Struct, error) { + var inputStruct InsertInput + err := base.ConvertFromStructpb(in, &inputStruct) + if err != nil { + return nil, err + } + + sqlStatement, values := buildSQLStatementInsert(inputStruct.TableName, &inputStruct.Data) + + // Prepare and execute the statement using NamedExec + _, err = e.client.NamedExec(sqlStatement, values) + + if err != nil { + return nil, err + } + + outputStruct := InsertOutput{ + Status: "Successfully inserted document", + } + + output, err := base.ConvertToStructpb(outputStruct) + if err != nil { + return nil, err + } + return output, nil +} + +func (e *execution) update(in *structpb.Struct) (*structpb.Struct, error) { + var inputStruct UpdateInput + err := base.ConvertFromStructpb(in, &inputStruct) + if err != nil { + return nil, err + } + + sqlStatement, values := buildSQLStatementUpdate(inputStruct.TableName, inputStruct.Update, inputStruct.Criteria, *e) + + // Prepare and execute the statement using NamedExec + _, err = e.client.NamedExec(sqlStatement, values) + + if err != nil { + return nil, err + } + + outputStruct := UpdateOutput{ + Status: "Successfully updated document", + } + + output, err := base.ConvertToStructpb(outputStruct) + if err != nil { + return nil, err + } + return output, nil +} + +func (e *execution) selects(in *structpb.Struct) (*structpb.Struct, error) { + var inputStruct SelectInput + err := base.ConvertFromStructpb(in, &inputStruct) + if err != nil { + return nil, err + } + + sqlStatement := buildSQLStatementSelect(inputStruct.TableName, &inputStruct.Criteria, inputStruct.To, inputStruct.From) + + // Prepare and execute the statement + rows, err := e.client.Queryx(sqlStatement) + if err != nil { + return nil, err + } + defer rows.Close() + + // Prepare the result slice of maps + var result []map[string]any + + // Iterate over the rows + for rows.Next() { + // Create a map to hold the row data + rowMap := make(map[string]any) + + // Load the row data into the map + err := rows.MapScan(rowMap) + if err != nil { + return nil, fmt.Errorf("failed to scan row: %v", err) + } + + // Convert each value in the map to the appropriate type + for key, value := range rowMap { + switch v := value.(type) { + case []byte: + // Convert byte slices to strings + rowMap[key] = string(v) + } + } + + // Add the row map to the result slice + result = append(result, rowMap) + } + + outputStruct := SelectOutput{ + Rows: result, + Status: "Successfully selected document", + } + + output, err := base.ConvertToStructpb(outputStruct) + if err != nil { + return nil, err + } + return output, nil +} + +func (e *execution) delete(in *structpb.Struct) (*structpb.Struct, error) { + var inputStruct DeleteInput + err := base.ConvertFromStructpb(in, &inputStruct) + if err != nil { + return nil, err + } + + sqlStatement, values := buildSQLStatementDelete(inputStruct.TableName, &inputStruct.Criteria) + + // Prepare and execute the statement using NamedExec + _, err = e.client.NamedExec(sqlStatement, values) + + if err != nil { + return nil, err + } + + outputStruct := DeleteOutput{ + Status: "Successfully deleted document", + } + + output, err := base.ConvertToStructpb(outputStruct) + if err != nil { + return nil, err + } + return output, nil +} diff --git a/store/store.go b/store/store.go index 16330696..de654bc2 100644 --- a/store/store.go +++ b/store/store.go @@ -27,6 +27,7 @@ import ( "github.com/instill-ai/component/data/googlecloudstorage/v0" "github.com/instill-ai/component/data/pinecone/v0" "github.com/instill-ai/component/data/redis/v0" + "github.com/instill-ai/component/data/sql/v0" "github.com/instill-ai/component/operator/base64/v0" "github.com/instill-ai/component/operator/document/v0" "github.com/instill-ai/component/operator/image/v0" @@ -113,6 +114,7 @@ func Init( compStore.Import(googlesearch.Init(baseComp)) compStore.Import(pinecone.Init(baseComp)) compStore.Import(redis.Init(baseComp)) + compStore.Import(sql.Init(baseComp)) compStore.Import(restapi.Init(baseComp)) compStore.Import(website.Init(baseComp)) compStore.Import(slack.Init(baseComp)) From 1cb97138b01e65153a1070f8e265996d576460c4 Mon Sep 17 00:00:00 2001 From: Yazidane Date: Tue, 9 Jul 2024 15:49:29 +0800 Subject: [PATCH 2/8] fix: fix dependencies --- data/sql/v0/config/definition.json | 2 +- go.mod | 15 ++++++++++ go.sum | 46 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/data/sql/v0/config/definition.json b/data/sql/v0/config/definition.json index da16e1c2..e75744e7 100644 --- a/data/sql/v0/config/definition.json +++ b/data/sql/v0/config/definition.json @@ -13,7 +13,7 @@ "description": "Access the SQL database of your choice", "tombstone": false, "type": "COMPONENT_TYPE_DATA", - "uid": "f12d6db7-45be-4f76-96aa-497bd7c0e1d6", + "uid": "5861fc8f-1a07-42f6-a6b8-0e5a2664de00", "version": "0.1.0", "sourceUrl": "https://github.com/instill-ai/component/blob/main/data/sql/v0", "releaseStage": "RELEASE_STAGE_ALPHA" diff --git a/go.mod b/go.mod index 3a227ded..e06a7a9f 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,17 @@ require ( cloud.google.com/go/iam v1.1.6 cloud.google.com/go/storage v1.38.0 code.sajari.com/docconv v1.3.8 + github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/JohannesKaufmann/html-to-markdown v1.5.0 github.com/PuerkitoBio/goquery v1.9.1 + github.com/denisenkom/go-mssqldb v0.12.3 github.com/emersion/go-imap/v2 v2.0.0-beta.3 github.com/emersion/go-message v0.18.1 github.com/fogleman/gg v1.3.0 github.com/frankban/quicktest v1.14.6 github.com/gabriel-vasile/mimetype v1.4.3 github.com/go-resty/resty/v2 v2.12.0 + github.com/go-sql-driver/mysql v1.8.1 github.com/gocolly/colly/v2 v2.1.0 github.com/gofrs/uuid v4.4.0+incompatible github.com/gojuno/minimock/v3 v3.3.6 @@ -22,16 +25,20 @@ require ( github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240530065422-d384f728a1e2 github.com/instill-ai/x v0.4.0-alpha github.com/itchyny/gojq v0.12.14 + github.com/jmoiron/sqlx v1.4.0 github.com/json-iterator/go v1.1.12 github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c github.com/lestrrat-go/jsref v0.0.0-20211028120858-c0bcbb5abf20 github.com/lestrrat-go/option v1.0.0 github.com/lestrrat-go/pdebug v0.0.0-20210111095411-35b07dbf089b github.com/lestrrat-go/structinfo v0.0.0-20210312050401-7f8bd69d6acb + github.com/lib/pq v1.10.9 + github.com/nakagami/firebirdsql v0.9.10 github.com/pkg/errors v0.9.1 github.com/pkoukk/tiktoken-go v0.1.6 github.com/redis/go-redis/v9 v9.5.1 github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 + github.com/sijms/go-ora/v2 v2.8.19 github.com/slack-go/slack v0.12.5 github.com/stretchr/testify v1.9.0 github.com/tmc/langchaingo v0.1.10 @@ -48,6 +55,7 @@ require ( cloud.google.com/go/compute v1.24.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/longrunning v0.5.6 // indirect + filippo.io/edwards25519 v1.1.0 // indirect github.com/JalfResi/justext v0.0.0-20170829062021-c0282dea7198 // indirect github.com/advancedlogic/GoOse v0.0.0-20191112112754-e742535969c1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect @@ -68,6 +76,8 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect + github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -81,6 +91,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 // indirect + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.17.2 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect @@ -94,11 +105,13 @@ require ( github.com/otiai10/gosseract/v2 v2.2.4 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.3 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect + github.com/shopspring/decimal v1.2.0 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/temoto/robotstxt v1.1.2 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect @@ -107,6 +120,7 @@ require ( gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a // indirect gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 // indirect gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f // indirect + gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect @@ -130,4 +144,5 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/mathutil v1.5.0 // indirect ) diff --git a/go.sum b/go.sum index 0007b727..c5b868d7 100644 --- a/go.sum +++ b/go.sum @@ -17,7 +17,14 @@ cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkp cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= code.sajari.com/docconv v1.3.8 h1:sT6s2TcjAF+aTNFxxHHhut2T5uoCIHpjG+BCtmMgRvU= code.sajari.com/docconv v1.3.8/go.mod h1:q2Wj80d67JJ4VVZCNv3fTht0fJ6eMFajQBsa+G1pKaw= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/JalfResi/justext v0.0.0-20170829062021-c0282dea7198 h1:8P+AjBhGByCuCX2zTkAf6UY+dj0JczX+t6cSdCSyvfw= github.com/JalfResi/justext v0.0.0-20170829062021-c0282dea7198/go.mod h1:0SURuH1rsE8aVWvutuMZghRNrNrYEUzibzJfhEYR8L0= github.com/JohannesKaufmann/html-to-markdown v1.5.0 h1:cEAcqpxk0hUJOXEVGrgILGW76d1GpyGY7PCnAaWQyAI= @@ -66,10 +73,13 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= +github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/emersion/go-imap/v2 v2.0.0-beta.3 h1:z0TLMfYnDsFupXLhzRXgOzXenD3uPvNniQSu5fN1teg= github.com/emersion/go-imap/v2 v2.0.0-beta.3/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk= github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK3QA2E= @@ -100,6 +110,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-resty/resty/v2 v2.0.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA= github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= @@ -113,6 +125,10 @@ github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gojuno/minimock/v3 v3.3.6 h1:tZQQaDgKSxsKiVia9vt6zZ/qsKNGBw2D0ubHQPr+mHc= github.com/gojuno/minimock/v3 v3.3.6/go.mod h1:kjvubEBVT8aUQ9e+g8x/hPfAhiOoqW7WinzzJgzr4ws= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -179,10 +195,15 @@ github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6Pyu github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 h1:g0fAGBisHaEQ0TRq1iBvemFRf+8AEWEmBESSiWB3Vsc= github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= @@ -206,15 +227,22 @@ github.com/lestrrat-go/structinfo v0.0.0-20210312050401-7f8bd69d6acb h1:DDg5u5lk github.com/lestrrat-go/structinfo v0.0.0-20210312050401-7f8bd69d6acb/go.mod h1:i+E8Uf04vf2QjOWyJdGY75vmG+4rxiZW2kIj1lTB5mo= github.com/levigross/exp-html v0.0.0-20120902181939-8df60c69a8f5 h1:W7p+m/AECTL3s/YR5RpQ4hz5SjNeKzZBl1q36ws12s0= github.com/levigross/exp-html v0.0.0-20120902181939-8df60c69a8f5/go.mod h1:QMe2wuKJ0o7zIVE8AqiT8rd8epmm6WDIZ2wyuBqYPzM= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/nakagami/firebirdsql v0.9.10 h1:7Y73BiH3j/f8faIaryZvDZ3nEo0L7c6S5pg+qWoZ91c= +github.com/nakagami/firebirdsql v0.9.10/go.mod h1:ei91eXUYcMkWJOr4rK6Sta+BVmi3K+WvYR4yASlq/kY= github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= @@ -225,6 +253,7 @@ github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -236,6 +265,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/richardlehane/mscfb v1.0.3 h1:rD8TBkYWkObWO0oLDFCbwMeZ4KoalxQy+QgniCj3nKI= github.com/richardlehane/mscfb v1.0.3/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= @@ -259,6 +291,10 @@ github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvK github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sijms/go-ora/v2 v2.8.19 h1:7LoKZatDYGi18mkpQTR/gQvG9yOdtc7hPAex96Bqisc= +github.com/sijms/go-ora/v2 v2.8.19/go.mod h1:EHxlY6x7y9HAsdfumurRfTd+v8NrEOTR3Xl4FWlH6xk= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= github.com/slack-go/slack v0.12.5 h1:ddZ6uz6XVaB+3MTDhoW04gG+Vc/M/X1ctC+wssy2cqs= github.com/slack-go/slack v0.12.5/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= @@ -300,6 +336,8 @@ gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 h1:qqjvoVX gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84/go.mod h1:IJZ+fdMvbW2qW6htJx7sLJ04FEs4Ldl/MDsJtMKywfw= gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f h1:Wku8eEdeJqIOFHtrfkYUByc4bCaTeA6fL0UJgfEiFMI= gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f/go.mod h1:Tiuhl+njh/JIg0uS/sOJVYi0x2HEa5rc1OAaVsb5tAs= +gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs= +gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638 h1:uPZaMiz6Sz0PZs3IZJWpU5qHKGNy///1pacZC9txiUI= gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638/go.mod h1:EGRJaqe2eO9XGmFtQCvV3Lm9NLico3UhFwUpCG/+mVU= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= @@ -328,7 +366,9 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= @@ -360,7 +400,9 @@ golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -485,10 +527,14 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= From c1086be42fd8c448dd67f1cb47b1eb965fe14e40 Mon Sep 17 00:00:00 2001 From: Yazidane Date: Thu, 11 Jul 2024 00:15:53 +0800 Subject: [PATCH 3/8] chore: adjust status string output --- data/sql/v0/component_test.go | 8 ++++---- data/sql/v0/tasks.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/data/sql/v0/component_test.go b/data/sql/v0/component_test.go index 359be1e3..e23f754f 100644 --- a/data/sql/v0/component_test.go +++ b/data/sql/v0/component_test.go @@ -103,7 +103,7 @@ func TestInsertUser(t *testing.T) { TableName: "users", }, wantResp: InsertOutput{ - Status: "Successfully inserted document", + Status: "Successfully inserted rows", }, }, } @@ -172,7 +172,7 @@ func TestUpdateUser(t *testing.T) { TableName: "users", }, wantResp: UpdateOutput{ - Status: "Successfully updated document", + Status: "Successfully updated rows", }, }, } @@ -240,7 +240,7 @@ func TestSelectUser(t *testing.T) { To: 1, }, wantResp: SelectOutput{ - Status: "Successfully selected document", + Status: "Successfully selected rows", Rows: []map[string]any{ {"id": "1", "name": "john", "email": "john@example.com"}, }, @@ -308,7 +308,7 @@ func TestDeleteUser(t *testing.T) { TableName: "users", }, wantResp: DeleteOutput{ - Status: "Successfully deleted document", + Status: "Successfully deleted rows", }, }, } diff --git a/data/sql/v0/tasks.go b/data/sql/v0/tasks.go index a1469610..ff7e1eee 100644 --- a/data/sql/v0/tasks.go +++ b/data/sql/v0/tasks.go @@ -184,7 +184,7 @@ func (e *execution) insert(in *structpb.Struct) (*structpb.Struct, error) { } outputStruct := InsertOutput{ - Status: "Successfully inserted document", + Status: "Successfully inserted rows", } output, err := base.ConvertToStructpb(outputStruct) @@ -211,7 +211,7 @@ func (e *execution) update(in *structpb.Struct) (*structpb.Struct, error) { } outputStruct := UpdateOutput{ - Status: "Successfully updated document", + Status: "Successfully updated rows", } output, err := base.ConvertToStructpb(outputStruct) @@ -266,7 +266,7 @@ func (e *execution) selects(in *structpb.Struct) (*structpb.Struct, error) { outputStruct := SelectOutput{ Rows: result, - Status: "Successfully selected document", + Status: "Successfully selected rows", } output, err := base.ConvertToStructpb(outputStruct) @@ -293,7 +293,7 @@ func (e *execution) delete(in *structpb.Struct) (*structpb.Struct, error) { } outputStruct := DeleteOutput{ - Status: "Successfully deleted document", + Status: "Successfully deleted rows", } output, err := base.ConvertToStructpb(outputStruct) From 885794b7e7d1b4316572f2603447e8e5e2e24822 Mon Sep 17 00:00:00 2001 From: Yazidane Date: Wed, 17 Jul 2024 04:18:33 +0800 Subject: [PATCH 4/8] feat: add create table and drop table task --- data/sql/v0/README.mdx | 45 +++++++++ data/sql/v0/component_test.go | 152 ++++++++++++++++++++++++++++- data/sql/v0/config/definition.json | 4 +- data/sql/v0/config/tasks.json | 107 ++++++++++++++++++++ data/sql/v0/main.go | 14 ++- data/sql/v0/tasks.go | 91 +++++++++++++++++ go.mod | 4 +- 7 files changed, 409 insertions(+), 8 deletions(-) diff --git a/data/sql/v0/README.mdx b/data/sql/v0/README.mdx index c860e224..c0181a6f 100644 --- a/data/sql/v0/README.mdx +++ b/data/sql/v0/README.mdx @@ -12,6 +12,8 @@ It can carry out the following tasks: - [Update](#update) - [Select](#select) - [Delete](#delete) +- [Create Table](#create-table) +- [Drop Table](#drop-table) @@ -137,4 +139,47 @@ Perform a delete operation based on specified criteria +### Create Table + +Create a table in the database + + +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_CREATE_TABLE` | +| Table Name (required) | `table-name` | string | The table name in the database to be created | +| Columns (required) | `columns` | semi-structured/json | The columns to be created in the table | + + + +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Status | `status` | string | Create table status | + + + + + + +### Drop Table + +Drop a table in the database + + +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_DROP_TABLE` | +| Table Name (required) | `table-name` | string | The table name in the database to be dropped | + + + +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Status | `status` | string | Drop table status | + + + + + + diff --git a/data/sql/v0/component_test.go b/data/sql/v0/component_test.go index e23f754f..d556e4ef 100644 --- a/data/sql/v0/component_test.go +++ b/data/sql/v0/component_test.go @@ -62,7 +62,7 @@ func (m *MockSQLClient) NamedExec(query string, arg interface{}) (sql.Result, er return sqlxDB.NamedExec("DELETE FROM users WHERE id = :id AND name = :name", arg) - } else { + } else if strings.Contains(query, "UPDATE") { mockDB, mock, _ := sqlmock.New() defer mockDB.Close() @@ -76,7 +76,34 @@ func (m *MockSQLClient) NamedExec(query string, arg interface{}) (sql.Result, er WithArgs("1", "John Doe Updated", "1", "John Doe Updated").WillReturnResult(sqlmock.NewResult(1, 1)) return sqlxDB.NamedExec("UPDATE users SET id = :id, name = :name WHERE id = :id AND name = :name", arg) + } else if strings.Contains(query, "CREATE") { + mockDB, mock, _ := sqlmock.New() + defer mockDB.Close() + + sqlxDB := sqlx.NewDb(mockDB, "sqlmock") + arg = map[string]interface{}{ + "id": "INT", + "name": "VARCHAR(255)", + } + + mock.ExpectExec("CREATE TABLE users \\(id INT, name VARCHAR\\(255\\)\\)"). + WillReturnResult(sqlmock.NewResult(1, 1)) + + return sqlxDB.NamedExec("CREATE TABLE users (id INT, name VARCHAR(255))", arg) + } else if strings.Contains(query, "DROP") { + mockDB, mock, _ := sqlmock.New() + defer mockDB.Close() + + sqlxDB := sqlx.NewDb(mockDB, "sqlmock") + arg = map[string]interface{}{} + + mock.ExpectExec("DROP TABLE users"). + WillReturnResult(sqlmock.NewResult(1, 1)) + + return sqlxDB.NamedExec("DROP TABLE users", arg) } + + return nil, nil } func TestInsertUser(t *testing.T) { @@ -348,3 +375,126 @@ func TestDeleteUser(t *testing.T) { }) } } + +func TestCreateTable(t *testing.T) { + c := qt.New(t) + ctx := context.Background() + bc := base.Component{Logger: zap.NewNop()} + connector := Init(bc) + + testcases := []struct { + name string + tableName string + input CreateTableInput + wantResp CreateTableOutput + wantErr string + }{ + { + name: "create table", + input: CreateTableInput{ + Columns: map[string]string{ + "id": "INT", + "name": "VARCHAR(255)", + }, + TableName: "users", + }, + wantResp: CreateTableOutput{ + Status: "Successfully created table", + }, + }, + } + + for _, tc := range testcases { + c.Run(tc.name, func(c *qt.C) { + setup, err := structpb.NewStruct(map[string]any{ + "user": "test_user", + "password": "test_pass", + "name": "test_db", + "host": "localhost", + "port": "3306", + "region": "us-west-2", + }) + c.Assert(err, qt.IsNil) + + e := &execution{ + ComponentExecution: base.ComponentExecution{Component: connector, SystemVariables: nil, Setup: setup, Task: TaskCreateTable}, + client: &MockSQLClient{}, + } + e.execute = e.createTable + exec := &base.ExecutionWrapper{Execution: e} + + pbIn, err := base.ConvertToStructpb(tc.input) + c.Assert(err, qt.IsNil) + + got, err := exec.Execution.Execute(ctx, []*structpb.Struct{pbIn}) + + if tc.wantErr != "" { + c.Assert(err, qt.ErrorMatches, tc.wantErr) + return + } + + wantJSON, err := json.Marshal(tc.wantResp) + c.Assert(err, qt.IsNil) + c.Check(wantJSON, qt.JSONEquals, got[0].AsMap()) + }) + } +} + +func TestDropTable(t *testing.T) { + c := qt.New(t) + ctx := context.Background() + bc := base.Component{Logger: zap.NewNop()} + connector := Init(bc) + + testcases := []struct { + name string + input DropTableInput + wantResp DropTableOutput + wantErr string + }{ + { + name: "drop table", + input: DropTableInput{ + TableName: "users", + }, + wantResp: DropTableOutput{ + Status: "Successfully dropped table", + }, + }, + } + + for _, tc := range testcases { + c.Run(tc.name, func(c *qt.C) { + setup, err := structpb.NewStruct(map[string]any{ + "user": "test_user", + "password": "test_pass", + "name": "test_db", + "host": "localhost", + "port": "3306", + "region": "us-west-2", + }) + c.Assert(err, qt.IsNil) + + e := &execution{ + ComponentExecution: base.ComponentExecution{Component: connector, SystemVariables: nil, Setup: setup, Task: TaskDropTable}, + client: &MockSQLClient{}, + } + e.execute = e.dropTable + exec := &base.ExecutionWrapper{Execution: e} + + pbIn, err := base.ConvertToStructpb(tc.input) + c.Assert(err, qt.IsNil) + + got, err := exec.Execution.Execute(ctx, []*structpb.Struct{pbIn}) + + if tc.wantErr != "" { + c.Assert(err, qt.ErrorMatches, tc.wantErr) + return + } + + wantJSON, err := json.Marshal(tc.wantResp) + c.Assert(err, qt.IsNil) + c.Check(wantJSON, qt.JSONEquals, got[0].AsMap()) + }) + } +} diff --git a/data/sql/v0/config/definition.json b/data/sql/v0/config/definition.json index e75744e7..b48f95d6 100644 --- a/data/sql/v0/config/definition.json +++ b/data/sql/v0/config/definition.json @@ -3,7 +3,9 @@ "TASK_INSERT", "TASK_UPDATE", "TASK_SELECT", - "TASK_DELETE" + "TASK_DELETE", + "TASK_CREATE_TABLE", + "TASK_DROP_TABLE" ], "documentationUrl": "https://www.instill.tech/docs/component/data/sql", "icon": "assets/sql.svg", diff --git a/data/sql/v0/config/tasks.json b/data/sql/v0/config/tasks.json index 6d9b1609..8b501a8a 100644 --- a/data/sql/v0/config/tasks.json +++ b/data/sql/v0/config/tasks.json @@ -315,5 +315,112 @@ "title": "Output", "type": "object" } + }, + "TASK_CREATE_TABLE":{ + "instillShortDescription": "Create a table in the database", + "input": { + "instillUIOrder": 0, + "properties": { + "table-name": { + "description": "The table name in the database to be created", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "Database Table Name", + "instillUIOrder": 0, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Table Name", + "type":"string" + }, + "columns": { + "description": "The columns to be created in the table", + "instillAcceptFormats": [ + "semi-structured/*","structured/*","object" + ], + "instillShortDescription": "JSON Data", + "instillUIOrder": 1, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Columns" + } + }, + "required": [ + "table-name", + "columns" + ], + "title": "Input", + "type": "object" + }, + "output": { + "instillUIOrder": 0, + "properties": { + "status": { + "description": "Create table status", + "instillFormat": "string", + "required": [], + "instillUIOrder": 0, + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_DROP_TABLE":{ + "instillShortDescription": "Drop a table in the database", + "input": { + "instillUIOrder": 0, + "properties": { + "table-name": { + "description": "The table name in the database to be dropped", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "Database Table Name", + "instillUIOrder": 0, + "instillUpstreamTypes": [ + "reference", + "template", + "value" + ], + "title": "Table Name", + "type":"string" + } + }, + "required": [ + "table-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "instillUIOrder": 0, + "properties": { + "status": { + "description": "Drop table status", + "instillFormat": "string", + "required": [], + "instillUIOrder": 0, + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "Output", + "type": "object" + } } } diff --git a/data/sql/v0/main.go b/data/sql/v0/main.go index 804096a1..b16591f7 100644 --- a/data/sql/v0/main.go +++ b/data/sql/v0/main.go @@ -15,10 +15,12 @@ import ( ) const ( - TaskInsert = "TASK_INSERT" - TaskUpdate = "TASK_UPDATE" - TaskSelect = "TASK_SELECT" - TaskDelete = "TASK_DELETE" + TaskInsert = "TASK_INSERT" + TaskUpdate = "TASK_UPDATE" + TaskSelect = "TASK_SELECT" + TaskDelete = "TASK_DELETE" + TaskCreateTable = "TASK_CREATE_TABLE" + TaskDropTable = "TASK_DROP_TABLE" ) //go:embed config/definition.json @@ -75,6 +77,10 @@ func (c *component) CreateExecution(sysVars map[string]any, setup *structpb.Stru e.execute = e.selects case TaskDelete: e.execute = e.delete + case TaskCreateTable: + e.execute = e.createTable + case TaskDropTable: + e.execute = e.dropTable default: return nil, errmsg.AddMessage( fmt.Errorf("not supported task: %s", task), diff --git a/data/sql/v0/tasks.go b/data/sql/v0/tasks.go index ff7e1eee..5d588af7 100644 --- a/data/sql/v0/tasks.go +++ b/data/sql/v0/tasks.go @@ -49,6 +49,23 @@ type DeleteOutput struct { Status string `json:"status"` } +type CreateTableInput struct { + TableName string `json:"table-name"` + Columns map[string]string `json:"columns"` +} + +type CreateTableOutput struct { + Status string `json:"status"` +} + +type DropTableInput struct { + TableName string `json:"table-name"` +} + +type DropTableOutput struct { + Status string `json:"status"` +} + func buildSQLStatementInsert(tableName string, data *map[string]any) (string, map[string]any) { sqlStatement := "INSERT INTO " + tableName + " (" var columns []string @@ -167,6 +184,26 @@ func buildSQLStatementDelete(tableName string, criteria *map[string]any) (string return sqlStatement, values } +func buildSQLStatementCreateTable(tableName string, columns map[string]string) (string, map[string]any) { + sqlStatement := "CREATE TABLE " + tableName + " (" + var columnDefs []string + values := make(map[string]any) + + for colName, colType := range columns { + columnDefs = append(columnDefs, fmt.Sprintf("%s %s", colName, colType)) + values[colName] = colType + } + + sqlStatement += strings.Join(columnDefs, ", ") + ");" + return sqlStatement, values +} + +func buildSQLStatementDropTable(tableName string) (string, map[string]any) { + sqlStatement := "DROP TABLE " + tableName + ";" + values := map[string]any{"table_name": tableName} + return sqlStatement, values +} + func (e *execution) insert(in *structpb.Struct) (*structpb.Struct, error) { var inputStruct InsertInput err := base.ConvertFromStructpb(in, &inputStruct) @@ -302,3 +339,57 @@ func (e *execution) delete(in *structpb.Struct) (*structpb.Struct, error) { } return output, nil } + +func (e *execution) createTable(in *structpb.Struct) (*structpb.Struct, error) { + var inputStruct CreateTableInput + err := base.ConvertFromStructpb(in, &inputStruct) + if err != nil { + return nil, err + } + + sqlStatement, values := buildSQLStatementCreateTable(inputStruct.TableName, inputStruct.Columns) + + // Prepare and execute the statement using NamedExec + _, err = e.client.NamedExec(sqlStatement, values) + + if err != nil { + return nil, err + } + + outputStruct := CreateTableOutput{ + Status: "Successfully created table", + } + + output, err := base.ConvertToStructpb(outputStruct) + if err != nil { + return nil, err + } + return output, nil +} + +func (e *execution) dropTable(in *structpb.Struct) (*structpb.Struct, error) { + var inputStruct DropTableInput + err := base.ConvertFromStructpb(in, &inputStruct) + if err != nil { + return nil, err + } + + sqlStatement, values := buildSQLStatementDropTable(inputStruct.TableName) + + // Prepare and execute the statement using NamedExec + _, err = e.client.NamedExec(sqlStatement, values) + + if err != nil { + return nil, err + } + + outputStruct := DropTableOutput{ + Status: "Successfully dropped table", + } + + output, err := base.ConvertToStructpb(outputStruct) + if err != nil { + return nil, err + } + return output, nil +} diff --git a/go.mod b/go.mod index 51f9e2b2..676fe0d2 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/JohannesKaufmann/html-to-markdown v1.5.0 github.com/PuerkitoBio/goquery v1.9.1 - github.com/denisenkom/go-mssqldb v0.12.3 - github.com/cohere-ai/cohere-go/v2 v2.8.5 + github.com/denisenkom/go-mssqldb v0.12.3 + github.com/cohere-ai/cohere-go/v2 v2.8.5 github.com/emersion/go-imap/v2 v2.0.0-beta.3 github.com/emersion/go-message v0.18.1 github.com/fogleman/gg v1.3.0 From 611a35fe028e6270e50e1301a0f2b7fc46b8f3eb Mon Sep 17 00:00:00 2001 From: Yazidane Date: Wed, 17 Jul 2024 04:32:56 +0800 Subject: [PATCH 5/8] chore: change sql icon svg --- data/sql/v0/assets/sql.svg | 11 +++++------ go.mod | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/data/sql/v0/assets/sql.svg b/data/sql/v0/assets/sql.svg index f5be6803..1ba1685d 100644 --- a/data/sql/v0/assets/sql.svg +++ b/data/sql/v0/assets/sql.svg @@ -1,7 +1,6 @@ - - - - - - + + + + + diff --git a/go.mod b/go.mod index 676fe0d2..b385773e 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/JohannesKaufmann/html-to-markdown v1.5.0 github.com/PuerkitoBio/goquery v1.9.1 - github.com/denisenkom/go-mssqldb v0.12.3 github.com/cohere-ai/cohere-go/v2 v2.8.5 + github.com/denisenkom/go-mssqldb v0.12.3 github.com/emersion/go-imap/v2 v2.0.0-beta.3 github.com/emersion/go-message v0.18.1 github.com/fogleman/gg v1.3.0 From 9f130a788b163e80c2491fbef739694a8988b707 Mon Sep 17 00:00:00 2001 From: Yazidane Date: Sun, 21 Jul 2024 16:44:39 +0800 Subject: [PATCH 6/8] chore: make limit params to be optional --- data/sql/v0/README.mdx | 35 ++- data/sql/v0/client.go | 35 +-- data/sql/v0/component_test.go | 3 +- data/sql/v0/config/setup.json | 71 +----- data/sql/v0/config/tasks.json | 436 +++++++++++++++++++++++++++++++--- data/sql/v0/main.go | 25 +- data/sql/v0/tasks.go | 40 +--- 7 files changed, 465 insertions(+), 180 deletions(-) diff --git a/data/sql/v0/README.mdx b/data/sql/v0/README.mdx index c0181a6f..f82eb58f 100644 --- a/data/sql/v0/README.mdx +++ b/data/sql/v0/README.mdx @@ -35,12 +35,8 @@ The component configuration is defined and maintained [here](https://github.com/ | Field | Field ID | Type | Note | | :--- | :--- | :--- | :--- | -| Engine (required) | `engine` | string | Choose the engine of your database | | Username (required) | `user` | string | Fill in your account username | -| Password | `password` | string | Fill in your account password | -| Database Name (required) | `name` | string | Fill in your database name | -| Host (required) | `host` | string | Fill in your database host | -| Port (required) | `port` | number | Fill in your database port | +| Password (required) | `password` | string | Fill in your account password | @@ -55,7 +51,11 @@ Perform an insert operation based on specified criteria | Input | ID | Type | Description | | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_INSERT` | +| Engine (required) | `engine` | string | Choose the engine of your database | +| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to insert data into | +| Host (required) | `host` | string | The host of your database | +| Port (required) | `port` | number | The port of your database | | Data (required) | `data` | semi-structured/json | The data to be inserted | @@ -77,7 +77,11 @@ Perform an update operation based on specified criteria | Input | ID | Type | Description | | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_UPDATE` | +| Engine (required) | `engine` | string | Choose the engine of your database | +| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to update data into | +| Host (required) | `host` | string | The host of your database | +| Port (required) | `port` | number | The port of your database | | Criteria (required) | `criteria` | semi-structured/json | The data criteria to be updated | | Update (required) | `update` | semi-structured/json | The new data to be updated to | @@ -100,10 +104,13 @@ Perform a select operation based on specified criteria | Input | ID | Type | Description | | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_SELECT` | +| Engine (required) | `engine` | string | Choose the engine of your database | +| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to be selected | +| Host (required) | `host` | string | The host of your database | +| Port (required) | `port` | number | The port of your database | | Criteria (required) | `criteria` | semi-structured/json | The data criteria to be selected, if JSON value of a key is null e.g \{'name':null\}, then name column will be selected without any criteria, if JSON is empty e.g \{\}, then all columns will be selected | -| From (required) | `from` | number | The starting row to be selected, if both 'from' and 'to' equals 0 then all rows will be selected | -| To (required) | `to` | number | The starting row to be selected, if both 'from' and 'to' equals 0 then all rows will be selected | +| Limit | `limit` | integer | The limit of rows to be selected, optional (empty for all rows) | @@ -125,7 +132,11 @@ Perform a delete operation based on specified criteria | Input | ID | Type | Description | | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_DELETE` | +| Engine (required) | `engine` | string | Choose the engine of your database | +| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to be deleted | +| Host (required) | `host` | string | The host of your database | +| Port (required) | `port` | number | The port of your database | | Criteria (required) | `criteria` | semi-structured/json | The data criteria to be deleted | @@ -147,8 +158,12 @@ Create a table in the database | Input | ID | Type | Description | | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_CREATE_TABLE` | +| Engine (required) | `engine` | string | Choose the engine of your database | +| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to be created | -| Columns (required) | `columns` | semi-structured/json | The columns to be created in the table | +| Host (required) | `host` | string | The host of your database | +| Port (required) | `port` | number | The port of your database | +| Columns (required) | `columns` | semi-structured/json | The columns to be created in the table, json with value string, e.g \{'name': 'VARCHAR(255'), 'age': 'INT not null'\} | @@ -169,7 +184,11 @@ Drop a table in the database | Input | ID | Type | Description | | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_DROP_TABLE` | +| Engine (required) | `engine` | string | Choose the engine of your database | +| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to be dropped | +| Host (required) | `host` | string | The host of your database | +| Port (required) | `port` | number | The port of your database | diff --git a/data/sql/v0/client.go b/data/sql/v0/client.go index fcbbdc7c..d631da18 100644 --- a/data/sql/v0/client.go +++ b/data/sql/v0/client.go @@ -2,7 +2,6 @@ package sql import ( "fmt" - "strconv" "github.com/jmoiron/sqlx" "google.golang.org/protobuf/types/known/structpb" @@ -26,7 +25,7 @@ var engines = map[string]string{ var enginesType = map[string]string{ "PostgreSQL": "postgres", // PostgreSQL "SQL Server": "sqlserver", // SQL Server - "Oracle": "godror", // Oracle + "Oracle": "oracle", // Oracle "MySQL": "mysql", // MySQL and MariaDB "Firebird": "firebirdsql", // Firebird } @@ -34,38 +33,30 @@ var enginesType = map[string]string{ type Config struct { DBUser string DBPassword string - DBName string - DBHost string - DBPort string - DBEngine string } func LoadConfig(setup *structpb.Struct) *Config { return &Config{ DBUser: getUser(setup), DBPassword: getPassword(setup), - DBName: getName(setup), - DBHost: getHost(setup), - DBPort: getPort(setup), - DBEngine: getEngine(setup), } } -func newClient(setup *structpb.Struct) SQLClient { +func newClient(setup *structpb.Struct, inputSetup *SetupNoSecret) SQLClient { cfg := LoadConfig(setup) - DBEndpoint := fmt.Sprintf("%v:%v", cfg.DBHost, cfg.DBPort) + DBEndpoint := fmt.Sprintf("%v:%v", inputSetup.DBHost, inputSetup.DBPort) // Test every engines to find the correct one var db *sqlx.DB var err error // Get the correct engine - engine := engines[cfg.DBEngine] - engineType := enginesType[cfg.DBEngine] + engine := engines[inputSetup.DBEngine] + engineType := enginesType[inputSetup.DBEngine] dsn := fmt.Sprintf(engine, - cfg.DBUser, cfg.DBPassword, DBEndpoint, cfg.DBName, + cfg.DBUser, cfg.DBPassword, DBEndpoint, inputSetup.DBName, ) db, err = sqlx.Open(engineType, dsn) @@ -82,17 +73,3 @@ func getUser(setup *structpb.Struct) string { func getPassword(setup *structpb.Struct) string { return setup.GetFields()["password"].GetStringValue() } -func getName(setup *structpb.Struct) string { - return setup.GetFields()["name"].GetStringValue() -} -func getHost(setup *structpb.Struct) string { - return setup.GetFields()["host"].GetStringValue() -} -func getPort(setup *structpb.Struct) string { - port := setup.GetFields()["port"].GetNumberValue() - portStr := strconv.FormatFloat(port, 'f', -1, 64) - return portStr -} -func getEngine(setup *structpb.Struct) string { - return setup.GetFields()["engine"].GetStringValue() -} diff --git a/data/sql/v0/component_test.go b/data/sql/v0/component_test.go index d556e4ef..8f18171e 100644 --- a/data/sql/v0/component_test.go +++ b/data/sql/v0/component_test.go @@ -263,8 +263,7 @@ func TestSelectUser(t *testing.T) { "email": "john@example.com", }, TableName: "users", - From: 0, - To: 1, + Limit: 0, }, wantResp: SelectOutput{ Status: "Successfully selected rows", diff --git a/data/sql/v0/config/setup.json b/data/sql/v0/config/setup.json index 9d2576a7..a78cf994 100644 --- a/data/sql/v0/config/setup.json +++ b/data/sql/v0/config/setup.json @@ -2,29 +2,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { - "engine": { - "description": "Choose the engine of your database", - "instillUpstreamTypes": [ - "value", - "reference", - "template" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 0, - "title": "Engine", - "enum": [ - "MySQL", - "PostgreSQL", - "SQL Server", - "Oracle", - "MariaDB", - "Firebird" - ], - "type": "string", - "x-oaiTypeLabel": "string" - }, "user": { "description": "Fill in your account username", "instillUpstreamTypes": [ @@ -33,6 +10,7 @@ "instillAcceptFormats": [ "string" ], + "instillSecret": true, "instillUIOrder": 1, "title": "Username", "type": "string" @@ -49,58 +27,15 @@ "instillUIOrder": 2, "title": "Password", "type": "string" - }, - "name": { - "description": "Fill in your database name", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 3, - "title": "Database Name", - "type": "string" - }, - "host": { - "description": "Fill in your database host", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 4, - "title": "Host", - "type": "string" - }, - "port": { - "description": "Fill in your database port", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "number" - ], - "instillUIOrder": 5, - "title": "Port", - "type": "number" } }, "required": [ - "engine", "user", - "name", - "host", - "port" + "password" ], "instillEditOnNodeFields": [ - "engine", "user", - "password", - "name", - "host", - "port" + "password" ], "title": "SQL Connection", "type": "object" diff --git a/data/sql/v0/config/tasks.json b/data/sql/v0/config/tasks.json index 8b501a8a..09ce10fe 100644 --- a/data/sql/v0/config/tasks.json +++ b/data/sql/v0/config/tasks.json @@ -4,13 +4,71 @@ "input": { "instillUIOrder": 0, "properties": { + "engine": { + "description": "Choose the engine of your database", + "instillUpstreamTypes": [ + "value", + "reference", + "template" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 0, + "title": "Engine", + "enum": [ + "MySQL", + "PostgreSQL", + "SQL Server", + "Oracle", + "MariaDB", + "Firebird" + ], + "type": "string" + }, + "database-name": { + "description": "The database name to insert data into", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 1, + "title": "Database Name", + "type": "string" + }, + "host": { + "description": "The host of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 3, + "title": "Host", + "type": "string" + }, + "port": { + "description": "The port of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "number" + ], + "instillUIOrder": 4, + "title": "Port", + "type": "number" + }, "data": { "description": "The data to be inserted", "instillAcceptFormats": [ "semi-structured/*","structured/*","object","array" ], "instillShortDescription": "JSON Data", - "instillUIOrder": 1, + "instillUIOrder": 5, "instillUpstreamTypes": [ "reference", "template", @@ -28,7 +86,7 @@ "string" ], "instillShortDescription": "Database Table Name", - "instillUIOrder": 0, + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", @@ -39,6 +97,10 @@ } }, "required": [ + "engine", + "database-name", + "host", + "port", "data", "table-name" ], @@ -69,13 +131,71 @@ "input": { "instillUIOrder": 0, "properties": { + "engine": { + "description": "Choose the engine of your database", + "instillUpstreamTypes": [ + "value", + "reference", + "template" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 0, + "title": "Engine", + "enum": [ + "MySQL", + "PostgreSQL", + "SQL Server", + "Oracle", + "MariaDB", + "Firebird" + ], + "type": "string" + }, + "database-name": { + "description": "The database name to insert data into", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 1, + "title": "Database Name", + "type": "string" + }, + "host": { + "description": "The host of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 3, + "title": "Host", + "type": "string" + }, + "port": { + "description": "The port of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "number" + ], + "instillUIOrder": 4, + "title": "Port", + "type": "number" + }, "criteria": { "description": "The data criteria to be updated", "instillAcceptFormats": [ "semi-structured/*","structured/*","object" ], "instillShortDescription": "JSON Data", - "instillUIOrder": 1, + "instillUIOrder": 5, "instillUpstreamTypes": [ "reference", "template", @@ -89,7 +209,7 @@ "semi-structured/*","structured/*","object","array" ], "instillShortDescription": "JSON Data", - "instillUIOrder": 2, + "instillUIOrder": 6, "instillUpstreamTypes": [ "reference", "template", @@ -107,7 +227,7 @@ "string" ], "instillShortDescription": "Database Table Name", - "instillUIOrder": 0, + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", @@ -118,6 +238,10 @@ } }, "required": [ + "engine", + "database-name", + "host", + "port", "criteria", "update", "table-name" @@ -149,49 +273,92 @@ "input": { "instillUIOrder": 0, "properties": { - "criteria": { - "description": "The data criteria to be selected, if JSON value of a key is null e.g {'name':null}, then name column will be selected without any criteria, if JSON is empty e.g {}, then all columns will be selected", + "engine": { + "description": "Choose the engine of your database", + "instillUpstreamTypes": [ + "value", + "reference", + "template" + ], "instillAcceptFormats": [ - "semi-structured/*","structured/*","object" + "string" + ], + "instillUIOrder": 0, + "title": "Engine", + "enum": [ + "MySQL", + "PostgreSQL", + "SQL Server", + "Oracle", + "MariaDB", + "Firebird" + ], + "type": "string" + }, + "database-name": { + "description": "The database name to insert data into", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" ], - "instillShortDescription": "JSON Data", "instillUIOrder": 1, + "title": "Database Name", + "type": "string" + }, + "host": { + "description": "The host of your database", "instillUpstreamTypes": [ - "reference", - "template", - "value" + "value","reference" ], - "title": "Criteria" + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 3, + "title": "Host", + "type": "string" }, - "from": { - "description": "The starting row to be selected, if both 'from' and 'to' equals 0 then all rows will be selected", + "port": { + "description": "The port of your database", + "instillUpstreamTypes": [ + "value","reference" + ], "instillAcceptFormats": [ "number" ], - "instillShortDescription": "Starting of rows (check description for more details)", - "instillUIOrder": 2, + "instillUIOrder": 4, + "title": "Port", + "type": "number" + }, + "criteria": { + "description": "The data criteria to be selected, if JSON value of a key is null e.g {'name':null}, then name column will be selected without any criteria, if JSON is empty e.g {}, then all columns will be selected", + "instillAcceptFormats": [ + "semi-structured/*","structured/*","object" + ], + "instillShortDescription": "JSON Data", + "instillUIOrder": 5, "instillUpstreamTypes": [ "reference", "template", "value" ], - "title": "From", - "type":"number" + "title": "Criteria" }, - "to": { - "description": "The starting row to be selected, if both 'from' and 'to' equals 0 then all rows will be selected", + "limit": { + "description": "The limit of rows to be selected, optional (empty for all rows)", "instillAcceptFormats": [ - "number" + "integer" ], - "instillShortDescription": "Ending of rows (check description for more details)", - "instillUIOrder": 3, + "instillShortDescription": "Limit Rows", + "instillUIOrder": 6, "instillUpstreamTypes": [ "reference", "template", "value" ], - "title": "To", - "type":"number" + "title": "Limit", + "type":"integer" }, "table-name": { "description": "The table name in the database to be selected", @@ -199,7 +366,7 @@ "string" ], "instillShortDescription": "Database Table Name", - "instillUIOrder": 0, + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", @@ -210,10 +377,21 @@ } }, "required": [ + "engine", + "database-name", + "host", + "port", + "table-name", + "criteria" + ], + "instillEditOnNodeFields": [ + "engine", + "database-name", + "host", + "port", "table-name", "criteria", - "from", - "to" + "limit" ], "title": "Input", "type": "object" @@ -260,13 +438,71 @@ "input": { "instillUIOrder": 0, "properties": { + "engine": { + "description": "Choose the engine of your database", + "instillUpstreamTypes": [ + "value", + "reference", + "template" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 0, + "title": "Engine", + "enum": [ + "MySQL", + "PostgreSQL", + "SQL Server", + "Oracle", + "MariaDB", + "Firebird" + ], + "type": "string" + }, + "database-name": { + "description": "The database name to insert data into", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 1, + "title": "Database Name", + "type": "string" + }, + "host": { + "description": "The host of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 3, + "title": "Host", + "type": "string" + }, + "port": { + "description": "The port of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "number" + ], + "instillUIOrder": 4, + "title": "Port", + "type": "number" + }, "criteria": { "description": "The data criteria to be deleted", "instillAcceptFormats": [ "semi-structured/*","structured/*","object" ], "instillShortDescription": "JSON Data", - "instillUIOrder": 1, + "instillUIOrder": 5, "instillUpstreamTypes": [ "reference", "template", @@ -280,7 +516,7 @@ "string" ], "instillShortDescription": "Database Table Name", - "instillUIOrder": 0, + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", @@ -291,6 +527,10 @@ } }, "required": [ + "engine", + "database-name", + "host", + "port", "criteria", "table-name" ], @@ -321,13 +561,71 @@ "input": { "instillUIOrder": 0, "properties": { + "engine": { + "description": "Choose the engine of your database", + "instillUpstreamTypes": [ + "value", + "reference", + "template" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 0, + "title": "Engine", + "enum": [ + "MySQL", + "PostgreSQL", + "SQL Server", + "Oracle", + "MariaDB", + "Firebird" + ], + "type": "string" + }, + "database-name": { + "description": "The database name to insert data into", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 1, + "title": "Database Name", + "type": "string" + }, + "host": { + "description": "The host of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 3, + "title": "Host", + "type": "string" + }, + "port": { + "description": "The port of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "number" + ], + "instillUIOrder": 4, + "title": "Port", + "type": "number" + }, "table-name": { "description": "The table name in the database to be created", "instillAcceptFormats": [ "string" ], "instillShortDescription": "Database Table Name", - "instillUIOrder": 0, + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", @@ -337,12 +635,12 @@ "type":"string" }, "columns": { - "description": "The columns to be created in the table", + "description": "The columns to be created in the table, json with value string, e.g {'name': 'VARCHAR(255'), 'age': 'INT not null'}", "instillAcceptFormats": [ "semi-structured/*","structured/*","object" ], - "instillShortDescription": "JSON Data", - "instillUIOrder": 1, + "instillShortDescription": "JSON Data, e.g {'name': 'VARCHAR(255'), 'age': 'INT not null'}", + "instillUIOrder": 5, "instillUpstreamTypes": [ "reference", "template", @@ -352,6 +650,10 @@ } }, "required": [ + "engine", + "database-name", + "host", + "port", "table-name", "columns" ], @@ -382,13 +684,71 @@ "input": { "instillUIOrder": 0, "properties": { + "engine": { + "description": "Choose the engine of your database", + "instillUpstreamTypes": [ + "value", + "reference", + "template" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 0, + "title": "Engine", + "enum": [ + "MySQL", + "PostgreSQL", + "SQL Server", + "Oracle", + "MariaDB", + "Firebird" + ], + "type": "string" + }, + "database-name": { + "description": "The database name to insert data into", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 1, + "title": "Database Name", + "type": "string" + }, + "host": { + "description": "The host of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 3, + "title": "Host", + "type": "string" + }, + "port": { + "description": "The port of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "number" + ], + "instillUIOrder": 4, + "title": "Port", + "type": "number" + }, "table-name": { "description": "The table name in the database to be dropped", "instillAcceptFormats": [ "string" ], "instillShortDescription": "Database Table Name", - "instillUIOrder": 0, + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", @@ -399,6 +759,10 @@ } }, "required": [ + "engine", + "database-name", + "host", + "port", "table-name" ], "title": "Input", diff --git a/data/sql/v0/main.go b/data/sql/v0/main.go index b16591f7..66fdf1b6 100644 --- a/data/sql/v0/main.go +++ b/data/sql/v0/main.go @@ -65,7 +65,6 @@ func Init(bc base.Component) *component { func (c *component) CreateExecution(sysVars map[string]any, setup *structpb.Struct, task string) (*base.ExecutionWrapper, error) { e := &execution{ ComponentExecution: base.ComponentExecution{Component: c, SystemVariables: sysVars, Setup: setup, Task: task}, - client: newClient(setup), } switch task { @@ -90,10 +89,29 @@ func (c *component) CreateExecution(sysVars map[string]any, setup *structpb.Stru return &base.ExecutionWrapper{Execution: e}, nil } +type SetupNoSecret struct { + DBName string `json:"database-name"` + DBHost string `json:"host"` + DBPort string `json:"port"` + DBEngine string `json:"engine"` +} + +// newClient being setup here in the Execute since all the input for connection string is in inputs +// therefore, every new inputs will create a new connection func (e *execution) Execute(_ context.Context, inputs []*structpb.Struct) ([]*structpb.Struct, error) { outputs := make([]*structpb.Struct, len(inputs)) for i, input := range inputs { + var inputStruct SetupNoSecret + err := base.ConvertFromStructpb(input, &inputStruct) + if err != nil { + return nil, err + } + + if e.client == nil { + e.client = newClient(e.Setup, &inputStruct) + } + output, err := e.execute(input) if err != nil { return nil, err @@ -104,8 +122,3 @@ func (e *execution) Execute(_ context.Context, inputs []*structpb.Struct) ([]*st return outputs, nil } - -func (c *component) Test(sysVars map[string]any, setup *structpb.Struct) error { - - return nil -} diff --git a/data/sql/v0/tasks.go b/data/sql/v0/tasks.go index 5d588af7..aab9c661 100644 --- a/data/sql/v0/tasks.go +++ b/data/sql/v0/tasks.go @@ -2,7 +2,6 @@ package sql import ( "fmt" - "strconv" "strings" "github.com/instill-ai/component/base" @@ -31,8 +30,7 @@ type UpdateOutput struct { type SelectInput struct { Criteria map[string]any `json:"criteria"` TableName string `json:"table-name"` - From int `json:"from"` - To int `json:"to"` + Limit int `json:"limit"` } type SelectOutput struct { @@ -84,20 +82,16 @@ func buildSQLStatementInsert(tableName string, data *map[string]any) (string, ma } func buildSQLStatementUpdate(tableName string, updateData map[string]interface{}, criteria map[string]interface{}, e execution) (string, map[string]interface{}) { - // Take all columns sqlStatementCols := "SELECT * FROM " + tableName - // Prepare and execute the statement rows, _ := e.client.Queryx(sqlStatementCols) defer rows.Close() sqlStatement := "UPDATE " + tableName + " SET " values := make(map[string]interface{}) - // Get column names from the query result columns, _ := rows.Columns() - // Build SET clauses var setClauses []string for _, col := range columns { if updateValue, found := updateData[col]; found { @@ -110,7 +104,6 @@ func buildSQLStatementUpdate(tableName string, updateData map[string]interface{} sqlStatement += strings.Join(setClauses, ", ") - // Build WHERE clauses var whereClauses []string for col, criteriaValue := range criteria { whereClauses = append(whereClauses, fmt.Sprintf("%s = :%s_criteria", col, col)) @@ -122,8 +115,8 @@ func buildSQLStatementUpdate(tableName string, updateData map[string]interface{} return sqlStatement, values } -func buildSQLStatementSelect(tableName string, criteria *map[string]any, to int, from int) string { - // Begin constructing SQL statement +// limit can be empty, but it will have default value 0 +func buildSQLStatementSelect(tableName string, criteria *map[string]any, limit int) string { sqlStatement := "SELECT " var where []string var columns []string @@ -132,13 +125,10 @@ func buildSQLStatementSelect(tableName string, criteria *map[string]any, to int, if criteriaValue != nil { switch criteriaValue.(type) { case string: - // If the value is a string, quote it in the SQL statement where = append(where, fmt.Sprintf("%s = '%v'", criteriaKey, criteriaValue)) case map[string]any: - // If the value is json, quote it in the SQL statement where = append(where, fmt.Sprintf("%s = '%v'", criteriaKey, criteriaValue)) default: - // If the value is a number or bool, use it directly without quotes where = append(where, fmt.Sprintf("%s = %v", criteriaKey, criteriaValue)) } } @@ -147,10 +137,10 @@ func buildSQLStatementSelect(tableName string, criteria *map[string]any, to int, } var notAll string - if to == 0 && from == 0 { + if limit == 0 { notAll = "" } else { - notAll = " LIMIT " + strconv.Itoa(to-from+1) + " OFFSET " + strconv.Itoa(from-1) + notAll = fmt.Sprintf(" LIMIT %d", limit) } if len(columns) > 0 { @@ -169,10 +159,9 @@ func buildSQLStatementSelect(tableName string, criteria *map[string]any, to int, } func buildSQLStatementDelete(tableName string, criteria *map[string]any) (string, map[string]any) { - // Begin constructing SQL statement sqlStatement := "DELETE FROM " + tableName + " WHERE " var where []string - values := make(map[string]any) // Initialize the map + values := make(map[string]any) for criteriaKey, criteriaValue := range *criteria { where = append(where, fmt.Sprintf("%s = :%s", criteriaKey, criteriaKey)) @@ -184,6 +173,7 @@ func buildSQLStatementDelete(tableName string, criteria *map[string]any) (string return sqlStatement, values } +// columns is a map of column name and column type and handled in json format to prevent sql injection func buildSQLStatementCreateTable(tableName string, columns map[string]string) (string, map[string]any) { sqlStatement := "CREATE TABLE " + tableName + " (" var columnDefs []string @@ -213,7 +203,6 @@ func (e *execution) insert(in *structpb.Struct) (*structpb.Struct, error) { sqlStatement, values := buildSQLStatementInsert(inputStruct.TableName, &inputStruct.Data) - // Prepare and execute the statement using NamedExec _, err = e.client.NamedExec(sqlStatement, values) if err != nil { @@ -240,7 +229,6 @@ func (e *execution) update(in *structpb.Struct) (*structpb.Struct, error) { sqlStatement, values := buildSQLStatementUpdate(inputStruct.TableName, inputStruct.Update, inputStruct.Criteria, *e) - // Prepare and execute the statement using NamedExec _, err = e.client.NamedExec(sqlStatement, values) if err != nil { @@ -258,6 +246,7 @@ func (e *execution) update(in *structpb.Struct) (*structpb.Struct, error) { return output, nil } +// Queryx is used since we need not only status but also result return func (e *execution) selects(in *structpb.Struct) (*structpb.Struct, error) { var inputStruct SelectInput err := base.ConvertFromStructpb(in, &inputStruct) @@ -265,39 +254,31 @@ func (e *execution) selects(in *structpb.Struct) (*structpb.Struct, error) { return nil, err } - sqlStatement := buildSQLStatementSelect(inputStruct.TableName, &inputStruct.Criteria, inputStruct.To, inputStruct.From) + sqlStatement := buildSQLStatementSelect(inputStruct.TableName, &inputStruct.Criteria, inputStruct.Limit) - // Prepare and execute the statement rows, err := e.client.Queryx(sqlStatement) if err != nil { return nil, err } defer rows.Close() - // Prepare the result slice of maps var result []map[string]any - // Iterate over the rows for rows.Next() { - // Create a map to hold the row data rowMap := make(map[string]any) - // Load the row data into the map err := rows.MapScan(rowMap) if err != nil { return nil, fmt.Errorf("failed to scan row: %v", err) } - // Convert each value in the map to the appropriate type for key, value := range rowMap { switch v := value.(type) { case []byte: - // Convert byte slices to strings rowMap[key] = string(v) } } - // Add the row map to the result slice result = append(result, rowMap) } @@ -322,7 +303,6 @@ func (e *execution) delete(in *structpb.Struct) (*structpb.Struct, error) { sqlStatement, values := buildSQLStatementDelete(inputStruct.TableName, &inputStruct.Criteria) - // Prepare and execute the statement using NamedExec _, err = e.client.NamedExec(sqlStatement, values) if err != nil { @@ -349,7 +329,6 @@ func (e *execution) createTable(in *structpb.Struct) (*structpb.Struct, error) { sqlStatement, values := buildSQLStatementCreateTable(inputStruct.TableName, inputStruct.Columns) - // Prepare and execute the statement using NamedExec _, err = e.client.NamedExec(sqlStatement, values) if err != nil { @@ -376,7 +355,6 @@ func (e *execution) dropTable(in *structpb.Struct) (*structpb.Struct, error) { sqlStatement, values := buildSQLStatementDropTable(inputStruct.TableName) - // Prepare and execute the statement using NamedExec _, err = e.client.NamedExec(sqlStatement, values) if err != nil { From 5c7863e6a3905fcde3662b51b79c916ca97484e1 Mon Sep 17 00:00:00 2001 From: Yazidane Date: Sun, 21 Jul 2024 20:09:36 +0800 Subject: [PATCH 7/8] fix: change port type from string to int --- data/sql/v0/README.mdx | 16 ++++++------- data/sql/v0/component_test.go | 6 ++--- data/sql/v0/config/tasks.json | 41 ++++++++++++++++---------------- data/sql/v0/main.go | 2 +- data/sql/v0/tasks.go | 44 +++++++++++++++++------------------ 5 files changed, 54 insertions(+), 55 deletions(-) diff --git a/data/sql/v0/README.mdx b/data/sql/v0/README.mdx index f82eb58f..5dd3e4f0 100644 --- a/data/sql/v0/README.mdx +++ b/data/sql/v0/README.mdx @@ -45,7 +45,7 @@ The component configuration is defined and maintained [here](https://github.com/ ### Insert -Perform an insert operation based on specified criteria +Perform an insert operation based on specified filter | Input | ID | Type | Description | @@ -71,7 +71,7 @@ Perform an insert operation based on specified criteria ### Update -Perform an update operation based on specified criteria +Perform an update operation based on specified filter | Input | ID | Type | Description | @@ -82,7 +82,7 @@ Perform an update operation based on specified criteria | Table Name (required) | `table-name` | string | The table name in the database to update data into | | Host (required) | `host` | string | The host of your database | | Port (required) | `port` | number | The port of your database | -| Criteria (required) | `criteria` | semi-structured/json | The data criteria to be updated | +| Filter (required) | `filter` | semi-structured/json | The data filter to be updated | | Update (required) | `update` | semi-structured/json | The new data to be updated to | @@ -98,7 +98,7 @@ Perform an update operation based on specified criteria ### Select -Perform a select operation based on specified criteria +Perform a select operation based on specified filter | Input | ID | Type | Description | @@ -109,7 +109,7 @@ Perform a select operation based on specified criteria | Table Name (required) | `table-name` | string | The table name in the database to be selected | | Host (required) | `host` | string | The host of your database | | Port (required) | `port` | number | The port of your database | -| Criteria (required) | `criteria` | semi-structured/json | The data criteria to be selected, if JSON value of a key is null e.g \{'name':null\}, then name column will be selected without any criteria, if JSON is empty e.g \{\}, then all columns will be selected | +| Filter | `filter` | semi-structured/json | The data filter to be selected, if JSON value of a key is null e.g \{"name":null\}, then name column will be selected without any filter. If empty, then all columns will be selected | | Limit | `limit` | integer | The limit of rows to be selected, optional (empty for all rows) | @@ -126,7 +126,7 @@ Perform a select operation based on specified criteria ### Delete -Perform a delete operation based on specified criteria +Perform a delete operation based on specified filter | Input | ID | Type | Description | @@ -137,7 +137,7 @@ Perform a delete operation based on specified criteria | Table Name (required) | `table-name` | string | The table name in the database to be deleted | | Host (required) | `host` | string | The host of your database | | Port (required) | `port` | number | The port of your database | -| Criteria (required) | `criteria` | semi-structured/json | The data criteria to be deleted | +| Filter (required) | `filter` | semi-structured/json | The data filter to be deleted | @@ -163,7 +163,7 @@ Create a table in the database | Table Name (required) | `table-name` | string | The table name in the database to be created | | Host (required) | `host` | string | The host of your database | | Port (required) | `port` | number | The port of your database | -| Columns (required) | `columns` | semi-structured/json | The columns to be created in the table, json with value string, e.g \{'name': 'VARCHAR(255'), 'age': 'INT not null'\} | +| Columns (required) | `columns` | semi-structured/json | The columns to be created in the table, json with value string, e.g \{"name": "VARCHAR(255)", "age": "INT not null"\} | diff --git a/data/sql/v0/component_test.go b/data/sql/v0/component_test.go index 8f18171e..6e57a306 100644 --- a/data/sql/v0/component_test.go +++ b/data/sql/v0/component_test.go @@ -188,7 +188,7 @@ func TestUpdateUser(t *testing.T) { name: "update user", tableName: "users", input: UpdateInput{ - Criteria: map[string]any{ + Filter: map[string]any{ "id": "1", "name": "John Doe", }, @@ -257,7 +257,7 @@ func TestSelectUser(t *testing.T) { name: "select users", tableName: "users", input: SelectInput{ - Criteria: map[string]any{ + Filter: map[string]any{ "id": "1", "name": "john", "email": "john@example.com", @@ -327,7 +327,7 @@ func TestDeleteUser(t *testing.T) { name: "delete user", tableName: "users", input: DeleteInput{ - Criteria: map[string]any{ + Filter: map[string]any{ "id": "1", "name": "john", }, diff --git a/data/sql/v0/config/tasks.json b/data/sql/v0/config/tasks.json index 09ce10fe..4e7cde6a 100644 --- a/data/sql/v0/config/tasks.json +++ b/data/sql/v0/config/tasks.json @@ -1,6 +1,6 @@ { "TASK_INSERT": { - "instillShortDescription": "Perform an insert operation based on specified criteria", + "instillShortDescription": "Perform an insert operation based on specified filter", "input": { "instillUIOrder": 0, "properties": { @@ -127,7 +127,7 @@ } }, "TASK_UPDATE": { - "instillShortDescription": "Perform an update operation based on specified criteria", + "instillShortDescription": "Perform an update operation based on specified filter", "input": { "instillUIOrder": 0, "properties": { @@ -189,8 +189,8 @@ "title": "Port", "type": "number" }, - "criteria": { - "description": "The data criteria to be updated", + "filter": { + "description": "The data filter to be updated", "instillAcceptFormats": [ "semi-structured/*","structured/*","object" ], @@ -201,7 +201,7 @@ "template", "value" ], - "title": "Criteria" + "title": "Filter" }, "update": { "description": "The new data to be updated to", @@ -242,7 +242,7 @@ "database-name", "host", "port", - "criteria", + "filter", "update", "table-name" ], @@ -269,7 +269,7 @@ } }, "TASK_SELECT":{ - "instillShortDescription": "Perform a select operation based on specified criteria", + "instillShortDescription": "Perform a select operation based on specified filter", "input": { "instillUIOrder": 0, "properties": { @@ -331,19 +331,19 @@ "title": "Port", "type": "number" }, - "criteria": { - "description": "The data criteria to be selected, if JSON value of a key is null e.g {'name':null}, then name column will be selected without any criteria, if JSON is empty e.g {}, then all columns will be selected", + "filter": { + "description": "The data filter to be selected, if JSON value of a key is null e.g {\"name\":null}, then name column will be selected without any filter. If empty, then all columns will be selected", "instillAcceptFormats": [ "semi-structured/*","structured/*","object" ], - "instillShortDescription": "JSON Data", + "instillShortDescription": "JSON Data, optional (empty for all rows)", "instillUIOrder": 5, "instillUpstreamTypes": [ "reference", "template", "value" ], - "title": "Criteria" + "title": "Filter" }, "limit": { "description": "The limit of rows to be selected, optional (empty for all rows)", @@ -381,8 +381,7 @@ "database-name", "host", "port", - "table-name", - "criteria" + "table-name" ], "instillEditOnNodeFields": [ "engine", @@ -390,7 +389,7 @@ "host", "port", "table-name", - "criteria", + "filter", "limit" ], "title": "Input", @@ -434,7 +433,7 @@ } }, "TASK_DELETE": { - "instillShortDescription": "Perform a delete operation based on specified criteria", + "instillShortDescription": "Perform a delete operation based on specified filter", "input": { "instillUIOrder": 0, "properties": { @@ -496,8 +495,8 @@ "title": "Port", "type": "number" }, - "criteria": { - "description": "The data criteria to be deleted", + "filter": { + "description": "The data filter to be deleted", "instillAcceptFormats": [ "semi-structured/*","structured/*","object" ], @@ -508,7 +507,7 @@ "template", "value" ], - "title": "Criteria" + "title": "Filter" }, "table-name": { "description": "The table name in the database to be deleted", @@ -531,7 +530,7 @@ "database-name", "host", "port", - "criteria", + "filter", "table-name" ], "title": "Input", @@ -635,11 +634,11 @@ "type":"string" }, "columns": { - "description": "The columns to be created in the table, json with value string, e.g {'name': 'VARCHAR(255'), 'age': 'INT not null'}", + "description": "The columns to be created in the table, json with value string, e.g {\"name\": \"VARCHAR(255)\", \"age\": \"INT not null\"}", "instillAcceptFormats": [ "semi-structured/*","structured/*","object" ], - "instillShortDescription": "JSON Data, e.g {'name': 'VARCHAR(255'), 'age': 'INT not null'}", + "instillShortDescription": "JSON Data, e.g {\"name\": \"VARCHAR(255)\", \"age\": \"INT not null\"}", "instillUIOrder": 5, "instillUpstreamTypes": [ "reference", diff --git a/data/sql/v0/main.go b/data/sql/v0/main.go index 66fdf1b6..9a3b4206 100644 --- a/data/sql/v0/main.go +++ b/data/sql/v0/main.go @@ -92,7 +92,7 @@ func (c *component) CreateExecution(sysVars map[string]any, setup *structpb.Stru type SetupNoSecret struct { DBName string `json:"database-name"` DBHost string `json:"host"` - DBPort string `json:"port"` + DBPort int `json:"port"` DBEngine string `json:"engine"` } diff --git a/data/sql/v0/tasks.go b/data/sql/v0/tasks.go index aab9c661..46a8f6cd 100644 --- a/data/sql/v0/tasks.go +++ b/data/sql/v0/tasks.go @@ -19,7 +19,7 @@ type InsertOutput struct { type UpdateInput struct { Update map[string]any `json:"update"` - Criteria map[string]any `json:"criteria"` + Filter map[string]any `json:"filter"` TableName string `json:"table-name"` } @@ -28,7 +28,7 @@ type UpdateOutput struct { } type SelectInput struct { - Criteria map[string]any `json:"criteria"` + Filter map[string]any `json:"filter"` TableName string `json:"table-name"` Limit int `json:"limit"` } @@ -39,7 +39,7 @@ type SelectOutput struct { } type DeleteInput struct { - Criteria map[string]any `json:"criteria"` + Filter map[string]any `json:"filter"` TableName string `json:"table-name"` } @@ -81,7 +81,7 @@ func buildSQLStatementInsert(tableName string, data *map[string]any) (string, ma return sqlStatement, values } -func buildSQLStatementUpdate(tableName string, updateData map[string]interface{}, criteria map[string]interface{}, e execution) (string, map[string]interface{}) { +func buildSQLStatementUpdate(tableName string, updateData map[string]interface{}, filter map[string]interface{}, e execution) (string, map[string]interface{}) { sqlStatementCols := "SELECT * FROM " + tableName rows, _ := e.client.Queryx(sqlStatementCols) @@ -105,9 +105,9 @@ func buildSQLStatementUpdate(tableName string, updateData map[string]interface{} sqlStatement += strings.Join(setClauses, ", ") var whereClauses []string - for col, criteriaValue := range criteria { - whereClauses = append(whereClauses, fmt.Sprintf("%s = :%s_criteria", col, col)) - values[col+"_criteria"] = criteriaValue + for col, filterValue := range filter { + whereClauses = append(whereClauses, fmt.Sprintf("%s = :%s_filter", col, col)) + values[col+"_filter"] = filterValue } sqlStatement += " WHERE " + strings.Join(whereClauses, " AND ") @@ -116,24 +116,24 @@ func buildSQLStatementUpdate(tableName string, updateData map[string]interface{} } // limit can be empty, but it will have default value 0 -func buildSQLStatementSelect(tableName string, criteria *map[string]any, limit int) string { +func buildSQLStatementSelect(tableName string, filter *map[string]any, limit int) string { sqlStatement := "SELECT " var where []string var columns []string - for criteriaKey, criteriaValue := range *criteria { - if criteriaValue != nil { - switch criteriaValue.(type) { + for filterKey, filterValue := range *filter { + if filterValue != nil { + switch filterValue.(type) { case string: - where = append(where, fmt.Sprintf("%s = '%v'", criteriaKey, criteriaValue)) + where = append(where, fmt.Sprintf("%s = '%v'", filterKey, filterValue)) case map[string]any: - where = append(where, fmt.Sprintf("%s = '%v'", criteriaKey, criteriaValue)) + where = append(where, fmt.Sprintf("%s = '%v'", filterKey, filterValue)) default: - where = append(where, fmt.Sprintf("%s = %v", criteriaKey, criteriaValue)) + where = append(where, fmt.Sprintf("%s = %v", filterKey, filterValue)) } } - columns = append(columns, criteriaKey) + columns = append(columns, filterKey) } var notAll string @@ -158,14 +158,14 @@ func buildSQLStatementSelect(tableName string, criteria *map[string]any, limit i return sqlStatement } -func buildSQLStatementDelete(tableName string, criteria *map[string]any) (string, map[string]any) { +func buildSQLStatementDelete(tableName string, filter *map[string]any) (string, map[string]any) { sqlStatement := "DELETE FROM " + tableName + " WHERE " var where []string values := make(map[string]any) - for criteriaKey, criteriaValue := range *criteria { - where = append(where, fmt.Sprintf("%s = :%s", criteriaKey, criteriaKey)) - values[criteriaKey] = criteriaValue + for filterKey, filterValue := range *filter { + where = append(where, fmt.Sprintf("%s = :%s", filterKey, filterKey)) + values[filterKey] = filterValue } sqlStatement += strings.Join(where, " AND ") @@ -227,7 +227,7 @@ func (e *execution) update(in *structpb.Struct) (*structpb.Struct, error) { return nil, err } - sqlStatement, values := buildSQLStatementUpdate(inputStruct.TableName, inputStruct.Update, inputStruct.Criteria, *e) + sqlStatement, values := buildSQLStatementUpdate(inputStruct.TableName, inputStruct.Update, inputStruct.Filter, *e) _, err = e.client.NamedExec(sqlStatement, values) @@ -254,7 +254,7 @@ func (e *execution) selects(in *structpb.Struct) (*structpb.Struct, error) { return nil, err } - sqlStatement := buildSQLStatementSelect(inputStruct.TableName, &inputStruct.Criteria, inputStruct.Limit) + sqlStatement := buildSQLStatementSelect(inputStruct.TableName, &inputStruct.Filter, inputStruct.Limit) rows, err := e.client.Queryx(sqlStatement) if err != nil { @@ -301,7 +301,7 @@ func (e *execution) delete(in *structpb.Struct) (*structpb.Struct, error) { return nil, err } - sqlStatement, values := buildSQLStatementDelete(inputStruct.TableName, &inputStruct.Criteria) + sqlStatement, values := buildSQLStatementDelete(inputStruct.TableName, &inputStruct.Filter) _, err = e.client.NamedExec(sqlStatement, values) From af7a38fa932bc739e9cffd27f5bf18db2a02ecf6 Mon Sep 17 00:00:00 2001 From: Yazidane Date: Thu, 25 Jul 2024 21:16:08 +0800 Subject: [PATCH 8/8] feat: change filter into sql to provide logical flexibility --- data/sql/v0/README.mdx | 34 +--- data/sql/v0/client.go | 22 +- data/sql/v0/component_test.go | 21 +- data/sql/v0/config/setup.json | 52 ++++- data/sql/v0/config/tasks.json | 374 +++++++--------------------------- data/sql/v0/main.go | 9 +- data/sql/v0/tasks.go | 126 ++++++------ 7 files changed, 219 insertions(+), 419 deletions(-) diff --git a/data/sql/v0/README.mdx b/data/sql/v0/README.mdx index 5dd3e4f0..0a2a6bde 100644 --- a/data/sql/v0/README.mdx +++ b/data/sql/v0/README.mdx @@ -37,6 +37,9 @@ The component configuration is defined and maintained [here](https://github.com/ | :--- | :--- | :--- | :--- | | Username (required) | `user` | string | Fill in your account username | | Password (required) | `password` | string | Fill in your account password | +| Database Name (required) | `database-name` | string | Fill in the name of your database | +| Host (required) | `host` | string | Fill in the host of your database | +| Port (required) | `port` | number | Fill in the port of your database | @@ -52,10 +55,7 @@ Perform an insert operation based on specified filter | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_INSERT` | | Engine (required) | `engine` | string | Choose the engine of your database | -| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to insert data into | -| Host (required) | `host` | string | The host of your database | -| Port (required) | `port` | number | The port of your database | | Data (required) | `data` | semi-structured/json | The data to be inserted | @@ -78,12 +78,9 @@ Perform an update operation based on specified filter | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_UPDATE` | | Engine (required) | `engine` | string | Choose the engine of your database | -| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to update data into | -| Host (required) | `host` | string | The host of your database | -| Port (required) | `port` | number | The port of your database | -| Filter (required) | `filter` | semi-structured/json | The data filter to be updated | -| Update (required) | `update` | semi-structured/json | The new data to be updated to | +| Filter (required) | `filter` | string | The filter to be applied to the data with SQL syntax, which starts with WHERE clause | +| Update (required) | `update-data` | semi-structured/json | The new data to be updated to | @@ -105,12 +102,10 @@ Perform a select operation based on specified filter | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_SELECT` | | Engine (required) | `engine` | string | Choose the engine of your database | -| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to be selected | -| Host (required) | `host` | string | The host of your database | -| Port (required) | `port` | number | The port of your database | -| Filter | `filter` | semi-structured/json | The data filter to be selected, if JSON value of a key is null e.g \{"name":null\}, then name column will be selected without any filter. If empty, then all columns will be selected | -| Limit | `limit` | integer | The limit of rows to be selected, optional (empty for all rows) | +| Filter | `filter` | string | The filter to be applied to the data with SQL syntax, which starts with WHERE clause, empty for all rows | +| Limit | `limit` | integer | The limit of rows to be selected, empty for all rows | +| Columns | `columns` | array[string] | The columns to return in the rows. If empty then all columns will be returned | @@ -133,11 +128,8 @@ Perform a delete operation based on specified filter | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_DELETE` | | Engine (required) | `engine` | string | Choose the engine of your database | -| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to be deleted | -| Host (required) | `host` | string | The host of your database | -| Port (required) | `port` | number | The port of your database | -| Filter (required) | `filter` | semi-structured/json | The data filter to be deleted | +| Filter (required) | `filter` | string | The filter to be applied to the data with SQL syntax, which starts with WHERE clause | @@ -159,11 +151,8 @@ Create a table in the database | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_CREATE_TABLE` | | Engine (required) | `engine` | string | Choose the engine of your database | -| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to be created | -| Host (required) | `host` | string | The host of your database | -| Port (required) | `port` | number | The port of your database | -| Columns (required) | `columns` | semi-structured/json | The columns to be created in the table, json with value string, e.g \{"name": "VARCHAR(255)", "age": "INT not null"\} | +| Columns (required) | `columns-structure` | semi-structured/json | The columns structure to be created in the table, json with value string, e.g \{"name": "VARCHAR(255)", "age": "INT not null"\} | @@ -185,10 +174,7 @@ Drop a table in the database | :--- | :--- | :--- | :--- | | Task ID (required) | `task` | string | `TASK_DROP_TABLE` | | Engine (required) | `engine` | string | Choose the engine of your database | -| Database Name (required) | `database-name` | string | The database name to insert data into | | Table Name (required) | `table-name` | string | The table name in the database to be dropped | -| Host (required) | `host` | string | The host of your database | -| Port (required) | `port` | number | The port of your database | diff --git a/data/sql/v0/client.go b/data/sql/v0/client.go index d631da18..27192073 100644 --- a/data/sql/v0/client.go +++ b/data/sql/v0/client.go @@ -2,6 +2,7 @@ package sql import ( "fmt" + "strconv" "github.com/jmoiron/sqlx" "google.golang.org/protobuf/types/known/structpb" @@ -33,19 +34,25 @@ var enginesType = map[string]string{ type Config struct { DBUser string DBPassword string + DBName string + DBHost string + DBPort string } func LoadConfig(setup *structpb.Struct) *Config { return &Config{ DBUser: getUser(setup), DBPassword: getPassword(setup), + DBName: getDatabaseName(setup), + DBHost: getHost(setup), + DBPort: getPort(setup), } } -func newClient(setup *structpb.Struct, inputSetup *SetupNoSecret) SQLClient { +func newClient(setup *structpb.Struct, inputSetup *Engine) SQLClient { cfg := LoadConfig(setup) - DBEndpoint := fmt.Sprintf("%v:%v", inputSetup.DBHost, inputSetup.DBPort) + DBEndpoint := fmt.Sprintf("%v:%v", cfg.DBHost, cfg.DBPort) // Test every engines to find the correct one var db *sqlx.DB @@ -56,7 +63,7 @@ func newClient(setup *structpb.Struct, inputSetup *SetupNoSecret) SQLClient { engineType := enginesType[inputSetup.DBEngine] dsn := fmt.Sprintf(engine, - cfg.DBUser, cfg.DBPassword, DBEndpoint, inputSetup.DBName, + cfg.DBUser, cfg.DBPassword, DBEndpoint, cfg.DBName, ) db, err = sqlx.Open(engineType, dsn) @@ -73,3 +80,12 @@ func getUser(setup *structpb.Struct) string { func getPassword(setup *structpb.Struct) string { return setup.GetFields()["password"].GetStringValue() } +func getDatabaseName(setup *structpb.Struct) string { + return setup.GetFields()["database-name"].GetStringValue() +} +func getHost(setup *structpb.Struct) string { + return setup.GetFields()["host"].GetStringValue() +} +func getPort(setup *structpb.Struct) string { + return strconv.Itoa(int(setup.GetFields()["port"].GetNumberValue())) +} diff --git a/data/sql/v0/component_test.go b/data/sql/v0/component_test.go index 6e57a306..b4ed9f28 100644 --- a/data/sql/v0/component_test.go +++ b/data/sql/v0/component_test.go @@ -61,7 +61,6 @@ func (m *MockSQLClient) NamedExec(query string, arg interface{}) (sql.Result, er WithArgs("1", "john").WillReturnResult(sqlmock.NewResult(1, 1)) return sqlxDB.NamedExec("DELETE FROM users WHERE id = :id AND name = :name", arg) - } else if strings.Contains(query, "UPDATE") { mockDB, mock, _ := sqlmock.New() defer mockDB.Close() @@ -188,11 +187,8 @@ func TestUpdateUser(t *testing.T) { name: "update user", tableName: "users", input: UpdateInput{ - Filter: map[string]any{ - "id": "1", - "name": "John Doe", - }, - Update: map[string]any{ + Filter: "id = 1 AND name = 'John Doe'", + UpdateData: map[string]any{ "id": "1", "name": "John Doe Updated", }, @@ -257,11 +253,7 @@ func TestSelectUser(t *testing.T) { name: "select users", tableName: "users", input: SelectInput{ - Filter: map[string]any{ - "id": "1", - "name": "john", - "email": "john@example.com", - }, + Filter: "id = 1 AND name = 'john' AND email = 'john@example.com'", TableName: "users", Limit: 0, }, @@ -327,10 +319,7 @@ func TestDeleteUser(t *testing.T) { name: "delete user", tableName: "users", input: DeleteInput{ - Filter: map[string]any{ - "id": "1", - "name": "john", - }, + Filter: "id = 1 AND name = 'john'", TableName: "users", }, wantResp: DeleteOutput{ @@ -391,7 +380,7 @@ func TestCreateTable(t *testing.T) { { name: "create table", input: CreateTableInput{ - Columns: map[string]string{ + ColumnsStructure: map[string]string{ "id": "INT", "name": "VARCHAR(255)", }, diff --git a/data/sql/v0/config/setup.json b/data/sql/v0/config/setup.json index a78cf994..1c4d1014 100644 --- a/data/sql/v0/config/setup.json +++ b/data/sql/v0/config/setup.json @@ -10,8 +10,7 @@ "instillAcceptFormats": [ "string" ], - "instillSecret": true, - "instillUIOrder": 1, + "instillUIOrder": 0, "title": "Username", "type": "string" }, @@ -24,18 +23,61 @@ "string" ], "instillSecret": true, - "instillUIOrder": 2, + "instillUIOrder": 1, "title": "Password", "type": "string" + }, + "database-name": { + "description": "Fill in the name of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 2, + "title": "Database Name", + "type": "string" + }, + "host": { + "description": "Fill in the host of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillUIOrder": 3, + "title": "Host", + "type": "string" + }, + "port": { + "description": "Fill in the port of your database", + "instillUpstreamTypes": [ + "value","reference" + ], + "default": 3306, + "instillAcceptFormats": [ + "number" + ], + "instillUIOrder": 4, + "title": "Port", + "type": "number" } }, "required": [ "user", - "password" + "password", + "database-name", + "host", + "port" ], "instillEditOnNodeFields": [ "user", - "password" + "password", + "database-name", + "host", + "port" ], "title": "SQL Connection", "type": "object" diff --git a/data/sql/v0/config/tasks.json b/data/sql/v0/config/tasks.json index 4e7cde6a..9f5afb35 100644 --- a/data/sql/v0/config/tasks.json +++ b/data/sql/v0/config/tasks.json @@ -26,49 +26,27 @@ ], "type": "string" }, - "database-name": { - "description": "The database name to insert data into", - "instillUpstreamTypes": [ - "value","reference" - ], + "table-name": { + "description": "The table name in the database to insert data into", "instillAcceptFormats": [ "string" ], + "instillShortDescription": "Database Table Name", "instillUIOrder": 1, - "title": "Database Name", - "type": "string" - }, - "host": { - "description": "The host of your database", "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 3, - "title": "Host", - "type": "string" - }, - "port": { - "description": "The port of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "number" + "reference", + "template", + "value" ], - "instillUIOrder": 4, - "title": "Port", - "type": "number" + "title": "Table Name", + "type":"string" }, "data": { "description": "The data to be inserted", "instillAcceptFormats": [ "semi-structured/*","structured/*","object","array" ], - "instillShortDescription": "JSON Data", - "instillUIOrder": 5, + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", @@ -79,28 +57,10 @@ "instillFormat": "semi-structured/json" }, "title": "Data" - }, - "table-name": { - "description": "The table name in the database to insert data into", - "instillAcceptFormats": [ - "string" - ], - "instillShortDescription": "Database Table Name", - "instillUIOrder": 2, - "instillUpstreamTypes": [ - "reference", - "template", - "value" - ], - "title": "Table Name", - "type":"string" } }, "required": [ "engine", - "database-name", - "host", - "port", "data", "table-name" ], @@ -153,63 +113,42 @@ ], "type": "string" }, - "database-name": { - "description": "The database name to insert data into", - "instillUpstreamTypes": [ - "value","reference" - ], + "table-name": { + "description": "The table name in the database to update data into", "instillAcceptFormats": [ "string" ], + "instillShortDescription": "Database Table Name", "instillUIOrder": 1, - "title": "Database Name", - "type": "string" - }, - "host": { - "description": "The host of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 3, - "title": "Host", - "type": "string" - }, - "port": { - "description": "The port of your database", "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "number" + "reference", + "template", + "value" ], - "instillUIOrder": 4, - "title": "Port", - "type": "number" + "title": "Table Name", + "type":"string" }, "filter": { - "description": "The data filter to be updated", - "instillAcceptFormats": [ - "semi-structured/*","structured/*","object" - ], - "instillShortDescription": "JSON Data", - "instillUIOrder": 5, + "instillShortDescription": "The filter to be applied to the data", + "description": "The filter to be applied to the data with SQL syntax, which starts with WHERE clause", + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", "value" ], - "title": "Filter" + "instillAcceptFormats":[ + "string" + ], + "title": "Filter", + "type": "string" }, - "update": { + "update-data": { "description": "The new data to be updated to", "instillAcceptFormats": [ "semi-structured/*","structured/*","object","array" ], - "instillShortDescription": "JSON Data", - "instillUIOrder": 6, + "instillUIOrder": 3, "instillUpstreamTypes": [ "reference", "template", @@ -220,30 +159,12 @@ "instillFormat": "semi-structured/json" }, "title": "Update" - }, - "table-name": { - "description": "The table name in the database to update data into", - "instillAcceptFormats": [ - "string" - ], - "instillShortDescription": "Database Table Name", - "instillUIOrder": 2, - "instillUpstreamTypes": [ - "reference", - "template", - "value" - ], - "title": "Table Name", - "type":"string" } }, "required": [ "engine", - "database-name", - "host", - "port", "filter", - "update", + "update-data", "table-name" ], "title": "Input", @@ -295,63 +216,43 @@ ], "type": "string" }, - "database-name": { - "description": "The database name to insert data into", - "instillUpstreamTypes": [ - "value","reference" - ], + "table-name": { + "description": "The table name in the database to be selected", "instillAcceptFormats": [ "string" ], + "instillShortDescription": "Database Table Name", "instillUIOrder": 1, - "title": "Database Name", - "type": "string" - }, - "host": { - "description": "The host of your database", "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 3, - "title": "Host", - "type": "string" - }, - "port": { - "description": "The port of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "number" + "reference", + "template", + "value" ], - "instillUIOrder": 4, - "title": "Port", - "type": "number" + "title": "Table Name", + "type":"string" }, "filter": { - "description": "The data filter to be selected, if JSON value of a key is null e.g {\"name\":null}, then name column will be selected without any filter. If empty, then all columns will be selected", - "instillAcceptFormats": [ - "semi-structured/*","structured/*","object" - ], - "instillShortDescription": "JSON Data, optional (empty for all rows)", - "instillUIOrder": 5, + "instillShortDescription": "The filter to be applied to the data. If empty, then all rows will be updated", + "description": "The filter to be applied to the data with SQL syntax, which starts with WHERE clause, empty for all rows", + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", "value" ], - "title": "Filter" + "instillAcceptFormats":[ + "string" + ], + "title": "Filter", + "type": "string" }, "limit": { - "description": "The limit of rows to be selected, optional (empty for all rows)", + "description": "The limit of rows to be selected, empty for all rows", "instillAcceptFormats": [ "integer" ], "instillShortDescription": "Limit Rows", - "instillUIOrder": 6, + "instillUIOrder": 3, "instillUpstreamTypes": [ "reference", "template", @@ -360,37 +261,34 @@ "title": "Limit", "type":"integer" }, - "table-name": { - "description": "The table name in the database to be selected", + "columns":{ + "description": "The columns to return in the rows. If empty then all columns will be returned", "instillAcceptFormats": [ - "string" + "array:string" ], - "instillShortDescription": "Database Table Name", - "instillUIOrder": 2, + "instillShortDescription": "Columns to be returned, empty for all columns", + "instillUIOrder": 4, "instillUpstreamTypes": [ "reference", "template", "value" ], - "title": "Table Name", - "type":"string" + "title": "Columns", + "type":"array", + "items": { + "title":"Column", + "type": "string" + } } }, "required": [ "engine", - "database-name", - "host", - "port", "table-name" ], "instillEditOnNodeFields": [ "engine", - "database-name", - "host", - "port", "table-name", - "filter", - "limit" + "filter" ], "title": "Input", "type": "object" @@ -459,77 +357,39 @@ ], "type": "string" }, - "database-name": { - "description": "The database name to insert data into", - "instillUpstreamTypes": [ - "value","reference" - ], + "table-name": { + "description": "The table name in the database to be deleted", "instillAcceptFormats": [ "string" ], + "instillShortDescription": "Database Table Name", "instillUIOrder": 1, - "title": "Database Name", - "type": "string" - }, - "host": { - "description": "The host of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 3, - "title": "Host", - "type": "string" - }, - "port": { - "description": "The port of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "number" - ], - "instillUIOrder": 4, - "title": "Port", - "type": "number" - }, - "filter": { - "description": "The data filter to be deleted", - "instillAcceptFormats": [ - "semi-structured/*","structured/*","object" - ], - "instillShortDescription": "JSON Data", - "instillUIOrder": 5, "instillUpstreamTypes": [ "reference", "template", "value" ], - "title": "Filter" + "title": "Table Name", + "type":"string" }, - "table-name": { - "description": "The table name in the database to be deleted", - "instillAcceptFormats": [ - "string" - ], - "instillShortDescription": "Database Table Name", + "filter": { + "instillShortDescription": "The filter to be applied to the data", + "description": "The filter to be applied to the data with SQL syntax, which starts with WHERE clause", "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", "value" ], - "title": "Table Name", - "type":"string" + "instillAcceptFormats":[ + "string" + ], + "title": "Filter", + "type": "string" } }, "required": [ "engine", - "database-name", - "host", - "port", "filter", "table-name" ], @@ -582,49 +442,13 @@ ], "type": "string" }, - "database-name": { - "description": "The database name to insert data into", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 1, - "title": "Database Name", - "type": "string" - }, - "host": { - "description": "The host of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 3, - "title": "Host", - "type": "string" - }, - "port": { - "description": "The port of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "number" - ], - "instillUIOrder": 4, - "title": "Port", - "type": "number" - }, "table-name": { "description": "The table name in the database to be created", "instillAcceptFormats": [ "string" ], "instillShortDescription": "Database Table Name", - "instillUIOrder": 2, + "instillUIOrder": 1, "instillUpstreamTypes": [ "reference", "template", @@ -633,13 +457,13 @@ "title": "Table Name", "type":"string" }, - "columns": { - "description": "The columns to be created in the table, json with value string, e.g {\"name\": \"VARCHAR(255)\", \"age\": \"INT not null\"}", + "columns-structure": { + "description": "The columns structure to be created in the table, json with value string, e.g {\"name\": \"VARCHAR(255)\", \"age\": \"INT not null\"}", "instillAcceptFormats": [ "semi-structured/*","structured/*","object" ], - "instillShortDescription": "JSON Data, e.g {\"name\": \"VARCHAR(255)\", \"age\": \"INT not null\"}", - "instillUIOrder": 5, + "instillShortDescription": "Columns Structure, e.g {\"name\": \"VARCHAR(255)\", \"age\": \"INT not null\"}", + "instillUIOrder": 2, "instillUpstreamTypes": [ "reference", "template", @@ -650,11 +474,8 @@ }, "required": [ "engine", - "database-name", - "host", - "port", "table-name", - "columns" + "columns-structure" ], "title": "Input", "type": "object" @@ -705,49 +526,13 @@ ], "type": "string" }, - "database-name": { - "description": "The database name to insert data into", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 1, - "title": "Database Name", - "type": "string" - }, - "host": { - "description": "The host of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "string" - ], - "instillUIOrder": 3, - "title": "Host", - "type": "string" - }, - "port": { - "description": "The port of your database", - "instillUpstreamTypes": [ - "value","reference" - ], - "instillAcceptFormats": [ - "number" - ], - "instillUIOrder": 4, - "title": "Port", - "type": "number" - }, "table-name": { "description": "The table name in the database to be dropped", "instillAcceptFormats": [ "string" ], "instillShortDescription": "Database Table Name", - "instillUIOrder": 2, + "instillUIOrder": 1, "instillUpstreamTypes": [ "reference", "template", @@ -759,9 +544,6 @@ }, "required": [ "engine", - "database-name", - "host", - "port", "table-name" ], "title": "Input", diff --git a/data/sql/v0/main.go b/data/sql/v0/main.go index 9a3b4206..8b8863e4 100644 --- a/data/sql/v0/main.go +++ b/data/sql/v0/main.go @@ -89,20 +89,17 @@ func (c *component) CreateExecution(sysVars map[string]any, setup *structpb.Stru return &base.ExecutionWrapper{Execution: e}, nil } -type SetupNoSecret struct { - DBName string `json:"database-name"` - DBHost string `json:"host"` - DBPort int `json:"port"` +type Engine struct { DBEngine string `json:"engine"` } -// newClient being setup here in the Execute since all the input for connection string is in inputs +// newClient being setup here in the Execute since engine is part of the input // therefore, every new inputs will create a new connection func (e *execution) Execute(_ context.Context, inputs []*structpb.Struct) ([]*structpb.Struct, error) { outputs := make([]*structpb.Struct, len(inputs)) for i, input := range inputs { - var inputStruct SetupNoSecret + var inputStruct Engine err := base.ConvertFromStructpb(input, &inputStruct) if err != nil { return nil, err diff --git a/data/sql/v0/tasks.go b/data/sql/v0/tasks.go index 46a8f6cd..8e58d531 100644 --- a/data/sql/v0/tasks.go +++ b/data/sql/v0/tasks.go @@ -2,6 +2,7 @@ package sql import ( "fmt" + "regexp" "strings" "github.com/instill-ai/component/base" @@ -18,9 +19,9 @@ type InsertOutput struct { } type UpdateInput struct { - Update map[string]any `json:"update"` - Filter map[string]any `json:"filter"` - TableName string `json:"table-name"` + UpdateData map[string]any `json:"update-data"` + Filter string `json:"filter"` + TableName string `json:"table-name"` } type UpdateOutput struct { @@ -28,9 +29,10 @@ type UpdateOutput struct { } type SelectInput struct { - Filter map[string]any `json:"filter"` - TableName string `json:"table-name"` - Limit int `json:"limit"` + Filter string `json:"filter"` + TableName string `json:"table-name"` + Limit int `json:"limit"` + Columns []string `json:"columns"` } type SelectOutput struct { @@ -39,8 +41,8 @@ type SelectOutput struct { } type DeleteInput struct { - Filter map[string]any `json:"filter"` - TableName string `json:"table-name"` + Filter string `json:"filter"` + TableName string `json:"table-name"` } type DeleteOutput struct { @@ -48,8 +50,8 @@ type DeleteOutput struct { } type CreateTableInput struct { - TableName string `json:"table-name"` - Columns map[string]string `json:"columns"` + TableName string `json:"table-name"` + ColumnsStructure map[string]string `json:"columns-structure"` } type CreateTableOutput struct { @@ -64,6 +66,16 @@ type DropTableOutput struct { Status string `json:"status"` } +func isValidWhereClause(whereClause string) error { + // Extended regex pattern for logical operators and additional conditions + regex := `^(?:\w+ (?:=|!=|>|<|>=|<=|LIKE|MATCH|IS NULL|IS NOT NULL|BETWEEN|IN|EXISTS|NOT|REGEXP|RLIKE|IS DISTINCT FROM|IS NOT DISTINCT FROM|COALESCE\(.*\)|NULLIF\(.*\)) (?:[\w'%]+|\d+|\([\w\s,']+\)|(?:CASE .* END)|(?:\w+\s+\w+)))(?: (?:AND|OR) (?:\w+ (?:=|!=|>|<|>=|<=|LIKE|MATCH|IS NULL|IS NOT NULL|BETWEEN|IN|EXISTS|NOT|REGEXP|RLIKE|IS DISTINCT FROM|IS NOT DISTINCT FROM|COALESCE\(.*\)|NULLIF\(.*\)) (?:[\w'%]+|\d+|\([\w\s,']+\)|(?:CASE .* END)|(?:\w+\s+\w+))))*$` + matched, err := regexp.MatchString(regex, whereClause) + if err != nil || !matched { + return err + } + return nil +} + func buildSQLStatementInsert(tableName string, data *map[string]any) (string, map[string]any) { sqlStatement := "INSERT INTO " + tableName + " (" var columns []string @@ -81,60 +93,29 @@ func buildSQLStatementInsert(tableName string, data *map[string]any) (string, ma return sqlStatement, values } -func buildSQLStatementUpdate(tableName string, updateData map[string]interface{}, filter map[string]interface{}, e execution) (string, map[string]interface{}) { - sqlStatementCols := "SELECT * FROM " + tableName - - rows, _ := e.client.Queryx(sqlStatementCols) - defer rows.Close() - +func buildSQLStatementUpdate(tableName string, updateData map[string]any, filter string) (string, map[string]any) { sqlStatement := "UPDATE " + tableName + " SET " - values := make(map[string]interface{}) - - columns, _ := rows.Columns() + values := make(map[string]any) var setClauses []string - for _, col := range columns { - if updateValue, found := updateData[col]; found { - setClauses = append(setClauses, fmt.Sprintf("%s = :%s", col, col)) - values[col] = updateValue - } else { - setClauses = append(setClauses, fmt.Sprintf("%s = NULL", col)) - } + for col, updateValue := range updateData { + setClauses = append(setClauses, fmt.Sprintf("%s = :%s", col, col)) + values[col] = updateValue } sqlStatement += strings.Join(setClauses, ", ") - var whereClauses []string - for col, filterValue := range filter { - whereClauses = append(whereClauses, fmt.Sprintf("%s = :%s_filter", col, col)) - values[col+"_filter"] = filterValue + if filter != "" { + sqlStatement += " WHERE " + filter } - sqlStatement += " WHERE " + strings.Join(whereClauses, " AND ") - return sqlStatement, values } // limit can be empty, but it will have default value 0 -func buildSQLStatementSelect(tableName string, filter *map[string]any, limit int) string { +// columns can be empty, if empty it will select all columns +func buildSQLStatementSelect(tableName string, filter string, limit int, columns []string) string { sqlStatement := "SELECT " - var where []string - var columns []string - - for filterKey, filterValue := range *filter { - if filterValue != nil { - switch filterValue.(type) { - case string: - where = append(where, fmt.Sprintf("%s = '%v'", filterKey, filterValue)) - case map[string]any: - where = append(where, fmt.Sprintf("%s = '%v'", filterKey, filterValue)) - default: - where = append(where, fmt.Sprintf("%s = %v", filterKey, filterValue)) - } - } - - columns = append(columns, filterKey) - } var notAll string if limit == 0 { @@ -150,36 +131,31 @@ func buildSQLStatementSelect(tableName string, filter *map[string]any, limit int } sqlStatement += " FROM " + tableName - if len(where) > 0 { - sqlStatement += " WHERE " + strings.Join(where, " AND ") + if filter != "" { + sqlStatement += " WHERE " + filter } sqlStatement += notAll return sqlStatement } -func buildSQLStatementDelete(tableName string, filter *map[string]any) (string, map[string]any) { - sqlStatement := "DELETE FROM " + tableName + " WHERE " - var where []string - values := make(map[string]any) +func buildSQLStatementDelete(tableName string, filter string) string { + sqlStatement := "DELETE FROM " + tableName - for filterKey, filterValue := range *filter { - where = append(where, fmt.Sprintf("%s = :%s", filterKey, filterKey)) - values[filterKey] = filterValue + if filter != "" { + sqlStatement += " WHERE " + filter } - sqlStatement += strings.Join(where, " AND ") - - return sqlStatement, values + return sqlStatement } // columns is a map of column name and column type and handled in json format to prevent sql injection -func buildSQLStatementCreateTable(tableName string, columns map[string]string) (string, map[string]any) { +func buildSQLStatementCreateTable(tableName string, columnsStructure map[string]string) (string, map[string]any) { sqlStatement := "CREATE TABLE " + tableName + " (" var columnDefs []string values := make(map[string]any) - for colName, colType := range columns { + for colName, colType := range columnsStructure { columnDefs = append(columnDefs, fmt.Sprintf("%s %s", colName, colType)) values[colName] = colType } @@ -226,8 +202,12 @@ func (e *execution) update(in *structpb.Struct) (*structpb.Struct, error) { if err != nil { return nil, err } + err = isValidWhereClause(inputStruct.Filter) + if err != nil { + return nil, err + } - sqlStatement, values := buildSQLStatementUpdate(inputStruct.TableName, inputStruct.Update, inputStruct.Filter, *e) + sqlStatement, values := buildSQLStatementUpdate(inputStruct.TableName, inputStruct.UpdateData, inputStruct.Filter) _, err = e.client.NamedExec(sqlStatement, values) @@ -253,8 +233,12 @@ func (e *execution) selects(in *structpb.Struct) (*structpb.Struct, error) { if err != nil { return nil, err } + err = isValidWhereClause(inputStruct.Filter) + if err != nil { + return nil, err + } - sqlStatement := buildSQLStatementSelect(inputStruct.TableName, &inputStruct.Filter, inputStruct.Limit) + sqlStatement := buildSQLStatementSelect(inputStruct.TableName, inputStruct.Filter, inputStruct.Limit, inputStruct.Columns) rows, err := e.client.Queryx(sqlStatement) if err != nil { @@ -300,10 +284,14 @@ func (e *execution) delete(in *structpb.Struct) (*structpb.Struct, error) { if err != nil { return nil, err } + err = isValidWhereClause(inputStruct.Filter) + if err != nil { + return nil, err + } - sqlStatement, values := buildSQLStatementDelete(inputStruct.TableName, &inputStruct.Filter) + sqlStatement := buildSQLStatementDelete(inputStruct.TableName, inputStruct.Filter) - _, err = e.client.NamedExec(sqlStatement, values) + _, err = e.client.Queryx(sqlStatement) if err != nil { return nil, err @@ -327,7 +315,7 @@ func (e *execution) createTable(in *structpb.Struct) (*structpb.Struct, error) { return nil, err } - sqlStatement, values := buildSQLStatementCreateTable(inputStruct.TableName, inputStruct.Columns) + sqlStatement, values := buildSQLStatementCreateTable(inputStruct.TableName, inputStruct.ColumnsStructure) _, err = e.client.NamedExec(sqlStatement, values)