Skip to content

Commit

Permalink
feat: default value for mysql source (#963)
Browse files Browse the repository at this point in the history 
* source dv

* test fix

* change

* comment changes

* test fix

* change

* fix github v

* change
  • Loading branch information
@asthamohta
asthamohta authored Dec 20, 2024
1 parent 1b5cf7d commit e1f6d28
Show file tree
Hide file tree
Showing 20 changed files with 158 additions and 34 deletions.
1 change: 1 addition & 0 deletions common/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,5 +125,6 @@ const (
//VerifyExpresions API
CHECK_EXPRESSION = "CHECK"
DEFAUT_EXPRESSION = "DEFAULT"
DEFAULT_GENERATED = "DEFAULT_GENERATED"
TEMP_DB = "smt-staging-db"
)
2 changes: 1 addition & 1 deletion internal/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func GenerateSequenceId() string {
return GenerateId("s")
}
func GenerateExpressionId() string {
return GenerateId("i")
return GenerateId("e")
}

func GetSrcColNameIdMap(srcs schema.Table) map[string]string {
Expand Down
12 changes: 11 additions & 1 deletion sources/common/infoschema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,10 +257,20 @@ func (is *InfoSchemaImpl) GetIncludedSrcTablesFromConv(conv *internal.Conv) (sch

// SanitizeDefaultValue removes extra characters added to Default Value in information schema in MySQL.
func SanitizeDefaultValue(defaultValue string, ty string, generated bool) string {
types := []string{"char", "varchar", "text", "varbinary", "tinyblob", "tinytext", "text",
"blob", "mediumtext", "mediumblob", "longtext", "longblob", "STRING"}
// Check if ty exists in the types array
stringType := false
for _, t := range types {
if t == ty {
stringType = true
break
}
}
defaultValue = strings.ReplaceAll(defaultValue, "_utf8mb4", "")
defaultValue = strings.ReplaceAll(defaultValue, "\\\\", "\\")
defaultValue = strings.ReplaceAll(defaultValue, "\\'", "'")
if !generated && (ty == "char" || ty == "varchar" || ty == "text" || ty == "STRING") && !strings.HasPrefix(defaultValue, "'") && !strings.HasSuffix(defaultValue, "'") {
if !generated && stringType && !strings.HasPrefix(defaultValue, "'") && !strings.HasSuffix(defaultValue, "'") {
defaultValue = "'" + defaultValue + "'"
}
return defaultValue
Expand Down
47 changes: 47 additions & 0 deletions sources/common/infoschema_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
"testing"

"github.com/stretchr/testify/assert"
)


func TestSanitizeDefaultValue(t *testing.T) {
tests := []struct {
inputString string
ty string
generated bool
expectedString string
}{
{"a", "char", true, "a"},
{"b", "char", false, "'b'"},
{"c", "int", true, "c"},
{"d", "int", false, "d"},
{"_utf8mb4\\'hello world\\'", "char", false, "'hello world'"},
{"week(_utf8mb4\\'2024-06-20\\',0)", "char", true, "week('2024-06-20',0)"},
{"_utf8mb4\\'This is a message \\\\nwith a newline\\\\rand a carriage return.\\'", "char", false, "'This is a message \\nwith a newline\\rand a carriage return.'"},
{"strcmp(_utf8mb4\\'abc\\',_utf8mb4\\'abcd\\')", "char", true, "strcmp('abc','abcd')"},
{"_utf8mb4\\'John\\\\\\'s Jack\\'", "char", false, "'John\\'s Jack'"},
{"_utf8mb4\\'This product has\tmultiple features.\\'", "char", false, "'This product has\tmultiple features.'"},
{"_utf8mb4\\'C:\\\\\\\\Users\\\\\\\\johndoe\\\\\\\\Documents\\\\\\\\myfile.txt\\'", "char", false, "'C:\\\\Users\\\\johndoe\\\\Documents\\\\myfile.txt'"},
}
for _, test := range tests {
result := SanitizeDefaultValue(test.inputString, test.ty, test.generated)
assert.Equal(t, test.expectedString, result)
}
}
25 changes: 19 additions & 6 deletions sources/mysql/infoschema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,13 +219,26 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
} else {
colAutoGen = ddl.AutoGenCol{}
}

defaultVal := ddl.DefaultValue{
IsPresent: colDefault.Valid,
Value: ddl.Expression{},
}
if colDefault.Valid {
defaultVal.Value = ddl.Expression{
ExpressionId: internal.GenerateExpressionId(),
Statement: common.SanitizeDefaultValue(colDefault.String, dataType, colExtra.String == constants.DEFAULT_GENERATED),
}
}

c := schema.Column{
Id: colId,
Name: colName,
Type: toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
NotNull: common.ToNotNull(conv, isNullable),
Ignored: ignored,
AutoGen: colAutoGen,
Id: colId,
Name: colName,
Type: toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
NotNull: common.ToNotNull(conv, isNullable),
Ignored: ignored,
AutoGen: colAutoGen,
DefaultValue: defaultVal,
}
colDefs[colId] = c
colIds = append(colIds, colId)
Expand Down
16 changes: 8 additions & 8 deletions sources/mysql/infoschema_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ func TestProcessSchemaMYSQL(t *testing.T) {
args: []driver.Value{"test", "user"},
cols: []string{"column_name", "data_type", "column_type", "is_nullable", "column_default", "character_maximum_length", "numeric_precision", "numeric_scale", "extra"},
rows: [][]driver.Value{
{"user_id", "text", "text", "NO", nil, nil, nil, nil, nil},
{"name", "text", "text", "NO", nil, nil, nil, nil, nil},
{"user_id", "text", "text", "NO", "uuid()", nil, nil, nil, constants.DEFAULT_GENERATED},
{"name", "text", "text", "NO", "default_name", nil, nil, nil, nil},
{"ref", "bigint", "bigint", "NO", nil, nil, nil, nil, nil}},
},
// db call to fetch index happens after fetching of column
Expand Down Expand Up @@ -233,19 +233,19 @@ func TestProcessSchemaMYSQL(t *testing.T) {
"test": schema.Table{Name: "test", Schema: "test", ColIds: []string{"id", "s", "txt", "b", "bs", "bl", "c", "c8", "d", "dec", "f8", "f4", "i8", "i4", "i2", "si", "ts", "tz", "vc", "vc6"}, ColDefs: map[string]schema.Column{
"b": schema.Column{Name: "b", Type: schema.Type{Name: "boolean", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"bl": schema.Column{Name: "bl", Type: schema.Type{Name: "blob", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"bs": schema.Column{Name: "bs", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"bs": schema.Column{Name: "bs", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", DefaultValue: ddl.DefaultValue{IsPresent: true, Value: ddl.Expression{ExpressionId: "e27", Statement: "nextval('test11_bs_seq'::regclass)"}}},
"c": schema.Column{Name: "c", Type: schema.Type{Name: "char", Mods: []int64{1}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"c8": schema.Column{Name: "c8", Type: schema.Type{Name: "char", Mods: []int64{8}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"d": schema.Column{Name: "d", Type: schema.Type{Name: "date", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"dec": schema.Column{Name: "dec", Type: schema.Type{Name: "decimal", Mods: []int64{20, 5}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"f4": schema.Column{Name: "f4", Type: schema.Type{Name: "float", Mods: []int64{24}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"f8": schema.Column{Name: "f8", Type: schema.Type{Name: "double", Mods: []int64{53}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"i2": schema.Column{Name: "i2", Type: schema.Type{Name: "smallint", Mods: []int64{16}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"i4": schema.Column{Name: "i4", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", AutoGen: ddl.AutoGenCol{Name: "Sequence34", GenerationType: constants.AUTO_INCREMENT}},
"i4": schema.Column{Name: "i4", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", AutoGen: ddl.AutoGenCol{Name: "Sequence37", GenerationType: constants.AUTO_INCREMENT}},
"i8": schema.Column{Name: "i8", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"id": schema.Column{Name: "id", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"s": schema.Column{Name: "s", Type: schema.Type{Name: "set", Mods: []int64(nil), ArrayBounds: []int64{-1}}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"si": schema.Column{Name: "si", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"si": schema.Column{Name: "si", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", DefaultValue: ddl.DefaultValue{IsPresent: true, Value: ddl.Expression{ExpressionId: "e40", Statement: "nextval('test11_s_seq'::regclass)"}}},
"ts": schema.Column{Name: "ts", Type: schema.Type{Name: "datetime", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"txt": schema.Column{Name: "txt", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"tz": schema.Column{Name: "tz", Type: schema.Type{Name: "timestamp", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
Expand All @@ -262,9 +262,9 @@ func TestProcessSchemaMYSQL(t *testing.T) {
ForeignKeys: []schema.ForeignKey(nil),
Indexes: []schema.Index(nil), Id: ""},
"user": schema.Table{Name: "user", Schema: "test", ColIds: []string{"user_id", "name", "ref"}, ColDefs: map[string]schema.Column{
"name": schema.Column{Name: "name", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"name": schema.Column{Name: "name", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", DefaultValue: ddl.DefaultValue{Value: ddl.Expression{ExpressionId: "e6", Statement: "'default_name'"}, IsPresent: true}},
"ref": schema.Column{Name: "ref", Type: schema.Type{Name: "bigint", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
"user_id": schema.Column{Name: "user_id", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}},
"user_id": schema.Column{Name: "user_id", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", DefaultValue: ddl.DefaultValue{Value: ddl.Expression{ExpressionId: "e4", Statement: "uuid()"}, IsPresent: true}}},
PrimaryKeys: []schema.Key{schema.Key{ColId: "user_id", Desc: false, Order: 0}},
ForeignKeys: []schema.ForeignKey{schema.ForeignKey{Name: "fk_test", ColIds: []string{"ref"}, ReferTableId: "test", ReferColumnIds: []string{"id"}, OnUpdate: constants.FK_CASCADE, OnDelete: constants.FK_SET_NULL, Id: ""}},
Indexes: []schema.Index(nil), Id: ""}}
Expand Down Expand Up @@ -395,7 +395,7 @@ func TestProcessData_MultiCol(t *testing.T) {
}
internal.AssertSpSchema(conv, t, expectedSchema, stripSchemaComments(conv.SpSchema))
columnLevelIssues := map[string][]internal.SchemaIssue{
"c49": []internal.SchemaIssue{
"c53": []internal.SchemaIssue{
2,
},
}
Expand Down
Loading

0 comments on commit e1f6d28

Please sign in to comment.