Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/googleps04 71 support set datatype #5

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
Open

Feat/googleps04 71 support set datatype #5

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bb2ff12
Support StringSET
Dec 4, 2024
fe02ba5
Support StringSET
Dec 4, 2024
0c7fcaa
Add negative test cases
Dec 4, 2024
ea241a7
fix UT
Dec 4, 2024
ceb3b1b
Handling of Binary and Number SET
Dec 18, 2024
bd51299
Handling of Binary and Number SET
Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions api/v1/condition.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,7 @@ func ChangeResponseColumn(obj map[string]interface{}) map[string]interface{} {
// ChangeColumnToSpanner converts original column name to spanner supported column names
func ChangeColumnToSpanner(obj map[string]interface{}) map[string]interface{} {
rs := make(map[string]interface{})

for k, v := range obj {

if k1, ok := models.ColumnToOriginalCol[k]; ok {
Expand Down Expand Up @@ -519,9 +519,13 @@ func convertFrom(a *dynamodb.AttributeValue, tableName string) interface{} {
return a.B
}
if a.SS != nil {
l := make([]interface{}, len(a.SS))
for index, v := range a.SS {
l[index] = *v
l := []string{}
stringMap := make(map[string]struct{})
for _, v := range a.SS {
if _, exists := stringMap[*v]; !exists {
stringMap[*v] = struct{}{}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So this change only handles deduplication.

l = append(l, *v)
}
}
return l
}
Expand Down Expand Up @@ -631,7 +635,7 @@ func ChangeQueryResponseColumn(tableName string, obj map[string]interface{}) map
return obj
}

//ChangeMaptoDynamoMap converts simple map into dynamo map
// ChangeMaptoDynamoMap converts simple map into dynamo map
func ChangeMaptoDynamoMap(in interface{}) (map[string]interface{}, error) {
if in == nil {
return nil, nil
Expand Down Expand Up @@ -699,6 +703,14 @@ func convertSlice(output map[string]interface{}, v reflect.Value) error {
return nil
}
output["B"] = append([]byte{}, b...)
case reflect.String:
listVal := []string{}
count := 0
for i := 0; i < v.Len(); i++ {
listVal = append(listVal, v.Index(i).String())
count++
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

count is not used.

}
output["SS"] = listVal
default:
listVal := make([]map[string]interface{}, 0, v.Len())

Expand Down
2 changes: 2 additions & 0 deletions api/v1/condition_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,11 +327,13 @@ func TestConvertDynamoToMap(t *testing.T) {
"address": {S: aws.String("Ney York")},
"first_name": {S: aws.String("Catalina")},
"last_name": {S: aws.String("Smith")},
"titles": {SS: aws.StringSlice([]string{"Mr", "Dr"})},
},
map[string]interface{}{
"address": "Ney York",
"first_name": "Catalina",
"last_name": "Smith",
"titles": []string{"Mr", "Dr"},
},
},
{
Expand Down
2 changes: 1 addition & 1 deletion api/v1/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func RouteRequest(c *gin.Context) {
case "UpdateItem":
Update(c)
default:
c.JSON(errors.New("ValidationException", "Invalid X-Amz-Target header value of" + amzTarget).
c.JSON(errors.New("ValidationException", "Invalid X-Amz-Target header value of"+amzTarget).
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a space after "of" before concatenating amzTarget

HTTPResponse("X-Amz-Target Header not supported"))
}
}
Expand Down
255 changes: 139 additions & 116 deletions storage/spanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"math"
"reflect"
"regexp"
Expand Down Expand Up @@ -612,7 +613,6 @@ func (s Storage) performPutOperation(ctx context.Context, t *spanner.ReadWriteTr
m[k] = ba
}
}

mutation := spanner.InsertOrUpdateMap(table, m)
mutations := []*spanner.Mutation{mutation}
err := t.BufferWrite(mutations)
Expand Down Expand Up @@ -733,7 +733,7 @@ func evaluateStatementFromRowMap(conditionalExpression, colName string, rowMap m
return true
}
_, ok := rowMap[colName]
return !ok
return !ok
}
if strings.HasPrefix(conditionalExpression, "attribute_exists") || strings.HasPrefix(conditionalExpression, "if_exists") {
if len(rowMap) == 0 {
Expand All @@ -745,7 +745,6 @@ func evaluateStatementFromRowMap(conditionalExpression, colName string, rowMap m
return rowMap[conditionalExpression]
}

//parseRow - Converts Spanner row and datatypes to a map removing null columns from the result.
func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, error) {
singleRow := make(map[string]interface{})
if r == nil {
Expand All @@ -759,133 +758,157 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{},
}
v, ok := colDDL[k]
if !ok {
fmt.Println(k)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this just left over debug logging?

return nil, errors.New("ResourceNotFoundException", k)
}

var err error
switch v {
case "STRING(MAX)":
var s spanner.NullString
err := r.Column(i, &s)
if err != nil {
if strings.Contains(err.Error(), "ambiguous column name") {
continue
}
return nil, errors.New("ValidationException", err, k)
}
if !s.IsNull() {
singleRow[k] = s.StringVal
}
err = parseStringColumn(r, i, k, singleRow)
case "BYTES(MAX)":
var s []byte
err := r.Column(i, &s)
if err != nil {
if strings.Contains(err.Error(), "ambiguous column name") {
continue
}
return nil, errors.New("ValidationException", err, k)
}
if len(s) > 0 {
var m interface{}
err := json.Unmarshal(s, &m)
if err != nil {
logger.LogError(err, string(s))
singleRow[k] = string(s)
continue
}
val1, ok := m.(string)
if ok {
if base64Regexp.MatchString(val1) {
ba, err := base64.StdEncoding.DecodeString(val1)
if err == nil {
var sample interface{}
err = json.Unmarshal(ba, &sample)
if err == nil {
singleRow[k] = sample
continue
} else {
singleRow[k] = string(s)
continue
}
}
}
}

if mp, ok := m.(map[string]interface{}); ok {
for k, v := range mp {
if val, ok := v.(string); ok {
if base64Regexp.MatchString(val) {
ba, err := base64.StdEncoding.DecodeString(val)
if err == nil {
var sample interface{}
err = json.Unmarshal(ba, &sample)
if err == nil {
mp[k] = sample
m = mp
}
}
}
}
}
}
singleRow[k] = m
}
err = parseBytesColumn(r, i, k, singleRow)
case "INT64":
var s spanner.NullInt64
err := r.Column(i, &s)
if err != nil {
if strings.Contains(err.Error(), "ambiguous column name") {
continue
}
return nil, errors.New("ValidationException", err, k)
}
if !s.IsNull() {
singleRow[k] = s.Int64
}
err = parseInt64Column(r, i, k, singleRow)
case "FLOAT64":
var s spanner.NullFloat64
err := r.Column(i, &s)
if err != nil {
if strings.Contains(err.Error(), "ambiguous column name") {
continue
}
return nil, errors.New("ValidationException", err, k)

}
if !s.IsNull() {
singleRow[k] = s.Float64
}
err = parseFloat64Column(r, i, k, singleRow)
case "NUMERIC":
var s spanner.NullNumeric
err := r.Column(i, &s)
if err != nil {
if strings.Contains(err.Error(), "ambiguous column name") {
continue
}
return nil, errors.New("ValidationException", err, k)
}
if !s.IsNull() {
if s.Numeric.IsInt() {
tmp, _ := s.Numeric.Float64()
singleRow[k] = int64(tmp)
} else {
singleRow[k], _ = s.Numeric.Float64()
}
}
err = parseNumericColumn(r, i, k, singleRow)
case "BOOL":
var s spanner.NullBool
err := r.Column(i, &s)
if err != nil {
if strings.Contains(err.Error(), "ambiguous column name") {
continue
}
return nil, errors.New("ValidationException", err, k)
err = parseBoolColumn(r, i, k, singleRow)
case "ARRAY<STRING(MAX)>":
err = parseStringArrayColumn(r, i, k, singleRow)
}

if err != nil {
return nil, errors.New("ValidationException", err, k)
}
}
return singleRow, nil
}

func parseStringColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error {
var s spanner.NullString
err := r.Column(idx, &s)
if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
return err
}
if !s.IsNull() {
row[col] = s.StringVal
}
return nil
}

func parseBytesColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error {
var s []byte
err := r.Column(idx, &s)
if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
return err
}
if len(s) > 0 {
var m interface{}
if err := json.Unmarshal(s, &m); err != nil {
logger.LogError(err, string(s))
row[col] = string(s)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment here as you are falling back to raw string.

return nil
}
m = processDecodedData(m)
row[col] = m
}
return nil
}

func parseInt64Column(r *spanner.Row, idx int, col string, row map[string]interface{}) error {
var s spanner.NullInt64
err := r.Column(idx, &s)
if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
return err
}
if !s.IsNull() {
row[col] = s.Int64
}
return nil
}

func parseFloat64Column(r *spanner.Row, idx int, col string, row map[string]interface{}) error {
var s spanner.NullFloat64
err := r.Column(idx, &s)
if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
return err
}
if !s.IsNull() {
row[col] = s.Float64
}
return nil
}

func parseNumericColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error {
var s spanner.NullNumeric
err := r.Column(idx, &s)
if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
return err
}
if !s.IsNull() {
val, _ := s.Numeric.Float64()
if s.Numeric.IsInt() {
row[col] = int64(val)
} else {
row[col] = val
}
}
return nil
}

func parseBoolColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error {
var s spanner.NullBool
err := r.Column(idx, &s)
if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
return err
}
if !s.IsNull() {
row[col] = s.Bool
}
return nil
}

func parseStringArrayColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error {
var s []spanner.NullString
err := r.Column(idx, &s)
if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
return err
}
var temp []string
for _, val := range s {
temp = append(temp, val.StringVal)
}
if len(s) > 0 {
row[col] = temp
}
return nil
}

func processDecodedData(m interface{}) interface{} {
if val, ok := m.(string); ok && base64Regexp.MatchString(val) {
if ba, err := base64.StdEncoding.DecodeString(val); err == nil {
var sample interface{}
if err := json.Unmarshal(ba, &sample); err == nil {
return sample
}
if !s.IsNull() {
singleRow[k] = s.Bool
}
}
if mp, ok := m.(map[string]interface{}); ok {
for k, v := range mp {
if val, ok := v.(string); ok && base64Regexp.MatchString(val) {
if ba, err := base64.StdEncoding.DecodeString(val); err == nil {
var sample interface{}
if err := json.Unmarshal(ba, &sample); err == nil {
mp[k] = sample
}
}
}
}
}
return singleRow, nil
return m
}

func checkInifinty(value float64, logData interface{}) error {
Expand Down
Loading