From 2c8592a5e8642a3e67e0ce9233204be245e4b681 Mon Sep 17 00:00:00 2001 From: Dan Hansen Date: Tue, 23 Jan 2024 15:22:17 -0800 Subject: [PATCH] Split ToApiString() / ToString() --- internal/encoder.go | 20 --------- internal/rows.go | 33 +++++++-------- internal/value.go | 100 +++++++++++++++++++++++++++++++++++++++----- query_test.go | 62 +++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 49 deletions(-) diff --git a/internal/encoder.go b/internal/encoder.go index a4218bc..4f343d6 100644 --- a/internal/encoder.go +++ b/internal/encoder.go @@ -379,18 +379,6 @@ func CastValue(t types.Type, v Value) (Value, error) { } return FloatValue(f64), nil case types.STRING: - switch v.(type) { - // If this is coming from a date/time, the format is slightly different - // than when just writing the value out as a string. - case DateValue: - return convertTimeValueToStringWithFormat(v, "2006-01-02") - case DatetimeValue: - return convertTimeValueToStringWithFormat(v, "2006-01-02 15:04:05.999999") - case TimeValue: - return convertTimeValueToStringWithFormat(v, "15:04:05.999999") - case TimestampValue: - return convertTimeValueToStringWithFormat(v, "2006-01-02 15:04:05.999999-07") - } s, err := v.ToString() if err != nil { return nil, err @@ -525,14 +513,6 @@ func CastValue(t types.Type, v Value) (Value, error) { return nil, fmt.Errorf("unsupported cast %s value", t.Kind()) } -func convertTimeValueToStringWithFormat(v Value, format string) (Value, error) { - valueTime, err := v.ToTime() - if err != nil { - return nil, err - } - return StringValue(valueTime.UTC().Format(format)), nil -} - func ValueFromGoValue(v interface{}) (Value, error) { if isNullValue(v) { return nil, nil diff --git a/internal/rows.go b/internal/rows.go index 80944d9..b4f21dd 100644 --- a/internal/rows.go +++ b/internal/rows.go @@ -5,12 +5,10 @@ import ( "database/sql" "database/sql/driver" "fmt" - "io" - "reflect" - "time" - "github.com/goccy/go-json" "github.com/goccy/go-zetasql/types" + "io" + "reflect" ) type Rows struct { @@ -232,68 +230,65 @@ func (r *Rows) assignInterfaceValue(src Value, dst reflect.Value, typ *Type) err } dst.Set(reflect.ValueOf(f64)) case types.BYTES: - s, err := src.ToString() + s, err := src.ToApiString() if err != nil { return err } dst.Set(reflect.ValueOf(s)) case types.STRING: - s, err := src.ToString() + s, err := src.ToApiString() if err != nil { return err } dst.Set(reflect.ValueOf(s)) case types.NUMERIC: - s, err := src.ToString() + s, err := src.ToApiString() if err != nil { return err } dst.Set(reflect.ValueOf(s)) case types.BIG_NUMERIC: - s, err := src.ToString() + s, err := src.ToApiString() if err != nil { return err } dst.Set(reflect.ValueOf(s)) case types.DATE: - date, err := src.ToJSON() + date, err := src.ToApiString() if err != nil { return err } dst.Set(reflect.ValueOf(date)) case types.DATETIME: - datetime, err := src.ToJSON() + datetime, err := src.ToApiString() if err != nil { return err } dst.Set(reflect.ValueOf(datetime)) case types.TIME: - time, err := src.ToJSON() + time, err := src.ToApiString() if err != nil { return err } dst.Set(reflect.ValueOf(time)) case types.TIMESTAMP: - t, err := src.ToTime() + t, err := src.ToApiString() if err != nil { return err } - unixmicro := t.UnixMicro() - sec := unixmicro / int64(time.Millisecond) - nsec := unixmicro - sec*int64(time.Millisecond) - dst.Set(reflect.ValueOf(fmt.Sprintf("%d.%d", sec, nsec))) + dst.Set(reflect.ValueOf(t)) case types.INTERVAL: - s, err := src.ToString() + s, err := src.ToApiString() if err != nil { return err } dst.Set(reflect.ValueOf(s)) case types.JSON: - json, err := src.ToJSON() + j, err := src.ToApiString() if err != nil { return err } - dst.Set(reflect.ValueOf(json)) + dst.Set(reflect.ValueOf(j)) case types.STRUCT: s, err := src.ToStruct() if err != nil { diff --git a/internal/value.go b/internal/value.go index fd9d288..fbdc248 100644 --- a/internal/value.go +++ b/internal/value.go @@ -26,6 +26,7 @@ type Value interface { GTE(Value) (bool, error) LT(Value) (bool, error) LTE(Value) (bool, error) + ToApiString() (string, error) ToInt64() (int64, error) ToString() (string, error) ToBytes() ([]byte, error) @@ -125,6 +126,10 @@ func (iv IntValue) ToString() (string, error) { return fmt.Sprint(iv), nil } +func (iv IntValue) ToApiString() (string, error) { + return iv.ToString() +} + func (iv IntValue) ToBytes() ([]byte, error) { return []byte(fmt.Sprint(iv)), nil } @@ -251,6 +256,10 @@ func (sv StringValue) ToString() (string, error) { return string(sv), nil } +func (sv StringValue) ToApiString() (string, error) { + return sv.ToString() +} + func (sv StringValue) ToBytes() ([]byte, error) { return []byte(string(sv)), nil } @@ -401,6 +410,10 @@ func (bv BytesValue) ToInt64() (int64, error) { } func (bv BytesValue) ToString() (string, error) { + return string([]byte(bv)[:]), nil +} + +func (bv BytesValue) ToApiString() (string, error) { return base64.StdEncoding.EncodeToString([]byte(bv)), nil } @@ -583,6 +596,10 @@ func (fv FloatValue) ToString() (string, error) { return fmt.Sprint(fv), nil } +func (fv FloatValue) ToApiString() (string, error) { + return fv.ToString() +} + func (fv FloatValue) ToBytes() ([]byte, error) { return []byte(fmt.Sprint(fv)), nil } @@ -752,6 +769,10 @@ func (nv *NumericValue) ToString() (string, error) { return nv.toString(), nil } +func (nv *NumericValue) ToApiString() (string, error) { + return nv.toString(), nil +} + func (nv *NumericValue) ToBytes() ([]byte, error) { return []byte(nv.toString()), nil } @@ -852,6 +873,10 @@ func (bv BoolValue) ToString() (string, error) { return fmt.Sprint(bv), nil } +func (bv BoolValue) ToApiString() (string, error) { + return bv.ToString() +} + func (bv BoolValue) ToBytes() ([]byte, error) { return []byte(fmt.Sprint(bv)), nil } @@ -947,6 +972,10 @@ func (jv JsonValue) ToString() (string, error) { return string(jv), nil } +func (jv JsonValue) ToApiString() (string, error) { + return jv.ToString() +} + func (jv JsonValue) ToBytes() ([]byte, error) { return []byte(string(jv)), nil } @@ -1178,6 +1207,10 @@ func (av *ArrayValue) ToString() (string, error) { return fmt.Sprintf("[%s]", strings.Join(elems, ",")), nil } +func (av *ArrayValue) ToApiString() (string, error) { + return "", fmt.Errorf("arrays do not have string-based API representations %v", av) +} + func (av *ArrayValue) ToBytes() ([]byte, error) { v, err := av.ToString() if err != nil { @@ -1388,6 +1421,10 @@ func (sv *StructValue) ToString() (string, error) { return fmt.Sprintf("{%s}", strings.Join(fields, ",")), nil } +func (sv *StructValue) ToApiString() (string, error) { + return "", fmt.Errorf("Structs do not have string-based API representations %v", "") +} + func (sv *StructValue) ToBytes() ([]byte, error) { v, err := sv.ToString() if err != nil { @@ -1575,6 +1612,8 @@ func (d DateValue) ToString() (string, error) { return time.Time(d).Format("2006-01-02"), nil } +func (d DateValue) ToApiString() (string, error) { return d.ToString() } + func (d DateValue) ToBytes() ([]byte, error) { v, err := d.ToString() if err != nil { @@ -1600,7 +1639,8 @@ func (d DateValue) ToStruct() (*StructValue, error) { } func (d DateValue) ToJSON() (string, error) { - return d.ToString() + val, err := d.ToString() + return strconv.Quote(val), err } func (d DateValue) ToTime() (time.Time, error) { @@ -1627,7 +1667,8 @@ func (d DateValue) Interface() interface{} { } const ( - datetimeFormat = "2006-01-02T15:04:05.999999" + datetimeDefaultFormat = "2006-01-02T15:04:05.999999" + datetimeAsStringFormat = "2006-01-02 15:04:05.999999" ) type DatetimeValue time.Time @@ -1724,7 +1765,11 @@ func (d DatetimeValue) ToInt64() (int64, error) { } func (d DatetimeValue) ToString() (string, error) { - return time.Time(d).Format(datetimeFormat), nil + return time.Time(d).Format(datetimeAsStringFormat), nil +} + +func (d DatetimeValue) ToApiString() (string, error) { + return time.Time(d).Format(datetimeDefaultFormat), nil } func (d DatetimeValue) ToBytes() ([]byte, error) { @@ -1752,7 +1797,7 @@ func (d DatetimeValue) ToStruct() (*StructValue, error) { } func (d DatetimeValue) ToJSON() (string, error) { - return d.ToString() + return strconv.Quote(time.Time(d).Format(datetimeDefaultFormat)), nil } func (d DatetimeValue) ToTime() (time.Time, error) { @@ -1764,7 +1809,7 @@ func (d DatetimeValue) ToRat() (*big.Rat, error) { } func (d DatetimeValue) Format(verb rune) string { - formatted := time.Time(d).Format(datetimeFormat) + formatted := time.Time(d).Format(datetimeDefaultFormat) switch verb { case 't': return formatted @@ -1775,7 +1820,7 @@ func (d DatetimeValue) Format(verb rune) string { } func (d DatetimeValue) Interface() interface{} { - return time.Time(d).Format(datetimeFormat) + return time.Time(d).Format(datetimeDefaultFormat) } type TimeValue time.Time @@ -1844,6 +1889,10 @@ func (t TimeValue) ToString() (string, error) { return time.Time(t).Format("15:04:05.999999"), nil } +func (t TimeValue) ToApiString() (string, error) { + return t.ToString() +} + func (t TimeValue) ToBytes() ([]byte, error) { v, err := t.ToString() if err != nil { @@ -1869,7 +1918,11 @@ func (t TimeValue) ToStruct() (*StructValue, error) { } func (t TimeValue) ToJSON() (string, error) { - return t.ToString() + val, err := t.ToString() + if err != nil { + return "", err + } + return strconv.Quote(val), nil } func (t TimeValue) ToTime() (time.Time, error) { @@ -1897,6 +1950,10 @@ func (t TimeValue) Interface() interface{} { type TimestampValue time.Time +func (t TimestampValue) GetFormatString() string { + return "2006-01-02 15:04:05.999999-07" +} + func (t TimestampValue) AddValueWithPart(v time.Duration, part string) (Value, error) { switch part { case "MICROSECOND": @@ -2008,7 +2065,18 @@ func (t TimestampValue) ToInt64() (int64, error) { } func (t TimestampValue) ToString() (string, error) { - return time.Time(t).Format(time.RFC3339Nano), nil + return time.Time(t).Format("2006-01-02 15:04:05.999999-07"), nil +} + +func (t TimestampValue) ToApiString() (string, error) { + ti, err := t.ToTime() + if err != nil { + return "", err + } + unixmicro := ti.UnixMicro() + sec := unixmicro / int64(time.Millisecond) + nsec := unixmicro - sec*int64(time.Millisecond) + return fmt.Sprintf("%d.%d", sec, nsec), nil } func (t TimestampValue) ToBytes() ([]byte, error) { @@ -2036,7 +2104,7 @@ func (t TimestampValue) ToStruct() (*StructValue, error) { } func (t TimestampValue) ToJSON() (string, error) { - return t.ToString() + return strconv.Quote(time.Time(t).UTC().Format("2006-01-02T15:04:05.999999Z")), nil } func (t TimestampValue) ToTime() (time.Time, error) { @@ -2114,6 +2182,10 @@ func (iv *IntervalValue) ToString() (string, error) { return iv.String(), nil } +func (iv *IntervalValue) ToApiString() (string, error) { + return iv.ToString() +} + func (iv *IntervalValue) ToBytes() ([]byte, error) { s, err := iv.ToString() if err != nil { @@ -2262,6 +2334,14 @@ func (v *SafeValue) ToString() (string, error) { return ret, nil } +func (v *SafeValue) ToApiString() (string, error) { + ret, err := v.value.ToApiString() + if err != nil { + return "", nil + } + return ret, nil +} + func (v *SafeValue) ToBytes() ([]byte, error) { ret, err := v.value.ToBytes() if err != nil { @@ -2368,7 +2448,7 @@ func parseDate(date string) (time.Time, error) { } func parseDatetime(datetime string) (time.Time, error) { - if t, err := time.Parse(datetimeFormat, datetime); err == nil { + if t, err := time.Parse(datetimeDefaultFormat, datetime); err == nil { return t, nil } return time.Parse("2006-01-02 15:04:05.999999", datetime) diff --git a/query_test.go b/query_test.go index 90bb831..810ade3 100644 --- a/query_test.go +++ b/query_test.go @@ -1751,6 +1751,11 @@ SELECT * FROM Employees`, {"Jose", int64(2), "2013-03-17"}, }, }, + { + name: "date to json", + query: `SELECT TO_JSON_STRING(DATE(2024, 1, 1))`, + expectedRows: [][]interface{}{{`"2024-01-01"`}}, + }, { name: "window rank", query: ` @@ -2144,6 +2149,14 @@ FROM UNNEST([ }, // array functions + { + name: "array formatting", + query: `SELECT [0, 1, 1, 2, 3, 5] AS some_numbers;`, + expectedRows: [][]interface{}{ + {[]interface{}{int64(0), int64(1), int64(1), int64(2), int64(3), int64(5)}}, + }, + }, + { name: "make_array", query: `SELECT a, b FROM UNNEST([STRUCT(DATE(2022, 1, 1) AS a, 1 AS b)])`, @@ -3660,6 +3673,16 @@ SELECT date, EXTRACT(ISOYEAR FROM date), EXTRACT(YEAR FROM date), EXTRACT(MONTH query: `SELECT DATE_DIFF(DATE '2021-06-06', DATE '2017-11-12', DAY) AS days_diff`, expectedRows: [][]interface{}{{int64(1302)}}, }, + { + name: "date_diff with month", + query: `SELECT DATE_DIFF(DATE '2018-01-01', DATE '2017-10-30', MONTH) AS months_diff`, + expectedRows: [][]interface{}{{int64(3)}}, + }, + { + name: "date_diff with day", + query: `SELECT DATE_DIFF(DATE '2021-06-06', DATE '2017-11-12', DAY) AS days_diff`, + expectedRows: [][]interface{}{{int64(1302)}}, + }, { name: "date_from_unix_date", query: `SELECT DATE_FROM_UNIX_DATE(14238) AS date_from_epoch`, @@ -3882,6 +3905,16 @@ SELECT date, EXTRACT(ISOYEAR FROM date), EXTRACT(YEAR FROM date), EXTRACT(MONTH query: `SELECT FORMAT_DATETIME("%E4Y", DATETIME "2008-12-25 15:30:12.345678")`, expectedRows: [][]interface{}{{"2008"}}, }, + { + name: "datetime literal", + query: `SELECT DATETIME '2023-01-01 12:01:00'`, + expectedRows: [][]interface{}{{"2023-01-01T12:01:00"}}, + }, + { + name: "datetime to json", + query: `SELECT TO_JSON_STRING(STRUCT(DATETIME "2006-01-02T15:04:05.999999" AS DT))`, + expectedRows: [][]interface{}{{`{"DT":"2006-01-02T15:04:05.999999"}`}}, + }, { name: "cast datetime as string", query: `SELECT CAST(DATETIME(TIMESTAMP("2022-08-01 06:47:51.123456-07:00")) AS STRING)`, @@ -3973,6 +4006,11 @@ SELECT date, EXTRACT(ISOYEAR FROM date), EXTRACT(YEAR FROM date), EXTRACT(MONTH query: `SELECT CAST(TIME("2022-08-01 06:47:51.123456-04:00") AS STRING)`, expectedRows: [][]interface{}{{"10:47:51.123456"}}, }, + { + name: "time to json string", + query: `SELECT TO_JSON_STRING(TIME("2022-08-01 06:47:51.123456-04:00"))`, + expectedRows: [][]interface{}{{`"10:47:51.123456"`}}, + }, { name: "cast time with timezone as string", query: `SELECT CAST(TIME("2022-08-01 06:47:51.123456-04:00", "America/Los_Angeles") AS STRING)`, @@ -4042,6 +4080,17 @@ SELECT date, EXTRACT(ISOYEAR FROM date), EXTRACT(YEAR FROM date), EXTRACT(MONTH query: `SELECT TIMESTAMP(DATE "2008-12-25")`, expectedRows: [][]interface{}{{createTimestampFormatFromString("2008-12-25 00:00:00+00")}}, }, + { + name: "timestamp to json", + query: `SELECT TO_JSON_STRING(STRUCT(TIMESTAMP "2006-01-02T15:04:05.999999-07" AS TIMESTAMP))`, + expectedRows: [][]interface{}{{`{"TIMESTAMP":"2006-01-02T22:04:05.999999Z"}`}}, + }, + + { + name: "timestamp to string", + query: `WITH tmp AS ( SELECT TIMESTAMP "2006-01-02T15:04:05-07" AS TS ) SELECT ts, CAST(ts AS STRING) FROM tmp`, + expectedRows: [][]interface{}{{createTimestampFormatFromString(`2006-01-02 22:04:05+00`), `2006-01-02 22:04:05+00`}}, + }, { name: "timestamp_add", query: `SELECT TIMESTAMP_ADD(TIMESTAMP "2008-12-25 15:30:00+00", INTERVAL 10 MINUTE)`, @@ -4130,6 +4179,11 @@ SELECT date, EXTRACT(ISOYEAR FROM date), EXTRACT(YEAR FROM date), EXTRACT(MONTH query: `SELECT CAST(TIMESTAMP("2022-08-01 06:47:51.123456-07:00") AS STRING);`, expectedRows: [][]interface{}{{"2022-08-01 13:47:51.123456+00"}}, }, + { + name: "timestamp parses with T value", + query: `SELECT CAST(TIMESTAMP("2022-08-01T06:47:51.123456-07:00") AS STRING);`, + expectedRows: [][]interface{}{{"2022-08-01 13:47:51.123456+00"}}, + }, { name: "parse timestamp with %a %b %e %I:%M:%S %Y", query: `SELECT PARSE_TIMESTAMP("%a %b %e %I:%M:%S %Y", "Thu Dec 25 07:30:00 2008")`, @@ -4210,6 +4264,7 @@ FROM Input`, query: `SELECT DATE "2020-09-22" + val FROM UNNEST([INTERVAL 1 DAY,INTERVAL -1 DAY,INTERVAL 2 YEAR,CAST('1-2 3 18:1:55' AS INTERVAL)]) as val`, expectedRows: [][]interface{}{{"2020-09-23T00:00:00"}, {"2020-09-21T00:00:00"}, {"2022-09-22T00:00:00"}, {"2021-11-25T18:01:55"}}, }, + { name: "interval from sub operator", query: ` @@ -4275,6 +4330,13 @@ SELECT expectedRows: [][]interface{}{{"123.456", "123.456"}}, }, + // bytes formatting + { + name: "bytes_formatting", + query: `SELECT b"abc", CAST(b"abc" AS STRING)`, + expectedRows: [][]interface{}{{"YWJj", "abc"}}, + }, + // security functions { name: "session_user",