From f8682e1eb9f1965da066b01454a2e94d750711ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=9B=E5=93=A5?= Date: Mon, 15 Jun 2020 09:07:14 +0800 Subject: [PATCH] merge for the release of v1.6.0 (#195) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add license checker (#175) * add release note for v1.5.0 (#178) Co-authored-by: Xin.Zh * Imp: cache in reflection (#179) * benchmark result * use cache in findField * encode benchmark * call field() once * remove version * fix import sync * cache in registerPOJO * add json bench result * prune unneccessary rtype.Field(index) * cache comment * rename cache * switch to if * remove return value name * findFieldWithCache * remove if check when fieldStruct is nil Co-authored-by: 望哥 * update dependency * rename serialize arg name * Create .asf.yaml * 优化hessian解码string性能，提升54% * optimize code. * optimize code. * fix code review. * optimize codes. * optimize cods. * optimize code. * update license * go.sum * ci go version * testify -> 1.4.0 * testcase * travis.yml * decode value before reflect find * setvalue * decode nilPtr to nilPtr * fix get attachment lost nil key * manually import package * add ToMapStringString unit test * rename test function name with issue * setmap * support for decode emoji. * refactor code * add unit test. * add unit tests. * refactor tests. * Update travis/main.sh (#200) - Remove duplicate key 'webhooks' - Key 'matrix' is an alias for `jobs`, using `jobs` - Specify the os and dist explicitly * Mod: modify * Code format (#199) * .gitignore * code clean * code clean * remove length check * Fix: comments * Fix: format package * Fix #181: float32 accuracy issue (#196) * Fix #181: float32 accuracy issue * Fix go fmt failure * Add the unit test case for Issue181 * Add encFloat32 in double.go to encode float32 type - Call encFloat32 to encode float32 while encoding - Add unit test case to test float32 encoding * Improve encFloat32 of double.go * Fix git fmt failure * add release note for v1.6.0 (#202) * add release note for v1.5.1 * add release note for v1.5.1 * add notice * update notice * =fix release note for v1.6.0 Co-authored-by: Joe Zou Co-authored-by: Xin.Zh Co-authored-by: huiren Co-authored-by: Huang YunKun Co-authored-by: zonghaishang Co-authored-by: fangyincheng Co-authored-by: champly Co-authored-by: wilson chen Co-authored-by: fangyincheng Co-authored-by: gaoxinge --- .asf.yaml | 5 + .gitignore | 1 + .travis.yml | 14 +- CHANGE.md | 13 + LICENSE | 25 ++ README.md | 7 + before_validate_license.sh | 26 ++ binary_test.go | 2 - decode.go | 41 +++ decode_test.go | 1 + double.go | 33 ++ double_test.go | 27 ++ encode.go | 2 +- go.mod | 4 +- go.sum | 6 +- int_test.go | 5 +- object.go | 87 ++++-- object_test.go | 109 +++++++ pojo.go | 5 + request.go | 5 + request_test.go | 35 +++ serialize.go | 8 +- string.go | 283 +++++++++++++++--- string_test.go | 37 +++ .../src/main/java/test/TestCustomDecode.java | 5 + .../src/main/java/test/TestCustomReply.java | 28 ++ .../src/main/java/test/TestString.java | 5 + 27 files changed, 739 insertions(+), 80 deletions(-) create mode 100644 .asf.yaml create mode 100644 before_validate_license.sh diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 00000000..8d84e695 --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,5 @@ +notifications: + commits: commits@dubbo.apache.org + issues: notifications@dubbo.apache.org + pullrequests: notifications@dubbo.apache.org + jira_options: link label link label diff --git a/.gitignore b/.gitignore index 13637208..fc62e8a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea +vendor coverage.txt diff --git a/.travis.yml b/.travis.yml index a711aab9..5442ab18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go -matrix: +jobs: include: - language: java jdk: openjdk8 @@ -8,15 +8,23 @@ matrix: go: - "1.12" +os: linux + +dist: xenial + script: - mvn clean package -f test_hessian/pom.xml - mvn clean package -f test_dubbo/pom.xml - go fmt && [[ -z `git status -s` ]] + - sh before_validate_license.sh + - chmod u+x /tmp/tools/license/license-header-checker + - /tmp/tools/license/license-header-checker -v -a -r -i vendor /tmp/tools/license/license.txt . go && [[ -z `git status -s` ]] - GO111MODULE=on && go mod vendor && go test -race -v && go test -bench . -race -coverprofile=coverage.txt after_success: - bash <(curl -s https://codecov.io/bash) notifications: - webhooks: https://oapi.dingtalk.com/robot/send?access_token=27a5eb4510c8cf913b67a72832549b123a8c44655483d20443515604669de0ae - webhooks: https://oapi.dingtalk.com/robot/send?access_token=8250008579ed1defda3a44fb8608a38d81a55700fdfb15466315a90a7dd2045f + webhooks: + - https://oapi.dingtalk.com/robot/send?access_token=27a5eb4510c8cf913b67a72832549b123a8c44655483d20443515604669de0ae + - https://oapi.dingtalk.com/robot/send?access_token=8250008579ed1defda3a44fb8608a38d81a55700fdfb15466315a90a7dd2045f diff --git a/CHANGE.md b/CHANGE.md index a3f32e46..492220cd 100644 --- a/CHANGE.md +++ b/CHANGE.md @@ -1,5 +1,18 @@ # Release Notes +## v1.6.0 + +### New Features +- ignore non-exist fields when decoding. [#201](https://github.com/apache/dubbo-go-hessian2/pull/201) + +### Enhancement +- add cache in reflection to improve performance. [#179](https://github.com/apache/dubbo-go-hessian2/pull/179) +- string decode performance improvement. [#188](https://github.com/apache/dubbo-go-hessian2/pull/188) + +### Bugfixes +- fix attachment lost for nil value. [#191](https://github.com/apache/dubbo-go-hessian2/pull/191) +- fix float32 accuracy issue. [#196](https://github.com/apache/dubbo-go-hessian2/pull/196) + ## v1.5.0 ### New Features diff --git a/LICENSE b/LICENSE index 43bbf609..7dd1e2be 100755 --- a/LICENSE +++ b/LICENSE @@ -174,3 +174,28 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 9c129417..6bc9b78e 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,15 @@ --- +> **Notice: When decoding, the java version of hessian will default skip and ignore non-exist fields.** +> **From the version of v1.6.0 , dubbo-go-hessian2 will skip non-exist fields too, while that before v1.6.0 will return errors.** + It's a golang hessian library used by [Apache/dubbo-go](https://github.com/apache/dubbo-go). +There is a big performance improvement, and some bugs fix for v1.6.0, +thanks to [micln](https://github.com/micln), [pantianying](https://github.com/pantianying), [zonghaishang](https://github.com/zonghaishang), + [willson-chen](https://github.com/willson-chen), [champly](https://github.com/champly). + ## Feature List * [All JDK Exceptions](https://github.com/apache/dubbo-go-hessian2/issues/59) diff --git a/before_validate_license.sh b/before_validate_license.sh new file mode 100644 index 00000000..8fa6e381 --- /dev/null +++ b/before_validate_license.sh @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +remoteLicenseCheckerPath="https://github.com/dubbogo/resources/raw/master/tools/license" +remoteLicenseCheckerName="license-header-checker" +remoteLicenseCheckerURL="${remoteLicenseCheckerPath}/${remoteLicenseCheckerName}" +remoteLicenseName="license.txt" +remoteLicenseURL="${remoteLicenseCheckerPath}/${remoteLicenseName}" + +licensePath="/tmp/tools/license" +mkdir -p ${licensePath} +wget -P "${licensePath}" ${remoteLicenseCheckerURL} +wget -P "${licensePath}" ${remoteLicenseURL} diff --git a/binary_test.go b/binary_test.go index dafda34d..c145e130 100644 --- a/binary_test.go +++ b/binary_test.go @@ -20,8 +20,6 @@ package hessian import ( "bytes" "fmt" - - // "fmt" "testing" ) diff --git a/decode.go b/decode.go index c5aafac9..f10456c8 100644 --- a/decode.go +++ b/decode.go @@ -50,15 +50,54 @@ func NewDecoder(b []byte) *Decoder { return &Decoder{reader: bufio.NewReader(bytes.NewReader(b)), typeRefs: &TypeRefs{records: map[string]bool{}}} } +// NewDecoder generate a decoder instance +func NewDecoderSize(b []byte, size int) *Decoder { + return &Decoder{reader: bufio.NewReaderSize(bytes.NewReader(b), size), typeRefs: &TypeRefs{records: map[string]bool{}}} +} + // NewDecoder generate a decoder instance with skip func NewDecoderWithSkip(b []byte) *Decoder { return &Decoder{reader: bufio.NewReader(bytes.NewReader(b)), typeRefs: &TypeRefs{records: map[string]bool{}}, isSkip: true} } +// NewCheapDecoderWithSkip generate a decoder instance with skip, +// only for cache pool, before decode Reset should be called. +// For example, with pooling use, will effectively improve performance +// +// var hessianPool = &sync.Pool{ +// New: func() interface{} { +// return hessian.NewCheapDecoderWithSkip([]byte{}) +// }, +// } +// +// decoder := hessianPool.Get().(*hessian.Decoder) +// fill decode data +// decoder.Reset(data[:]) +// decode anything ... +// hessianPool.Put(decoder) +func NewCheapDecoderWithSkip(b []byte) *Decoder { + return &Decoder{reader: bufio.NewReader(bytes.NewReader(b)), isSkip: true} +} + ///////////////////////////////////////// // utilities ///////////////////////////////////////// +func (d *Decoder) Reset(b []byte) *Decoder { + // reuse reader buf, avoid allocate + d.reader.Reset(bytes.NewReader(b)) + d.typeRefs = &TypeRefs{records: map[string]bool{}} + + if d.refs != nil { + d.refs = nil + } + if d.classInfoList != nil { + d.classInfoList = nil + } + + return d +} + // peek a byte func (d *Decoder) peekByte() byte { return d.peek(1)[0] @@ -150,6 +189,8 @@ func (d *Decoder) Decode() (interface{}, error) { return EnsureInterface(d.DecodeValue()) } +func (d *Decoder) Buffered() int { return d.reader.Buffered() } + // DecodeValue parse hessian data, the return value maybe a reflection value when it's a map, list, object, or ref. func (d *Decoder) DecodeValue() (interface{}, error) { var ( diff --git a/decode_test.go b/decode_test.go index f91aca05..3728fecc 100644 --- a/decode_test.go +++ b/decode_test.go @@ -33,6 +33,7 @@ import ( const ( hessianJar = "test_hessian/target/test_hessian-1.0.0.jar" + testString = "hello, world! 你好，世界！" ) func isFileExist(file string) bool { diff --git a/double.go b/double.go index d70c6821..376a1f41 100644 --- a/double.go +++ b/double.go @@ -20,6 +20,7 @@ package hessian import ( "encoding/binary" "math" + "strconv" ) import ( @@ -62,6 +63,38 @@ END: byte(bits>>32), byte(bits>>24), byte(bits>>16), byte(bits>>8), byte(bits)) } +func encFloat32(b []byte, v float32) []byte { + fv := float32(int32(v)) + if fv == v { + iv := int32(v) + switch iv { + case 0: + return encByte(b, BC_DOUBLE_ZERO) + case 1: + return encByte(b, BC_DOUBLE_ONE) + } + if iv >= -0x80 && iv < 0x80 { + return encByte(b, BC_DOUBLE_BYTE, byte(iv)) + } else if iv >= -0x8000 && iv < 0x8000 { + return encByte(b, BC_DOUBLE_SHORT, byte(iv>>8), byte(iv)) + } + + goto END + } + +END: + if float32(int32(v*1000)) == v*1000 { + iv := int32(v * 1000) + return encByte(b, BC_DOUBLE_MILL, byte(iv>>24), byte(iv>>16), byte(iv>>8), byte(iv)) + } else { + str := strconv.FormatFloat(float64(v), 'f', -1, 32) + d, _ := strconv.ParseFloat(str, 64) + bits := math.Float64bits(d) + return encByte(b, BC_DOUBLE, byte(bits>>56), byte(bits>>48), byte(bits>>40), + byte(bits>>32), byte(bits>>24), byte(bits>>16), byte(bits>>8), byte(bits)) + } +} + ///////////////////////////////////////// // Double ///////////////////////////////////////// diff --git a/double_test.go b/double_test.go index 111124b5..43e93912 100644 --- a/double_test.go +++ b/double_test.go @@ -42,6 +42,33 @@ func TestEncDouble(t *testing.T) { t.Logf("decode(%v) = %v, %v\n", v, res, err) } +func TestIssue181(t *testing.T) { + var ( + v float32 + err error + e *Encoder + d *Decoder + res interface{} + ) + + e = NewEncoder() + v = 99.8 + e.Encode(v) + if len(e.Buffer()) == 0 { + t.Fail() + } + + // res would be '99.800003' without patches in PR #196 + d = NewDecoder(e.Buffer()) + res, err = d.Decode() + f := res.(float64) + if float32(f) != v { + t.Errorf("decode(%v) = %v, %v\n", v, res, err) + return + } + t.Logf("decode(%v) = %v\n", v, res) +} + func TestDouble(t *testing.T) { testDecodeFramework(t, "replyDouble_0_0", 0.0) testDecodeFramework(t, "replyDouble_0_001", 0.001) diff --git a/encode.go b/encode.go index fbeae99c..ad9d5c69 100644 --- a/encode.go +++ b/encode.go @@ -112,7 +112,7 @@ func (e *Encoder) Encode(v interface{}) error { } case float32: - e.buffer = encFloat(e.buffer, float64(val)) + e.buffer = encFloat32(e.buffer, val) case float64: e.buffer = encFloat(e.buffer, val) diff --git a/go.mod b/go.mod index 8497c835..fb0a269b 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/apache/dubbo-go-hessian2 require ( - github.com/dubbogo/gost v1.5.1 - github.com/pkg/errors v0.8.1 + github.com/dubbogo/gost v1.9.0 + github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.4.0 ) diff --git a/go.sum b/go.sum index 6c93958e..f799fe33 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,11 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dubbogo/gost v1.5.1 h1:oG5dzaWf1KYynBaBoUIOkgT+YD0niHV6xxI0Odq7hDg= -github.com/dubbogo/gost v1.5.1/go.mod h1:pPTjVyoJan3aPxBPNUX0ADkXjPibLo+/Ib0/fADXSG8= +github.com/dubbogo/gost v1.9.0 h1:UT+dWwvLyJiDotxJERO75jB3Yxgsdy10KztR5ycxRAk= +github.com/dubbogo/gost v1.9.0/go.mod h1:pPTjVyoJan3aPxBPNUX0ADkXjPibLo+/Ib0/fADXSG8= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/int_test.go b/int_test.go index 726a11fa..082f3991 100644 --- a/int_test.go +++ b/int_test.go @@ -18,10 +18,13 @@ package hessian import ( - "github.com/stretchr/testify/assert" "testing" ) +import ( + "github.com/stretchr/testify/assert" +) + func TestEncInt32Len1B(t *testing.T) { var ( v int32 diff --git a/object.go b/object.go index 4025dace..e2ef7b3f 100644 --- a/object.go +++ b/object.go @@ -21,6 +21,7 @@ import ( "io" "reflect" "strings" + "sync" ) import ( @@ -164,21 +165,23 @@ func (e *Encoder) encObject(v POJO) error { structs := []reflect.Value{vv} for len(structs) > 0 { vv := structs[0] + vvt := vv.Type() num = vv.NumField() for i = 0; i < num; i++ { + tf := vvt.Field(i) // skip unexported anonymous field - if vv.Type().Field(i).PkgPath != "" { + if tf.PkgPath != "" { continue } // skip ignored field - if tag, _ := vv.Type().Field(i).Tag.Lookup(tagIdentifier); tag == `-` { + if tag, _ := tf.Tag.Lookup(tagIdentifier); tag == `-` { continue } field := vv.Field(i) - if vv.Type().Field(i).Anonymous && field.Kind() == reflect.Struct { - structs = append(structs, vv.Field(i)) + if tf.Anonymous && field.Kind() == reflect.Struct { + structs = append(structs, field) continue } @@ -292,36 +295,71 @@ func (d *Decoder) decClassDef() (interface{}, error) { return classInfo{javaName: clsName, fieldNameList: fieldList}, nil } -func findField(name string, typ reflect.Type) ([]int, error) { +type fieldInfo struct { + indexes []int + field *reflect.StructField +} + +// map[rType][fieldName]indexes +var fieldIndexCache sync.Map + +func findFieldWithCache(name string, typ reflect.Type) ([]int, *reflect.StructField, error) { + typCache, _ := fieldIndexCache.Load(typ) + if typCache == nil { + typCache = &sync.Map{} + fieldIndexCache.Store(typ, typCache) + } + + iindexes, existCache := typCache.(*sync.Map).Load(name) + if existCache && iindexes != nil { + finfo := iindexes.(*fieldInfo) + var err error + if len(finfo.indexes) == 0 { + err = perrors.Errorf("failed to find field %s", name) + } + return finfo.indexes, finfo.field, err + } + + indexes, field, err := findField(name, typ) + typCache.(*sync.Map).Store(name, &fieldInfo{indexes: indexes, field: field}) + return indexes, field, err +} + +// findField find structField in rType +// +// return +// indexes []int +// field reflect.StructField +// err error +func findField(name string, typ reflect.Type) ([]int, *reflect.StructField, error) { for i := 0; i < typ.NumField(); i++ { // matching tag first, then lowerCamelCase, SameCase, lowerCase typField := typ.Field(i) - if val, has := typField.Tag.Lookup(tagIdentifier); has && strings.Compare(val, name) == 0 { - return []int{i}, nil - } + tagVal, hasTag := typField.Tag.Lookup(tagIdentifier) fieldName := typField.Name - switch { - case strings.Compare(lowerCamelCase(fieldName), name) == 0: - return []int{i}, nil - case strings.Compare(fieldName, name) == 0: - return []int{i}, nil - case strings.Compare(strings.ToLower(fieldName), name) == 0: - return []int{i}, nil + if hasTag && tagVal == name || + fieldName == name || + lowerCamelCase(fieldName) == name || + strings.ToLower(fieldName) == name { + + return []int{i}, &typField, nil } if typField.Anonymous && typField.Type.Kind() == reflect.Struct { - next, _ := findField(name, typField.Type) + next, field, _ := findField(name, typField.Type) if len(next) > 0 { - pos := []int{i} - return append(pos, next...), nil + indexes := []int{i} + indexes = append(indexes, next...) + + return indexes, field, nil } } } - return []int{}, perrors.Errorf("failed to find field %s", name) + return []int{}, nil, perrors.Errorf("failed to find field %s", name) } func (d *Decoder) decInstance(typ reflect.Type, cls classInfo) (interface{}, error) { @@ -337,13 +375,14 @@ func (d *Decoder) decInstance(typ reflect.Type, cls classInfo) (interface{}, err for i := 0; i < len(cls.fieldNameList); i++ { fieldName := cls.fieldNameList[i] - index, err := findField(fieldName, typ) + index, fieldStruct, err := findFieldWithCache(fieldName, typ) if err != nil { - return nil, perrors.Errorf("can not find field %s", fieldName) + d.DecodeValue() + continue } // skip unexported anonymous field - if vv.Type().FieldByIndex(index).PkgPath != "" { + if fieldStruct.PkgPath != "" { continue } @@ -357,8 +396,8 @@ func (d *Decoder) decInstance(typ reflect.Type, cls classInfo) (interface{}, err // unpack pointer to enable value setting fldRawValue := UnpackPtrValue(field) - kind := fldTyp.Kind() + switch kind { case reflect.String: str, err := d.decString(TAG_READ) @@ -483,7 +522,7 @@ func (d *Decoder) decInstance(typ reflect.Type, cls classInfo) (interface{}, err } default: - return nil, perrors.Errorf("unknown struct member type: %v %v", kind, typ.Name()+"."+typ.FieldByIndex(index).Name) + return nil, perrors.Errorf("unknown struct member type: %v %v", kind, typ.Name()+"."+fieldStruct.Name) } } // end for diff --git a/object_test.go b/object_test.go index 2484d07a..db665961 100644 --- a/object_test.go +++ b/object_test.go @@ -18,9 +18,15 @@ package hessian import ( + "encoding/json" "math" "reflect" "testing" + "time" +) + +import ( + "github.com/stretchr/testify/assert" ) type Department struct { @@ -659,3 +665,106 @@ func TestIssue150_EmbedStructJavaDecode(t *testing.T) { testJavaDecode(t, "customArgTypedFixed_Extends", dog) } + +type Mix struct { + A int + B string + CA time.Time + CB int64 + CC string + CD []float64 + D map[string]interface{} +} + +func (m Mix) JavaClassName() string { + return `test.mix` +} + +func init() { + RegisterPOJO(new(Mix)) +} + +// +// BenchmarkJsonEncode-8 217354 4799 ns/op 832 B/op 15 allocs/op +func BenchmarkJsonEncode(b *testing.B) { + m := Mix{A: int('a'), B: `hello`} + m.CD = []float64{1, 2, 3} + m.D = map[string]interface{}{`floats`: m.CD, `A`: m.A, `m`: m} + + for i := 0; i < b.N; i++ { + _, err := json.Marshal(&m) + if err != nil { + b.Error(err) + } + } +} + +// +// BenchmarkEncode-8 211452 5560 ns/op 1771 B/op 51 allocs/op +func BenchmarkEncode(b *testing.B) { + m := Mix{A: int('a'), B: `hello`} + m.CD = []float64{1, 2, 3} + m.D = map[string]interface{}{`floats`: m.CD, `A`: m.A, `m`: m} + + for i := 0; i < b.N; i++ { + _, err := encodeTarget(&m) + if err != nil { + b.Error(err) + } + } +} + +// +// BenchmarkJsonDecode-8 123922 8549 ns/op 1776 B/op 51 allocs/op +func BenchmarkJsonDecode(b *testing.B) { + m := Mix{A: int('a'), B: `hello`} + m.CD = []float64{1, 2, 3} + m.D = map[string]interface{}{`floats`: m.CD, `A`: m.A, `m`: m} + bytes, err := json.Marshal(&m) + if err != nil { + b.Error(err) + } + + for i := 0; i < b.N; i++ { + m := &Mix{} + err := json.Unmarshal(bytes, m) + if err != nil { + b.Error(err) + } + } +} + +// +// BenchmarkDecode-8 104196 10924 ns/op 6424 B/op 98 allocs/op +func BenchmarkDecode(b *testing.B) { + m := Mix{A: int('a'), B: `hello`} + m.CD = []float64{1, 2, 3} + m.D = map[string]interface{}{`floats`: m.CD, `A`: m.A, `m`: m} + bytes, err := encodeTarget(&m) + if err != nil { + b.Error(err) + } + + for i := 0; i < b.N; i++ { + d := NewDecoder(bytes) + _, err := d.Decode() + if err != nil { + b.Error(err) + } + } +} + +type Person183 struct { + Name string +} + +func (Person183) JavaClassName() string { + return `test.Person183` +} + +func TestIssue183_DecodeExcessStructField(t *testing.T) { + RegisterPOJO(&Person183{}) + got, err := decodeJavaResponse(`customReplyPerson183`, ``, false) + assert.NoError(t, err) + t.Logf("%T %+v", got, got) +} diff --git a/pojo.go b/pojo.go index 0c9f9aa6..be74bacf 100644 --- a/pojo.go +++ b/pojo.go @@ -120,6 +120,11 @@ func RegisterPOJO(o POJO) int { pojoRegistry.Lock() defer pojoRegistry.Unlock() + if goName, ok := pojoRegistry.j2g[o.JavaClassName()]; ok { + return pojoRegistry.registry[goName].index + } + + // JavaClassName shouldn't equal to goName if _, ok := pojoRegistry.registry[o.JavaClassName()]; ok { return -1 } diff --git a/request.go b/request.go index 807f73f8..020c7897 100644 --- a/request.go +++ b/request.go @@ -337,6 +337,11 @@ func ToMapStringString(origin map[interface{}]interface{}) map[string]string { dest := make(map[string]string) for k, v := range origin { if kv, ok := k.(string); ok { + if v == nil { + dest[kv] = "" + continue + } + if vv, ok := v.(string); ok { dest[kv] = vv } diff --git a/request_test.go b/request_test.go index c64e013c..b5886db3 100644 --- a/request_test.go +++ b/request_test.go @@ -18,6 +18,7 @@ package hessian import ( + "reflect" "testing" "time" ) @@ -80,3 +81,37 @@ func TestDescRegex(t *testing.T) { assert.Equal(t, "[Ljava/lang/String;", results[0]) assert.Equal(t, "[I", results[1]) } + +func TestIssue192(t *testing.T) { + type args struct { + origin map[interface{}]interface{} + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "not null", + args: args{ + origin: map[interface{}]interface{}{ + "1": nil, + "2": "3", + "": "", + }, + }, + want: map[string]string{ + "1": "", + "2": "3", + "": "", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToMapStringString(tt.args.origin); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToMapStringString() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/serialize.go b/serialize.go index bb5860fa..2584d108 100644 --- a/serialize.go +++ b/serialize.go @@ -42,12 +42,12 @@ type Serializer interface { var serializerMap = make(map[string]Serializer, 16) -func SetSerializer(key string, codec Serializer) { - serializerMap[key] = codec +func SetSerializer(javaClassName string, codec Serializer) { + serializerMap[javaClassName] = codec } -func GetSerializer(key string) (Serializer, bool) { - codec, ok := serializerMap[key] +func GetSerializer(javaClassName string) (Serializer, bool) { + codec, ok := serializerMap[javaClassName] return codec, ok } diff --git a/string.go b/string.go index f692438d..a2a24cef 100644 --- a/string.go +++ b/string.go @@ -217,46 +217,46 @@ func encString(b []byte, v string) []byte { // ::= 'S' b1 b0 # string of length 0-65535 // ::= [x00-x1f] # string of length 0-31 // ::= [x30-x34] # string of length 0-1023 -func (d *Decoder) getStringLength(tag byte) (int32, error) { - var ( - err error - buf [2]byte - length int32 - ) +func (d *Decoder) getStringLength(tag byte) (int, error) { + var length int switch { case tag >= BC_STRING_DIRECT && tag <= STRING_DIRECT_MAX: - return int32(tag - 0x00), nil + return int(tag - 0x00), nil case tag >= 0x30 && tag <= 0x33: - _, err = io.ReadFull(d.reader, buf[:1]) + b, err := d.readByte() if err != nil { return -1, perrors.WithStack(err) } - length = int32(tag-0x30)<<8 + int32(buf[0]) + length = int(tag-0x30)<<8 + int(b) return length, nil case tag == BC_STRING_CHUNK || tag == BC_STRING: - _, err = io.ReadFull(d.reader, buf[:2]) + b0, err := d.readByte() + if err != nil { + return -1, perrors.WithStack(err) + } + + b1, err := d.readByte() if err != nil { return -1, perrors.WithStack(err) } - length = int32(buf[0])<<8 + int32(buf[1]) + + length = int(b0)<<8 + int(b1) return length, nil default: - return -1, perrors.WithStack(err) + return -1, perrors.Errorf("string decode: unknown tag %b", tag) } } func (d *Decoder) decString(flag int32) (string, error) { var ( - tag byte - charTotal int32 - last bool - s string - r rune + tag byte + last bool + s string ) if flag != TAG_READ { @@ -311,24 +311,18 @@ func (d *Decoder) decString(flag int32) (string, error) { last = true } - l, err := d.getStringLength(tag) + chunkLen, err := d.getStringLength(tag) if err != nil { return s, perrors.WithStack(err) } - charTotal = l - charCount := 0 - - runeData := make([]rune, charTotal) - runeIndex := 0 - - byteCount := 0 - byteLen := 0 - charLen := 0 + bytesBuf := make([]byte, chunkLen<<2) + offset := 0 for { - if int32(charCount) == charTotal { + if chunkLen <= 0 { if last { - return string(runeData[:runeIndex]), nil + b := bytesBuf[:offset] + return *(*string)(unsafe.Pointer(&b)), nil } b, _ := d.readByte() @@ -343,21 +337,184 @@ func (d *Decoder) decString(flag int32) (string, error) { last = true } - l, err := d.getStringLength(b) + chunkLen, err = d.getStringLength(b) if err != nil { return s, perrors.WithStack(err) } - charTotal += l - bs := make([]rune, charTotal) - copy(bs, runeData) - runeData = bs - + remain, cap := len(bytesBuf)-offset, chunkLen<<2 + if remain < cap { + grow := len(bytesBuf) + cap + bs := make([]byte, grow) + copy(bs, bytesBuf) + bytesBuf = bs + } default: return s, perrors.New("expect string tag") } } - r, charLen, byteLen, err = decodeUcs4Rune(d.reader) + if chunkLen > 0 { + nread, err := d.next(bytesBuf[offset : offset+chunkLen]) + if err != nil { + if err == io.EOF { + break + } + return s, perrors.WithStack(err) + } + + // quickly detect the actual number of bytes + prev, i := offset, offset + len := offset + nread + copied := false + for r, r1 := len-1, len-2; i < len; chunkLen-- { + ch := bytesBuf[offset] + if ch < 0x80 { + i++ + offset++ + } else if (ch & 0xe0) == 0xc0 { + i += 2 + offset += 2 + } else if (ch & 0xf0) == 0xe0 { + // handle the 3-byte right edge + // case: + // 1. Expect 3 bytes, but the current byte is on the right + // 2. Expect 3 bytes, but the current byte is second to last to the right + if i == r { + bytesBuf[i+1], err = d.reader.ReadByte() + if err != nil { + return s, perrors.WithStack(err) + } + bytesBuf[i+2], err = d.reader.ReadByte() + if err != nil { + return s, perrors.WithStack(err) + } + nread += 2 + len += 2 + } else if i == r1 { + bytesBuf[i+2], err = d.reader.ReadByte() + if err != nil { + return s, perrors.WithStack(err) + } + nread++ + len++ + } + + // we detect emoji first + c1 := ((uint32(ch) & 0x0f) << 12) + ((uint32(bytesBuf[i+1]) & 0x3f) << 6) + (uint32(bytesBuf[i+2]) & 0x3f) + if c1 >= 0xD800 && c1 <= 0xDBFF { + + var ( + c2 rune + n2 int + err error + ch0 byte + ) + + // more cache byte available + if i+3 < len { + ch0 = bytesBuf[i+3] + } else { + ch0, err = d.reader.ReadByte() + if err != nil { + return s, perrors.WithStack(err) + } + // update accumulates read bytes, + // because it reads more than thunk bytes + nread++ + len++ + } + + if ch0 < 0x80 { + c2, n2 = rune(ch0), 1 + } else if (ch0 & 0xe0) == 0xc0 { + var ch1 byte + if i+4 < len { + ch1 = bytesBuf[i+4] + } else { + // out of the chunk byte data + bytesBuf[i+4], err = d.reader.ReadByte() + ch1 = bytesBuf[i+4] + nread++ + len++ + } + c2, n2 = rune(((uint32(ch0)&0x1f)<<6)+(uint32(ch1)&0x3f)), 2 + } else if (ch0 & 0xf0) == 0xe0 { + var ch1, ch2 byte + if i+5 < len { + ch1 = bytesBuf[i+4] + ch2 = bytesBuf[i+5] + } else { + ch1, err = d.reader.ReadByte() + if err != nil { + return s, perrors.WithStack(err) + } + ch2, err = d.reader.ReadByte() + len += 2 + nread += 2 + } + c := ((uint32(ch0) & 0x0f) << 12) + ((uint32(ch1) & 0x3f) << 6) + (uint32(ch2) & 0x3f) + c2, n2 = rune(c), 3 + } + + c := rune(c1-0xD800)<<10 + (c2 - 0xDC00) + 0x10000 + n3 := utf8.EncodeRune(bytesBuf[i:], c) + if copied = n3 > 0 && n3 < /** front three byte */ 3+n2; copied { + // We need to move the bytes, + // for example, less bytes after decoding + offset = i + n3 + copy(bytesBuf[offset:], bytesBuf[i+3+n2:len]) + } + + i += n2 + chunkLen-- + } + i += 3 + + // fix read the next byte index + if copied { + copied = false + continue + } + + offset += 3 + } else { + return s, perrors.Errorf("bad utf-8 encoding") + } + } + + if remain := offset - prev - nread; remain > 0 { + if remain == 1 { + ch, err := d.readByte() + if err != nil { + return s, perrors.WithStack(err) + } + bytesBuf[offset-1] = ch + } else { + var err error + if buffed := d.Buffered(); buffed < remain { + // trigger fill data if required + copy(bytesBuf[offset-remain:offset], d.peek(remain)) + _, err = d.reader.Discard(remain) + } else { + // copy remaining bytes. + _, err = d.next(bytesBuf[offset-remain : offset]) + } + + if err != nil { + return s, perrors.WithStack(err) + } + } + } + + // the expected length string has been processed. + if chunkLen <= 0 { + // we need to detect next chunk + continue + } + } + + // decode byte + ch, err := d.readByte() if err != nil { if err == io.EOF { break @@ -365,14 +522,58 @@ func (d *Decoder) decString(flag int32) (string, error) { return s, perrors.WithStack(err) } - runeData[runeIndex] = r - runeIndex++ + if ch < 0x80 { + bytesBuf[offset] = ch + offset++ + } else if (ch & 0xe0) == 0xc0 { + ch1, err := d.readByte() + if err != nil { + return s, perrors.WithStack(err) + } + bytesBuf[offset] = ch + bytesBuf[offset+1] = ch1 + offset += 2 + } else if (ch & 0xf0) == 0xe0 { + var err error + if buffed := d.Buffered(); buffed < 2 { + // trigger fill data if required + copy(bytesBuf[offset+1:offset+3], d.peek(2)) + _, err = d.reader.Discard(2) + } else { + _, err = d.next(bytesBuf[offset+1 : offset+3]) + } + if err != nil { + return s, perrors.WithStack(err) + } - charCount += charLen - byteCount += byteLen + bytesBuf[offset] = ch + + // we detect emoji first + c1 := ((uint32(ch) & 0x0f) << 12) + ((uint32(bytesBuf[offset+1]) & 0x3f) << 6) + (uint32(bytesBuf[offset+2]) & 0x3f) + if c1 >= 0xD800 && c1 <= 0xDBFF { + c2, n2, err := decodeUcs2Rune(d.reader) + if err != nil { + return s, perrors.WithStack(err) + } + + c := rune(c1-0xD800)<<10 + (c2 - 0xDC00) + 0x10000 + utf8.EncodeRune(bytesBuf[offset:], c) + + // update next rune + offset += n2 + chunkLen-- + } + + offset += 3 + } else { + return s, perrors.Errorf("bad utf-8 encoding, offset=%d\n", offset) + } + + chunkLen-- } - return string(runeData[:runeIndex]), nil + b := bytesBuf[:offset] + return *(*string)(unsafe.Pointer(&b)), nil } return s, perrors.Errorf("unknown string tag %#x\n", tag) diff --git a/string_test.go b/string_test.go index 57513321..ae8d5c1c 100644 --- a/string_test.go +++ b/string_test.go @@ -19,6 +19,7 @@ package hessian import ( "fmt" + "sync" "testing" ) @@ -158,6 +159,34 @@ func TestStringEncode(t *testing.T) { testJavaDecode(t, "argString_65536", s65560[:65536]) } +var decodePool = &sync.Pool{ + New: func() interface{} { + return NewCheapDecoderWithSkip([]byte{}) + }, +} + +func TestStringWithPool(t *testing.T) { + e := NewEncoder() + e.Encode(testString) + buf := e.buffer + + for i := 0; i < 3; i++ { + d := decodePool.Get().(*Decoder) + d.Reset(buf) + + v, err := d.Decode() + if err != nil { + t.Errorf("err:%s", err.Error()) + } + if v != testString { + t.Errorf("excpect decode %v, actual %v", testString, v) + } + + decodePool.Put(d) + } + +} + func TestStringEmoji(t *testing.T) { // see: test_hessian/src/main/java/test/TestString.java s0 := "emoji🤣" @@ -166,3 +195,11 @@ func TestStringEmoji(t *testing.T) { testDecodeFramework(t, "customReplyStringEmoji", s0) testJavaDecode(t, "customArgString_emoji", s0) } + +func TestStringComplex(t *testing.T) { + // see: test_hessian/src/main/java/test/TestString.java + s0 := "킐\u0088中国你好!\u0088\u0088\u0088\u0088\u0088\u0088" + + testDecodeFramework(t, "customReplyComplexString", s0) + testJavaDecode(t, "customArgComplexString", s0) +} diff --git a/test_hessian/src/main/java/test/TestCustomDecode.java b/test_hessian/src/main/java/test/TestCustomDecode.java index e2394f5d..1c4c2761 100644 --- a/test_hessian/src/main/java/test/TestCustomDecode.java +++ b/test_hessian/src/main/java/test/TestCustomDecode.java @@ -200,6 +200,11 @@ public Object customArgString_emoji() throws Exception { return TestString.getEmojiTestString().equals(o); } + public Object customArgComplexString() throws Exception { + String o = (String) input.readObject(); + return TestString.getComplexString().equals(o); + } + public Object customArgTypedFixedList_HashSet() throws Exception { HashSet o = (HashSet) input.readObject(); return o.contains(0) && o.contains(1); diff --git a/test_hessian/src/main/java/test/TestCustomReply.java b/test_hessian/src/main/java/test/TestCustomReply.java index 813d7eb9..12fc411c 100644 --- a/test_hessian/src/main/java/test/TestCustomReply.java +++ b/test_hessian/src/main/java/test/TestCustomReply.java @@ -425,6 +425,23 @@ public void customReplyStringEmoji() throws Exception { output.flush(); } + public void customReplyPerson183() throws Exception { + Person183 p = new Person183(); + p.name = "pname"; + p.age = 13; + InnerPerson innerPerson = new InnerPerson(); + innerPerson.name = "pname2"; + innerPerson.age = 132; + p.innerPerson = innerPerson; + output.writeObject(p); + output.flush(); + } + + public void customReplyComplexString() throws Exception { + output.writeObject(TestString.getComplexString()); + output.flush(); + } + public void customReplyExtendClass() throws Exception { Dog dog = new Dog(); dog.name = "a dog"; @@ -486,3 +503,14 @@ class TypedListTest implements Serializable { } } + +class Person183 implements Serializable { + public String name; + public Integer age; + public InnerPerson innerPerson; +} + +class InnerPerson implements Serializable { + public String name; + public Integer age; +} diff --git a/test_hessian/src/main/java/test/TestString.java b/test_hessian/src/main/java/test/TestString.java index 2ffbdf3c..c0267963 100644 --- a/test_hessian/src/main/java/test/TestString.java +++ b/test_hessian/src/main/java/test/TestString.java @@ -29,4 +29,9 @@ public static String getEmojiTestString() { return s + ",max" + maxUnicode; } + + public static String getComplexString() { + String s = "킐\u0088中国你好!\u0088\u0088\u0088\u0088\u0088\u0088"; + return s; + } }