Merge pull request #116 from heetch/sixstone-003/upgrade-libs

Upgrade actgardner/gogen-avro library

cmd/avsc2avdl/main.go

@@ -233,6 +233,8 @@ func (g *generator) writeMetadata(d interface{}, indent string) {
 	if m.doc != "" {
 		g.printf("%s", indent)
 		if strings.Contains(m.doc, "\n") {
+			// This is dead-code by now https://github.com/actgardner/gogen-avro/pull/177
+			// as gogen-avro removes the whitespaces.
 			// idl2schemata strips leading and trailing whitespace,
 			// so put some back again if it's a multiline comment.
 			g.printf("/**\n%s %s\n%s */\n", indent, m.doc, indent)

cmd/avsc2avdl/testdata/simple.txt

@@ -18,7 +18,7 @@ cmp stdout expect.avdl
   }, {
     "name" : "fstrWithDefault",
     "type" : "string",
-    "doc" : "* doc comment on field with\n\t\t * newline",
+    "doc" : "* doc comment on field with\n * newline",
     "default" : ""
   }, {
     "name" : "fStrNoDefault",
@@ -90,10 +90,7 @@ protocol _ {
 		/** doc comment on field */
 		@fieldMetadata(245)
 		map<string> fmap;
-		/**
-		 * doc comment on field with
-		 * newline
-		 */
+		/** * doc comment on field with  * newline */
 		string fstrWithDefault = "";
 		string fStrNoDefault = "";
 		@logicalType("timestamp-micros")

go.mod

@@ -3,11 +3,11 @@ module github.com/heetch/avro
 go 1.15
 
 require (
-	github.com/actgardner/gogen-avro/v10 v10.1.0
+	github.com/actgardner/gogen-avro/v10 v10.2.1
 	github.com/frankban/quicktest v1.14.0
 	github.com/google/uuid v1.3.0
 	github.com/kr/pretty v0.3.0
-	github.com/linkedin/goavro/v2 v2.10.1
+	github.com/linkedin/goavro/v2 v2.11.1
 	github.com/rogpeppe/go-internal v1.8.0
 	gopkg.in/httprequest.v1 v1.2.1
 	gopkg.in/retry.v1 v1.0.3

go.sum

@@ -1,9 +1,5 @@
-github.com/actgardner/gogen-avro/v10 v10.1.0 h1:AetfQINMHgGiycb+jHqiP/+dc/Tz2McS+cqMKEesxZU=
-github.com/actgardner/gogen-avro/v10 v10.1.0/go.mod h1:o+ybmVjEa27AAr35FRqU98DJu1fXES56uXniYFv4yDA=
-github.com/actgardner/gogen-avro/v9 v9.1.0/go.mod h1:nyTj6wPqDJoxM3qdnjcLv+EnMDSDFqE0qDpva2QRmKc=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/actgardner/gogen-avro/v10 v10.2.1 h1:z3pOGblRjAJCYpkIJ8CmbMJdksi4rAhaygw0dyXZ930=
+github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -14,21 +10,15 @@ github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq
 github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
 github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8=
-github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/juju/qthttptest v0.1.1 h1:JPju5P5CDMCy8jmBJV2wGLjDItUsx2KKL514EfOYueM=
 github.com/juju/qthttptest v0.1.1/go.mod h1:aTlAv8TYaflIiTDIQYzxnl1QdPjAg8Q8qJMErpKy6A4=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
@@ -41,16 +31,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/linkedin/goavro v2.1.0+incompatible h1:DV2aUlj2xZiuxQyvag8Dy7zjY69ENjS66bWkSfdpddY=
-github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM=
-github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
-github.com/linkedin/goavro/v2 v2.10.1 h1:ExVurHDnf0eyUocILs48kiZ4pGvaEbDvBOQcfLruA/0=
-github.com/linkedin/goavro/v2 v2.10.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g=
+github.com/linkedin/goavro/v2 v2.11.1 h1:4cuAtbDfqkKnBXp9E+tRkIJGa6W6iAjwonwt8O1f4U0=
+github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -61,9 +43,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -78,15 +59,13 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

singledecoder_test.go

@@ -11,9 +11,11 @@ import (
 )
 
 //go:generate avrogo testschema1.avsc
+//go:generate avrogo testschema2.avsc
 
-func TestSingleDecoder(t *testing.T) {
+func TestSingleDecoder_CompabilitySameRecord(t *testing.T) {
 	c := qt.New(t)
+	ctx := context.Background()
 	dec := avro.NewSingleDecoder(memRegistry{
 		1: mustParseType(`{
 	"name": "TestRecord",
@@ -28,6 +30,17 @@ func TestSingleDecoder(t *testing.T) {
 		"type": {
 		    "type": "int"
 		}
+	}, {
+		"name": "C",
+		"type": [
+                    "null",
+                    {
+                       "name": "EnumC",
+                       "type": "enum",
+                       "symbols": ["x", "y", "z"]
+                    }
+                ],
+		"default": "null"
 	}]
 }`),
 		2: mustParseType(`{
@@ -69,23 +82,86 @@ func TestSingleDecoder(t *testing.T) {
 	// 	1: the schema id
 	//	40: B=20 (zig-zag encoded)
 	//	80: A=40 (ditto)
-	_, err = dec.Unmarshal(context.Background(), []byte{1, 40, 80}, &x)
+	//       0: C=null Choose null value
+	_, err = dec.Unmarshal(ctx, []byte{1, 40, 80, 0}, &x)
 	c.Assert(err, qt.Equals, nil)
-	c.Assert(x, qt.Equals, TestRecord{A: 40, B: 20})
+	c.Assert(x, qt.DeepEquals, TestRecord{A: 40, B: 20})
 
 	// Check the record compatibility stuff is working by reading from a
 	// record written with less fields (note: the default value for A is 42).
 	var x1 TestRecord
-	_, err = dec.Unmarshal(context.Background(), []byte{2, 80}, &x1)
+	_, err = dec.Unmarshal(ctx, []byte{2, 80}, &x1)
 	c.Assert(err, qt.Equals, nil)
 	c.Assert(x1, qt.Equals, TestRecord{A: 42, B: 40})
 
 	// There's no default value for A, so it doesn't work that way around.
 	var x2 TestRecord
-	_, err = dec.Unmarshal(context.Background(), []byte{3, 80}, &x2)
+	_, err = dec.Unmarshal(ctx, []byte{3, 80}, &x2)
 	c.Assert(err, qt.ErrorMatches, `cannot unmarshal: cannot create decoder: Incompatible schemas: field B in reader is not present in writer and has no default value`)
 }
 
+func TestSingleDecoder_CompabilityDifferentRecord(t *testing.T) {
+	c := qt.New(t)
+	ctx := context.Background()
+	dec := avro.NewSingleDecoder(memRegistry{
+		1: mustParseType(`{
+	"name": "TestRecord",
+	"type": "record",
+	"fields": [{
+		"name": "B",
+		"type": {
+		    "type": "int"
+		}
+	}, {
+		"name": "A",
+		"type": {
+		    "type": "int"
+		}
+	}]
+}`),
+		2: mustParseType(`{
+	"name": "TestRecord",
+	"type": "record",
+	"fields": [{
+		"name": "B",
+		"type": {
+		    "type": "int"
+		}
+	}, {
+		"name": "A",
+		"type": {
+		    "type": "int"
+		}
+	}, {
+		"name": "C",
+		"type": [
+                    "null",
+                    {
+                       "name": "EnumC",
+                       "type": "enum",
+                       "symbols": ["x", "y", "z"]
+                    }
+                ],
+		"default": "null"
+	}]
+}`),
+	}, nil)
+	cVal := EnumCY
+	data, _, err := avro.Marshal(TestNewRecord{A: 40, B: 20, C: &cVal})
+	c.Assert(err, qt.IsNil)
+	c.Logf("data: %d", data)
+	var x TestRecord
+	// In the byte slice below:
+	// 	1: the schema id
+	//	40: B=20 (zig-zag encoded)
+	//	80: A=40 (ditto)
+	//       2: C=Choose EnumC type
+	//       2  C="y"
+	_, err = dec.Unmarshal(ctx, []byte{2, 40, 80, 2, 2}, &x)
+	c.Assert(err, qt.IsNil)
+	c.Assert(x, qt.Equals, TestRecord{A: 40, B: 20})
+}
+
 // memRegistry implements DecodingRegistry and EncodingRegistry by associating a single-byte
 // schema ID with schemas.
 type memRegistry map[int64]*avro.Type

testschema2.avsc

@@ -0,0 +1,27 @@
+{
+	"name": "TestNewRecord",
+	"type": "record",
+	"fields": [{
+		"name": "A",
+		"type": {
+		    "type": "int"
+		},
+		"default": 42
+	}, {
+		"name": "B",
+		"type": {
+		    "type": "int"
+		}
+	}, {
+		"name": "C",
+		"type": [
+                    "null",
+                    {
+                       "name": "EnumC",
+                       "type": "enum",
+                       "symbols": ["x", "y", "z"]
+                    }
+                ],
+		"default": null
+	}]
+}