diff --git a/cmd/avsc2avdl/main.go b/cmd/avsc2avdl/main.go index 93d115d..e753c5a 100644 --- a/cmd/avsc2avdl/main.go +++ b/cmd/avsc2avdl/main.go @@ -233,6 +233,8 @@ func (g *generator) writeMetadata(d interface{}, indent string) { if m.doc != "" { g.printf("%s", indent) if strings.Contains(m.doc, "\n") { + // This is dead-code by now https://github.com/actgardner/gogen-avro/pull/177 + // as gogen-avro removes the whitespaces. // idl2schemata strips leading and trailing whitespace, // so put some back again if it's a multiline comment. g.printf("/**\n%s %s\n%s */\n", indent, m.doc, indent) diff --git a/cmd/avsc2avdl/testdata/simple.txt b/cmd/avsc2avdl/testdata/simple.txt index 3701b13..c2790b4 100644 --- a/cmd/avsc2avdl/testdata/simple.txt +++ b/cmd/avsc2avdl/testdata/simple.txt @@ -18,7 +18,7 @@ cmp stdout expect.avdl }, { "name" : "fstrWithDefault", "type" : "string", - "doc" : "* doc comment on field with\n\t\t * newline", + "doc" : "* doc comment on field with\n * newline", "default" : "" }, { "name" : "fStrNoDefault", @@ -90,10 +90,7 @@ protocol _ { /** doc comment on field */ @fieldMetadata(245) map fmap; - /** - * doc comment on field with - * newline - */ + /** * doc comment on field with * newline */ string fstrWithDefault = ""; string fStrNoDefault = ""; @logicalType("timestamp-micros") diff --git a/go.mod b/go.mod index b9c88b9..ff8269c 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/heetch/avro go 1.15 require ( - github.com/actgardner/gogen-avro/v10 v10.1.0 + github.com/actgardner/gogen-avro/v10 v10.2.1 github.com/frankban/quicktest v1.14.0 github.com/google/uuid v1.3.0 github.com/kr/pretty v0.3.0 - github.com/linkedin/goavro/v2 v2.10.1 + github.com/linkedin/goavro/v2 v2.11.1 github.com/rogpeppe/go-internal v1.8.0 gopkg.in/httprequest.v1 v1.2.1 gopkg.in/retry.v1 v1.0.3 diff --git a/go.sum b/go.sum index ea89213..592f8de 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ -github.com/actgardner/gogen-avro/v10 v10.1.0 h1:AetfQINMHgGiycb+jHqiP/+dc/Tz2McS+cqMKEesxZU= -github.com/actgardner/gogen-avro/v10 v10.1.0/go.mod h1:o+ybmVjEa27AAr35FRqU98DJu1fXES56uXniYFv4yDA= -github.com/actgardner/gogen-avro/v9 v9.1.0/go.mod h1:nyTj6wPqDJoxM3qdnjcLv+EnMDSDFqE0qDpva2QRmKc= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/actgardner/gogen-avro/v10 v10.2.1 h1:z3pOGblRjAJCYpkIJ8CmbMJdksi4rAhaygw0dyXZ930= +github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,7 +10,6 @@ github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -22,13 +17,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8= -github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/juju/qthttptest v0.1.1 h1:JPju5P5CDMCy8jmBJV2wGLjDItUsx2KKL514EfOYueM= github.com/juju/qthttptest v0.1.1/go.mod h1:aTlAv8TYaflIiTDIQYzxnl1QdPjAg8Q8qJMErpKy6A4= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= @@ -41,16 +31,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/linkedin/goavro v2.1.0+incompatible h1:DV2aUlj2xZiuxQyvag8Dy7zjY69ENjS66bWkSfdpddY= -github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= -github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= -github.com/linkedin/goavro/v2 v2.10.1 h1:ExVurHDnf0eyUocILs48kiZ4pGvaEbDvBOQcfLruA/0= -github.com/linkedin/goavro/v2 v2.10.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g= +github.com/linkedin/goavro/v2 v2.11.1 h1:4cuAtbDfqkKnBXp9E+tRkIJGa6W6iAjwonwt8O1f4U0= +github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -61,9 +43,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -78,7 +59,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -86,7 +66,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/singledecoder_test.go b/singledecoder_test.go index 9d93c9a..5073977 100644 --- a/singledecoder_test.go +++ b/singledecoder_test.go @@ -11,9 +11,11 @@ import ( ) //go:generate avrogo testschema1.avsc +//go:generate avrogo testschema2.avsc -func TestSingleDecoder(t *testing.T) { +func TestSingleDecoder_CompabilitySameRecord(t *testing.T) { c := qt.New(t) + ctx := context.Background() dec := avro.NewSingleDecoder(memRegistry{ 1: mustParseType(`{ "name": "TestRecord", @@ -28,6 +30,17 @@ func TestSingleDecoder(t *testing.T) { "type": { "type": "int" } + }, { + "name": "C", + "type": [ + "null", + { + "name": "EnumC", + "type": "enum", + "symbols": ["x", "y", "z"] + } + ], + "default": "null" }] }`), 2: mustParseType(`{ @@ -69,23 +82,86 @@ func TestSingleDecoder(t *testing.T) { // 1: the schema id // 40: B=20 (zig-zag encoded) // 80: A=40 (ditto) - _, err = dec.Unmarshal(context.Background(), []byte{1, 40, 80}, &x) + // 0: C=null Choose null value + _, err = dec.Unmarshal(ctx, []byte{1, 40, 80, 0}, &x) c.Assert(err, qt.Equals, nil) - c.Assert(x, qt.Equals, TestRecord{A: 40, B: 20}) + c.Assert(x, qt.DeepEquals, TestRecord{A: 40, B: 20}) // Check the record compatibility stuff is working by reading from a // record written with less fields (note: the default value for A is 42). var x1 TestRecord - _, err = dec.Unmarshal(context.Background(), []byte{2, 80}, &x1) + _, err = dec.Unmarshal(ctx, []byte{2, 80}, &x1) c.Assert(err, qt.Equals, nil) c.Assert(x1, qt.Equals, TestRecord{A: 42, B: 40}) // There's no default value for A, so it doesn't work that way around. var x2 TestRecord - _, err = dec.Unmarshal(context.Background(), []byte{3, 80}, &x2) + _, err = dec.Unmarshal(ctx, []byte{3, 80}, &x2) c.Assert(err, qt.ErrorMatches, `cannot unmarshal: cannot create decoder: Incompatible schemas: field B in reader is not present in writer and has no default value`) } +func TestSingleDecoder_CompabilityDifferentRecord(t *testing.T) { + c := qt.New(t) + ctx := context.Background() + dec := avro.NewSingleDecoder(memRegistry{ + 1: mustParseType(`{ + "name": "TestRecord", + "type": "record", + "fields": [{ + "name": "B", + "type": { + "type": "int" + } + }, { + "name": "A", + "type": { + "type": "int" + } + }] +}`), + 2: mustParseType(`{ + "name": "TestRecord", + "type": "record", + "fields": [{ + "name": "B", + "type": { + "type": "int" + } + }, { + "name": "A", + "type": { + "type": "int" + } + }, { + "name": "C", + "type": [ + "null", + { + "name": "EnumC", + "type": "enum", + "symbols": ["x", "y", "z"] + } + ], + "default": "null" + }] +}`), + }, nil) + cVal := EnumCY + data, _, err := avro.Marshal(TestNewRecord{A: 40, B: 20, C: &cVal}) + c.Assert(err, qt.IsNil) + c.Logf("data: %d", data) + var x TestRecord + // In the byte slice below: + // 1: the schema id + // 40: B=20 (zig-zag encoded) + // 80: A=40 (ditto) + // 2: C=Choose EnumC type + // 2 C="y" + _, err = dec.Unmarshal(ctx, []byte{2, 40, 80, 2, 2}, &x) + c.Assert(err, qt.IsNil) + c.Assert(x, qt.Equals, TestRecord{A: 40, B: 20}) +} + // memRegistry implements DecodingRegistry and EncodingRegistry by associating a single-byte // schema ID with schemas. type memRegistry map[int64]*avro.Type diff --git a/testschema2.avsc b/testschema2.avsc new file mode 100644 index 0000000..e596878 --- /dev/null +++ b/testschema2.avsc @@ -0,0 +1,27 @@ +{ + "name": "TestNewRecord", + "type": "record", + "fields": [{ + "name": "A", + "type": { + "type": "int" + }, + "default": 42 + }, { + "name": "B", + "type": { + "type": "int" + } + }, { + "name": "C", + "type": [ + "null", + { + "name": "EnumC", + "type": "enum", + "symbols": ["x", "y", "z"] + } + ], + "default": null + }] +} diff --git a/testschema2_gen_test.go b/testschema2_gen_test.go new file mode 100644 index 0000000..c1834de --- /dev/null +++ b/testschema2_gen_test.go @@ -0,0 +1,74 @@ +// Code generated by avrogen. DO NOT EDIT. + +package avro_test + +import ( + "fmt" + "github.com/heetch/avro/avrotypegen" + "strconv" +) + +type EnumC int + +const ( + EnumCX EnumC = iota + EnumCY + EnumCZ +) + +var _EnumC_strings = []string{ + "x", + "y", + "z", +} + +// String returns the textual representation of EnumC. +func (e EnumC) String() string { + if e < 0 || int(e) >= len(_EnumC_strings) { + return "EnumC(" + strconv.FormatInt(int64(e), 10) + ")" + } + return _EnumC_strings[e] +} + +// MarshalText implements encoding.TextMarshaler +// by returning the textual representation of EnumC. +func (e EnumC) MarshalText() ([]byte, error) { + if e < 0 || int(e) >= len(_EnumC_strings) { + return nil, fmt.Errorf("EnumC value %d is out of bounds", e) + } + return []byte(_EnumC_strings[e]), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler +// by expecting the textual representation of EnumC. +func (e *EnumC) UnmarshalText(data []byte) error { + // Note for future: this could be more efficient. + for i, s := range _EnumC_strings { + if string(data) == s { + *e = EnumC(i) + return nil + } + } + return fmt.Errorf("unknown value %q for EnumC", data) +} + +type TestNewRecord struct { + A int + B int + C *EnumC +} + +// AvroRecord implements the avro.AvroRecord interface. +func (TestNewRecord) AvroRecord() avrotypegen.RecordInfo { + return avrotypegen.RecordInfo{ + Schema: `{"fields":[{"default":42,"name":"A","type":{"type":"int"}},{"name":"B","type":{"type":"int"}},{"default":null,"name":"C","type":["null",{"name":"EnumC","symbols":["x","y","z"],"type":"enum"}]}],"name":"TestNewRecord","type":"record"}`, + Required: []bool{ + 1: true, + }, + Defaults: []func() interface{}{ + 0: func() interface{} { + return 42 + }, + }, + } +}