diff --git a/go.mod b/go.mod index 1fe84f3..b91534c 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/golangci/golangci-lint v1.59.1 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 + github.com/hamba/avro/v2 v2.22.1 github.com/matryer/is v1.4.1 github.com/mitchellh/mapstructure v1.5.0 go.uber.org/goleak v1.3.0 @@ -128,6 +129,7 @@ require ( github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect github.com/jjti/go-spancheck v0.6.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/julz/importas v0.1.0 // indirect github.com/karamaru-alpha/copyloopvar v1.1.0 // indirect github.com/kisielk/errcheck v1.7.0 // indirect @@ -155,6 +157,8 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/moricho/tparallel v0.3.1 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/nakabonne/nestif v0.3.1 // indirect diff --git a/go.sum b/go.sum index 06f8c64..e2c7d08 100644 --- a/go.sum +++ b/go.sum @@ -367,6 +367,8 @@ github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoIS github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/hamba/avro/v2 v2.22.1 h1:q1rAbfJsrbMaZPDLQvwUQMfQzp6H+hGXvckmU/lXemk= +github.com/hamba/avro/v2 v2.22.1/go.mod h1:HOeTrE3kvWnBAgsufqhAzDDV5gvS0QXs65Z6BHfGgbg= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -398,6 +400,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -478,9 +481,11 @@ github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6U github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA= github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI= diff --git a/schema/avro_builder.go b/schema/avro_builder.go new file mode 100644 index 0000000..7657af2 --- /dev/null +++ b/schema/avro_builder.go @@ -0,0 +1,88 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import ( + "errors" + "fmt" + + "github.com/hamba/avro/v2" +) + +// AvroBuilder builds avro.RecordSchema instances and marshals them into JSON. +// AvroBuilder accepts arguments for creating fields and creates them internally +// (i.e. a user doesn't need to create the fields). +// All errors will be returned as a joined error when marshaling the schema to JSON. +type AvroBuilder struct { + errs []error + fields []*avro.Field + name string + namespace string +} + +// NewAvroBuilder constructs a new AvroBuilder and initializes it +// with the given name and namespace. +func NewAvroBuilder(name, namespace string) *AvroBuilder { + return &AvroBuilder{ + name: name, + namespace: namespace, + } +} + +// AddField adds a new field with the given name, schema and schema options. +// If creating the field returns an error, the error is saved, joined with +// other errors (if any), and returned when marshaling to JSON. +func (b *AvroBuilder) AddField(name string, typ avro.Schema, opts ...avro.SchemaOption) *AvroBuilder { + f, err := avro.NewField(name, typ, opts...) + if err != nil { + b.errs = append(b.errs, fmt.Errorf("field %v: %w", name, err)) + } else { + b.fields = append(b.fields, f) + } + + return b +} + +// Build builds the underlying schema. +// Errors that occurred while creating fields or constructing +// the schema will be returned as a joined error. +func (b *AvroBuilder) Build() (*avro.RecordSchema, error) { + if b.errs != nil { + return nil, errors.Join(b.errs...) + } + + schema, err := avro.NewRecordSchema(b.name, b.namespace, b.fields) + if err != nil { + return nil, fmt.Errorf("failed building schema: %w", err) + } + + return schema, nil +} + +// MarshalJSON marshals the underlying schema to JSON. +// Errors that occurred while creating fields, constructing +// the schema or marshaling it will be returned as a joined error. +func (b *AvroBuilder) MarshalJSON() ([]byte, error) { + schema, err := b.Build() + if err != nil { + return nil, err + } + + bytes, err := schema.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed marshaling schema to JSON: %w", err) + } + return bytes, nil +} diff --git a/schema/avro_builder_example_test.go b/schema/avro_builder_example_test.go new file mode 100644 index 0000000..dd2676f --- /dev/null +++ b/schema/avro_builder_example_test.go @@ -0,0 +1,77 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import ( + "fmt" + + "github.com/goccy/go-json" + "github.com/hamba/avro/v2" +) + +func ExampleAvroBuilder() { + enumSchema, err := avro.NewEnumSchema("enum_schema", "enum_namespace", []string{"val1", "val2", "val3"}) + if err != nil { + panic(err) + } + bytes, err := NewAvroBuilder("schema_name", "schema_namespace"). + AddField("int_field", avro.NewPrimitiveSchema(avro.Int, nil), avro.WithDefault(100)). + AddField("enum_field", enumSchema). + MarshalJSON() + if err != nil { + panic(err) + } + + prettyPrint(bytes) + // Output: + // { + // "fields": [ + // { + // "default": 100, + // "name": "int_field", + // "type": "int" + // }, + // { + // "name": "enum_field", + // "type": { + // "name": "enum_namespace.enum_schema", + // "symbols": [ + // "val1", + // "val2", + // "val3" + // ], + // "type": "enum" + // } + // } + // ], + // "name": "schema_namespace.schema_name", + // "type": "record" + // } +} + +func prettyPrint(bytes []byte) { + m := map[string]interface{}{} + err := json.Unmarshal(bytes, &m) + if err != nil { + panic(err) + } + + pretty, err := json.MarshalIndent(m, "", " ") + if err != nil { + panic(err) + } + + fmt.Println(string(pretty)) +} diff --git a/schema/avro_builder_test.go b/schema/avro_builder_test.go new file mode 100644 index 0000000..6628322 --- /dev/null +++ b/schema/avro_builder_test.go @@ -0,0 +1,53 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import ( + "testing" + + "github.com/hamba/avro/v2" + "github.com/matryer/is" +) + +func TestAvroBuilder_Build(t *testing.T) { + is := is.New(t) + + enumSchema, err := avro.NewEnumSchema("enum_schema", "enum_namespace", []string{"val1", "val2", "val3"}) + is.NoErr(err) + + idField, err := avro.NewField("int_field", avro.NewPrimitiveSchema(avro.Int, nil), avro.WithDefault(100)) + is.NoErr(err) + + enumField, err := avro.NewField("enum_field", enumSchema) + is.NoErr(err) + + wantSchema, err := avro.NewRecordSchema( + "schema_name", + "schema_namespace", + []*avro.Field{idField, enumField}, + ) + is.NoErr(err) + + want, err := wantSchema.MarshalJSON() + is.NoErr(err) + + got, err := NewAvroBuilder("schema_name", "schema_namespace"). + AddField("int_field", avro.NewPrimitiveSchema(avro.Int, nil), avro.WithDefault(100)). + AddField("enum_field", enumSchema). + MarshalJSON() + is.NoErr(err) + + is.Equal(want, got) +}