Add Avro schema builder

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/golangci/golangci-lint v1.59.1
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
+	github.com/hamba/avro/v2 v2.22.1
 	github.com/matryer/is v1.4.1
 	github.com/mitchellh/mapstructure v1.5.0
 	go.uber.org/goleak v1.3.0
@@ -127,6 +128,7 @@ require (
 	github.com/jingyugao/rowserrcheck v1.1.1 // indirect
 	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
 	github.com/jjti/go-spancheck v0.6.1 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/julz/importas v0.1.0 // indirect
 	github.com/karamaru-alpha/copyloopvar v1.1.0 // indirect
 	github.com/kisielk/errcheck v1.7.0 // indirect
@@ -154,6 +156,8 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/term v0.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/moricho/tparallel v0.3.1 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nakabonne/nestif v0.3.1 // indirect

go.sum

@@ -369,6 +369,8 @@ github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoIS
 github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
+github.com/hamba/avro/v2 v2.22.1 h1:q1rAbfJsrbMaZPDLQvwUQMfQzp6H+hGXvckmU/lXemk=
+github.com/hamba/avro/v2 v2.22.1/go.mod h1:HOeTrE3kvWnBAgsufqhAzDDV5gvS0QXs65Z6BHfGgbg=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
@@ -400,6 +402,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -480,9 +483,11 @@ github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6U
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA=
 github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI=

schema/avro_builder.go

@@ -0,0 +1,91 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/hamba/avro/v2"
+)
+
+// Builder builds avro.RecordSchema instances and marshals them into JSON.
+// Builder accepts arguments for creating fields and creates them internally
+// (i.e. a user doesn't need to create the fields).
+// All errors will be returned as a joined error when marshaling the schema to JSON.
+type Builder struct {
+	errs      error
+	fields    []*avro.Field
+	name      string
+	namespace string
+}
+
+// NewBuilder constructs a new Builder and initializes it
+// with the given name and namespace.
+func NewBuilder(name, namespace string) *Builder {
+	return &Builder{
+		name:      name,
+		namespace: namespace,
+	}
+}
+
+// AddField adds a new field with the given name, schema and schema options.
+// If creating the field returns an error, the error is saved, joined with
+// other errors (if any), and returned when marshaling to JSON.
+func (b *Builder) AddField(name string, typ avro.Schema, opts ...avro.SchemaOption) *Builder {
+	f, err := avro.NewField(name, typ, opts...)
+	if err != nil {
+		b.errs = errors.Join(b.errs, fmt.Errorf("field %v: %w", name, err))
+	} else {
+		b.fields = append(b.fields, f)
+	}
+
+	return b
+}
+
+// Build builds the underlying schema.
+// Errors that occurred while creating fields or constructing
+// the schema will be returned as a joined error.
+func (b *Builder) Build() (*avro.RecordSchema, error) {
+	if b.errs != nil {
+		return nil, b.errs
+	}
+
+	schema, err := avro.NewRecordSchema(b.name, b.namespace, b.fields)
+	if err != nil {
+		return nil, fmt.Errorf("failed building schema: %w", err)
+	}
+	return schema, nil
+}
+
+// MarshalJSON marshals the underlying schema to JSON.
+// Errors that occurred while creating fields, constructing
+// the schema or marshaling it will be returned as a joined error.
+func (b *Builder) MarshalJSON() ([]byte, error) {
+	if b.errs != nil {
+		return nil, b.errs
+	}
+
+	schema, err := avro.NewRecordSchema(b.name, b.namespace, b.fields)
+	if err != nil {
+		return nil, fmt.Errorf("failed building schema: %w", err)
+	}
+
+	bytes, err := schema.MarshalJSON()
+	if err != nil {
+		return nil, fmt.Errorf("failed marshaling schema to JSON: %w", err)
+	}
+	return bytes, nil
+}

schema/avro_builder_example_test.go

@@ -0,0 +1,78 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+	"fmt"
+
+	"github.com/goccy/go-json"
+	"github.com/hamba/avro/v2"
+)
+
+//nolint:govet // example usage of multiple functions, not only one
+func ExampleAvroBuilder() {
+	enumSchema, err := avro.NewEnumSchema("enum_schema", "enum_namespace", []string{"val1", "val2", "val3"})
+	if err != nil {
+		panic(err)
+	}
+	bytes, err := NewBuilder("schema_name", "schema_namespace").
+		AddField("int_field", avro.NewPrimitiveSchema(avro.Int, nil), avro.WithDefault(100)).
+		AddField("enum_field", enumSchema).
+		MarshalJSON()
+	if err != nil {
+		panic(err)
+	}
+
+	prettyPrint(bytes)
+	// Output:
+	// {
+	//   "fields": [
+	//     {
+	//       "default": 100,
+	//       "name": "int_field",
+	//       "type": "int"
+	//     },
+	//     {
+	//       "name": "enum_field",
+	//       "type": {
+	//         "name": "enum_namespace.enum_schema",
+	//         "symbols": [
+	//           "val1",
+	//           "val2",
+	//           "val3"
+	//         ],
+	//         "type": "enum"
+	//       }
+	//     }
+	//   ],
+	//   "name": "schema_namespace.schema_name",
+	//   "type": "record"
+	// }
+}
+
+func prettyPrint(bytes []byte) {
+	m := map[string]interface{}{}
+	err := json.Unmarshal(bytes, &m)
+	if err != nil {
+		panic(err)
+	}
+
+	pretty, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println(string(pretty))
+}

schema/avro_builder_test.go

@@ -0,0 +1,53 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+	"testing"
+
+	"github.com/hamba/avro/v2"
+	"github.com/matryer/is"
+)
+
+func TestAvroBuilder_Build(t *testing.T) {
+	is := is.New(t)
+
+	enumSchema, err := avro.NewEnumSchema("enum_schema", "enum_namespace", []string{"val1", "val2", "val3"})
+	is.NoErr(err)
+
+	idField, err := avro.NewField("int_field", avro.NewPrimitiveSchema(avro.Int, nil), avro.WithDefault(100))
+	is.NoErr(err)
+
+	enumField, err := avro.NewField("enum_field", enumSchema)
+	is.NoErr(err)
+
+	wantSchema, err := avro.NewRecordSchema(
+		"schema_name",
+		"schema_namespace",
+		[]*avro.Field{idField, enumField},
+	)
+	is.NoErr(err)
+
+	want, err := wantSchema.MarshalJSON()
+	is.NoErr(err)
+
+	got, err := NewBuilder("schema_name", "schema_namespace").
+		AddField("int_field", avro.NewPrimitiveSchema(avro.Int, nil), avro.WithDefault(100)).
+		AddField("enum_field", enumSchema).
+		MarshalJSON()
+	is.NoErr(err)
+
+	is.Equal(want, got)
+}