feat: add optional protobuf formatter

Protobuf formatter is written in go and based on the buf formatter. It is installed as Extension and introduces go as dependency. If enabled it will run on top and independent of normalization.

MANIFEST.in

@@ -9,5 +9,6 @@ include setup.py
 include setup.cfg
 include LICENSE
 include MANIFEST.in
+include *.so
 
 recursive-exclude examples *~ *.pyc \.*

README.rst

@@ -468,6 +468,9 @@ Keys to take special care are the ones needed to configure Kafka and advertised_
    * - ``master_election_strategy``
      - ``lowest``
      - Decides on what basis the Karapace cluster master is chosen (only relevant in a multi node setup)
+   * - ``use_protobuf_formatter``
+     - ``false``
+     - If protobuf formatter should be used on protobuf schemas in order to normalize schemas. The formatter is used on top and independent of reglar normalization and schemas will be persisted in a formatted state.
 
 
 Authentication and authorization of Karapace Schema Registry REST API

go/protopace/.gitignore

@@ -0,0 +1,25 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env

go/protopace/Makefile

@@ -0,0 +1,69 @@
+# Change these variables as necessary.
+MAIN_PACKAGE_PATH := .
+BINARY_NAME := protopace
+BUILD_DIR := ../../karapace/protobuf/protopace/bin
+
+# ==================================================================================== #
+# HELPERS
+# ==================================================================================== #
+
+## help: print this help message
+.PHONY: help
+help:
+	@echo 'Usage:'
+	@sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' |  sed -e 's/^/ /'
+
+.PHONY: confirm
+confirm:
+	@echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ]
+
+.PHONY: no-dirty
+no-dirty:
+	git diff --exit-code
+
+
+# ==================================================================================== #
+# QUALITY CONTROL
+# ==================================================================================== #
+
+## format: format code
+.PHONY: format
+format:
+	go fmt ./...
+
+## tidy: format code and tidy modfile
+.PHONY: tidy
+tidy:
+	go fmt ./...
+	go mod tidy -v
+
+## audit: run quality control checks
+.PHONY: audit
+audit:
+	go mod verify
+	go vet ./...
+	go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./...
+	go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+	go test -race -buildvcs -vet=off ./...
+
+
+# ==================================================================================== #
+# DEVELOPMENT
+# ==================================================================================== #
+
+## test: run all tests
+.PHONY: test
+test:
+	go test -v -race -buildvcs ./...
+
+## test/cover: run all tests and display coverage
+.PHONY: test/cover
+test/cover:
+	go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./...
+	go tool cover -html=/tmp/coverage.out
+
+## build: build the application
+.PHONY: build
+build:
+	# Include additional build steps, like TypeScript, SCSS or Tailwind compilation here...
+	go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH}

go/protopace/compatibility.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"context"
+
+	"github.com/Aiven-Open/karapace/go/protopace/schema"
+
+	"github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking"
+	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
+	"github.com/bufbuild/buf/private/pkg/tracing"
+	"go.uber.org/zap"
+)
+
+func Check(schema schema.Schema, previousSchema schema.Schema) error {
+	handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer)
+	ctx := context.Background()
+	image, err := schema.CompileBufImage()
+	if err != nil {
+		return err
+	}
+	previousImage, err := previousSchema.CompileBufImage()
+	if err != nil {
+		return err
+	}
+	checkConfig, _ := bufconfig.NewEnabledCheckConfig(
+		bufconfig.FileVersionV2,
+		nil,
+		[]string{
+			"FIELD_NO_DELETE",
+			"FILE_SAME_PACKAGE",
+			"FIELD_SAME_NAME",
+			"FIELD_SAME_JSON_NAME",
+			"FILE_NO_DELETE",
+			"ENUM_NO_DELETE",
+		},
+		nil,
+		nil,
+	)
+	config := bufconfig.NewBreakingConfig(checkConfig, false)
+	return handler.Check(ctx, config, previousImage, image)
+}

go/protopace/compatibility_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	s "github.com/Aiven-Open/karapace/go/protopace/schema"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCompatibility(t *testing.T) {
+	assert := assert.New(t)
+
+	data, _ := os.ReadFile("./fixtures/dependency.proto")
+	dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil)
+	assert.NoError(err)
+	assert.NotNil(dependencySchema)
+
+	data, _ = os.ReadFile("./fixtures/test.proto")
+	testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
+	assert.NoError(err)
+	assert.NotNil(testSchema)
+
+	data, _ = os.ReadFile("./fixtures/test_previous.proto")
+	previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
+	assert.NoError(err)
+	assert.NotNil(previousSchema)
+
+	err = Check(*testSchema, *testSchema)
+	assert.NoError(err)
+
+	err = Check(*testSchema, *previousSchema)
+	assert.ErrorContains(err, "Field \"5\" with name \"foo\" on message \"EventValue\" changed type from \"string\" to \"int32\".")
+}

go/protopace/fixtures/dependency.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+package my.awesome.customer.v1;
+
+message NestedValue {
+    string value = 1;
+}
+
+enum Status {
+    UNKNOWN = 0;
+    ACTIVE = 1;
+    INACTIVE = 2;
+}

go/protopace/fixtures/test.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package my.awesome.customer.v1;
+
+import "my/awesome/customer/v1/nested_value.proto";
+import "google/protobuf/timestamp.proto";
+
+option ruby_package = "My::Awesome::Customer::V1";
+option csharp_namespace = "my.awesome.customer.V1";
+option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
+option java_multiple_files = true;
+option java_outer_classname = "EventValueProto";
+option java_package = "com.my.awesome.customer.v1";
+option objc_class_prefix = "TDD";
+option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
+option php_namespace = "My\\Awesome\\Customer\\V1";
+
+message Local {
+    message NestedValue {
+        string foo = 1;
+    }
+}
+
+message EventValue {
+  NestedValue nested_value = 1;
+  google.protobuf.Timestamp created_at = 2;
+  Status status = 3;
+  Local.NestedValue local_nested_value = 4;
+  int32 foo = 5;
+}

go/protopace/fixtures/test_previous.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package my.awesome.customer.v1;
+
+import "my/awesome/customer/v1/nested_value.proto";
+import "google/protobuf/timestamp.proto";
+
+option ruby_package = "My::Awesome::Customer::V1";
+option csharp_namespace = "my.awesome.customer.V1";
+option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
+option java_multiple_files = true;
+option java_outer_classname = "EventValueProto";
+option java_package = "com.my.awesome.customer.v1";
+option objc_class_prefix = "TDD";
+option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
+option php_namespace = "My\\Awesome\\Customer\\V1";
+
+message Local {
+    message NestedValue {
+        string foo = 1;
+    }
+}
+
+message EventValue {
+  NestedValue nested_value = 1;
+  google.protobuf.Timestamp created_at = 2;
+  Status status = 3;
+  Local.NestedValue local_nested_value = 4;
+  string foo = 5;
+}