From b5b10f946d11786e74737f25e76b6ec42eee3d0f Mon Sep 17 00:00:00 2001 From: Jonas Keeling Date: Wed, 25 Sep 2024 11:59:42 +0200 Subject: [PATCH 1/2] feat: add optional protobuf formatter Protobuf formatter is written in go and based on the buf formatter. It is installed as Extension and introduces go as dependency. If enabled it will run on top and independent of normalization. --- MANIFEST.in | 1 + README.rst | 3 + go/protopace/.gitignore | 25 + go/protopace/Makefile | 69 + go/protopace/compatibility.go | 41 + go/protopace/compatibility_test.go | 34 + go/protopace/fixtures/dependency.proto | 12 + go/protopace/fixtures/test.proto | 30 + go/protopace/fixtures/test_previous.proto | 30 + go/protopace/formatter.go | 2469 +++++++++++++++++ go/protopace/formatter_test.go | 169 ++ go/protopace/go.mod | 42 + go/protopace/go.sum | 83 + go/protopace/main.go | 86 + go/protopace/schema/compiler.go | 10 + go/protopace/schema/google/type/README.md | 7 + .../schema/google/type/calendar_period.proto | 56 + go/protopace/schema/google/type/color.proto | 174 ++ go/protopace/schema/google/type/date.proto | 52 + .../schema/google/type/datetime.proto | 104 + .../schema/google/type/dayofweek.proto | 50 + go/protopace/schema/google/type/decimal.proto | 95 + go/protopace/schema/google/type/expr.proto | 73 + .../schema/google/type/fraction.proto | 33 + .../schema/google/type/interval.proto | 46 + go/protopace/schema/google/type/latlng.proto | 37 + .../schema/google/type/localized_text.proto | 36 + go/protopace/schema/google/type/money.proto | 42 + go/protopace/schema/google/type/month.proto | 65 + .../schema/google/type/phone_number.proto | 113 + .../schema/google/type/postal_address.proto | 134 + .../schema/google/type/quaternion.proto | 94 + .../schema/google/type/timeofday.proto | 44 + go/protopace/schema/google/type/type.yaml | 40 + go/protopace/schema/google_types.go | 23 + go/protopace/schema/resolver.go | 39 + go/protopace/schema/schema.go | 81 + karapace/config.py | 2 + karapace/dependency.py | 13 + karapace/protobuf/protopace/__init__.py | 6 + karapace/protobuf/protopace/protopace.py | 189 ++ karapace/schema_models.py | 28 +- karapace/schema_registry_apis.py | 3 + requirements/requirements-dev.in | 3 + requirements/requirements-dev.txt | 56 +- requirements/requirements-typing.txt | 14 +- requirements/requirements.in | 4 +- requirements/requirements.txt | 32 +- setup.py | 11 +- tests/integration/conftest.py | 3 +- tests/integration/test_schema_protobuf.py | 21 +- .../protobuf/test_protobuf_normalization.py | 502 +++- 52 files changed, 5292 insertions(+), 137 deletions(-) create mode 100644 go/protopace/.gitignore create mode 100644 go/protopace/Makefile create mode 100644 go/protopace/compatibility.go create mode 100644 go/protopace/compatibility_test.go create mode 100644 go/protopace/fixtures/dependency.proto create mode 100644 go/protopace/fixtures/test.proto create mode 100644 go/protopace/fixtures/test_previous.proto create mode 100644 go/protopace/formatter.go create mode 100644 go/protopace/formatter_test.go create mode 100644 go/protopace/go.mod create mode 100644 go/protopace/go.sum create mode 100644 go/protopace/main.go create mode 100644 go/protopace/schema/compiler.go create mode 100644 go/protopace/schema/google/type/README.md create mode 100644 go/protopace/schema/google/type/calendar_period.proto create mode 100644 go/protopace/schema/google/type/color.proto create mode 100644 go/protopace/schema/google/type/date.proto create mode 100644 go/protopace/schema/google/type/datetime.proto create mode 100644 go/protopace/schema/google/type/dayofweek.proto create mode 100644 go/protopace/schema/google/type/decimal.proto create mode 100644 go/protopace/schema/google/type/expr.proto create mode 100644 go/protopace/schema/google/type/fraction.proto create mode 100644 go/protopace/schema/google/type/interval.proto create mode 100644 go/protopace/schema/google/type/latlng.proto create mode 100644 go/protopace/schema/google/type/localized_text.proto create mode 100644 go/protopace/schema/google/type/money.proto create mode 100644 go/protopace/schema/google/type/month.proto create mode 100644 go/protopace/schema/google/type/phone_number.proto create mode 100644 go/protopace/schema/google/type/postal_address.proto create mode 100644 go/protopace/schema/google/type/quaternion.proto create mode 100644 go/protopace/schema/google/type/timeofday.proto create mode 100644 go/protopace/schema/google/type/type.yaml create mode 100644 go/protopace/schema/google_types.go create mode 100644 go/protopace/schema/resolver.go create mode 100644 go/protopace/schema/schema.go create mode 100644 karapace/protobuf/protopace/__init__.py create mode 100644 karapace/protobuf/protopace/protopace.py diff --git a/MANIFEST.in b/MANIFEST.in index fa15133f4..0b55e5636 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,5 +9,6 @@ include setup.py include setup.cfg include LICENSE include MANIFEST.in +include *.so recursive-exclude examples *~ *.pyc \.* diff --git a/README.rst b/README.rst index 15f9dd52f..650b17165 100644 --- a/README.rst +++ b/README.rst @@ -475,6 +475,9 @@ Keys to take special care are the ones needed to configure Kafka and advertised_ - ``true`` - If enabled, kafka errors which can be retried or custom errors specififed for the service will not be raised, instead, a warning log is emitted. This will denoise issue tracking systems, i.e. sentry + * - ``use_protobuf_formatter`` + - ``false`` + - If protobuf formatter should be used on protobuf schemas in order to normalize schemas. The formatter is used on top and independent of regular normalization and schemas will be persisted in a formatted state. Authentication and authorization of Karapace Schema Registry REST API diff --git a/go/protopace/.gitignore b/go/protopace/.gitignore new file mode 100644 index 000000000..6f72f8926 --- /dev/null +++ b/go/protopace/.gitignore @@ -0,0 +1,25 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env diff --git a/go/protopace/Makefile b/go/protopace/Makefile new file mode 100644 index 000000000..f6acf75db --- /dev/null +++ b/go/protopace/Makefile @@ -0,0 +1,69 @@ +# Change these variables as necessary. +MAIN_PACKAGE_PATH := . +BINARY_NAME := protopace +BUILD_DIR := ../../karapace/protobuf/protopace/bin + +# ==================================================================================== # +# HELPERS +# ==================================================================================== # + +## help: print this help message +.PHONY: help +help: + @echo 'Usage:' + @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /' + +.PHONY: confirm +confirm: + @echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ] + +.PHONY: no-dirty +no-dirty: + git diff --exit-code + + +# ==================================================================================== # +# QUALITY CONTROL +# ==================================================================================== # + +## format: format code +.PHONY: format +format: + go fmt ./... + +## tidy: format code and tidy modfile +.PHONY: tidy +tidy: + go fmt ./... + go mod tidy -v + +## audit: run quality control checks +.PHONY: audit +audit: + go mod verify + go vet ./... + go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./... + go run golang.org/x/vuln/cmd/govulncheck@latest ./... + go test -race -buildvcs -vet=off ./... + + +# ==================================================================================== # +# DEVELOPMENT +# ==================================================================================== # + +## test: run all tests +.PHONY: test +test: + go test -v -race -buildvcs ./... + +## test/cover: run all tests and display coverage +.PHONY: test/cover +test/cover: + go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./... + go tool cover -html=/tmp/coverage.out + +## build: build the application +.PHONY: build +build: + # Include additional build steps, like TypeScript, SCSS or Tailwind compilation here... + go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH} diff --git a/go/protopace/compatibility.go b/go/protopace/compatibility.go new file mode 100644 index 000000000..ee90df999 --- /dev/null +++ b/go/protopace/compatibility.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + + "github.com/Aiven-Open/karapace/go/protopace/schema" + + "github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/pkg/tracing" + "go.uber.org/zap" +) + +func Check(schema schema.Schema, previousSchema schema.Schema) error { + handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer) + ctx := context.Background() + image, err := schema.CompileBufImage() + if err != nil { + return err + } + previousImage, err := previousSchema.CompileBufImage() + if err != nil { + return err + } + checkConfig, _ := bufconfig.NewEnabledCheckConfig( + bufconfig.FileVersionV2, + nil, + []string{ + "FIELD_NO_DELETE", + "FILE_SAME_PACKAGE", + "FIELD_SAME_NAME", + "FIELD_SAME_JSON_NAME", + "FILE_NO_DELETE", + "ENUM_NO_DELETE", + }, + nil, + nil, + ) + config := bufconfig.NewBreakingConfig(checkConfig, false) + return handler.Check(ctx, config, previousImage, image) +} diff --git a/go/protopace/compatibility_test.go b/go/protopace/compatibility_test.go new file mode 100644 index 000000000..a9a5dd16e --- /dev/null +++ b/go/protopace/compatibility_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "os" + "testing" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/stretchr/testify/assert" +) + +func TestCompatibility(t *testing.T) { + assert := assert.New(t) + + data, _ := os.ReadFile("./fixtures/dependency.proto") + dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil) + assert.NoError(err) + assert.NotNil(dependencySchema) + + data, _ = os.ReadFile("./fixtures/test.proto") + testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(testSchema) + + data, _ = os.ReadFile("./fixtures/test_previous.proto") + previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(previousSchema) + + err = Check(*testSchema, *testSchema) + assert.NoError(err) + + err = Check(*testSchema, *previousSchema) + assert.ErrorContains(err, "Field \"5\" with name \"foo\" on message \"EventValue\" changed type from \"string\" to \"int32\".") +} diff --git a/go/protopace/fixtures/dependency.proto b/go/protopace/fixtures/dependency.proto new file mode 100644 index 000000000..45b94a1de --- /dev/null +++ b/go/protopace/fixtures/dependency.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} diff --git a/go/protopace/fixtures/test.proto b/go/protopace/fixtures/test.proto new file mode 100644 index 000000000..e973b579d --- /dev/null +++ b/go/protopace/fixtures/test.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; +option php_namespace = "My\\Awesome\\Customer\\V1"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; + int32 foo = 5; +} diff --git a/go/protopace/fixtures/test_previous.proto b/go/protopace/fixtures/test_previous.proto new file mode 100644 index 000000000..6ca7b7c06 --- /dev/null +++ b/go/protopace/fixtures/test_previous.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; +option php_namespace = "My\\Awesome\\Customer\\V1"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; + string foo = 5; +} diff --git a/go/protopace/formatter.go b/go/protopace/formatter.go new file mode 100644 index 000000000..749fa16a6 --- /dev/null +++ b/go/protopace/formatter.go @@ -0,0 +1,2469 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Modifications made by Aiven Ltd, 2024: +// - Use fully qualified name for field types +// - Always sort options by name + +package main + +import ( + "errors" + "fmt" + "io" + "reflect" + "sort" + "strings" + "unicode" + "unicode/utf8" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/bufbuild/protocompile/ast" + "github.com/bufbuild/protocompile/walk" + "go.uber.org/multierr" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +func Format(schema s.Schema) (s.Schema, error) { + all, err := schema.Compile() + if err != nil { + return schema, err + } + res := all[0] + + astNodeMapping := map[ast.Node]protoreflect.FullName{} + + walk.DescriptorProtos(res.FileDescriptorProto(), func(fn protoreflect.FullName, m proto.Message) error { + astNode := res.Node(m) + astNodeMapping[astNode] = fn + return nil + }) + + fieldTypeMapping := map[protoreflect.FullName]string{} + + walk.Descriptors(res, func(d protoreflect.Descriptor) error { + fd, ok := d.(protoreflect.FieldDescriptor) + if ok { + message := fd.Message() + if message != nil { + fieldTypeMapping[fd.FullName()] = string(message.FullName()) + return nil + } + enum := fd.Enum() + if enum != nil { + fieldTypeMapping[fd.FullName()] = string(enum.FullName()) + return nil + } + } + return nil + }) + writer := strings.Builder{} + fileNode := res.FileNode().(*ast.FileNode) + f := newFormatter(&writer, fileNode, astNodeMapping, fieldTypeMapping) + f.Run() + newSchema := schema + newSchema.Schema = writer.String() + return newSchema, nil +} + +type formatter struct { + writer io.Writer + fileNode *ast.FileNode + astNodeMapping map[ast.Node]protoreflect.FullName + fieldTypeMapping map[protoreflect.FullName]string + + // Used to adjust comments when we remove superfluous + // separators tp canonicalize message literals + overrideTrailingComments map[ast.Node]ast.Comments + + // Current level of indentation. + indent int + // The last character written to writer. + lastWritten rune + + // The last node written. This must be updated from all functions + // that write comments with a node. This flag informs how the next + // node's leading comments and whitespace should be written. + previousNode ast.Node + + // If true, a space will be written to the output unless the next character + // written is a newline (don't wait errant trailing spaces). + pendingSpace bool + // If true, the formatter is in the middle of printing compact options. + inCompactOptions bool + + // Track runes that open blocks/scopes and are expected to increase indention + // level. For example, when runes "{" "[" "(" ")" are written, the pending + // value is 2 (increment three times for "{" "[" "("; decrement once for ")"). + // If it's greater than zero at the end of a line, we call In() so that + // subsequent lines are indented. If it's less than zero at the end of a line, + // we call Out(). This minimizes the amount of explicit indent/unindent code + // that is needed and makes it less error-prone. + pendingIndent int + // If true, an inline node/sequence is being written. We treat whitespace a + // little differently for when blocks are printed inline vs. across multiple + // lines. So this flag informs the logic that makes those whitespace decisions. + inline bool + + // Records all errors that occur during the formatting process. Nearly any + // non-nil error represents a bug in the implementation. + err error +} + +// newFormatter returns a new formatter for the given file. +func newFormatter( + writer io.Writer, + fileNode *ast.FileNode, + astNodeMapping map[ast.Node]protoreflect.FullName, + fieldTypeMapping map[protoreflect.FullName]string, +) *formatter { + return &formatter{ + writer: writer, + fileNode: fileNode, + astNodeMapping: astNodeMapping, + fieldTypeMapping: fieldTypeMapping, + overrideTrailingComments: map[ast.Node]ast.Comments{}, + } +} + +// Run runs the formatter and writes the file's content to the formatter's writer. +func (f *formatter) Run() error { + f.writeFile() + return f.err +} + +// P prints a line to the generated output. +// +// This will emit a newline and proper indentation. If you do not +// want to emit a newline and want to write a raw string, use +// WriteString (which P calls). +// +// If strings.TrimSpace(elem) is empty, no indentation is produced. +func (f *formatter) P(elem string) { + if len(strings.TrimSpace(elem)) > 0 { + // We only want to write an indent if we're + // writing a non-empty string (not just a newline). + f.Indent(nil) + f.WriteString(elem) + } + f.WriteString("\n") + + if f.pendingIndent > 0 { + f.In() + } else if f.pendingIndent < 0 { + f.Out() + } + f.pendingIndent = 0 +} + +// Space adds a space to the generated output. +func (f *formatter) Space() { + f.pendingSpace = true +} + +// In increases the current level of indentation. +func (f *formatter) In() { + f.indent++ +} + +// Out reduces the current level of indentation. +func (f *formatter) Out() { + if f.indent <= 0 { + // Unreachable. + f.err = multierr.Append( + f.err, + errors.New("internal error: attempted to decrement indentation at zero"), + ) + return + } + f.indent-- +} + +// Indent writes the number of spaces associated +// with the current level of indentation. +func (f *formatter) Indent(nextNode ast.Node) { + // only indent at beginning of line + if f.lastWritten != '\n' { + return + } + indent := f.indent + if rn, ok := nextNode.(*ast.RuneNode); ok && indent > 0 { + if strings.ContainsRune("}])>", rn.Rune) { + indent-- + } + } + f.WriteString(strings.Repeat(" ", indent)) +} + +// WriteString writes the given element to the generated output. +// +// This will not write indentation or newlines. Use P if you +// want to emit identation or newlines. +func (f *formatter) WriteString(elem string) { + if f.pendingSpace { + f.pendingSpace = false + first, _ := utf8.DecodeRuneInString(elem) + + // We don't want "dangling spaces" before certain characters: + // newlines, commas, and semicolons. Also, when writing + // elements inline, we don't want spaces before close parens + // and braces. Similarly, we don't want extra/doubled spaces + // or dangling spaces after certain characters when printing + // inline, like open parens/braces. So only print the space + // if the previous and next character don't match above + // conditions. + + prevBlockList := "\x00 \t\n" + nextBlockList := "\n;," + if f.inline { + prevBlockList = "\x00 \t\n<[{(" + nextBlockList = "\n;,)]}>" + } + + if !strings.ContainsRune(prevBlockList, f.lastWritten) && + !strings.ContainsRune(nextBlockList, first) { + if _, err := f.writer.Write([]byte{' '}); err != nil { + f.err = multierr.Append(f.err, err) + return + } + } + } + if len(elem) == 0 { + return + } + f.lastWritten, _ = utf8.DecodeLastRuneInString(elem) + if _, err := f.writer.Write([]byte(elem)); err != nil { + f.err = multierr.Append(f.err, err) + } +} + +// SetPreviousNode sets the previously written node. This should +// be called in all of the comment writing functions. +func (f *formatter) SetPreviousNode(node ast.Node) { + f.previousNode = node +} + +// writeFile writes the file node. +func (f *formatter) writeFile() { + f.writeFileHeader() + f.writeFileTypes() + if f.fileNode.EOF != nil { + info := f.nodeInfo(f.fileNode.EOF) + f.writeMultilineComments(info.LeadingComments()) + } + if f.lastWritten != 0 && f.lastWritten != '\n' { + // If anything was written, we always conclude with + // a newline. + f.P("") + } +} + +// writeFileHeader writes the header of a .proto file. This includes the syntax, +// package, imports, and options (in that order). The imports and options are +// sorted. All other file elements are handled by f.writeFileTypes. +// +// For example, +// +// syntax = "proto3"; +// +// package acme.v1.weather; +// +// import "acme/payment/v1/payment.proto"; +// import "google/type/datetime.proto"; +// +// option cc_enable_arenas = true; +// option optimize_for = SPEED; +func (f *formatter) writeFileHeader() { + var ( + packageNode *ast.PackageNode + importNodes []*ast.ImportNode + optionNodes []*ast.OptionNode + ) + for _, fileElement := range f.fileNode.Decls { + switch node := fileElement.(type) { + case *ast.PackageNode: + packageNode = node + case *ast.ImportNode: + importNodes = append(importNodes, node) + case *ast.OptionNode: + optionNodes = append(optionNodes, node) + default: + continue + } + } + if f.fileNode.Syntax == nil && f.fileNode.Edition == nil && + packageNode == nil && importNodes == nil && optionNodes == nil { + // There aren't any header values, so we can return early. + return + } + editionNode := f.fileNode.Edition + if editionNode != nil { + f.writeEdition(editionNode) + } + if syntaxNode := f.fileNode.Syntax; syntaxNode != nil && editionNode == nil { + f.writeSyntax(syntaxNode) + } + if packageNode != nil { + f.writePackage(packageNode) + } + sort.Slice(importNodes, func(i, j int) bool { + iName := importNodes[i].Name.AsString() + jName := importNodes[j].Name.AsString() + // sort by public > None > weak + iOrder := importSortOrder(importNodes[i]) + jOrder := importSortOrder(importNodes[j]) + + if iName < jName { + return true + } + if iName > jName { + return false + } + if iOrder > jOrder { + return true + } + if iOrder < jOrder { + return false + } + + // put commented import first + return !f.importHasComment(importNodes[j]) + }) + for i, importNode := range importNodes { + if i == 0 && f.previousNode != nil && !f.leadingCommentsContainBlankLine(importNode) { + f.P("") + } + + // since the imports are sorted, this will skip write imports + // if they have appear before and dont have comment + if i > 0 && importNode.Name.AsString() == importNodes[i-1].Name.AsString() && + !f.importHasComment(importNode) { + continue + } + + f.writeImport(importNode, i > 0) + } + sort.Sort(ByOptionName(optionNodes)) + for i, optionNode := range optionNodes { + if i == 0 && f.previousNode != nil && !f.leadingCommentsContainBlankLine(optionNode) { + f.P("") + } + f.writeFileOption(optionNode, i > 0) + } +} + +// writeFileTypes writes the types defined in a .proto file. This includes the messages, enums, +// services, etc. All other elements are ignored since they are handled by f.writeFileHeader. +func (f *formatter) writeFileTypes() { + for i, fileElement := range f.fileNode.Decls { + switch node := fileElement.(type) { + case *ast.PackageNode, *ast.OptionNode, *ast.ImportNode, *ast.EmptyDeclNode: + // These elements have already been written by f.writeFileHeader. + continue + default: + info := f.nodeInfo(node) + wantNewline := f.previousNode != nil && (i == 0 || info.LeadingComments().Len() > 0) + if wantNewline && !f.leadingCommentsContainBlankLine(node) { + f.P("") + } + f.writeNode(node) + } + } +} + +// writeSyntax writes the syntax. +// +// For example, +// +// syntax = "proto3"; +func (f *formatter) writeSyntax(syntaxNode *ast.SyntaxNode) { + f.writeStart(syntaxNode.Keyword) + f.Space() + f.writeInline(syntaxNode.Equals) + f.Space() + f.writeInline(syntaxNode.Syntax) + f.writeLineEnd(syntaxNode.Semicolon) +} + +// writeEdition writes the edition. +// +// For example, +// +// edition = "2023"; +func (f *formatter) writeEdition(editionNode *ast.EditionNode) { + f.writeStart(editionNode.Keyword) + f.Space() + f.writeInline(editionNode.Equals) + f.Space() + f.writeInline(editionNode.Edition) + f.writeLineEnd(editionNode.Semicolon) +} + +// writePackage writes the package. +// +// For example, +// +// package acme.weather.v1; +func (f *formatter) writePackage(packageNode *ast.PackageNode) { + f.writeStart(packageNode.Keyword) + f.Space() + f.writeInline(packageNode.Name) + f.writeLineEnd(packageNode.Semicolon) +} + +// writeImport writes an import statement. +// +// For example, +// +// import "google/protobuf/descriptor.proto"; +func (f *formatter) writeImport(importNode *ast.ImportNode, forceCompact bool) { + f.writeStartMaybeCompact(importNode.Keyword, forceCompact) + f.Space() + // We don't want to write the "public" and "weak" nodes + // if they aren't defined. One could be set, but never both. + switch { + case importNode.Public != nil: + f.writeInline(importNode.Public) + f.Space() + case importNode.Weak != nil: + f.writeInline(importNode.Weak) + f.Space() + } + f.writeInline(importNode.Name) + f.writeLineEnd(importNode.Semicolon) +} + +// writeFileOption writes a file option. This function is slightly +// different than f.writeOption because file options are sorted at +// the top of the file, and leading comments are adjusted accordingly. +func (f *formatter) writeFileOption(optionNode *ast.OptionNode, forceCompact bool) { + f.writeStartMaybeCompact(optionNode.Keyword, forceCompact) + f.Space() + f.writeNode(optionNode.Name) + f.Space() + f.writeInline(optionNode.Equals) + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // Compound string literals are written across multiple lines + // immediately after the '=', so we don't need a trailing + // space in the option prefix. + f.writeCompoundStringLiteralIndentEndInline(node) + f.writeLineEnd(optionNode.Semicolon) + return + } + f.Space() + f.writeInline(optionNode.Val) + f.writeLineEnd(optionNode.Semicolon) +} + +// writeOption writes an option. +// +// For example, +// +// option go_package = "github.com/foo/bar"; +func (f *formatter) writeOption(optionNode *ast.OptionNode) { + f.writeOptionPrefix(optionNode) + if optionNode.Semicolon != nil { + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // Compound string literals are written across multiple lines + // immediately after the '=', so we don't need a trailing + // space in the option prefix. + f.writeCompoundStringLiteralIndentEndInline(node) + f.writeLineEnd(optionNode.Semicolon) + return + } + f.writeInline(optionNode.Val) + f.writeLineEnd(optionNode.Semicolon) + return + } + + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + f.writeCompoundStringLiteralIndent(node) + return + } + f.writeInline(optionNode.Val) +} + +// writeLastCompactOption writes a compact option but preserves its the +// trailing end comments. This is only used for the last compact option +// since it's the only time a trailing ',' will be omitted. +// +// For example, +// +// [ +// deprecated = true, +// json_name = "something" // Trailing comment on the last element. +// ] +func (f *formatter) writeLastCompactOption(optionNode *ast.OptionNode) { + f.writeOptionPrefix(optionNode) + f.writeLineEnd(optionNode.Val) +} + +// writeOptionValue writes the option prefix, which makes up all of the +// option's definition, excluding the final token(s). +// +// For example, +// +// deprecated = +func (f *formatter) writeOptionPrefix(optionNode *ast.OptionNode) { + if optionNode.Keyword != nil { + // Compact options don't have the keyword. + f.writeStart(optionNode.Keyword) + f.Space() + f.writeNode(optionNode.Name) + } else { + f.writeStart(optionNode.Name) + } + f.Space() + f.writeInline(optionNode.Equals) + f.Space() +} + +// writeOptionName writes an option name. +// +// For example, +// +// go_package +// (custom.thing) +// (custom.thing).bridge.(another.thing) +func (f *formatter) writeOptionName(optionNameNode *ast.OptionNameNode) { + for i := 0; i < len(optionNameNode.Parts); i++ { + if f.inCompactOptions && i == 0 { + // The leading comments of the first token (either open rune or the + // name) will have already been written, so we need to handle this + // case specially. + fieldReferenceNode := optionNameNode.Parts[0] + if fieldReferenceNode.Open != nil { + f.writeNode(fieldReferenceNode.Open) + if info := f.nodeInfo(fieldReferenceNode.Open); info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } + f.writeInline(fieldReferenceNode.Name) + } else { + f.writeNode(fieldReferenceNode.Name) + if info := f.nodeInfo(fieldReferenceNode.Name); info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } + } + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } + continue + } + if i > 0 { + // The length of this slice must be exactly len(Parts)-1. + f.writeInline(optionNameNode.Dots[i-1]) + } + f.writeNode(optionNameNode.Parts[i]) + } +} + +// writeMessage writes the message node. +// +// For example, +// +// message Foo { +// option deprecated = true; +// reserved 50 to 100; +// extensions 150 to 200; +// +// message Bar { +// string name = 1; +// } +// enum Baz { +// BAZ_UNSPECIFIED = 0; +// } +// extend Bar { +// string value = 2; +// } +// +// Bar bar = 1; +// Baz baz = 2; +// } +func (f *formatter) writeMessage(messageNode *ast.MessageNode) { + var elementWriterFunc func() + if len(messageNode.Decls) != 0 { + elementWriterFunc = func() { + writeNodes(messageNode.Decls, f) + } + } + f.writeStart(messageNode.Keyword) + f.Space() + f.writeInline(messageNode.Name) + f.Space() + f.writeCompositeTypeBody( + messageNode.OpenBrace, + messageNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeMessageLiteral writes a message literal. +// +// For example, +// +// { +// foo: 1 +// foo: 2 +// foo: 3 +// bar: < +// name:"abc" +// id:123 +// > +// } +func (f *formatter) writeMessageLiteral(messageLiteralNode *ast.MessageLiteralNode) { + if f.maybeWriteCompactMessageLiteral(messageLiteralNode, false) { + return + } + var elementWriterFunc func() + if len(messageLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + f.writeMessageLiteralElements(messageLiteralNode) + } + } + f.writeCompositeValueBody( + messageLiteralOpen(messageLiteralNode), + messageLiteralClose(messageLiteralNode), + elementWriterFunc, + ) +} + +// writeMessageLiteral writes a message literal suitable for +// an element in an array literal. +func (f *formatter) writeMessageLiteralForArray( + messageLiteralNode *ast.MessageLiteralNode, + lastElement bool, +) { + if f.maybeWriteCompactMessageLiteral(messageLiteralNode, true) { + return + } + var elementWriterFunc func() + if len(messageLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + f.writeMessageLiteralElements(messageLiteralNode) + } + } + closeWriter := f.writeBodyEndInline + if lastElement { + closeWriter = f.writeBodyEnd + } + f.writeBody( + messageLiteralOpen(messageLiteralNode), + messageLiteralClose(messageLiteralNode), + elementWriterFunc, + f.writeOpenBracePrefixForArray, + closeWriter, + ) +} + +func (f *formatter) maybeWriteCompactMessageLiteral( + messageLiteralNode *ast.MessageLiteralNode, + inArrayLiteral bool, +) bool { + if len(messageLiteralNode.Elements) == 0 || len(messageLiteralNode.Elements) > 1 || + f.hasInteriorComments(messageLiteralNode.Children()...) || + messageLiteralHasNestedMessageOrArray(messageLiteralNode) { + return false + } + // messages with a single scalar field and no comments can be + // printed all on one line + openNode := messageLiteralOpen(messageLiteralNode) + closeNode := messageLiteralClose(messageLiteralNode) + if inArrayLiteral { + f.Indent(openNode) + } + f.writeInline(openNode) + fieldNode := messageLiteralNode.Elements[0] + f.writeInline(fieldNode.Name) + if fieldNode.Sep != nil { + f.writeInline(fieldNode.Sep) + } else { + f.WriteString(":") + } + f.Space() + if messageLiteralNode.Seps[0] != nil { + // We are dropping the optional trailing separator. If it had + // trailing comments and the value does not, move the separator's + // trailing comment to the value. + sepTrailingComments := f.nodeInfo(messageLiteralNode.Seps[0]).TrailingComments() + if sepTrailingComments.Len() > 0 && + f.nodeInfo(fieldNode.Val).TrailingComments().Len() == 0 { + f.setTrailingComments(fieldNode.Val, sepTrailingComments) + } + } + f.writeInline(fieldNode.Val) + f.writeInline(closeNode) + return true +} + +func messageLiteralHasNestedMessageOrArray(messageLiteralNode *ast.MessageLiteralNode) bool { + for _, elem := range messageLiteralNode.Elements { + switch elem.Val.(type) { + case *ast.ArrayLiteralNode, *ast.MessageLiteralNode: + return true + } + } + return false +} + +func arrayLiteralHasNestedMessageOrArray(arrayLiteralNode *ast.ArrayLiteralNode) bool { + for _, elem := range arrayLiteralNode.Elements { + switch elem.(type) { + case *ast.ArrayLiteralNode, *ast.MessageLiteralNode: + return true + } + } + return false +} + +// writeMessageLiteralElements writes the message literal's elements. +// +// For example, +// +// foo: 1 +// foo: 2 +func (f *formatter) writeMessageLiteralElements(messageLiteralNode *ast.MessageLiteralNode) { + for i := 0; i < len(messageLiteralNode.Elements); i++ { + // Separators ("," or ";") are optional. To avoid inconsistent formatted output, + // we suppress them, since they aren't needed. So we just write the element and + // ignore any optional separator in the AST. + if messageLiteralNode.Seps[i] != nil { + // Since we are dropping the optional trailing separator, we should + // possibly move its trailing comment to the element value so we don't + // lose it. Skip this step if the value already has a trailing comment. + sepTrailingComments := f.nodeInfo(messageLiteralNode.Seps[i]).TrailingComments() + if sepTrailingComments.Len() > 0 && + f.nodeInfo(messageLiteralNode.Elements[i].Val).TrailingComments().Len() == 0 { + f.setTrailingComments(messageLiteralNode.Elements[i].Val, sepTrailingComments) + } + } + f.writeNode(messageLiteralNode.Elements[i]) + } +} + +// writeMessageField writes the message field node, and concludes the +// line without leaving room for a trailing separator in the parent +// message literal. +func (f *formatter) writeMessageField(messageFieldNode *ast.MessageFieldNode) { + f.writeMessageFieldPrefix(messageFieldNode) + if compoundStringLiteral, ok := messageFieldNode.Val.(*ast.CompoundStringLiteralNode); ok { + f.writeCompoundStringLiteralIndent(compoundStringLiteral) + return + } + f.writeLineEnd(messageFieldNode.Val) +} + +// writeMessageFieldPrefix writes the message field node as a single line. +// +// For example, +// +// foo:"bar" +func (f *formatter) writeMessageFieldPrefix(messageFieldNode *ast.MessageFieldNode) { + // The comments need to be written as a multiline comment above + // the message field name. + // + // Note that this is different than how field reference nodes are + // normally formatted in-line (i.e. as option name components). + fieldReferenceNode := messageFieldNode.Name + if fieldReferenceNode.Open != nil { + f.writeStart(fieldReferenceNode.Open) + if fieldReferenceNode.URLPrefix != nil { + f.writeInline(fieldReferenceNode.URLPrefix) + f.writeInline(fieldReferenceNode.Slash) + } + f.writeInline(fieldReferenceNode.Name) + } else { + f.writeStart(fieldReferenceNode.Name) + } + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } + // The colon separator is optional sometimes, but we don't have enough + // information here to know whether it's necessary. For more consistent + // output, just always include it. + if messageFieldNode.Sep != nil { + f.writeInline(messageFieldNode.Sep) + } else { + f.WriteString(":") + } + f.Space() +} + +// writeEnum writes the enum node. +// +// For example, +// +// enum Foo { +// option deprecated = true; +// reserved 1 to 5; +// +// FOO_UNSPECIFIED = 0; +// } +func (f *formatter) writeEnum(enumNode *ast.EnumNode) { + var elementWriterFunc func() + if len(enumNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(enumNode.Decls, f) + } + } + f.writeStart(enumNode.Keyword) + f.Space() + f.writeInline(enumNode.Name) + f.Space() + f.writeCompositeTypeBody( + enumNode.OpenBrace, + enumNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeEnumValue writes the enum value as a single line. If the enum has +// compact options, it will be written across multiple lines. +// +// For example, +// +// FOO_UNSPECIFIED = 1 [ +// deprecated = true +// ]; +func (f *formatter) writeEnumValue(enumValueNode *ast.EnumValueNode) { + f.writeStart(enumValueNode.Name) + f.Space() + f.writeInline(enumValueNode.Equals) + f.Space() + f.writeInline(enumValueNode.Number) + if enumValueNode.Options != nil { + f.Space() + f.writeNode(enumValueNode.Options) + } + f.writeLineEnd(enumValueNode.Semicolon) +} + +// writeField writes the field node as a single line. If the field has +// compact options, it will be written across multiple lines. +// +// For example, +// +// repeated string name = 1 [ +// deprecated = true, +// json_name = "name" +// ]; +func (f *formatter) writeField(fieldNode *ast.FieldNode) { + // We need to handle the comments for the field label specially since + // a label might not be defined, but it has the leading comments attached + // to it. + n := f.astNodeMapping[fieldNode] + fullType := f.fieldTypeMapping[n] + var t ast.IdentValueNode + if fullType == "" { + t = fieldNode.FldType + } else { + t = ast.NewIdentNode(fullType, fieldNode.FldType.Start()) + } + + if fieldNode.Label.KeywordNode != nil { + f.writeStart(fieldNode.Label) + f.Space() + //f.writeInline(fieldNode.FldType) + f.writeInline(t) + } else { + // If a label was not written, the multiline comments will be + // attached to the type. + //if compoundIdentNode, ok := fieldNode.FldType.(*ast.CompoundIdentNode); ok { + // f.writeCompountIdentForFieldName(compoundIdentNode) + //} else { + //f.writeStart(fieldNode.FldType) + //} + f.writeStart(t) + } + f.Space() + f.writeInline(fieldNode.Name) + f.Space() + f.writeInline(fieldNode.Equals) + f.Space() + f.writeInline(fieldNode.Tag) + if fieldNode.Options != nil { + f.Space() + f.writeNode(fieldNode.Options) + } + f.writeLineEnd(fieldNode.Semicolon) +} + +// writeMapField writes a map field (e.g. 'map pairs = 1;'). +func (f *formatter) writeMapField(mapFieldNode *ast.MapFieldNode) { + f.writeNode(mapFieldNode.MapType) + f.Space() + f.writeInline(mapFieldNode.Name) + f.Space() + f.writeInline(mapFieldNode.Equals) + f.Space() + f.writeInline(mapFieldNode.Tag) + if mapFieldNode.Options != nil { + f.Space() + f.writeNode(mapFieldNode.Options) + } + f.writeLineEnd(mapFieldNode.Semicolon) +} + +// writeMapType writes a map type (e.g. 'map'). +func (f *formatter) writeMapType(mapTypeNode *ast.MapTypeNode) { + f.writeStart(mapTypeNode.Keyword) + f.writeInline(mapTypeNode.OpenAngle) + f.writeInline(mapTypeNode.KeyType) + f.writeInline(mapTypeNode.Comma) + f.Space() + f.writeInline(mapTypeNode.ValueType) + f.writeInline(mapTypeNode.CloseAngle) +} + +// writeFieldReference writes a field reference (e.g. '(foo.bar)'). +func (f *formatter) writeFieldReference(fieldReferenceNode *ast.FieldReferenceNode) { + if fieldReferenceNode.Open != nil { + f.writeInline(fieldReferenceNode.Open) + } + f.writeInline(fieldReferenceNode.Name) + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } +} + +// writeExtend writes the extend node. +// +// For example, +// +// extend google.protobuf.FieldOptions { +// bool redacted = 33333; +// } +func (f *formatter) writeExtend(extendNode *ast.ExtendNode) { + var elementWriterFunc func() + if len(extendNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(extendNode.Decls, f) + } + } + f.writeStart(extendNode.Keyword) + f.Space() + f.writeInline(extendNode.Extendee) + f.Space() + f.writeCompositeTypeBody( + extendNode.OpenBrace, + extendNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeService writes the service node. +// +// For example, +// +// service FooService { +// option deprecated = true; +// +// rpc Foo(FooRequest) returns (FooResponse) {}; +func (f *formatter) writeService(serviceNode *ast.ServiceNode) { + var elementWriterFunc func() + if len(serviceNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(serviceNode.Decls, f) + } + } + f.writeStart(serviceNode.Keyword) + f.Space() + f.writeInline(serviceNode.Name) + f.Space() + f.writeCompositeTypeBody( + serviceNode.OpenBrace, + serviceNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeRPC writes the RPC node. RPCs are formatted in +// the following order: +// +// For example, +// +// rpc Foo(FooRequest) returns (FooResponse) { +// option deprecated = true; +// }; +func (f *formatter) writeRPC(rpcNode *ast.RPCNode) { + var elementWriterFunc func() + if len(rpcNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(rpcNode.Decls, f) + } + } + f.writeStart(rpcNode.Keyword) + f.Space() + f.writeInline(rpcNode.Name) + f.writeInline(rpcNode.Input) + f.Space() + f.writeInline(rpcNode.Returns) + f.Space() + f.writeInline(rpcNode.Output) + if rpcNode.OpenBrace == nil { + // This RPC doesn't have any elements, so we prefer the + // ';' form. + // + // rpc Ping(PingRequest) returns (PingResponse); + // + f.writeLineEnd(rpcNode.Semicolon) + return + } + f.Space() + f.writeCompositeTypeBody( + rpcNode.OpenBrace, + rpcNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeRPCType writes the RPC type node (e.g. (stream foo.Bar)). +func (f *formatter) writeRPCType(rpcTypeNode *ast.RPCTypeNode) { + f.writeInline(rpcTypeNode.OpenParen) + if rpcTypeNode.Stream != nil { + f.writeInline(rpcTypeNode.Stream) + f.Space() + } + f.writeInline(rpcTypeNode.MessageType) + f.writeInline(rpcTypeNode.CloseParen) +} + +// writeOneOf writes the oneof node. +// +// For example, +// +// oneof foo { +// option deprecated = true; +// +// string name = 1; +// int number = 2; +// } +func (f *formatter) writeOneOf(oneOfNode *ast.OneofNode) { + var elementWriterFunc func() + if len(oneOfNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(oneOfNode.Decls, f) + } + } + f.writeStart(oneOfNode.Keyword) + f.Space() + f.writeInline(oneOfNode.Name) + f.Space() + f.writeCompositeTypeBody( + oneOfNode.OpenBrace, + oneOfNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeGroup writes the group node. +// +// For example, +// +// optional group Key = 4 [ +// deprecated = true, +// json_name = "key" +// ] { +// optional uint64 id = 1; +// optional string name = 2; +// } +func (f *formatter) writeGroup(groupNode *ast.GroupNode) { + var elementWriterFunc func() + if len(groupNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(groupNode.Decls, f) + } + } + // We need to handle the comments for the group label specially since + // a label might not be defined, but it has the leading comments attached + // to it. + if groupNode.Label.KeywordNode != nil { + f.writeStart(groupNode.Label) + f.Space() + f.writeInline(groupNode.Keyword) + } else { + // If a label was not written, the multiline comments will be + // attached to the keyword. + f.writeStart(groupNode.Keyword) + } + f.Space() + f.writeInline(groupNode.Name) + f.Space() + f.writeInline(groupNode.Equals) + f.Space() + f.writeInline(groupNode.Tag) + if groupNode.Options != nil { + f.Space() + f.writeNode(groupNode.Options) + } + f.Space() + f.writeCompositeTypeBody( + groupNode.OpenBrace, + groupNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeExtensionRange writes the extension range node. +// +// For example, +// +// extensions 5-10, 100 to max [ +// deprecated = true +// ]; +func (f *formatter) writeExtensionRange(extensionRangeNode *ast.ExtensionRangeNode) { + f.writeStart(extensionRangeNode.Keyword) + f.Space() + for i := 0; i < len(extensionRangeNode.Ranges); i++ { + if i > 0 { + // The length of this slice must be exactly len(Ranges)-1. + f.writeInline(extensionRangeNode.Commas[i-1]) + f.Space() + } + f.writeNode(extensionRangeNode.Ranges[i]) + } + if extensionRangeNode.Options != nil { + f.Space() + f.writeNode(extensionRangeNode.Options) + } + f.writeLineEnd(extensionRangeNode.Semicolon) +} + +// writeReserved writes a reserved node. +// +// For example, +// +// reserved 5-10, 100 to max; +func (f *formatter) writeReserved(reservedNode *ast.ReservedNode) { + f.writeStart(reservedNode.Keyword) + // Either names or ranges will be set, but never both. + elements := make([]ast.Node, 0, len(reservedNode.Names)+len(reservedNode.Ranges)) + switch { + case reservedNode.Names != nil: + for _, nameNode := range reservedNode.Names { + elements = append(elements, nameNode) + } + case reservedNode.Identifiers != nil: + for _, identNode := range reservedNode.Identifiers { + elements = append(elements, identNode) + } + case reservedNode.Ranges != nil: + for _, rangeNode := range reservedNode.Ranges { + elements = append(elements, rangeNode) + } + } + f.Space() + for i := 0; i < len(elements); i++ { + if i > 0 { + // The length of this slice must be exactly len({Names,Ranges})-1. + f.writeInline(reservedNode.Commas[i-1]) + f.Space() + } + f.writeInline(elements[i]) + } + f.writeLineEnd(reservedNode.Semicolon) +} + +// writeRange writes the given range node (e.g. '1 to max'). +func (f *formatter) writeRange(rangeNode *ast.RangeNode) { + f.writeInline(rangeNode.StartVal) + if rangeNode.To != nil { + f.Space() + f.writeInline(rangeNode.To) + } + // Either EndVal or Max will be set, but never both. + switch { + case rangeNode.EndVal != nil: + f.Space() + f.writeInline(rangeNode.EndVal) + case rangeNode.Max != nil: + f.Space() + f.writeInline(rangeNode.Max) + } +} + +// writeCompactOptions writes a compact options node. +// +// For example, +// +// [ +// deprecated = true, +// json_name = "something" +// ] +func (f *formatter) writeCompactOptions(compactOptionsNode *ast.CompactOptionsNode) { + f.inCompactOptions = true + defer func() { + f.inCompactOptions = false + }() + if len(compactOptionsNode.Options) == 1 && + !f.hasInteriorComments(compactOptionsNode.OpenBracket, compactOptionsNode.Options[0].Name) { + // If there's only a single compact scalar option without comments, we can write it + // in-line. For example: + // + // [deprecated = true] + // + // However, this does not include the case when the '[' has trailing comments, + // or the option name has leading comments. In those cases, we write the option + // across multiple lines. For example: + // + // [ + // // This type is deprecated. + // deprecated = true + // ] + // + optionNode := compactOptionsNode.Options[0] + f.writeInline(compactOptionsNode.OpenBracket) + f.writeInline(optionNode.Name) + f.Space() + f.writeInline(optionNode.Equals) + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // If there's only a single compact option, the value needs to + // write its comments (if any) in a way that preserves the closing ']'. + f.writeCompoundStringLiteralNoIndentEndInline(node) + f.writeInline(compactOptionsNode.CloseBracket) + return + } + f.Space() + f.writeInline(optionNode.Val) + f.writeInline(compactOptionsNode.CloseBracket) + return + } + var elementWriterFunc func() + if len(compactOptionsNode.Options) > 0 { + elementWriterFunc = func() { + sort.Sort(ByOptionName(compactOptionsNode.Options)) + for i, opt := range compactOptionsNode.Options { + if i == len(compactOptionsNode.Options)-1 { + // The last element won't have a trailing comma. + f.writeLastCompactOption(opt) + return + } + f.writeNode(opt) + f.writeLineEnd(compactOptionsNode.Commas[i]) + } + } + } + f.writeCompositeValueBody( + compactOptionsNode.OpenBracket, + compactOptionsNode.CloseBracket, + elementWriterFunc, + ) +} + +func (f *formatter) hasInteriorComments(nodes ...ast.Node) bool { + for i, n := range nodes { + // interior comments mean we ignore leading comments on first + // token and trailing comments on the last one + info := f.nodeInfo(n) + if i > 0 && info.LeadingComments().Len() > 0 { + return true + } + if i < len(nodes)-1 && info.TrailingComments().Len() > 0 { + return true + } + } + return false +} + +// writeArrayLiteral writes an array literal across multiple lines. +// +// For example, +// +// [ +// "foo", +// "bar" +// ] +func (f *formatter) writeArrayLiteral(arrayLiteralNode *ast.ArrayLiteralNode) { + if len(arrayLiteralNode.Elements) == 1 && + !f.hasInteriorComments(arrayLiteralNode.Children()...) && + !arrayLiteralHasNestedMessageOrArray(arrayLiteralNode) { + // arrays with a single scalar value and no comments can be + // printed all on one line + valueNode := arrayLiteralNode.Elements[0] + f.writeInline(arrayLiteralNode.OpenBracket) + f.writeInline(valueNode) + f.writeInline(arrayLiteralNode.CloseBracket) + return + } + + var elementWriterFunc func() + if len(arrayLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + for i := 0; i < len(arrayLiteralNode.Elements); i++ { + lastElement := i == len(arrayLiteralNode.Elements)-1 + if compositeNode, ok := arrayLiteralNode.Elements[i].(ast.CompositeNode); ok { + f.writeCompositeValueForArrayLiteral(compositeNode, lastElement) + if !lastElement { + f.writeLineEnd(arrayLiteralNode.Commas[i]) + } + continue + } + if lastElement { + // The last element won't have a trailing comma. + f.writeLineElement(arrayLiteralNode.Elements[i]) + return + } + f.writeStart(arrayLiteralNode.Elements[i]) + f.writeLineEnd(arrayLiteralNode.Commas[i]) + } + } + } + f.writeCompositeValueBody( + arrayLiteralNode.OpenBracket, + arrayLiteralNode.CloseBracket, + elementWriterFunc, + ) +} + +// writeCompositeForArrayLiteral writes the composite node in a way that's suitable +// for array literals. In general, signed integers and compound strings should have their +// comments written in-line because they are one of many components in a single line. +// +// However, each of these composite types occupy a single line in an array literal, +// so they need their comments to be formatted like a standalone node. +// +// For example, +// +// option (value) = /* In-line comment for '-42' */ -42; +// +// option (thing) = { +// values: [ +// // Leading comment on -42. +// -42, // Trailing comment on -42. +// ] +// } +// +// The lastElement boolean is used to signal whether or not the composite value +// should be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeCompositeValueForArrayLiteral( + compositeNode ast.CompositeNode, + lastElement bool, +) { + switch node := compositeNode.(type) { + case *ast.CompoundStringLiteralNode: + f.writeCompoundStringLiteralForArray(node, lastElement) + case *ast.NegativeIntLiteralNode: + f.writeNegativeIntLiteralForArray(node, lastElement) + case *ast.SignedFloatLiteralNode: + f.writeSignedFloatLiteralForArray(node, lastElement) + case *ast.MessageLiteralNode: + f.writeMessageLiteralForArray(node, lastElement) + default: + f.err = multierr.Append(f.err, fmt.Errorf("unexpected array value node %T", node)) + } +} + +// writeCompositeTypeBody writes the body of a composite type, e.g. message, enum, extend, oneof, etc. +func (f *formatter) writeCompositeTypeBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), +) { + f.writeBody( + openBrace, + closeBrace, + elementWriterFunc, + f.writeOpenBracePrefix, + f.writeBodyEnd, + ) +} + +// writeCompositeValueBody writes the body of a composite value, e.g. compact options, +// array literal, etc. We need to handle the ']' different than composite types because +// there could be more tokens following the final ']'. +func (f *formatter) writeCompositeValueBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), +) { + f.writeBody( + openBrace, + closeBrace, + elementWriterFunc, + f.writeOpenBracePrefix, + f.writeBodyEndInline, + ) +} + +// writeBody writes the body of a type or value, e.g. message, enum, compact options, etc. +// The elementWriterFunc is used to write the declarations within the composite type (e.g. +// fields in a message). The openBraceWriterFunc and closeBraceWriterFunc functions are used +// to customize how the '{' and '} nodes are written, respectively. +func (f *formatter) writeBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), + openBraceWriterFunc func(ast.Node), + closeBraceWriterFunc func(ast.Node, bool), +) { + if elementWriterFunc == nil && !f.hasInteriorComments(openBrace, closeBrace) { + // completely empty body + f.writeInline(openBrace) + closeBraceWriterFunc(closeBrace, true) + return + } + + openBraceWriterFunc(openBrace) + if elementWriterFunc != nil { + elementWriterFunc() + } + closeBraceWriterFunc(closeBrace, false) +} + +// writeOpenBracePrefix writes the open brace with its leading comments in-line. +// This is used for nearly every use case of f.writeBody, excluding the instances +// in array literals. +func (f *formatter) writeOpenBracePrefix(openBrace ast.Node) { + defer f.SetPreviousNode(openBrace) + info := f.nodeInfo(openBrace) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(openBrace) + if info.TrailingComments().Len() > 0 { + f.writeTrailingEndComments(info.TrailingComments()) + } else { + f.P("") + } +} + +// writeOpenBracePrefixForArray writes the open brace with its leading comments +// on multiple lines. This is only used for message literals in arrays. +func (f *formatter) writeOpenBracePrefixForArray(openBrace ast.Node) { + defer f.SetPreviousNode(openBrace) + info := f.nodeInfo(openBrace) + if info.LeadingComments().Len() > 0 { + f.writeMultilineComments(info.LeadingComments()) + } + f.Indent(openBrace) + f.writeNode(openBrace) + if info.TrailingComments().Len() > 0 { + f.writeTrailingEndComments(info.TrailingComments()) + } else { + f.P("") + } +} + +// writeCompoundIdent writes a compound identifier (e.g. '.com.foo.Bar'). +func (f *formatter) writeCompoundIdent(compoundIdentNode *ast.CompoundIdentNode) { + if compoundIdentNode.LeadingDot != nil { + f.writeInline(compoundIdentNode.LeadingDot) + } + for i := 0; i < len(compoundIdentNode.Components); i++ { + if i > 0 { + // The length of this slice must be exactly len(Components)-1. + f.writeInline(compoundIdentNode.Dots[i-1]) + } + f.writeInline(compoundIdentNode.Components[i]) + } +} + +// writeCompountIdentForFieldName writes a compound identifier, but handles comments +// specially for field names. +// +// For example, +// +// message Foo { +// // These are comments attached to bar. +// bar.v1.Bar bar = 1; +// } +func (f *formatter) writeCompountIdentForFieldName(compoundIdentNode *ast.CompoundIdentNode) { + if compoundIdentNode.LeadingDot != nil { + f.writeStart(compoundIdentNode.LeadingDot) + } + for i := 0; i < len(compoundIdentNode.Components); i++ { + if i == 0 && compoundIdentNode.LeadingDot == nil { + f.writeStart(compoundIdentNode.Components[i]) + continue + } + if i > 0 { + // The length of this slice must be exactly len(Components)-1. + f.writeInline(compoundIdentNode.Dots[i-1]) + } + f.writeInline(compoundIdentNode.Components[i]) + } +} + +// writeFieldLabel writes the field label node. +// +// For example, +// +// optional +// repeated +// required +func (f *formatter) writeFieldLabel(fieldLabel ast.FieldLabel) { + f.WriteString(fieldLabel.Val) +} + +// writeCompoundStringLiteral writes a compound string literal value. +// +// For example, +// +// "one," +// "two," +// "three" +func (f *formatter) writeCompoundStringLiteral( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, + needsIndent bool, + hasTrailingPunctuation bool, +) { + f.P("") + if needsIndent { + f.In() + } + for i, child := range compoundStringLiteralNode.Children() { + if hasTrailingPunctuation && i == len(compoundStringLiteralNode.Children())-1 { + // inline because there may be a subsequent comma or punctuation from enclosing element + f.writeStart(child) + break + } + f.writeLineElement(child) + } + if needsIndent { + f.Out() + } +} + +func (f *formatter) writeCompoundStringLiteralIndent( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, true, false) +} + +func (f *formatter) writeCompoundStringLiteralIndentEndInline( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, true, true) +} + +func (f *formatter) writeCompoundStringLiteralNoIndentEndInline( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, false, true) +} + +// writeCompoundStringLiteralForArray writes a compound string literal value, +// but writes its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeCompoundStringLiteralForArray( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, + lastElement bool, +) { + for i, child := range compoundStringLiteralNode.Children() { + if !lastElement && i == len(compoundStringLiteralNode.Children())-1 { + f.writeStart(child) + return + } + f.writeLineElement(child) + } +} + +// writeFloatLiteral writes a float literal value (e.g. '42.2'). +func (f *formatter) writeFloatLiteral(floatLiteralNode *ast.FloatLiteralNode) { + f.writeRaw(floatLiteralNode) +} + +// writeSignedFloatLiteral writes a signed float literal value (e.g. '-42.2'). +func (f *formatter) writeSignedFloatLiteral(signedFloatLiteralNode *ast.SignedFloatLiteralNode) { + f.writeInline(signedFloatLiteralNode.Sign) + f.writeInline(signedFloatLiteralNode.Float) +} + +// writeSignedFloatLiteralForArray writes a signed float literal value, but writes +// its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeSignedFloatLiteralForArray( + signedFloatLiteralNode *ast.SignedFloatLiteralNode, + lastElement bool, +) { + f.writeStart(signedFloatLiteralNode.Sign) + if lastElement { + f.writeLineEnd(signedFloatLiteralNode.Float) + return + } + f.writeInline(signedFloatLiteralNode.Float) +} + +// writeSpecialFloatLiteral writes a special float literal value (e.g. "nan" or "inf"). +func (f *formatter) writeSpecialFloatLiteral(specialFloatLiteralNode *ast.SpecialFloatLiteralNode) { + f.WriteString(specialFloatLiteralNode.KeywordNode.Val) +} + +// writeStringLiteral writes a string literal value (e.g. "foo"). +// Note that the raw string is written as-is so that it preserves +// the quote style used in the original source. +func (f *formatter) writeStringLiteral(stringLiteralNode *ast.StringLiteralNode) { + f.writeRaw(stringLiteralNode) +} + +// writeUintLiteral writes a uint literal (e.g. '42'). +func (f *formatter) writeUintLiteral(uintLiteralNode *ast.UintLiteralNode) { + f.writeRaw(uintLiteralNode) +} + +// writeNegativeIntLiteral writes a negative int literal (e.g. '-42'). +func (f *formatter) writeNegativeIntLiteral(negativeIntLiteralNode *ast.NegativeIntLiteralNode) { + f.writeInline(negativeIntLiteralNode.Minus) + f.writeInline(negativeIntLiteralNode.Uint) +} + +func (f *formatter) writeRaw(n ast.Node) { + info := f.nodeInfo(n) + f.WriteString(info.RawText()) +} + +// writeNegativeIntLiteralForArray writes a negative int literal value, but writes +// its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeNegativeIntLiteralForArray( + negativeIntLiteralNode *ast.NegativeIntLiteralNode, + lastElement bool, +) { + f.writeStart(negativeIntLiteralNode.Minus) + if lastElement { + f.writeLineEnd(negativeIntLiteralNode.Uint) + return + } + f.writeInline(negativeIntLiteralNode.Uint) +} + +// writeIdent writes an identifier (e.g. 'foo'). +func (f *formatter) writeIdent(identNode *ast.IdentNode) { + f.WriteString(identNode.Val) +} + +// writeKeyword writes a keyword (e.g. 'syntax'). +func (f *formatter) writeKeyword(keywordNode *ast.KeywordNode) { + f.WriteString(keywordNode.Val) +} + +// writeRune writes a rune (e.g. '='). +func (f *formatter) writeRune(runeNode *ast.RuneNode) { + if strings.ContainsRune("{[(<", runeNode.Rune) { + f.pendingIndent++ + } else if strings.ContainsRune("}])>", runeNode.Rune) { + f.pendingIndent-- + } + f.WriteString(string(runeNode.Rune)) +} + +// writeNode writes the node by dispatching to a function tailored to its concrete type. +// +// Comments are handled in each respective write function so that it can determine whether +// to write the comments in-line or not. +func (f *formatter) writeNode(node ast.Node) { + switch element := node.(type) { + case *ast.ArrayLiteralNode: + f.writeArrayLiteral(element) + case *ast.CompactOptionsNode: + f.writeCompactOptions(element) + case *ast.CompoundIdentNode: + f.writeCompoundIdent(element) + case *ast.CompoundStringLiteralNode: + f.writeCompoundStringLiteralIndent(element) + case *ast.EnumNode: + f.writeEnum(element) + case *ast.EnumValueNode: + f.writeEnumValue(element) + case *ast.ExtendNode: + f.writeExtend(element) + case *ast.ExtensionRangeNode: + f.writeExtensionRange(element) + case ast.FieldLabel: + f.writeFieldLabel(element) + case *ast.FieldNode: + f.writeField(element) + case *ast.FieldReferenceNode: + f.writeFieldReference(element) + case *ast.FloatLiteralNode: + f.writeFloatLiteral(element) + case *ast.GroupNode: + f.writeGroup(element) + case *ast.IdentNode: + f.writeIdent(element) + case *ast.ImportNode: + f.writeImport(element, false) + case *ast.KeywordNode: + f.writeKeyword(element) + case *ast.MapFieldNode: + f.writeMapField(element) + case *ast.MapTypeNode: + f.writeMapType(element) + case *ast.MessageNode: + f.writeMessage(element) + case *ast.MessageFieldNode: + f.writeMessageField(element) + case *ast.MessageLiteralNode: + f.writeMessageLiteral(element) + case *ast.NegativeIntLiteralNode: + f.writeNegativeIntLiteral(element) + case *ast.OneofNode: + f.writeOneOf(element) + case *ast.OptionNode: + f.writeOption(element) + case *ast.OptionNameNode: + f.writeOptionName(element) + case *ast.PackageNode: + f.writePackage(element) + case *ast.RangeNode: + f.writeRange(element) + case *ast.ReservedNode: + f.writeReserved(element) + case *ast.RPCNode: + f.writeRPC(element) + case *ast.RPCTypeNode: + f.writeRPCType(element) + case *ast.RuneNode: + f.writeRune(element) + case *ast.ServiceNode: + f.writeService(element) + case *ast.SignedFloatLiteralNode: + f.writeSignedFloatLiteral(element) + case *ast.SpecialFloatLiteralNode: + f.writeSpecialFloatLiteral(element) + case *ast.StringLiteralNode: + f.writeStringLiteral(element) + case *ast.SyntaxNode: + f.writeSyntax(element) + case *ast.UintLiteralNode: + f.writeUintLiteral(element) + case *ast.EmptyDeclNode: + // Nothing to do here. + default: + f.err = multierr.Append(f.err, fmt.Errorf("unexpected node: %T", node)) + } +} + +// writeStart writes the node across as the start of a line. +// Start nodes have their leading comments written across +// multiple lines, but their trailing comments must be written +// in-line to preserve the line structure. +// +// For example, +// +// // Leading comment on 'message'. +// // Spread across multiple lines. +// message /* This is a trailing comment on 'message' */ Foo {} +// +// Newlines are preserved, so that any logical grouping of elements +// is maintained in the formatted result. +// +// For example, +// +// // Type represents a set of different types. +// enum Type { +// // Unspecified is the naming convention for default enum values. +// TYPE_UNSPECIFIED = 0; +// +// // The following elements are the real values. +// TYPE_ONE = 1; +// TYPE_TWO = 2; +// } +// +// Start nodes are always indented according to the formatter's +// current level of indentation (e.g. nested messages, fields, etc). +// +// Note that this is one of the most complex component of the formatter - it +// controls how each node should be separated from one another and preserves +// newlines in the original source. +func (f *formatter) writeStart(node ast.Node) { + f.writeStartMaybeCompact(node, true) +} + +func (f *formatter) writeStartMaybeCompact(node ast.Node, forceCompact bool) { + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + var ( + nodeNewlineCount = newlineCount(info.LeadingWhitespace()) + compact = forceCompact || isOpenBrace(f.previousNode) + ) + if length := info.LeadingComments().Len(); length > 0 { + // If leading comments are defined, the whitespace we care about + // is attached to the first comment. + f.writeMultilineCommentsMaybeCompact(info.LeadingComments(), forceCompact) + if !forceCompact && nodeNewlineCount > 1 { + // At this point, we're looking at the lines between + // a comment and the node its attached to. + // + // If the last comment is a standard comment, a single newline + // character is sufficient to warrant a separation of the + // two. + // + // If the last comment is a C-style comment, multiple newline + // characters are required because C-style comments don't consume + // a newline. + f.P("") + } + } else if !compact && nodeNewlineCount > 1 { + // If the previous node is an open brace, this is the first element + // in the body of a composite type, so we don't want to write a + // newline. This makes it so that trailing newlines are removed. + // + // For example, + // + // message Foo { + // + // string bar = 1; + // } + // + // Is formatted into the following: + // + // message Foo { + // string bar = 1; + // } + f.P("") + } + f.Indent(node) + f.writeNode(node) + if info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } +} + +// writeInline writes the node and its surrounding comments in-line. +// +// This is useful for writing individual nodes like keywords, runes, +// string literals, etc. +// +// For example, +// +// // This is a leading comment on the syntax keyword. +// syntax = /* This is a leading comment on 'proto3' */" proto3"; +func (f *formatter) writeInline(node ast.Node) { + f.inline = true + defer func() { + f.inline = false + }() + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(node) + f.writeInlineComments(info.TrailingComments()) +} + +// writeBodyEnd writes the node as the end of a body. +// Leading comments are written above the token across +// multiple lines, whereas the trailing comments are +// written in-line and preserve their format. +// +// Body end nodes are always indented according to the +// formatter's current level of indentation (e.g. nested +// messages). +// +// This is useful for writing a node that concludes a +// composite node: ']', '}', '>', etc. +// +// For example, +// +// message Foo { +// string bar = 1; +// // Leading comment on '}'. +// } // Trailing comment on '}. +func (f *formatter) writeBodyEnd(node ast.Node, leadingEndline bool) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + if f.lastWritten != '\n' { + f.P("") + } + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if leadingEndline { + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + } else { + f.writeMultilineComments(info.LeadingComments()) + f.Indent(node) + } + f.writeNode(node) + f.writeTrailingEndComments(info.TrailingComments()) +} + +func (f *formatter) writeLineElement(node ast.Node) { + f.writeBodyEnd(node, false) +} + +// writeBodyEndInline writes the node as the end of a body. +// Leading comments are written above the token across +// multiple lines, whereas the trailing comments are +// written in-line and adapt their comment style if they +// exist. +// +// Body end nodes are always indented according to the +// formatter's current level of indentation (e.g. nested +// messages). +// +// This is useful for writing a node that concludes either +// compact options or an array literal. +// +// This is behaviorally similar to f.writeStart, but it ignores +// the preceding newline logic because these body ends should +// always be compact. +// +// For example, +// +// message Foo { +// string bar = 1 [ +// deprecated = true +// +// // Leading comment on ']'. +// ] /* Trailing comment on ']' */ ; +// } +func (f *formatter) writeBodyEndInline(node ast.Node, leadingInline bool) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if leadingInline { + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + } else { + f.writeMultilineComments(info.LeadingComments()) + f.Indent(node) + } + f.writeNode(node) + if info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } +} + +// writeLineEnd writes the node so that it ends a line. +// +// This is useful for writing individual nodes like ';' and other +// tokens that conclude the end of a single line. In this case, we +// don't want to transform the trailing comment's from '//' to C-style +// because it's not necessary. +// +// For example, +// +// // This is a leading comment on the syntax keyword. +// syntax = " proto3" /* This is a leading comment on the ';'; // This is a trailing comment on the ';'. +func (f *formatter) writeLineEnd(node ast.Node) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + if f.lastWritten != '\n' { + f.P("") + } + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(node) + f.Space() + f.writeTrailingEndComments(info.TrailingComments()) +} + +// writeMultilineComments writes the given comments as a newline-delimited block. +// This is useful for both the beginning of a type (e.g. message, field, etc), as +// well as the trailing comments attached to the beginning of a body block (e.g. +// '{', '[', '<', etc). +// +// For example, +// +// // This is a comment spread across +// // multiple lines. +// message Foo {} +func (f *formatter) writeMultilineComments(comments ast.Comments) { + f.writeMultilineCommentsMaybeCompact(comments, false) +} + +func (f *formatter) writeMultilineCommentsMaybeCompact(comments ast.Comments, forceCompact bool) { + compact := forceCompact || isOpenBrace(f.previousNode) + for i := 0; i < comments.Len(); i++ { + comment := comments.Index(i) + if !compact && newlineCount(comment.LeadingWhitespace()) > 1 { + // Newlines between blocks of comments should be preserved. + // + // For example, + // + // // This is a license header + // // spread across multiple lines. + // + // // Package pet.v1 defines a PetStore API. + // package pet.v1; + // + f.P("") + } + compact = false + f.writeComment(comment.RawText()) + f.WriteString("\n") + } +} + +// writeInlineComments writes the given comments in-line. Standard comments are +// transformed to C-style comments so that we can safely write the comment in-line. +// +// Nearly all of these comments will already be C-style comments. The only cases we're +// preventing are when the type is defined across multiple lines. +// +// For example, given the following: +// +// extend . google. // in-line comment +// protobuf . +// ExtensionRangeOptions { +// optional string label = 20000; +// } +// +// The formatted result is shown below: +// +// extend .google.protobuf./* in-line comment */ExtensionRangeOptions { +// optional string label = 20000; +// } +func (f *formatter) writeInlineComments(comments ast.Comments) { + for i := 0; i < comments.Len(); i++ { + if i > 0 || comments.Index(i).LeadingWhitespace() != "" || f.lastWritten == ';' || f.lastWritten == '}' { + f.Space() + } + text := comments.Index(i).RawText() + if strings.HasPrefix(text, "//") { + text = strings.TrimSpace(strings.TrimPrefix(text, "//")) + text = "/* " + text + " */" + } else { + // no multi-line comments + lines := strings.Split(text, "\n") + for i := range lines { + lines[i] = strings.TrimSpace(lines[i]) + } + text = strings.Join(lines, " ") + } + f.WriteString(text) + } +} + +// writeTrailingEndComments writes the given comments at the end of a line and +// preserves the comment style. This is useful or writing comments attached to +// things like ';' and other tokens that conclude a type definition on a single +// line. +// +// If there is a newline between this trailing comment and the previous node, the +// comments are written immediately underneath the node on a newline. +// +// For example, +// +// enum Type { +// TYPE_UNSPECIFIED = 0; +// } +// // This comment is attached to the '}' +// // So is this one. +func (f *formatter) writeTrailingEndComments(comments ast.Comments) { + for i := 0; i < comments.Len(); i++ { + comment := comments.Index(i) + if i > 0 || comment.LeadingWhitespace() != "" { + f.Space() + } + f.writeComment(comment.RawText()) + } + f.P("") +} + +func (f *formatter) writeComment(comment string) { + if strings.HasPrefix(comment, "/*") && newlineCount(comment) > 0 { + lines := strings.Split(comment, "\n") + // find minimum indent, so we can make all other lines relative to that + minIndent := -1 // sentinel that means unset + // start at 1 because line at index zero starts with "/*", not whitespace + var prefix string + for i := 1; i < len(lines); i++ { + indent, ok := computeIndent(lines[i]) + if ok && (minIndent == -1 || indent < minIndent) { + minIndent = indent + } + if i > 1 && len(prefix) == 0 { + // no shared prefix + continue + } + line := strings.TrimSpace(lines[i]) + if line == "*/" { + continue + } + var linePrefix string + if len(line) > 0 && isCommentPrefix(line[0]) { + linePrefix = line[:1] + } + if i == 1 { + prefix = linePrefix + } else if linePrefix != prefix { + // they do not share prefix + prefix = "" + } + } + if minIndent < 0 { + // This shouldn't be necessary. + // But we do it just in case, to avoid possible panic + minIndent = 0 + } + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine == "" || trimmedLine == "*/" || len(prefix) > 0 { + line = trimmedLine + } else { + // we only trim space from the right; for the left, + // we unindent based on indentation found above. + line = unindent(line, minIndent) + line = strings.TrimRightFunc(line, unicode.IsSpace) + } + // If we have a block comment with no prefix, we'll format + // like so: + + /* + This is a multi-line comment example. + It has no comment prefix on each line. + */ + + // But if there IS a prefix, "|" for example, we'll left-align + // the prefix symbol under the asterisk of the comment start + // like this: + + /* + | This comment has a prefix before each line. + | Usually the prefix is asterisk, but it's a + | pipe in this example. + */ + + // Finally, if the comment prefix is an asterisk, we'll left-align + // the comment end so its asterisk also aligns, like so: + + /* + * This comment has a prefix before each line. + * Usually the prefix is asterisk, which is the + * case in this example. + */ + + if i > 0 && line != "*/" { + if len(prefix) == 0 { + line = " " + line + } else { + line = " " + line + } + } + if line == "*/" && prefix == "*" { + // align the comment end with the other asterisks + line = " " + line + } + + if i != len(lines)-1 { + f.P(line) + } else { + // for last line, we don't use P because we don't + // want to print a trailing newline + f.Indent(nil) + f.WriteString(line) + } + } + } else { + f.Indent(nil) + f.WriteString(strings.TrimSpace(comment)) + } +} + +func isCommentPrefix(ch byte) bool { + r := rune(ch) + // A multi-line comment prefix is *usually* an asterisk, like in the following + /* + * Foo + * Bar + * Baz + */ + // But we'll allow other prefixes. But if it's a letter or number, it's not a prefix. + return !unicode.IsLetter(r) && !unicode.IsNumber(r) +} + +func unindent(s string, unindent int) string { + pos := 0 + for i, r := range s { + if pos == unindent { + return s[i:] + } + if pos > unindent { + // removing tab-stop unindented too far, so we + // add back some spaces to compensate + return strings.Repeat(" ", pos-unindent) + s[i:] + } + + switch r { + case ' ': + pos++ + case '\t': + // jump to next tab stop + pos += 8 - (pos % 8) + default: + return s[i:] + } + } + // nothing but whitespace... + return "" +} + +func computeIndent(s string) (int, bool) { + if strings.TrimSpace(s) == "*/" { + return 0, false + } + indent := 0 + for _, r := range s { + switch r { + case ' ': + indent++ + case '\t': + // jump to next tab stop + indent += 8 - (indent % 8) + default: + return indent, true + } + } + // if we get here, line is nothing but whitespace + return 0, false +} + +func (f *formatter) leadingCommentsContainBlankLine(n ast.Node) bool { + info := f.nodeInfo(n) + comments := info.LeadingComments() + for i := 0; i < comments.Len(); i++ { + if newlineCount(comments.Index(i).LeadingWhitespace()) > 1 { + return true + } + } + return newlineCount(info.LeadingWhitespace()) > 1 +} + +func (f *formatter) importHasComment(importNode *ast.ImportNode) bool { + if f.nodeHasComment(importNode) { + return true + } + if importNode == nil { + return false + } + + return f.nodeHasComment(importNode.Keyword) || + f.nodeHasComment(importNode.Name) || + f.nodeHasComment(importNode.Semicolon) || + f.nodeHasComment(importNode.Public) || + f.nodeHasComment(importNode.Weak) +} + +func (f *formatter) nodeHasComment(node ast.Node) bool { + // when node != nil, node's value could be nil, see: https://go.dev/doc/faq#nil_error + if node == nil || reflect.ValueOf(node).IsNil() { + return false + } + + nodeinfo := f.nodeInfo(node) + return nodeinfo.LeadingComments().Len() > 0 || + nodeinfo.TrailingComments().Len() > 0 +} + +func (f *formatter) setTrailingComments(node ast.Node, comments ast.Comments) { + f.overrideTrailingComments[node] = comments +} + +func (f *formatter) nodeInfo(node ast.Node) nodeInfo { + info := f.fileNode.NodeInfo(node) + if trailingComments, ok := f.overrideTrailingComments[node]; ok { + return infoWithTrailingComments{info, trailingComments} + } + return info +} + +type nodeInfo interface { + Start() ast.SourcePos + End() ast.SourcePos + LeadingComments() ast.Comments + TrailingComments() ast.Comments + LeadingWhitespace() string + RawText() string +} + +type infoWithTrailingComments struct { + ast.NodeInfo + trailing ast.Comments +} + +func (n infoWithTrailingComments) TrailingComments() ast.Comments { + return n.trailing +} + +// importSortOrder maps import types to a sort order number, so it can be compared and sorted. +// `import`=3, `import public`=2, `import weak`=1 +func importSortOrder(node *ast.ImportNode) int { + switch { + case node.Public != nil: + return 2 + case node.Weak != nil: + return 1 + default: + return 3 + } +} + +// stringForOptionName returns the string representation of the given option name node. +// This is used for sorting file-level options. +func stringForOptionName(optionNameNode *ast.OptionNameNode) string { + var result string + for j, part := range optionNameNode.Parts { + if j > 0 { + // Add a dot between each of the parts. + result += "." + } + result += stringForFieldReference(part) + } + return result +} + +// stringForFieldReference returns the string representation of the given field reference. +// This is used for sorting file-level options. +func stringForFieldReference(fieldReference *ast.FieldReferenceNode) string { + var result string + if fieldReference.Open != nil { + result += "(" + } + result += string(fieldReference.Name.AsIdentifier()) + if fieldReference.Close != nil { + result += ")" + } + return result +} + +// isOpenBrace returns true if the given node represents one of the +// possible open brace tokens, namely '{', '[', or '<'. +func isOpenBrace(node ast.Node) bool { + if node == nil { + return false + } + runeNode, ok := node.(*ast.RuneNode) + if !ok { + return false + } + return runeNode.Rune == '{' || runeNode.Rune == '[' || runeNode.Rune == '<' +} + +// newlineCount returns the number of newlines in the given value. +// This is useful for determining whether or not we should preserve +// the newline between nodes. +// +// The newlines don't need to be adjacent to each other - all of the +// tokens between them are other whitespace characters, so we can +// safely ignore them. +func newlineCount(value string) int { + return strings.Count(value, "\n") +} + +func messageLiteralOpen(msg *ast.MessageLiteralNode) *ast.RuneNode { + node := msg.Open + if node.Rune == '{' { + return node + } + // If it's not "{" then this message literal used "<" and ">" to enclose it. + // For consistent formatted output, change it to "{". + return ast.NewRuneNode('{', node.Token()) +} + +func messageLiteralClose(msg *ast.MessageLiteralNode) *ast.RuneNode { + node := msg.Close + if node.Rune == '}' { + return node + } + // If it's not "}" then this message literal used "<" and ">" to enclose it. + // For consistent formatted output, change it to "}". + return ast.NewRuneNode('}', node.Token()) +} + +type ByOptionName []*ast.OptionNode + +func (a ByOptionName) Len() int { return len(a) } +func (a ByOptionName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByOptionName) Less(i, j int) bool { + // The default options (e.g. cc_enable_arenas) should always + // be sorted above custom options (which are identified by a + // leading '('). + left := stringForOptionName(a[i].Name) + right := stringForOptionName(a[j].Name) + if strings.HasPrefix(left, "(") && !strings.HasPrefix(right, "(") { + // Prefer the default option on the right. + return false + } + if !strings.HasPrefix(left, "(") && strings.HasPrefix(right, "(") { + // Prefer the default option on the left. + return true + } + // Both options are custom, so we defer to the standard sorting. + return left < right +} + +// writeNodes writes nodes with sorted options. +func writeNodes[T ast.Node](nodes []T, f *formatter) { + optionNodes := []*ast.OptionNode{} + for _, node := range nodes { + if option, ok := ast.Node(node).(*ast.OptionNode); ok { + optionNodes = append(optionNodes, option) + } + } + + sort.Sort(ByOptionName(optionNodes)) + + for _, node := range optionNodes { + f.writeNode(node) + } + + for _, node := range nodes { + if _, ok := ast.Node(node).(*ast.OptionNode); !ok { + f.writeNode(node) + } + } +} diff --git a/go/protopace/formatter_test.go b/go/protopace/formatter_test.go new file mode 100644 index 000000000..bdc960fd7 --- /dev/null +++ b/go/protopace/formatter_test.go @@ -0,0 +1,169 @@ +package main + +import ( + "testing" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/stretchr/testify/assert" +) + +func TestFormatEnum(t *testing.T) { + + testProto := + `syntax = "proto3"; + +package pkg; + +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +enum MyEnum { + option (my_option3) = "my_value3"; + option (my_option) = "my_value"; + option (my_option2) = "my_value2"; + + ACTIVE = 0; +}` + + expected := + `syntax = "proto3"; +package pkg; + +import "google/protobuf/descriptor.proto"; +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} +enum MyEnum { + option (my_option) = "my_value"; + option (my_option2) = "my_value2"; + option (my_option3) = "my_value3"; + ACTIVE = 0; +} +` + + assert := assert.New(t) + + testSchema, err := s.FromString("test.proto", testProto, []s.Schema{}) + assert.NoError(err) + assert.NotNil(testSchema) + + result, err := Format(*testSchema) + assert.NoError(err) + assert.Equal(result.Schema, expected) +} + +func TestFormatFullyQualifiedNames(t *testing.T) { + + dependency := + ` +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} +` + + dependencyV2 := + ` +syntax = "proto3"; +package my.awesome.customer.v2; + +enum Status { + ACTIVE = 0; + INACTIVE = 1; +} + +message NestedValue { + string value = 1; + Status status = 2; +} +` + + testProtoA := + `syntax = "proto3"; +package my.awesome.customer.v2; + +import "my/awesome/customer/v1/nested_value.proto"; +import "my/awesome/customer/v2/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option php_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option ruby_package = "My::Awesome::Customer::V1"; + +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; + + +message EventValue { + .my.awesome.customer.v2.NestedValue nested_value = 1; + .google.protobuf.Timestamp created_at = 2; + + + .my.awesome.customer.v2.Status status = 3; +} +` + + testProtoB := + `syntax = "proto3"; +package my.awesome.customer.v2; + +import "my/awesome/customer/v1/nested_value.proto"; +import "my/awesome/customer/v2/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option php_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option ruby_package = "My::Awesome::Customer::V1"; + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; +} +` + + assert := assert.New(t) + + testDependency, err := s.FromString("my/awesome/customer/v1/nested_value.proto", dependency, []s.Schema{}) + assert.NoError(err) + assert.NotNil(testDependency) + + testDependencyV2, err := s.FromString("my/awesome/customer/v2/nested_value.proto", dependencyV2, []s.Schema{}) + assert.NoError(err) + assert.NotNil(testDependencyV2) + + testSchemaA, err := s.FromString("test.proto", testProtoA, []s.Schema{*testDependency, *testDependencyV2}) + assert.NoError(err) + assert.NotNil(testSchemaA) + a, err := Format(*testSchemaA) + assert.NoError(err) + + testSchemaB, err := s.FromString("test.proto", testProtoB, []s.Schema{*testDependency, *testDependencyV2}) + assert.NoError(err) + assert.NotNil(testSchemaB) + b, err := Format(*testSchemaB) + assert.NoError(err) + + assert.Equal(a.Schema, b.Schema) + err = Check(b, a) + assert.NoError(err) +} diff --git a/go/protopace/go.mod b/go/protopace/go.mod new file mode 100644 index 000000000..ad7dccca3 --- /dev/null +++ b/go/protopace/go.mod @@ -0,0 +1,42 @@ +module github.com/Aiven-Open/karapace/go/protopace + +go 1.21.0 + +toolchain go1.21.10 + +replace github.com/bufbuild/buf => github.com/Aiven-Open/buf v0.0.0-20240829124012-099cfb8b4b31 + +require ( + github.com/bufbuild/buf v1.34.0 + github.com/bufbuild/protocompile v0.14.0 + github.com/gofrs/uuid/v5 v5.3.0 + github.com/stretchr/testify v1.9.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.27.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee // indirect + github.com/bufbuild/protovalidate-go v0.6.4 // indirect + github.com/bufbuild/protoyaml-go v0.1.11 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/cel-go v0.21.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect + golang.org/x/mod v0.20.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240823204242-4ba0660f739c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240823204242-4ba0660f739c // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go/protopace/go.sum b/go/protopace/go.sum new file mode 100644 index 000000000..5a2301354 --- /dev/null +++ b/go/protopace/go.sum @@ -0,0 +1,83 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2 h1:SZRVx928rbYZ6hEKUIN+vtGDkl7uotABRWGY4OAg5gM= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw= +github.com/Aiven-Open/buf v0.0.0-20240829124012-099cfb8b4b31 h1:sTDz5KNYsvDl4mjQ3o/grLdesCnu/YJilBmNVTNZs04= +github.com/Aiven-Open/buf v0.0.0-20240829124012-099cfb8b4b31/go.mod h1:1P0U+x/ky1KhpK7o7mGraDAYjQUG7710wk5lEZFWsTA= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/bufbuild/protocompile v0.14.0 h1:z3DW4IvXE5G/uTOnSQn+qwQQxvhckkTWLS/0No/o7KU= +github.com/bufbuild/protocompile v0.14.0/go.mod h1:N6J1NYzkspJo3ZwyL4Xjvli86XOj1xq4qAasUFxGups= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee h1:E6ET8YUcYJ1lAe6ctR3as7yqzW2BNItDFnaB5zQq/8M= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee/go.mod h1:HjGFxsck9RObrTJp2hXQZfWhPgZqnR6sR1U5fCA/Kus= +github.com/bufbuild/protovalidate-go v0.6.4 h1:QtNIz4LGclM3UArQv/R1AKNF7MO8wriT9v7b8Gnmqak= +github.com/bufbuild/protovalidate-go v0.6.4/go.mod h1:HlkVnkE/zVYZvHIG/a7QZuzqC9bSqHaOOTeRomYF0Q8= +github.com/bufbuild/protoyaml-go v0.1.11 h1:Iyixd6Y5dx6ws6Uh8APgC1lMyvXt710NayoY8cY0Vj8= +github.com/bufbuild/protoyaml-go v0.1.11/go.mod h1:KCBItkvZOK/zwGueLdH1Wx1RLyFn5rCH7YjQrdty2Wc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk= +github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= +github.com/google/cel-go v0.21.0 h1:cl6uW/gxN+Hy50tNYvI691+sXxioCnstFzLp2WO4GCI= +github.com/google/cel-go v0.21.0/go.mod h1:rHUlWCcBKgyEk+eV03RPdZUekPp6YcJwV0FxuUksYxc= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/api v0.0.0-20240823204242-4ba0660f739c h1:e0zB268kOca6FbuJkYUGxfwG4DKFZG/8DLyv9Zv66cE= +google.golang.org/genproto/googleapis/api v0.0.0-20240823204242-4ba0660f739c/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240823204242-4ba0660f739c h1:Kqjm4WpoWvwhMPcrAczoTyMySQmYa9Wy2iL6Con4zn8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240823204242-4ba0660f739c/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/protopace/main.go b/go/protopace/main.go new file mode 100644 index 000000000..84eb82d55 --- /dev/null +++ b/go/protopace/main.go @@ -0,0 +1,86 @@ +package main + +/* +#include + +struct result { + char* res; + char* err; +}; +*/ +import "C" +import ( + "unsafe" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" +) + +func result(schema string, err error) *C.struct_result { + res := (*C.struct_result)(C.malloc(C.size_t(unsafe.Sizeof(C.struct_result{})))) + res.res = C.CString(schema) + res.err = nil + if err != nil { + res.err = C.CString(err.Error()) + } + return res +} + +//export FreeResult +func FreeResult(result *C.struct_result) { + C.free(unsafe.Pointer(result.res)) + C.free(unsafe.Pointer(result.err)) + C.free(unsafe.Pointer(result)) +} + +func createSchema(cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int) (*s.Schema, error) { + depArray := unsafe.Slice(cDependencies, depsLenght) + depNamesArray := unsafe.Slice(cDependencyNames, depsLenght) + dependencies := []s.Schema{} + for i, dep := range depArray { + dependency, err := s.FromString(C.GoString(depNamesArray[i]), C.GoString(dep), []s.Schema{}) + if err != nil { + return nil, err + } + dependencies = append(dependencies, *dependency) + } + + schema, err := s.FromString(C.GoString(cSchemaName), C.GoString(cSchema), dependencies) + return schema, err +} + +//export FormatSchema +func FormatSchema(cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int) *C.struct_result { + schema, err := createSchema(cSchemaName, cSchema, cDependencyNames, cDependencies, depsLenght) + if err != nil { + return result("", err) + } + + res, err := Format(*schema) + if err != nil { + return result("", err) + } + return result(res.Schema, err) +} + +//export CheckCompatibility +func CheckCompatibility( + cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int, + cSchemaNamePrev *C.char, cSchemaPrev *C.char, cDependencyNamesPrev **C.char, cDependenciesPrev **C.char, depsLenghtPrev C.int) *C.char { + + schema, err := createSchema(cSchemaName, cSchema, cDependencyNames, cDependencies, depsLenght) + if err != nil { + return C.CString(err.Error()) + } + prevSchema, err := createSchema(cSchemaNamePrev, cSchemaPrev, cDependencyNamesPrev, cDependenciesPrev, depsLenghtPrev) + if err != nil { + return C.CString(err.Error()) + } + + err = Check(*schema, *prevSchema) + if err != nil { + return C.CString(err.Error()) + } + return nil +} + +func main() {} diff --git a/go/protopace/schema/compiler.go b/go/protopace/schema/compiler.go new file mode 100644 index 000000000..5311d022c --- /dev/null +++ b/go/protopace/schema/compiler.go @@ -0,0 +1,10 @@ +package schema + +import ( + "github.com/bufbuild/protocompile" +) + +func NewCompiler(resolver protocompile.Resolver) *protocompile.Compiler { + compiler := protocompile.Compiler{Resolver: resolver, RetainASTs: true} + return &compiler +} diff --git a/go/protopace/schema/google/type/README.md b/go/protopace/schema/google/type/README.md new file mode 100644 index 000000000..adf1563a8 --- /dev/null +++ b/go/protopace/schema/google/type/README.md @@ -0,0 +1,7 @@ +## Google Common Types + +This package contains definitions of common types for Google APIs. +All types defined in this package are suitable for different APIs to +exchange data, and will never break binary compatibility. They should +have design quality comparable to major programming languages like +Java and C#. diff --git a/go/protopace/schema/google/type/calendar_period.proto b/go/protopace/schema/google/type/calendar_period.proto new file mode 100644 index 000000000..25a8f6441 --- /dev/null +++ b/go/protopace/schema/google/type/calendar_period.proto @@ -0,0 +1,56 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/go/protopace/schema/google/type/color.proto b/go/protopace/schema/google/type/color.proto new file mode 100644 index 000000000..3e57c1fb2 --- /dev/null +++ b/go/protopace/schema/google/type/color.proto @@ -0,0 +1,174 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness. For example, the fields of this representation +// can be trivially provided to the constructor of `java.awt.Color` in Java; it +// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS `rgba()` string in JavaScript. +// +// This reference page doesn't carry information about the absolute color +// space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color +// space. +// +// When color equality needs to be decided, implementations, unless +// documented otherwise, treat two colors as equal if all their red, +// green, blue, and alpha values each differ by at most 1e-5. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is rendered as a solid color + // (as if the alpha value had been explicitly given a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/go/protopace/schema/google/type/date.proto b/go/protopace/schema/google/type/date.proto new file mode 100644 index 000000000..6370cd869 --- /dev/null +++ b/go/protopace/schema/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/go/protopace/schema/google/type/datetime.proto b/go/protopace/schema/google/type/datetime.proto new file mode 100644 index 000000000..a363a41ef --- /dev/null +++ b/go/protopace/schema/google/type/datetime.proto @@ -0,0 +1,104 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time (or occasionally physical time). +// +// This type can represent a civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type may also be used to represent a physical time if all the date and +// time fields are set and either case of the `time_offset` oneof is set. +// Consider using `Timestamp` message for physical time instead. If your use +// case also would like to store the user's timezone, that can be done in +// another field. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/go/protopace/schema/google/type/dayofweek.proto b/go/protopace/schema/google/type/dayofweek.proto new file mode 100644 index 000000000..e16c19469 --- /dev/null +++ b/go/protopace/schema/google/type/dayofweek.proto @@ -0,0 +1,50 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of the week. +enum DayOfWeek { + // The day of the week is unspecified. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // Monday + MONDAY = 1; + + // Tuesday + TUESDAY = 2; + + // Wednesday + WEDNESDAY = 3; + + // Thursday + THURSDAY = 4; + + // Friday + FRIDAY = 5; + + // Saturday + SATURDAY = 6; + + // Sunday + SUNDAY = 7; +} diff --git a/go/protopace/schema/google/type/decimal.proto b/go/protopace/schema/google/type/decimal.proto new file mode 100644 index 000000000..293d08273 --- /dev/null +++ b/go/protopace/schema/google/type/decimal.proto @@ -0,0 +1,95 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; +option java_multiple_files = true; +option java_outer_classname = "DecimalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A representation of a decimal value, such as 2.5. Clients may convert values +// into language-native decimal formats, such as Java's [BigDecimal][] or +// Python's [decimal.Decimal][]. +// +// [BigDecimal]: +// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html +// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html +message Decimal { + // The decimal value, as a string. + // + // The string representation consists of an optional sign, `+` (`U+002B`) + // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits + // ("the integer"), optionally followed by a fraction, optionally followed + // by an exponent. + // + // The fraction consists of a decimal point followed by zero or more decimal + // digits. The string must contain at least one digit in either the integer + // or the fraction. The number formed by the sign, the integer and the + // fraction is referred to as the significand. + // + // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) + // followed by one or more decimal digits. + // + // Services **should** normalize decimal values before storing them by: + // + // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). + // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). + // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). + // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). + // + // Services **may** perform additional normalization based on its own needs + // and the internal decimal implementation selected, such as shifting the + // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). + // Additionally, services **may** preserve trailing zeroes in the fraction + // to indicate increased precision, but are not required to do so. + // + // Note that only the `.` character is supported to divide the integer + // and the fraction; `,` **should not** be supported regardless of locale. + // Additionally, thousand separators **should not** be supported. If a + // service does support them, values **must** be normalized. + // + // The ENBF grammar is: + // + // DecimalString = + // [Sign] Significand [Exponent]; + // + // Sign = '+' | '-'; + // + // Significand = + // Digits ['.'] [Digits] | [Digits] '.' Digits; + // + // Exponent = ('e' | 'E') [Sign] Digits; + // + // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; + // + // Services **should** clearly document the range of supported values, the + // maximum supported precision (total number of digits), and, if applicable, + // the scale (number of digits after the decimal point), as well as how it + // behaves when receiving out-of-bounds values. + // + // Services **may** choose to accept values passed as input even when the + // value has a higher precision or scale than the service supports, and + // **should** round the value to fit the supported scale. Alternatively, the + // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) + // if precision would be lost. + // + // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in + // gRPC) if the service receives a value outside of the supported range. + string value = 1; +} diff --git a/go/protopace/schema/google/type/expr.proto b/go/protopace/schema/google/type/expr.proto new file mode 100644 index 000000000..544e66874 --- /dev/null +++ b/go/protopace/schema/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/go/protopace/schema/google/type/fraction.proto b/go/protopace/schema/google/type/fraction.proto new file mode 100644 index 000000000..06f072322 --- /dev/null +++ b/go/protopace/schema/google/type/fraction.proto @@ -0,0 +1,33 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The numerator in the fraction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/go/protopace/schema/google/type/interval.proto b/go/protopace/schema/google/type/interval.proto new file mode 100644 index 000000000..fcf94c866 --- /dev/null +++ b/go/protopace/schema/google/type/interval.proto @@ -0,0 +1,46 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/timestamp.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; +option java_multiple_files = true; +option java_outer_classname = "IntervalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time interval, encoded as a Timestamp start (inclusive) and a +// Timestamp end (exclusive). +// +// The start must be less than or equal to the end. +// When the start equals the end, the interval is empty (matches no time). +// When both start and end are unspecified, the interval matches any time. +message Interval { + // Optional. Inclusive start of the interval. + // + // If specified, a Timestamp matching this interval will have to be the same + // or after the start. + google.protobuf.Timestamp start_time = 1; + + // Optional. Exclusive end of the interval. + // + // If specified, a Timestamp matching this interval will have to be before the + // end. + google.protobuf.Timestamp end_time = 2; +} diff --git a/go/protopace/schema/google/type/latlng.proto b/go/protopace/schema/google/type/latlng.proto new file mode 100644 index 000000000..daeba48b4 --- /dev/null +++ b/go/protopace/schema/google/type/latlng.proto @@ -0,0 +1,37 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object that represents a latitude/longitude pair. This is expressed as a +// pair of doubles to represent degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/go/protopace/schema/google/type/localized_text.proto b/go/protopace/schema/google/type/localized_text.proto new file mode 100644 index 000000000..82d083c43 --- /dev/null +++ b/go/protopace/schema/google/type/localized_text.proto @@ -0,0 +1,36 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; +option java_multiple_files = true; +option java_outer_classname = "LocalizedTextProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Localized variant of a text in a particular language. +message LocalizedText { + // Localized string in the language corresponding to `language_code' below. + string text = 1; + + // The text's BCP-47 language code, such as "en-US" or "sr-Latn". + // + // For more information, see + // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. + string language_code = 2; +} diff --git a/go/protopace/schema/google/type/money.proto b/go/protopace/schema/google/type/money.proto new file mode 100644 index 000000000..c61094336 --- /dev/null +++ b/go/protopace/schema/google/type/money.proto @@ -0,0 +1,42 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The three-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/go/protopace/schema/google/type/month.proto b/go/protopace/schema/google/type/month.proto new file mode 100644 index 000000000..19982cb51 --- /dev/null +++ b/go/protopace/schema/google/type/month.proto @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecified month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/go/protopace/schema/google/type/phone_number.proto b/go/protopace/schema/google/type/phone_number.proto new file mode 100644 index 000000000..370d1623d --- /dev/null +++ b/go/protopace/schema/google/type/phone_number.proto @@ -0,0 +1,113 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; +option java_multiple_files = true; +option java_outer_classname = "PhoneNumberProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a phone number, suitable as an API wire format. +// +// This representation: +// +// - should not be used for locale-specific formatting of a phone number, such +// as "+1 (650) 253-0000 ext. 123" +// +// - is not designed for efficient storage +// - may not be suitable for dialing - specialized libraries (see references) +// should be used to parse the number for that purpose +// +// To do something meaningful with this number, such as format it for various +// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. +// +// For instance, in Java this would be: +// +// com.google.type.PhoneNumber wireProto = +// com.google.type.PhoneNumber.newBuilder().build(); +// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = +// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); +// if (!wireProto.getExtension().isEmpty()) { +// phoneNumber.setExtension(wireProto.getExtension()); +// } +// +// Reference(s): +// - https://github.com/google/libphonenumber +message PhoneNumber { + // An object representing a short code, which is a phone number that is + // typically much shorter than regular phone numbers and can be used to + // address messages in MMS and SMS systems, as well as for abbreviated dialing + // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). + // + // Short codes are restricted to a region and are not internationally + // dialable, which means the same short code can exist in different regions, + // with different usage and pricing, even if those regions share the same + // country calling code (e.g. US and CA). + message ShortCode { + // Required. The BCP-47 region code of the location where calls to this + // short code can be made, such as "US" and "BB". + // + // Reference(s): + // - http://www.unicode.org/reports/tr35/#unicode_region_subtag + string region_code = 1; + + // Required. The short code digits, without a leading plus ('+') or country + // calling code, e.g. "611". + string number = 2; + } + + // Required. Either a regular number, or a short code. New fields may be + // added to the oneof below in the future, so clients should ignore phone + // numbers for which none of the fields they coded against are set. + oneof kind { + // The phone number, represented as a leading plus sign ('+'), followed by a + // phone number that uses a relaxed ITU E.164 format consisting of the + // country calling code (1 to 3 digits) and the subscriber number, with no + // additional spaces or formatting, e.g.: + // - correct: "+15552220123" + // - incorrect: "+1 (555) 222-01234 x123". + // + // The ITU E.164 format limits the latter to 12 digits, but in practice not + // all countries respect that, so we relax that restriction here. + // National-only numbers are not allowed. + // + // References: + // - https://www.itu.int/rec/T-REC-E.164-201011-I + // - https://en.wikipedia.org/wiki/E.164. + // - https://en.wikipedia.org/wiki/List_of_country_calling_codes + string e164_number = 1; + + // A short code. + // + // Reference(s): + // - https://en.wikipedia.org/wiki/Short_code + ShortCode short_code = 2; + } + + // The phone number's extension. The extension is not standardized in ITU + // recommendations, except for being defined as a series of numbers with a + // maximum length of 40 digits. Other than digits, some other dialing + // characters such as ',' (indicating a wait) or '#' may be stored here. + // + // Note that no regions currently use extensions with short codes, so this + // field is normally only set in conjunction with an E.164 number. It is held + // separately from the E.164 number to allow for short code extensions in the + // future. + string extension = 3; +} diff --git a/go/protopace/schema/google/type/postal_address.proto b/go/protopace/schema/google/type/postal_address.proto new file mode 100644 index 000000000..7023a9b3e --- /dev/null +++ b/go/protopace/schema/google/type/postal_address.proto @@ -0,0 +1,134 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/go/protopace/schema/google/type/quaternion.proto b/go/protopace/schema/google/type/quaternion.proto new file mode 100644 index 000000000..416de30cf --- /dev/null +++ b/go/protopace/schema/google/type/quaternion.proto @@ -0,0 +1,94 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/go/protopace/schema/google/type/timeofday.proto b/go/protopace/schema/google/type/timeofday.proto new file mode 100644 index 000000000..3735745a4 --- /dev/null +++ b/go/protopace/schema/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and +// `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} diff --git a/go/protopace/schema/google/type/type.yaml b/go/protopace/schema/google/type/type.yaml new file mode 100644 index 000000000..d5c71364d --- /dev/null +++ b/go/protopace/schema/google/type/type.yaml @@ -0,0 +1,40 @@ +type: google.api.Service +config_version: 3 +name: type.googleapis.com +title: Common Types + +types: +- name: google.type.Color +- name: google.type.Date +- name: google.type.DateTime +- name: google.type.Decimal +- name: google.type.Expr +- name: google.type.Fraction +- name: google.type.Interval +- name: google.type.LatLng +- name: google.type.LocalizedText +- name: google.type.Money +- name: google.type.PhoneNumber +- name: google.type.PostalAddress +- name: google.type.Quaternion +- name: google.type.TimeOfDay + +enums: +- name: google.type.CalendarPeriod +- name: google.type.DayOfWeek +- name: google.type.Month + +documentation: + summary: Defines common types for Google APIs. + overview: |- + # Google Common Types + + This package contains definitions of common types for Google APIs. + All types defined in this package are suitable for different APIs to + exchange data, and will never break binary compatibility. They should + have design quality comparable to major programming languages like + Java and C#. + + NOTE: Some common types are defined in the package `google.protobuf` + as they are directly supported by Protocol Buffers compiler and + runtime. Those types are called Well-Known Types. diff --git a/go/protopace/schema/google_types.go b/go/protopace/schema/google_types.go new file mode 100644 index 000000000..e81f5a533 --- /dev/null +++ b/go/protopace/schema/google_types.go @@ -0,0 +1,23 @@ +package schema + +import ( + "embed" + "io" + + "github.com/bufbuild/protocompile" +) + +//go:embed google/type/*.proto +var files embed.FS + +// WithGoogleTypeImports returns a new resolver that can provide the source code for google/types protos +func WithGoogleTypeImports(resolver protocompile.Resolver) protocompile.Resolver { + return protocompile.CompositeResolver{ + resolver, + &protocompile.SourceResolver{ + Accessor: func(path string) (io.ReadCloser, error) { + return files.Open(path) + }, + }, + } +} diff --git a/go/protopace/schema/resolver.go b/go/protopace/schema/resolver.go new file mode 100644 index 000000000..9ace3e55d --- /dev/null +++ b/go/protopace/schema/resolver.go @@ -0,0 +1,39 @@ +package schema + +import ( + "fmt" + "strings" + + "github.com/bufbuild/protocompile" +) + +type SchemaResolver struct { + schemas map[string]Schema +} + +func NewSchemaResolver(schemas []Schema) protocompile.Resolver { + schemasIndex := map[string]Schema{} + for _, schema := range schemas { + schemasIndex[schema.Name] = schema + } + resolver := SchemaResolver{schemas: schemasIndex} + return WithGoogleTypeImports(protocompile.WithStandardImports(&resolver)) +} + +func (s *SchemaResolver) AddSchema(schema Schema) { + s.schemas[schema.Name] = schema +} + +// FindFileByPath implements protocompile.Resolver. +func (s *SchemaResolver) FindFileByPath(path string) (protocompile.SearchResult, error) { + searchResult := protocompile.SearchResult{} + schema, ok := s.schemas[path] + if !ok { + return searchResult, fmt.Errorf("schema not found: %s", path) + } + searchResult.Source = strings.NewReader(schema.Schema) + searchResult.ParseResult = schema.ParserResult + return searchResult, nil +} + +var _ protocompile.Resolver = (*SchemaResolver)(nil) diff --git a/go/protopace/schema/schema.go b/go/protopace/schema/schema.go new file mode 100644 index 000000000..1a0bb822e --- /dev/null +++ b/go/protopace/schema/schema.go @@ -0,0 +1,81 @@ +package schema + +import ( + "context" + "strings" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/protocompile/linker" + "github.com/bufbuild/protocompile/parser" + "github.com/bufbuild/protocompile/reporter" + "github.com/gofrs/uuid/v5" +) + +var ( + handler = reporter.NewHandler(nil) +) + +type Schema struct { + Schema string + Name string + ParserResult parser.Result + Dependencies []Schema +} + +func FromString(name string, proto string, dependencies []Schema) (*Schema, error) { + fileNode, err := parser.Parse(name, strings.NewReader(proto), handler) + if err != nil { + return nil, err + } + result, err := parser.ResultFromAST(fileNode, true, handler) + if err != nil { + return nil, err + } + return &Schema{Schema: proto, Name: name, ParserResult: result, Dependencies: dependencies}, nil +} + +func (s Schema) Compile() ([]linker.Result, error) { + resolver := NewSchemaResolver(append(s.Dependencies, s)) + compiler := NewCompiler(resolver) + ctx := context.Background() + schemas := []string{s.Name} + for _, dep := range s.Dependencies { + schemas = append(schemas, dep.Name) + } + files, err := compiler.Compile(ctx, schemas...) + if err != nil { + return nil, err + } + res := make([]linker.Result, len(files)) + for i, f := range files { + res[i] = f.(linker.Result) + } + return res, nil +} + +func (s Schema) CompileBufImage() (bufimage.Image, error) { + res, err := s.Compile() + if err != nil { + return nil, err + } + files := make([]bufimage.ImageFile, len(res)) + for i, r := range res { + file, err := bufimage.NewImageFile( + r.FileDescriptorProto(), + nil, + uuid.Nil, + "", + "", + false, + false, + nil, + ) + if err != nil { + return nil, err + } + files[i] = file + } + + image, err := bufimage.NewImage(files) + return image, err +} diff --git a/karapace/config.py b/karapace/config.py index bbae62701..2618158a2 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -82,6 +82,7 @@ class Config(TypedDict): statsd_port: int kafka_schema_reader_strict_mode: bool kafka_retriable_errors_silenced: bool + use_protobuf_formatter: bool sentry: NotRequired[Mapping[str, object]] tags: NotRequired[Mapping[str, object]] @@ -158,6 +159,7 @@ class ConfigDefaults(Config, total=False): "statsd_port": 8125, "kafka_schema_reader_strict_mode": False, "kafka_retriable_errors_silenced": True, + "use_protobuf_formatter": False, } SECRET_CONFIG_OPTIONS = [SASL_PLAIN_PASSWORD] diff --git a/karapace/dependency.py b/karapace/dependency.py index a289e1f25..11c84781b 100644 --- a/karapace/dependency.py +++ b/karapace/dependency.py @@ -7,7 +7,10 @@ from __future__ import annotations +from karapace.errors import InvalidSchema +from karapace.protobuf.protopace.protopace import Proto from karapace.schema_references import Reference +from karapace.schema_type import SchemaType from karapace.typing import JsonData, Subject, Version from typing import TYPE_CHECKING @@ -41,6 +44,16 @@ def get_schema(self) -> ValidatedTypedSchema: def of(reference: Reference, target_schema: ValidatedTypedSchema) -> Dependency: return Dependency(reference.name, reference.subject, reference.version, target_schema) + def to_proto(self) -> Proto: + if self.schema.schema_type is not SchemaType.PROTOBUF: + raise InvalidSchema("Only supported for Protobuf") + + dependencies = [] + if self.schema.dependencies: + for _, dep in self.schema.dependencies.items(): + dependencies.append(dep.to_proto()) + return Proto(self.name, self.schema.schema_str, dependencies) + def to_dict(self) -> JsonData: return { "name": self.name, diff --git a/karapace/protobuf/protopace/__init__.py b/karapace/protobuf/protopace/__init__.py new file mode 100644 index 000000000..ec6c72f0b --- /dev/null +++ b/karapace/protobuf/protopace/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from .protopace import check_compatibility, format_proto, IncompatibleError, Proto # noqa: F401 diff --git a/karapace/protobuf/protopace/protopace.py b/karapace/protobuf/protopace/protopace.py new file mode 100644 index 000000000..a65f90582 --- /dev/null +++ b/karapace/protobuf/protopace/protopace.py @@ -0,0 +1,189 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from dataclasses import dataclass, field +from functools import cached_property +from karapace.errors import InvalidSchema +from typing import Dict, List + +import ctypes +import importlib.util +import timeit + +spec = importlib.util.find_spec("protopacelib") +if not spec: + raise FileNotFoundError("Unable to find protopace shared library") + +lib_file = spec.origin +lib = ctypes.CDLL(lib_file) + +lib.FormatSchema.argtypes = [ + ctypes.c_char_p, # schema name + ctypes.c_char_p, # schema string + ctypes.Array, # dependency names + ctypes.Array, # dependency schema strings + ctypes.c_int, # number of dependencies +] +lib.FormatSchema.restype = ctypes.c_void_p +lib.CheckCompatibility.restype = ctypes.c_char_p + + +class FormatResult(ctypes.Structure): + _fields_ = [ + ("res", ctypes.c_char_p), + ("err", ctypes.c_char_p), + ] + + +@dataclass +class Proto: + name: str + schema: str + dependencies: List["Proto"] = field(default_factory=list) + + @cached_property + def all_dependencies(self) -> List["Proto"]: + dependencies: Dict[str, "Proto"] = {} + for dep in self.dependencies: + if dep.dependencies: + dependencies.update([(d.name, d) for d in dep.all_dependencies]) + dependencies[dep.name] = dep + return list(dependencies.values()) + + +class IncompatibleError(Exception): + pass + + +def format_proto(proto: Proto) -> str: + length = len(proto.all_dependencies) + c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.all_dependencies]) + c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.all_dependencies]) + c_name = ctypes.c_char_p(proto.name.encode()) + c_schema = ctypes.c_char_p(proto.schema.encode()) + res_ptr = lib.FormatSchema(c_name, c_schema, c_dependency_names, c_dependencies, length) + res = FormatResult.from_address(res_ptr) + + if res.err: + err = res.err + msg = err.decode() + lib.FreeResult(ctypes.c_void_p(res_ptr)) + raise InvalidSchema(msg) + + result = res.res.decode() + lib.FreeResult(ctypes.c_void_p(res_ptr)) + return result + + +def check_compatibility(proto: Proto, prev_proto: Proto) -> None: + length = len(proto.all_dependencies) + c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.all_dependencies]) + c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.all_dependencies]) + + prev_length = len(prev_proto.all_dependencies) + prev_c_dependencies = (ctypes.c_char_p * prev_length)(*[d.schema.encode() for d in prev_proto.all_dependencies]) + prev_c_dependency_names = (ctypes.c_char_p * prev_length)(*[d.name.encode() for d in prev_proto.all_dependencies]) + + err = lib.CheckCompatibility( + proto.name.encode(), + proto.schema.encode(), + c_dependency_names, + c_dependencies, + length, + prev_proto.name.encode(), + prev_proto.schema.encode(), + prev_c_dependency_names, + prev_c_dependencies, + prev_length, + ) + + if err is not None: + msg = err.decode() + raise IncompatibleError(msg) + + +SCHEMA = """ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; +} +""" + +DEPENDENCY = """ +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} +""" + + +def _format_time(seconds: float) -> str: + units = [("s", 1), ("ms", 1e-3), ("µs", 1e-6), ("ns", 1e-9)] + for unit, factor in units: + if seconds >= factor: + return f"{seconds / factor:.3f} {unit}" + return f"{seconds:.3f} s" + + +def _time_format() -> None: + def test() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + format_proto(proto) + + number = 10000 + seconds = timeit.timeit(test, number=number) + + print("----- Format -----") + print(f"Total time: {_format_time(seconds)}") + print(f"Execution time per loop: {_format_time(seconds / number)}") + + +def _time_check_compatibility() -> None: + def test() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + check_compatibility(proto, proto) + + number = 10000 + seconds = timeit.timeit(test, number=number) + + print("----- Compatibility Check -----") + print(f"Total time: {_format_time(seconds)}") + print(f"Execution time per loop: {_format_time(seconds / number)}") + + +if __name__ == "__main__": + _time_format() + _time_check_compatibility() diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 896e73f26..eab1a5c9f 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -11,6 +11,7 @@ from jsonschema.exceptions import SchemaError from karapace.dependency import Dependency from karapace.errors import InvalidSchema, InvalidVersion, VersionNotFoundException +from karapace.protobuf import protopace from karapace.protobuf.exception import ( Error as ProtobufError, IllegalArgumentException, @@ -25,7 +26,7 @@ from karapace.schema_type import SchemaType from karapace.typing import JsonObject, SchemaId, Subject, Version, VersionTag from karapace.utils import assert_never, json_decode, json_encode, JSONDecodeError -from typing import Any, cast, Dict, Final, final, Mapping, Sequence +from typing import Any, cast, Collection, Dict, Final, final, Mapping, Sequence import hashlib import logging @@ -58,12 +59,19 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: return Draft7Validator(schema) # type: ignore[arg-type] +def _format_protobuf(schema: str, dependencies: Collection[Dependency], name: str = "schema.proto") -> str: + deps = [dep.to_proto() for dep in dependencies] + proto = protopace.Proto(name, schema, deps) + return protopace.format_proto(proto) + + def parse_protobuf_schema_definition( schema_definition: str, references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, validate_references: bool = True, normalize: bool = False, + use_protobuf_formatter: bool = False, ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -71,6 +79,12 @@ def parse_protobuf_schema_definition( ProtobufUnresolvedDependencyException if Protobuf dependency cannot be resolved. """ + if use_protobuf_formatter: + try: + schema_definition = _format_protobuf(schema_definition, dependencies.values() if dependencies else []) + except InvalidSchema as err: + LOG.error("Error formatting schema %s", err) + protobuf_schema = ( ProtobufSchema(schema_definition, references, dependencies) if not normalize @@ -188,6 +202,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protobuf_formatter: bool = False, ) -> ParsedTypedSchema: if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -213,7 +228,12 @@ def parse( elif schema_type is SchemaType.PROTOBUF: try: parsed_schema = parse_protobuf_schema_definition( - schema_str, references, dependencies, validate_references=True, normalize=normalize + schema_str, + references, + dependencies, + validate_references=True, + normalize=normalize, + use_protobuf_formatter=use_protobuf_formatter, ) except ( TypeError, @@ -282,6 +302,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protobuf_formatter: bool = False, ) -> ParsedTypedSchema: return parse( schema_type=schema_type, @@ -291,6 +312,7 @@ def parse( references=references, dependencies=dependencies, normalize=normalize, + use_protobuf_formatter=use_protobuf_formatter, ) def __str__(self) -> str: @@ -366,6 +388,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protobuf_formatter: bool = False, ) -> ValidatedTypedSchema: parsed_schema = parse( schema_type=schema_type, @@ -375,6 +398,7 @@ def parse( references=references, dependencies=dependencies, normalize=normalize, + use_protobuf_formatter=use_protobuf_formatter, ) return cast(ValidatedTypedSchema, parsed_schema) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index d3b90dac6..5a9196087 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -391,6 +391,7 @@ async def compatibility_check( schema_str=body["schema"], references=references, dependencies=new_schema_dependencies, + use_protobuf_formatter=self.config["use_protobuf_formatter"], ) except InvalidSchema: self.r( @@ -1121,6 +1122,7 @@ async def subjects_schema_post( references=references, dependencies=new_schema_dependencies, normalize=normalize, + use_protobuf_formatter=self.config["use_protobuf_formatter"], ) except InvalidSchema: self.log.warning("Invalid schema: %r", schema_str) @@ -1222,6 +1224,7 @@ async def subject_post( references=references, dependencies=resolved_dependencies, normalize=normalize, + use_protobuf_formatter=self.config["use_protobuf_formatter"], ) except (InvalidReferences, InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in index 73dfa6bad..b11b729cb 100644 --- a/requirements/requirements-dev.in +++ b/requirements/requirements-dev.in @@ -20,3 +20,6 @@ locust # Sentry SDK sentry-sdk + +# Golang +setuptools-golang diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index e9024431f..8a7f8ac58 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -6,7 +6,11 @@ # accept-types==0.4.1 # via -r requirements.txt -aiohttp==3.9.5 +aiohappyeyeballs==2.4.0 + # via + # -r requirements.txt + # aiohttp +aiohttp==3.10.5 # via -r requirements.txt aiokafka==0.10.0 # via -r requirements.txt @@ -23,7 +27,7 @@ async-timeout==4.0.3 # -r requirements.txt # aiohttp # aiokafka -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements.txt # aiohttp @@ -52,13 +56,13 @@ configargparse==1.7 # via locust confluent-kafka==2.4.0 # via -r requirements.txt -coverage[toml]==7.5.3 +coverage[toml]==7.6.1 # via pytest-cov cramjam==2.8.3 # via # -r requirements.txt # python-snappy -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via # -r requirements.txt # anyio @@ -68,7 +72,7 @@ execnet==2.1.1 # via pytest-xdist fancycompleter==0.9.1 # via pdbpp -filelock==3.14.0 +filelock==3.15.4 # via -r requirements-dev.in flask==3.0.3 # via @@ -92,17 +96,17 @@ geventhttpclient==2.0.12 # via locust greenlet==3.0.3 # via gevent -hypothesis==6.103.1 +hypothesis==6.111.2 # via -r requirements-dev.in -idna==3.7 +idna==3.8 # via # -r requirements.txt # anyio # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.4.0 # via flask -importlib-resources==6.4.0 +importlib-resources==6.4.4 # via # -r requirements.txt # jsonschema @@ -115,7 +119,7 @@ itsdangerous==2.2.0 # via flask jinja2==3.1.4 # via flask -jsonschema==4.22.0 +jsonschema==4.23.0 # via -r requirements.txt jsonschema-specifications==2023.12.1 # via @@ -146,7 +150,7 @@ multidict==6.0.5 # yarl networkx==3.1 # via -r requirements.txt -packaging==24.0 +packaging==24.1 # via # -r requirements.txt # aiokafka @@ -163,7 +167,7 @@ prometheus-client==0.20.0 # via -r requirements.txt protobuf==3.20.3 # via -r requirements.txt -psutil==5.9.8 +psutil==6.0.0 # via # -r requirements-dev.in # locust @@ -173,11 +177,11 @@ pygments==2.18.0 # -r requirements.txt # pdbpp # rich -pyjwt==2.8.0 +pyjwt==2.9.0 # via -r requirements.txt pyrepl==0.9.0 # via fancycompleter -pytest==8.2.2 +pytest==8.3.2 # via # -r requirements-dev.in # pytest-cov @@ -194,9 +198,9 @@ pytest-xdist[psutil]==3.6.1 # via -r requirements-dev.in python-dateutil==2.9.0.post0 # via -r requirements.txt -python-snappy==0.7.1 +python-snappy==0.7.2 # via -r requirements.txt -pyzmq==26.0.3 +pyzmq==26.2.0 # via locust referencing==0.35.1 # via @@ -211,12 +215,14 @@ rich==13.7.1 # via -r requirements.txt roundrobin==0.0.4 # via locust -rpds-py==0.18.1 +rpds-py==0.20.0 # via # -r requirements.txt # jsonschema # referencing -sentry-sdk==2.5.0 +sentry-sdk==2.13.0 + # via -r requirements-dev.in +setuptools-golang==2.9.0 # via -r requirements-dev.in six==1.16.0 # via @@ -237,7 +243,7 @@ tomli==2.0.1 # coverage # locust # pytest -typing-extensions==4.12.1 +typing-extensions==4.12.2 # via # -r requirements.txt # anyio @@ -248,31 +254,31 @@ urllib3==2.2.2 # via # requests # sentry-sdk -watchfiles==0.22.0 +watchfiles==0.23.0 # via -r requirements.txt -werkzeug==3.0.3 +werkzeug==3.0.4 # via # flask # flask-login # locust wmctrl==0.5 # via pdbpp -xxhash==3.4.1 +xxhash==3.5.0 # via -r requirements.txt yarl==1.9.4 # via # -r requirements.txt # aiohttp -zipp==3.19.2 +zipp==3.20.1 # via # -r requirements.txt # importlib-metadata # importlib-resources zope-event==5.0 # via gevent -zope-interface==6.4.post2 +zope-interface==7.0.2 # via gevent -zstandard==0.22.0 +zstandard==0.23.0 # via -r requirements.txt # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index 0e109ac7c..7b064f425 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -4,7 +4,7 @@ # # 'make requirements' # -attrs==23.2.0 +attrs==24.2.0 # via # -c requirements-dev.txt # -c requirements.txt @@ -13,7 +13,7 @@ certifi==2024.7.4 # via # -c requirements-dev.txt # sentry-sdk -mypy==1.11.1 +mypy==1.11.2 # via -r requirements-typing.in mypy-extensions==1.0.0 # via mypy @@ -22,12 +22,12 @@ referencing==0.35.1 # -c requirements-dev.txt # -c requirements.txt # types-jsonschema -rpds-py==0.18.1 +rpds-py==0.20.0 # via # -c requirements-dev.txt # -c requirements.txt # referencing -sentry-sdk==2.5.0 +sentry-sdk==2.13.0 # via # -c requirements-dev.txt # -r requirements-typing.in @@ -35,13 +35,13 @@ tomli==2.0.1 # via # -c requirements-dev.txt # mypy -types-cachetools==5.3.0.7 +types-cachetools==5.5.0.20240820 # via -r requirements-typing.in -types-jsonschema==4.23.0.20240712 +types-jsonschema==4.23.0.20240813 # via -r requirements-typing.in types-protobuf==3.20.4.6 # via -r requirements-typing.in -typing-extensions==4.12.1 +typing-extensions==4.12.2 # via # -c requirements-dev.txt # -c requirements.txt diff --git a/requirements/requirements.in b/requirements/requirements.in index a9401892e..fa61d5d54 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -7,9 +7,9 @@ confluent-kafka==2.4.0 isodate<1 jsonschema<5 lz4 -networkx<4 +networkx==3.1 protobuf<4 -pyjwt>=2.4.0<3 +pyjwt>=2.4.0,<3 python-dateutil<3 python-snappy rich~=13.7.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6f1a771f9..368ffb097 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -6,7 +6,9 @@ # accept-types==0.4.1 # via -r requirements.in -aiohttp==3.9.5 +aiohappyeyeballs==2.4.0 + # via aiohttp +aiohttp==3.10.5 # via -r requirements.in aiokafka==0.10.0 # via -r requirements.in @@ -18,7 +20,7 @@ async-timeout==4.0.3 # via # aiohttp # aiokafka -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema @@ -31,23 +33,23 @@ confluent-kafka==2.4.0 # via -r requirements.in cramjam==2.8.3 # via python-snappy -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via anyio frozenlist==1.4.1 # via # aiohttp # aiosignal -idna==3.7 +idna==3.8 # via # anyio # yarl -importlib-resources==6.4.0 +importlib-resources==6.4.4 # via # jsonschema # jsonschema-specifications isodate==0.6.1 # via -r requirements.in -jsonschema==4.22.0 +jsonschema==4.23.0 # via -r requirements.in jsonschema-specifications==2023.12.1 # via jsonschema @@ -63,7 +65,7 @@ multidict==6.0.5 # yarl networkx==3.1 # via -r requirements.in -packaging==24.0 +packaging==24.1 # via aiokafka pkgutil-resolve-name==1.3.10 # via jsonschema @@ -73,11 +75,11 @@ protobuf==3.20.3 # via -r requirements.in pygments==2.18.0 # via rich -pyjwt==2.8.0 +pyjwt==2.9.0 # via -r requirements.in python-dateutil==2.9.0.post0 # via -r requirements.in -python-snappy==0.7.1 +python-snappy==0.7.2 # via -r requirements.in referencing==0.35.1 # via @@ -85,7 +87,7 @@ referencing==0.35.1 # jsonschema-specifications rich==13.7.1 # via -r requirements.in -rpds-py==0.18.1 +rpds-py==0.20.0 # via # jsonschema # referencing @@ -97,20 +99,20 @@ sniffio==1.3.1 # via anyio tenacity==9.0.0 # via -r requirements.in -typing-extensions==4.12.1 +typing-extensions==4.12.2 # via # -r requirements.in # anyio # rich ujson==5.10.0 # via -r requirements.in -watchfiles==0.22.0 +watchfiles==0.23.0 # via -r requirements.in -xxhash==3.4.1 +xxhash==3.5.0 # via -r requirements.in yarl==1.9.4 # via aiohttp -zipp==3.19.2 +zipp==3.20.1 # via importlib-resources -zstandard==0.22.0 +zstandard==0.23.0 # via -r requirements.in diff --git a/setup.py b/setup.py index 3b50270d1..cd8bac10e 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ Copyright (c) 2023 Aiven Ltd See LICENSE for details """ -from setuptools import find_packages, setup +from setuptools import Extension, find_packages, setup import os import version @@ -21,6 +21,7 @@ version=version_for_setup_py, zip_safe=False, packages=find_packages(exclude=["test"]), + setup_requires=["setuptools-golang"], install_requires=[ "accept-types", "aiohttp", @@ -70,4 +71,12 @@ "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries", ], + include_package_data=True, + ext_modules=[ + Extension( + "protopacelib", + ["go/protopace/main.go"], + ), + ], + build_golang={"root": "go/protopace"}, ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 04fbd7aa1..ecc52470a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -510,8 +510,9 @@ async def fixture_registry_cluster( endpoint = RegistryEndpoint(registry.scheme, registry.hostname, registry.port) yield RegistryDescription(endpoint, "_schemas") return - + user_config = request.param.get("config", {}) if hasattr(request, "param") else {} config = {"bootstrap_uri": kafka_servers.bootstrap_servers} + config.update(user_config) async with start_schema_registry_cluster( config_templates=[config], data_dir=session_logdir / _clear_test_name(request.node.name), diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 716a03e12..9eae2b994 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -43,6 +43,7 @@ def add_slashes(text: str) -> str: # This test ProtoBuf schemas in subject registeration, compatibility of evolved version and querying the schema # w.r.t. normalization of whitespace and other minor differences to verify equality and inequality comparison of such schemas @pytest.mark.parametrize("trail", ["", "/"]) +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_schema_normalization(registry_async_client: Client, trail: str) -> None: subject = create_subject_name_factory(f"test_protobuf_schema_compatibility-{trail}")() @@ -175,6 +176,7 @@ async def test_protobuf_schema_normalization(registry_async_client: Client, trai assert evolved_id == res.json()["id"], "Check returns evolved id" +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_schema_references(registry_async_client: Client) -> None: customer_schema = """ |syntax = "proto3"; @@ -310,6 +312,7 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None assert res.status_code == 200 +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> None: no_ref = """ |syntax = "proto3"; @@ -384,6 +387,7 @@ async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> No assert "id" in res.json() +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: customer_schema = """ |syntax = "proto3"; @@ -963,6 +967,7 @@ class ReferenceTestCase(BaseTestCase): ], ids=str, ) +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) @pytest.mark.parametrize("metadata", [None, {}]) @pytest.mark.parametrize("rule_set", [None, {}]) async def test_references( @@ -1019,6 +1024,7 @@ async def test_references( assert fetch_schema_res.status_code == 200 +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_reference_update_creates_new_schema_version(registry_async_client: Client): test_schemas = [ TestCaseSchema( @@ -1079,6 +1085,7 @@ async def test_reference_update_creates_new_schema_version(registry_async_client assert res.json_result.get("id") == expected_schema_id +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_error(registry_async_client: Client) -> None: testdata = TestCaseSchema( schema_type=SchemaType.PROTOBUF, @@ -1129,6 +1136,7 @@ async def test_protobuf_error(registry_async_client: Client) -> None: assert res.status_code == 200 +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_missing_google_import(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_missing_google_import")() @@ -1149,6 +1157,7 @@ async def test_protobuf_missing_google_import(registry_async_client: Client) -> assert myjson["error_code"] == 42201 and '"google.type.PostalAddress" is not defined' in myjson["message"] +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_customer_update(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_customer_update")() @@ -1185,6 +1194,7 @@ async def test_protobuf_customer_update(registry_async_client: Client) -> None: assert res.status_code == 200 +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_binary_serialized(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_binary_serialized")() @@ -1236,6 +1246,7 @@ async def test_protobuf_binary_serialized(registry_async_client: Client) -> None assert schema_id == res.json()["id"] +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_update_ordering(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_update_ordering")() @@ -1315,7 +1326,12 @@ async def test_protobuf_update_ordering(registry_async_client: Client) -> None: """ -async def test_registering_normalized_schema(registry_async_client: Client) -> None: +@pytest.mark.parametrize( + "registry_cluster, status", + [({"config": {}}, 404), ({"config": {"use_protobuf_formatter": True}}, 200)], + indirect=["registry_cluster"], +) +async def test_registering_normalized_schema(registry_async_client: Client, status: int) -> None: subject = create_subject_name_factory("test_protobuf_normalization")() body = {"schemaType": "PROTOBUF", "schema": SCHEMA_WITH_OPTION_ORDERED} @@ -1327,7 +1343,7 @@ async def test_registering_normalized_schema(registry_async_client: Client) -> N body = {"schemaType": "PROTOBUF", "schema": SCHEMA_WITH_OPTION_UNORDERDERED} res = await registry_async_client.post(f"subjects/{subject}", json=body) - assert res.status_code == 404 + assert res.status_code == status res = await registry_async_client.post(f"subjects/{subject}?normalize=true", json=body) @@ -1336,6 +1352,7 @@ async def test_registering_normalized_schema(registry_async_client: Client) -> N assert original_schema_id == res.json()["id"] +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_normalized_schema_idempotence_produce_and_fetch(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_normalization")() diff --git a/tests/unit/protobuf/test_protobuf_normalization.py b/tests/unit/protobuf/test_protobuf_normalization.py index 0f752d563..0cc6b8184 100644 --- a/tests/unit/protobuf/test_protobuf_normalization.py +++ b/tests/unit/protobuf/test_protobuf_normalization.py @@ -68,10 +68,20 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + ACTIVE = 0; } """ @@ -80,10 +90,20 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + ACTIVE = 0; } """ @@ -92,6 +112,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.ServiceOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + service MyService { option (my_option) = "my_value"; option (my_option2) = "my_value2"; @@ -104,6 +132,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.ServiceOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + service MyService { option (my_option3) = "my_value3"; option (my_option) = "my_value"; @@ -116,6 +152,18 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +message Foo { + string res = 1; +} + service MyService { rpc MyRpc (Foo) returns (Foo) { option (my_option) = "my_value"; @@ -130,6 +178,18 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +message Foo { + string res = 1; +} + service MyService { rpc MyRpc (Foo) returns (Foo) { option (my_option3) = "my_value3"; @@ -139,48 +199,26 @@ } """ -PROTO_WITH_OPTIONS_IN_EXTEND_ORDERED = """\ -syntax = "proto3"; - -package pkg; - -message Foo { - string fieldA = 1; -} - -extend Foo { - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; -} -""" - -PROTO_WITH_OPTIONS_IN_EXTEND_UNORDERED = """\ +PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED = """\ syntax = "proto3"; package pkg; -message Foo { - string fieldA = 1; -} +import "google/protobuf/descriptor.proto"; -extend Foo { - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; +extend google.protobuf.OneofOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; } -""" - -PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED = """\ -syntax = "proto3"; - -package pkg; message Foo { oneof my_oneof { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + string test = 1; } } """ @@ -190,11 +228,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.OneofOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { oneof my_oneof { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + string test = 1; } } """ @@ -204,6 +252,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -214,6 +270,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -224,6 +288,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -234,6 +306,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -244,11 +324,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { enum MyEnum { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + ACTIVE = 0; } } """ @@ -258,11 +348,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { enum MyEnum { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + ACTIVE = 0; } } """ @@ -272,6 +372,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { message Bar { string fieldA = 1 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; @@ -284,6 +392,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { message Bar { string fieldA = 1 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; @@ -297,6 +413,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0; } @@ -311,6 +435,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0; } @@ -325,6 +457,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -339,6 +479,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -353,6 +501,50 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option_EnumValue = 50002; + string my_option2_EnumValue = 50003; + string my_option3_EnumValue = 50004; +} + +extend google.protobuf.EnumOptions { + string my_option_Enum = 50002; + string my_option2_Enum = 50003; + string my_option3_Enum = 50004; +} + +extend google.protobuf.FieldOptions { + string my_option_Field = 50002; + string my_option2_Field = 50003; + string my_option3_Field = 50004; +} + +extend google.protobuf.OneofOptions { + string my_option_Oneof = 50002; + string my_option2_Oneof = 50003; + string my_option3_Oneof = 50004; +} + +extend google.protobuf.MessageOptions { + string my_option_Message = 50002; + string my_option2_Message = 50003; + string my_option3_Message = 50004; +} + +extend google.protobuf.MethodOptions { + string my_option_Method = 50002; + string my_option2_Method = 50003; + string my_option3_Method = 50004; +} + +extend google.protobuf.ServiceOptions { + string my_option_Service = 50002; + string my_option2_Service = 50003; + string my_option3_Service = 50004; +} + option cc_generic_services = true; option java_generate_equals_and_hash = true; option java_generic_services = true; @@ -373,47 +565,45 @@ message NestedFoo { string fieldA = 1; - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; + option (my_option_Message) = "my_value"; + option (my_option2_Message) = "my_value2"; } - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Message) = "my_value3"; + option (my_option2_Message) = "my_value2"; + option (my_option_Message) = "my_value"; oneof my_oneof { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Oneof) = "my_value3"; + option (my_option2_Oneof) = "my_value2"; + option (my_option_Oneof) = "my_value"; + + string test = 5; } enum MyEnum { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Enum) = "my_value3"; + option (my_option2_Enum) = "my_value2"; + option (my_option_Enum) = "my_value"; + + ACTIVE = 0; } } -extend Foo { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; -} - service MyService { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Service) = "my_value3"; + option (my_option2_Service) = "my_value2"; + option (my_option_Service) = "my_value"; rpc MyRpc (Foo) returns (Foo) { - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; + option (my_option_Method) = "my_value"; + option (my_option2_Method) = "my_value2"; + option (my_option3_Method) = "my_value3"; } } @@ -425,6 +615,50 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option_EnumValue = 50002; + string my_option2_EnumValue = 50003; + string my_option3_EnumValue = 50004; +} + +extend google.protobuf.EnumOptions { + string my_option_Enum = 50002; + string my_option2_Enum = 50003; + string my_option3_Enum = 50004; +} + +extend google.protobuf.FieldOptions { + string my_option_Field = 50002; + string my_option2_Field = 50003; + string my_option3_Field = 50004; +} + +extend google.protobuf.OneofOptions { + string my_option_Oneof = 50002; + string my_option2_Oneof = 50003; + string my_option3_Oneof = 50004; +} + +extend google.protobuf.MessageOptions { + string my_option_Message = 50002; + string my_option2_Message = 50003; + string my_option3_Message = 50004; +} + +extend google.protobuf.MethodOptions { + string my_option_Method = 50002; + string my_option2_Method = 50003; + string my_option3_Method = 50004; +} + +extend google.protobuf.ServiceOptions { + string my_option_Service = 50002; + string my_option2_Service = 50003; + string my_option3_Service = 50004; +} + option cc_generic_services = true; option java_outer_classname = "FooProto"; option optimize_for = SPEED; @@ -445,45 +679,42 @@ message NestedFoo { string fieldA = 1; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option2_Message) = "my_value2"; + option (my_option_Message) = "my_value"; } - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + option (my_option2_Message) = "my_value2"; + option (my_option3_Message) = "my_value3"; + option (my_option_Message) = "my_value"; oneof my_oneof { - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; + option (my_option_Oneof) = "my_value"; + option (my_option3_Oneof) = "my_value3"; + option (my_option2_Oneof) = "my_value2"; + + string test = 5; } enum MyEnum { - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - } -} - + option (my_option_Enum) = "my_value"; + option (my_option3_Enum) = "my_value3"; + option (my_option2_Enum) = "my_value2"; -extend Foo { - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + ACTIVE = 0; + } } service MyService { - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; + option (my_option2_Service) = "my_value2"; + option (my_option_Service) = "my_value"; + option (my_option3_Service) = "my_value3"; rpc MyRpc (Foo) returns (Foo) { - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + option (my_option2_Method) = "my_value2"; + option (my_option3_Method) = "my_value3"; + option (my_option_Method) = "my_value"; } } @@ -498,7 +729,6 @@ (PROTO_WITH_OPTIONS_IN_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_UNORDERED), (PROTO_WITH_OPTIONS_IN_SERVICE_ORDERED, PROTO_WITH_OPTIONS_IN_SERVICE_UNORDERED), (PROTO_WITH_OPTIONS_IN_RPC_ORDERED, PROTO_WITH_OPTIONS_IN_RPC_UNORDERED), - (PROTO_WITH_OPTIONS_IN_EXTEND_ORDERED, PROTO_WITH_OPTIONS_IN_EXTEND_UNORDERED), (PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED, PROTO_WITH_OPTIONS_IN_ONEOF_UNORDERED), (PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_UNORDERED), (PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_UNORDERED), @@ -537,6 +767,53 @@ def test_differently_ordered_options_normalizes_equally(ordered_schema: str, uno assert normalize(ordered_proto).to_schema() == normalize(unordered_proto).to_schema() +@pytest.mark.parametrize( + ("ordered_schema", "unordered_schema"), + ( + (PROTO_WITH_OPTIONS_ORDERED, PROTO_WITH_OPTIONS_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_UNORDERED), + (PROTO_WITH_OPTIONS_IN_SERVICE_ORDERED, PROTO_WITH_OPTIONS_IN_SERVICE_UNORDERED), + (PROTO_WITH_OPTIONS_IN_RPC_ORDERED, PROTO_WITH_OPTIONS_IN_RPC_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED, PROTO_WITH_OPTIONS_IN_ONEOF_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_UNORDERED), + (PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_UNORDERED), + (PROTO_WITH_NEASTED_ENUM_IN_MESSAGE_WITH_OPTIONS_ORDERED, PROTO_WITH_NEASTED_ENUM_IN_MESSAGE_WITH_OPTIONS_UNORDERED), + ( + PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_WITH_OPTIONS_ORDERED, + PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_WITH_OPTIONS_UNORDERED, + ), + (PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_UNORDERED), + ( + PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_WITH_OPTIONS_ORDERED, + PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_WITH_OPTIONS_UNORDERED, + ), + (PROTO_WITH_COMPLEX_SCHEMA_ORDERED, PROTO_WITH_COMPLEX_SCHEMA_UNORDERED), + ), +) +def test_differently_ordered_options_normalizes_equally_with_formatter(ordered_schema: str, unordered_schema: str) -> None: + ordered_proto = parse_protobuf_schema_definition( + schema_definition=ordered_schema, + references=None, + dependencies=None, + validate_references=True, + normalize=True, + use_protobuf_formatter=True, + ) + unordered_proto = parse_protobuf_schema_definition( + schema_definition=unordered_schema, + references=None, + dependencies=None, + validate_references=True, + normalize=True, + use_protobuf_formatter=True, + ) + + result = CompareResult() + ordered_proto.compare(unordered_proto, result) + assert result.is_compatible() + assert ordered_proto.schema == unordered_proto.schema + + DEPENDENCY = """\ syntax = "proto3"; package my.awesome.customer.v1; @@ -560,8 +837,8 @@ def test_differently_ordered_options_normalizes_equally(ordered_schema: str, uno option java_outer_classname = "EventValueProto"; option java_package = "com.my.awesome.customer.v1"; option objc_class_prefix = "TDD"; -option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; -option php_namespace = "My\\Awesome\\Customer\\V1"; +option php_metadata_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option php_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; option ruby_package = "My::Awesome::Customer::V1"; message EventValue { @@ -584,8 +861,8 @@ def test_differently_ordered_options_normalizes_equally(ordered_schema: str, uno option java_outer_classname = "EventValueProto"; option java_package = "com.my.awesome.customer.v1"; option objc_class_prefix = "TDD"; -option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; -option php_namespace = "My\\Awesome\\Customer\\V1"; +option php_metadata_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option php_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; option ruby_package = "My::Awesome::Customer::V1"; message EventValue { @@ -667,6 +944,34 @@ def test_full_path_and_simple_names_are_equal() -> None: ), "also the string rendering shouldn't change a simple name notation protofile" +def test_full_path_and_simple_names_are_equal_with_formatter() -> None: + no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, DEPENDENCY, normalize=True) + dep = Dependency("my/awesome/customer/v1/nested_value.proto", Subject("nested_value"), Version(1), no_ref_schema) + dependencies = {"my/awesome/customer/v1/nested_value.proto": dep} + fully_qualitifed_simple_name_notation = parse_protobuf_schema_definition( + schema_definition=PROTO_WITH_SIMPLE_NAMES, + references=None, + dependencies=dependencies, + validate_references=True, + normalize=True, + use_protobuf_formatter=True, + ) + fully_qualitifed_dot_notation = parse_protobuf_schema_definition( + schema_definition=PROTO_WITH_FULLY_QUALIFIED_PATHS, + references=None, + dependencies=dependencies, + validate_references=True, + normalize=True, + use_protobuf_formatter=True, + ) + result = CompareResult() + fully_qualitifed_simple_name_notation.compare(fully_qualitifed_dot_notation, result) + assert result.is_compatible(), "normalized schemas are not compatible" + assert ( + fully_qualitifed_dot_notation.schema == fully_qualitifed_simple_name_notation.schema + ), "normalized schemas should match" + + TRICKY_DEPENDENCY = """\ syntax = "proto3"; package org.my.awesome.customer.v1; @@ -725,3 +1030,36 @@ def test_full_path_and_simple_names_are_not_equal_if_simple_name_is_not_unique() ).to_schema() assert normalized_schema == schema, "Since the simple name is not unique identifying the type isn't replacing the source" + + +def test_full_path_and_simple_names_are_not_equal_if_simple_name_is_not_unique_with_formatter() -> None: + no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, DEPENDENCY, normalize=True) + tricky_no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, TRICKY_DEPENDENCY, normalize=True) + dep = Dependency("my/awesome/customer/v1/nested_value.proto", Subject("nested_value"), Version(1), no_ref_schema) + tricky_dep = Dependency( + "org/my/awesome/customer/v1/nested_value.proto", Subject("tricky_nested_value"), Version(1), tricky_no_ref_schema + ) + dependencies = { + "my/awesome/customer/v1/nested_value.proto": dep, + "org/my/awesome/customer/v1/nested_value.proto": tricky_dep, + } + schema = parse_protobuf_schema_definition( + schema_definition=PROTO_WITH_FULLY_QUALIFIED_PATHS_AND_TRICKY_DEPENDENCY, + references=None, + dependencies=dependencies, + validate_references=True, + normalize=False, + use_protobuf_formatter=True, + ) + normalized_schema = parse_protobuf_schema_definition( + schema_definition=PROTO_WITH_FULLY_QUALIFIED_PATHS_AND_TRICKY_DEPENDENCY, + references=None, + dependencies=dependencies, + validate_references=True, + normalize=False, + use_protobuf_formatter=True, + ) + + assert ( + normalized_schema.schema == schema.schema + ), "Since the simple name is not unique identifying the type isn't replacing the source" From 6d954e6dda956c7d1791fa0b92839c94a8511ea3 Mon Sep 17 00:00:00 2001 From: Jonas Keeling Date: Wed, 25 Sep 2024 11:19:04 +0200 Subject: [PATCH 2/2] chore: enable go extensions --- .dockerignore | 1 + .github/workflows/schema.yml | 2 +- .github/workflows/tests.yml | 5 +++++ container/Dockerfile | 7 +++++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 6244185b0..2e8e40bc6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ # Include source directories and files required for building. !karapace +!go !requirements/*.txt !setup.py !version.py diff --git a/.github/workflows/schema.yml b/.github/workflows/schema.yml index 1ee9a8049..6665ab235 100644 --- a/.github/workflows/schema.yml +++ b/.github/workflows/schema.yml @@ -21,7 +21,7 @@ jobs: requirements.txt - name: Install libsnappy-dev run: sudo apt install libsnappy-dev - - run: pip install -r requirements/requirements.txt + - run: make install # Compare with latest release when running on main. - run: make schema against=$(git describe --abbrev=0 --tags) if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 029a3a011..1df7f9ca8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,11 @@ jobs: cache: pip python-version: ${{ matrix.python-version }} + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.21.0' + - run: make install version - run: make unit-tests env: diff --git a/container/Dockerfile b/container/Dockerfile index 61ad142b8..4fe2a46a5 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -8,6 +8,13 @@ RUN python3 -m venv /venv ENV PATH="/venv/bin:$PATH" ENV PIP_REQUIRE_VIRTUALENV=true +# Install golang needed by extensions +ENV GO_VERSION=1.21.0 +ENV PATH="/usr/local/go/bin:${PATH}" +RUN wget --progress=dot:giga "https://go.dev/dl/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \ + && tar -C /usr/local -xzf "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \ + && rm "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" + # Copy the requirements.txt and install dependencies in venv. Using a separate # command to use layer caching. #