From ac476da3e27cf79bd2156d7ef2f801125bcf4ffb Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:53:50 -0800 Subject: [PATCH 01/24] Finalizing CSFLE work with ruleSet and KMS setting --- go.mod | 57 +++++++- go.sum | 23 ++-- pkg/serdes/avro_deserialization_provider.go | 3 + pkg/serdes/avro_serialization_provider.go | 33 ++++- pkg/serdes/json_deserialization_provider.go | 3 + pkg/serdes/json_serialization_provider.go | 21 +++ .../protobuf_deserialization_provider.go | 3 + pkg/serdes/protobuf_serialization_provider.go | 21 +++ pkg/serdes/serdes_test.go | 124 ++++++++++++++++++ 9 files changed, 272 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 0d0fd7bb67..2cc8b0739e 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/confluentinc/ccloud-sdk-go-v2/sso v0.0.1 github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0 github.com/confluentinc/cmf-sdk-go v0.0.2 - github.com/confluentinc/confluent-kafka-go/v2 v2.6.0 + github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241113012127-14c9b6103407 github.com/confluentinc/go-editor v0.11.0 github.com/confluentinc/go-prompt v0.2.40 github.com/confluentinc/go-ps1 v1.0.2 @@ -122,19 +122,42 @@ require ( require ( cloud.google.com/go/auth v0.8.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect cloud.google.com/go/compute/metadata v0.5.1 // indirect dario.cat/mergo v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/alecthomas/chroma/v2 v2.8.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect + github.com/aws/smithy-go v1.20.2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/c-bata/go-prompt v0.2.6 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/charmbracelet/x/ansi v0.1.2 // indirect github.com/charmbracelet/x/input v0.1.0 // indirect github.com/charmbracelet/x/term v0.1.1 // indirect @@ -146,24 +169,37 @@ require ( github.com/dlclark/regexp2 v1.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gdamore/encoding v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/googleapis v1.4.1 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/cel-go v0.20.1 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/hamba/avro/v2 v2.24.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.6 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/vault/api v1.15.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.12.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -171,6 +207,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lyft/protoc-gen-star/v2 v2.0.3 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -178,6 +215,7 @@ require ( github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-tty v0.0.4 // indirect github.com/microcosm-cc/bluemonday v1.0.25 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -192,35 +230,46 @@ require ( github.com/pkg/term v1.2.0-beta.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/shopspring/decimal v1.3.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect - github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/afero v1.10.0 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/swaggest/jsonschema-go v0.3.39 // indirect github.com/swaggest/refl v1.1.0 // indirect github.com/tidwall/match v1.1.1 // indirect + github.com/tink-crypto/tink-go-gcpkms/v2 v2.1.0 // indirect + github.com/tink-crypto/tink-go-hcvault/v2 v2.1.0 // indirect + github.com/tink-crypto/tink-go/v2 v2.1.0 // indirect github.com/travisjeffery/mocker v1.1.1 // indirect github.com/travisjeffery/proto-go-sql v0.0.0-20190911121832-39ff47280e87 // indirect github.com/ugorji/go/codec v1.2.8 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xiatechs/jsonata-go v1.8.5 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.2 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/mod v0.19.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect + golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.23.0 // indirect google.golang.org/api v0.191.0 // indirect google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect + google.golang.org/grpc v1.64.1 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/launchdarkly/go-jsonstream.v1 v1.0.1 // indirect diff --git a/go.sum b/go.sum index 87eb5f6233..9c94a3a137 100644 --- a/go.sum +++ b/go.sum @@ -17,7 +17,6 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= cloud.google.com/go/auth v0.8.0 h1:y8jUJLl/Fg+qNBWxP/Hox2ezJvjkrPb952PC1p0G6A4= cloud.google.com/go/auth v0.8.0/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc= cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= @@ -267,8 +266,8 @@ github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0 h1:zI59UpVm88hQn github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0/go.mod h1:29HYKS91pkCre55OPNGRIWpkKSBLUDlzXbrxZ291Xhk= github.com/confluentinc/cmf-sdk-go v0.0.2 h1:Sn7j9UwOBrcUhC42l7aCrUh50S9CUF6qgHrEO0VxHcM= github.com/confluentinc/cmf-sdk-go v0.0.2/go.mod h1:VXiUwzPStdks1l355KHNaCy/oRCOIMMzTPZJAD4OyJM= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.0 h1:VKnMT71Tl0dCp3lfGBp2D8eqQwc+amoDY5EeUgFHDDE= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.0/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= +github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241113012127-14c9b6103407 h1:UMIu4QP4Qsd6EE8IGsOmM83Q0bbPQVNzuCxp+6Xf/ZQ= +github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241113012127-14c9b6103407/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= github.com/confluentinc/go-editor v0.11.0 h1:fcEALYHj7xV/fRSp54/IHi2DS4GlZMJWVgrYvi/llvU= github.com/confluentinc/go-editor v0.11.0/go.mod h1:nEjwqdqx8S7ZGjXsDvRgawsA04Fu2P/KAtA8fa5afMI= github.com/confluentinc/go-prompt v0.2.40 h1:tveghQJ+FVOVvF0dgQaZEm7YZSQ3r3tyuLMHy7w4PK0= @@ -359,8 +358,8 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= -github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= -github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -414,6 +413,8 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-viper/mapstructure/v2 v2.0.0 h1:dhn8MZ1gZ0mzeodTG3jt5Vj/o87xZKuNAprG2mQfMfc= github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= @@ -482,6 +483,7 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -776,8 +778,8 @@ github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= @@ -815,8 +817,8 @@ github.com/sourcegraph/jsonrpc2 v0.2.0 h1:KjN/dC4fP6aN9030MZCJs9WQbTOjWHhrtKVpzz github.com/sourcegraph/jsonrpc2 v0.2.0/go.mod h1:ZafdZgk/axhT1cvZAPOhw+95nz2I/Ra5qMlU4gTRwIo= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= -github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -976,6 +978,7 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= @@ -1006,7 +1009,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -1055,6 +1057,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= diff --git a/pkg/serdes/avro_deserialization_provider.go b/pkg/serdes/avro_deserialization_provider.go index 75edf157df..a27f42b892 100644 --- a/pkg/serdes/avro_deserialization_provider.go +++ b/pkg/serdes/avro_deserialization_provider.go @@ -44,6 +44,9 @@ func (a *AvroDeserializationProvider) InitDeserializer(srClientUrl, srClusterId, } serdeConfig := avrov2.NewDeserializerConfig() + serdeConfig.RuleConfig = map[string]string{ + "secret": "avro_secret", + } var serdeType serde.Type switch mode { diff --git a/pkg/serdes/avro_serialization_provider.go b/pkg/serdes/avro_serialization_provider.go index 86141009d6..6b977839c8 100644 --- a/pkg/serdes/avro_serialization_provider.go +++ b/pkg/serdes/avro_serialization_provider.go @@ -6,6 +6,14 @@ import ( "github.com/linkedin/goavro/v2" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/cel" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/awskms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/azurekms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/gcpkms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/hcvault" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/localkms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/jsonata" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avrov2" ) @@ -30,6 +38,16 @@ func (a *AvroSerializationProvider) InitSerializer(srClientUrl, srClusterId, mod serdeClient, err := schemaregistry.NewClient(serdeClientConfig) + // Register the KMS drivers and the field-level encryption executor + awskms.Register() + azurekms.Register() + gcpkms.Register() + hcvault.Register() + localkms.Register() + encryption.Register() + cel.Register() + jsonata.Register() + if err != nil { return fmt.Errorf("failed to create serializer-specific Schema Registry client: %w", err) } @@ -40,6 +58,9 @@ func (a *AvroSerializationProvider) InitSerializer(srClientUrl, srClusterId, mod serdeConfig := avrov2.NewSerializerConfig() serdeConfig.AutoRegisterSchemas = false serdeConfig.UseLatestVersion = true + serdeConfig.RuleConfig = map[string]string{ + "secret": "avro_secret", + } if schemaId > 0 { serdeConfig.UseSchemaID = schemaId serdeConfig.UseLatestVersion = false @@ -98,8 +119,16 @@ func (a *AvroSerializationProvider) Serialize(topic, message string) ([]byte, er return nil, fmt.Errorf("failed to serialize message: %w", err) } - // Step#4: Serialize the Go native data object with the confluent-kafka-go Serialize() library - payload, err := a.ser.Serialize(topic, &object) + // Step#4: Fetch the Go native data object, cast it into generic map for Serialize() + // Note: the suggested argument to pass to Serialize() library function is: + // - pointer to a generic map consistent with the schema during registration + // - a materialized object consistent with the schema during registration + // Passing the Go native object directly could cause issues during ruleSet execution + v, ok := object.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("failed to serialize message: unexpected message type asserion result") + } + payload, err := a.ser.Serialize(topic, &v) if err != nil { return nil, fmt.Errorf("failed to serialize message: %w", err) } diff --git a/pkg/serdes/json_deserialization_provider.go b/pkg/serdes/json_deserialization_provider.go index edb3046369..497d445c27 100644 --- a/pkg/serdes/json_deserialization_provider.go +++ b/pkg/serdes/json_deserialization_provider.go @@ -44,6 +44,9 @@ func (j *JsonDeserializationProvider) InitDeserializer(srClientUrl, srClusterId, serdeConfig := jsonschema.NewDeserializerConfig() serdeConfig.EnableValidation = true + serdeConfig.RuleConfig = map[string]string{ + "secret": "json_secret", + } var serdeType serde.Type switch mode { diff --git a/pkg/serdes/json_serialization_provider.go b/pkg/serdes/json_serialization_provider.go index 16cf0b333d..c5f75caa45 100644 --- a/pkg/serdes/json_serialization_provider.go +++ b/pkg/serdes/json_serialization_provider.go @@ -5,6 +5,14 @@ import ( "fmt" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/cel" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/awskms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/azurekms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/gcpkms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/hcvault" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/localkms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/jsonata" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/jsonschema" ) @@ -26,6 +34,16 @@ func (j *JsonSerializationProvider) InitSerializer(srClientUrl, srClusterId, mod } serdeClient, err := schemaregistry.NewClient(serdeClientConfig) + // Register the KMS drivers and the field-level encryption executor + awskms.Register() + azurekms.Register() + gcpkms.Register() + hcvault.Register() + localkms.Register() + encryption.Register() + cel.Register() + jsonata.Register() + if err != nil { return fmt.Errorf("failed to create serializer-specific Schema Registry client: %w", err) } @@ -37,6 +55,9 @@ func (j *JsonSerializationProvider) InitSerializer(srClientUrl, srClusterId, mod serdeConfig.AutoRegisterSchemas = false serdeConfig.UseLatestVersion = true serdeConfig.EnableValidation = true + serdeConfig.RuleConfig = map[string]string{ + "secret": "json_secret", + } if schemaId > 0 { serdeConfig.UseSchemaID = schemaId serdeConfig.UseLatestVersion = false diff --git a/pkg/serdes/protobuf_deserialization_provider.go b/pkg/serdes/protobuf_deserialization_provider.go index ff108b1b66..702d231e5e 100644 --- a/pkg/serdes/protobuf_deserialization_provider.go +++ b/pkg/serdes/protobuf_deserialization_provider.go @@ -47,6 +47,9 @@ func (p *ProtobufDeserializationProvider) InitDeserializer(srClientUrl, srCluste } serdeConfig := protobuf.NewDeserializerConfig() + serdeConfig.RuleConfig = map[string]string{ + "secret": "protobuf_secret", + } var serdeType serde.Type switch mode { diff --git a/pkg/serdes/protobuf_serialization_provider.go b/pkg/serdes/protobuf_serialization_provider.go index 805c9d1353..1ca8c15939 100644 --- a/pkg/serdes/protobuf_serialization_provider.go +++ b/pkg/serdes/protobuf_serialization_provider.go @@ -16,6 +16,14 @@ import ( "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/protobuf" "github.com/confluentinc/cli/v4/pkg/errors" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/cel" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/awskms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/azurekms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/gcpkms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/hcvault" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/localkms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/jsonata" ) type ProtobufSerializationProvider struct { @@ -37,6 +45,16 @@ func (p *ProtobufSerializationProvider) InitSerializer(srClientUrl, srClusterId, } serdeClient, err := schemaregistry.NewClient(serdeClientConfig) + // Register the KMS drivers and the field-level encryption executor + awskms.Register() + azurekms.Register() + gcpkms.Register() + hcvault.Register() + localkms.Register() + encryption.Register() + cel.Register() + jsonata.Register() + if err != nil { return fmt.Errorf("failed to create serializer-specific Schema Registry client: %w", err) } @@ -47,6 +65,9 @@ func (p *ProtobufSerializationProvider) InitSerializer(srClientUrl, srClusterId, serdeConfig := protobuf.NewSerializerConfig() serdeConfig.AutoRegisterSchemas = false serdeConfig.UseLatestVersion = true + serdeConfig.RuleConfig = map[string]string{ + "secret": "protobuf_secret", + } if schemaId > 0 { serdeConfig.UseSchemaID = schemaId serdeConfig.UseLatestVersion = false diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 89bf01da88..a48e39c0d3 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -203,6 +203,68 @@ func TestAvroSerdesNestedValid(t *testing.T) { req.NoError(os.RemoveAll(dir)) } +func TestAvroSerdesValidWithRuleSet(t *testing.T) { + req := require.New(t) + + dir, err := createTempDir() + req.Nil(err) + + schemaString := `{"type":"record","name":"myRecord","fields":[{"name":"f1","type":"string","confluent:tags": ["PII"]}]}` + schemaPath := filepath.Join(dir, "avro-schema.txt") + req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) + + expectedString := `{"f1":"this is a confidential message in AVRO schema"}` + + // Initialize the mock serializer and use latest schemaId + serializationProvider, _ := GetSerializationProvider(avroSchemaName) + err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + req.Nil(err) + err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) + req.Nil(err) + + // CSFLE specific rules during schema registration + encRule := schemaregistry.Rule{ + Name: "avro-encrypt", + Kind: "TRANSFORM", + Mode: "WRITEREAD", + Type: "ENCRYPT", + Tags: []string{"PII"}, + Params: map[string]string{ + "encrypt.kek.name": "kek1", + "encrypt.kms.type": "local-kms", + "encrypt.kms.key.id": "mykey", + }, + OnFailure: "ERROR,NONE", + } + ruleSet := schemaregistry.RuleSet{ + DomainRules: []schemaregistry.Rule{encRule}, + } + + // Explicitly register the schema to have a schemaId with mock SR client + client := serializationProvider.GetSchemaRegistryClient() + info := schemaregistry.SchemaInfo{ + Schema: schemaString, + SchemaType: "AVRO", + RuleSet: &ruleSet, + } + _, err = client.Register("topic1-value", info, false) + req.Nil(err) + + data, err := serializationProvider.Serialize("topic1", expectedString) + req.Nil(err) + + // Initialize the mock deserializer + deserializationProvider, _ := GetDeserializationProvider(avroSchemaName) + err = deserializationProvider.InitDeserializer(mockClientUrl, "", "value", "", "", "", client) + req.Nil(err) + + actualString, err := deserializationProvider.Deserialize("topic1", data) + req.Nil(err) + + req.Equal(expectedString, actualString) + req.NoError(os.RemoveAll(dir)) +} + func TestJsonSerdesValid(t *testing.T) { req := require.New(t) @@ -423,6 +485,68 @@ func TestJsonSerdesNestedValid(t *testing.T) { req.NoError(os.RemoveAll(dir)) } +func TestJsonSerdesValidWithRuleSet(t *testing.T) { + req := require.New(t) + + dir, err := createTempDir() + req.Nil(err) + + schemaString := `{"type":"object","properties":{"f1":{"type":"string","confluent:tags": ["PII"]}},"required":["f1"]}` + schemaPath := filepath.Join(dir, "json-schema-ruleset.json") + req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) + + expectedString := `{"f1":"this is a confidential message in JSON schema"}` + + // Initialize the mock serializer and use latest schemaId + serializationProvider, _ := GetSerializationProvider(jsonSchemaName) + err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + req.Nil(err) + err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) + req.Nil(err) + + // CSFLE specific rules during schema registration + encRule := schemaregistry.Rule{ + Name: "json-encrypt", + Kind: "TRANSFORM", + Mode: "WRITEREAD", + Type: "ENCRYPT", + Tags: []string{"PII"}, + Params: map[string]string{ + "encrypt.kek.name": "kek1", + "encrypt.kms.type": "local-kms", + "encrypt.kms.key.id": "mykey", + }, + OnFailure: "ERROR,NONE", + } + ruleSet := schemaregistry.RuleSet{ + DomainRules: []schemaregistry.Rule{encRule}, + } + + // Explicitly register the schema to have a schemaId with mock SR client + client := serializationProvider.GetSchemaRegistryClient() + info := schemaregistry.SchemaInfo{ + Schema: schemaString, + SchemaType: "JSON", + RuleSet: &ruleSet, + } + _, err = client.Register("topic1-value", info, false) + req.Nil(err) + + data, err := serializationProvider.Serialize("topic1", expectedString) + req.Nil(err) + + // Initialize the mock deserializer + deserializationProvider, _ := GetDeserializationProvider(jsonSchemaName) + err = deserializationProvider.InitDeserializer(mockClientUrl, "", "value", "", "", "", client) + req.Nil(err) + + actualString, err := deserializationProvider.Deserialize("topic1", data) + req.Nil(err) + + req.Equal(expectedString, actualString) + req.NoError(os.RemoveAll(dir)) +} + func TestProtobufSerdesValid(t *testing.T) { req := require.New(t) From f21c55ebb9e98cdd83ec381129bb657cef9a5d76 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:58:06 -0800 Subject: [PATCH 02/24] Fix the package order to meet linter standard --- pkg/serdes/protobuf_serialization_provider.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/serdes/protobuf_serialization_provider.go b/pkg/serdes/protobuf_serialization_provider.go index 1ca8c15939..a535017c96 100644 --- a/pkg/serdes/protobuf_serialization_provider.go +++ b/pkg/serdes/protobuf_serialization_provider.go @@ -12,10 +12,6 @@ import ( "google.golang.org/protobuf/types/dynamicpb" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" - "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde" - "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/protobuf" - - "github.com/confluentinc/cli/v4/pkg/errors" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/cel" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/awskms" @@ -24,6 +20,10 @@ import ( "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/hcvault" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/localkms" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/jsonata" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/protobuf" + + "github.com/confluentinc/cli/v4/pkg/errors" ) type ProtobufSerializationProvider struct { From bbb00709ba821324330ef0f093999d09dacc96da Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Fri, 15 Nov 2024 14:22:40 -0800 Subject: [PATCH 03/24] Update the ckgo version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2cc8b0739e..ef8d9e61dc 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/confluentinc/ccloud-sdk-go-v2/sso v0.0.1 github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0 github.com/confluentinc/cmf-sdk-go v0.0.2 - github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241113012127-14c9b6103407 + github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241028224416-25a0fd52bde3 github.com/confluentinc/go-editor v0.11.0 github.com/confluentinc/go-prompt v0.2.40 github.com/confluentinc/go-ps1 v1.0.2 diff --git a/go.sum b/go.sum index 9c94a3a137..56f3e8bbbd 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0 h1:zI59UpVm88hQn github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0/go.mod h1:29HYKS91pkCre55OPNGRIWpkKSBLUDlzXbrxZ291Xhk= github.com/confluentinc/cmf-sdk-go v0.0.2 h1:Sn7j9UwOBrcUhC42l7aCrUh50S9CUF6qgHrEO0VxHcM= github.com/confluentinc/cmf-sdk-go v0.0.2/go.mod h1:VXiUwzPStdks1l355KHNaCy/oRCOIMMzTPZJAD4OyJM= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241113012127-14c9b6103407 h1:UMIu4QP4Qsd6EE8IGsOmM83Q0bbPQVNzuCxp+6Xf/ZQ= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241113012127-14c9b6103407/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= +github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241028224416-25a0fd52bde3 h1:iGEzubsDYZAENqwawZRYezKw3z6kmFhjXNoGCwlBZ64= +github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241028224416-25a0fd52bde3/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= github.com/confluentinc/go-editor v0.11.0 h1:fcEALYHj7xV/fRSp54/IHi2DS4GlZMJWVgrYvi/llvU= github.com/confluentinc/go-editor v0.11.0/go.mod h1:nEjwqdqx8S7ZGjXsDvRgawsA04Fu2P/KAtA8fa5afMI= github.com/confluentinc/go-prompt v0.2.40 h1:tveghQJ+FVOVvF0dgQaZEm7YZSQ3r3tyuLMHy7w4PK0= From 2f8fdc5bf67d8fb17dabd8773ad4eff48327b26f Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:04:09 -0800 Subject: [PATCH 04/24] Add PROTOBUF test cases and proto schema for tags --- go.mod | 2 +- go.sum | 4 +- pkg/serdes/confluent/decimal.proto | 17 +++++++ pkg/serdes/confluent/meta.proto | 28 +++++++++++ pkg/serdes/serdes_test.go | 74 ++++++++++++++++++++++++++++++ 5 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 pkg/serdes/confluent/decimal.proto create mode 100644 pkg/serdes/confluent/meta.proto diff --git a/go.mod b/go.mod index ef8d9e61dc..5b6731eecc 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/confluentinc/ccloud-sdk-go-v2/sso v0.0.1 github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0 github.com/confluentinc/cmf-sdk-go v0.0.2 - github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241028224416-25a0fd52bde3 + github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 github.com/confluentinc/go-editor v0.11.0 github.com/confluentinc/go-prompt v0.2.40 github.com/confluentinc/go-ps1 v1.0.2 diff --git a/go.sum b/go.sum index 56f3e8bbbd..ff97954e17 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0 h1:zI59UpVm88hQn github.com/confluentinc/ccloud-sdk-go-v2/stream-designer v0.3.0/go.mod h1:29HYKS91pkCre55OPNGRIWpkKSBLUDlzXbrxZ291Xhk= github.com/confluentinc/cmf-sdk-go v0.0.2 h1:Sn7j9UwOBrcUhC42l7aCrUh50S9CUF6qgHrEO0VxHcM= github.com/confluentinc/cmf-sdk-go v0.0.2/go.mod h1:VXiUwzPStdks1l355KHNaCy/oRCOIMMzTPZJAD4OyJM= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241028224416-25a0fd52bde3 h1:iGEzubsDYZAENqwawZRYezKw3z6kmFhjXNoGCwlBZ64= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.1-0.20241028224416-25a0fd52bde3/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= +github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 h1:XFkytnGvk/ZcY2qU0ql4E4h+ftBaGqkLO7tlZ4kRbr4= +github.com/confluentinc/confluent-kafka-go/v2 v2.6.1/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= github.com/confluentinc/go-editor v0.11.0 h1:fcEALYHj7xV/fRSp54/IHi2DS4GlZMJWVgrYvi/llvU= github.com/confluentinc/go-editor v0.11.0/go.mod h1:nEjwqdqx8S7ZGjXsDvRgawsA04Fu2P/KAtA8fa5afMI= github.com/confluentinc/go-prompt v0.2.40 h1:tveghQJ+FVOVvF0dgQaZEm7YZSQ3r3tyuLMHy7w4PK0= diff --git a/pkg/serdes/confluent/decimal.proto b/pkg/serdes/confluent/decimal.proto new file mode 100644 index 0000000000..75d8b9b46f --- /dev/null +++ b/pkg/serdes/confluent/decimal.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package confluent.type; + +option go_package="../types"; + +message Decimal { + + // The two's-complement representation of the unscaled integer value in big-endian byte order + bytes value = 1; + + // The precision + uint32 precision = 2; + + // The scale + int32 scale = 3; +} \ No newline at end of file diff --git a/pkg/serdes/confluent/meta.proto b/pkg/serdes/confluent/meta.proto new file mode 100644 index 0000000000..6016459bea --- /dev/null +++ b/pkg/serdes/confluent/meta.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; +package confluent; + +import "google/protobuf/descriptor.proto"; + +option go_package="../confluent"; + +message Meta { + string doc = 1; + map params = 2; + repeated string tags = 3; +} + +extend google.protobuf.FileOptions { + Meta file_meta = 1088; +} +extend google.protobuf.MessageOptions { + Meta message_meta = 1088; +} +extend google.protobuf.FieldOptions { + Meta field_meta = 1088; +} +extend google.protobuf.EnumOptions { + Meta enum_meta = 1088; +} +extend google.protobuf.EnumValueOptions { + Meta enum_value_meta = 1088; +} diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index a48e39c0d3..9cb35e3e24 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -790,6 +790,80 @@ func TestProtobufSerdesNestedValid(t *testing.T) { req.NoError(os.RemoveAll(dir)) } +func TestProtobufSerdesValidWithRuleSet(t *testing.T) { + req := require.New(t) + + dir, err := createTempDir() + req.Nil(err) + + schemaString := ` + syntax = "proto3"; + + import "meta.proto"; + + message Person { + string name = 1 [ + (confluent.field_meta) = { + tags: ["PII"] + } + ]; + int32 page = 2; + double result = 3; + }` + schemaPath := filepath.Join(dir, "person-schema-ruleset.proto") + req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) + + expectedString := `{"name":"abc","page":1,"result":2.5}` + + serializationProvider, _ := GetSerializationProvider(protobufSchemaName) + err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + req.Nil(err) + err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) + req.Nil(err) + + // CSFLE specific rules during schema registration + encRule := schemaregistry.Rule{ + Name: "protobuf-encrypt", + Kind: "TRANSFORM", + Mode: "WRITEREAD", + Type: "ENCRYPT", + Tags: []string{"PII"}, + Params: map[string]string{ + "encrypt.kek.name": "kek1", + "encrypt.kms.type": "local-kms", + "encrypt.kms.key.id": "mykey", + }, + OnFailure: "ERROR,NONE", + } + ruleSet := schemaregistry.RuleSet{ + DomainRules: []schemaregistry.Rule{encRule}, + } + + // Explicitly register the schema to have a schemaId with mock SR client + client := serializationProvider.GetSchemaRegistryClient() + info := schemaregistry.SchemaInfo{ + Schema: schemaString, + SchemaType: "PROTOBUF", + RuleSet: &ruleSet, + } + _, err = client.Register("topic1-value", info, false) + req.Nil(err) + + data, err := serializationProvider.Serialize("topic1", expectedString) + req.Nil(err) + + deserializationProvider, _ := GetDeserializationProvider(protobufSchemaName) + err = deserializationProvider.InitDeserializer(mockClientUrl, "", "value", "", "", "", client) + req.Nil(err) + err = deserializationProvider.LoadSchema(schemaPath, map[string]string{}) + req.Nil(err) + actualString, err := deserializationProvider.Deserialize("topic1", data) + req.Nil(err) + req.JSONEq(expectedString, actualString) + + req.NoError(os.RemoveAll(dir)) +} + func createTempDir() (string, error) { dir := filepath.Join(os.TempDir(), "ccloud-schema") err := os.MkdirAll(dir, 0755) From 9896eda14cf59f8e1b5edafa758e4d347c39b8f7 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:37:36 -0800 Subject: [PATCH 05/24] temp --- pkg/serdes/google/descriptor.proto | 911 +++++++++++++++++++++++++++++ pkg/serdes/serdes_test.go | 12 +- 2 files changed, 916 insertions(+), 7 deletions(-) create mode 100644 pkg/serdes/google/descriptor.proto diff --git a/pkg/serdes/google/descriptor.proto b/pkg/serdes/google/descriptor.proto new file mode 100644 index 0000000000..f307be7e0a --- /dev/null +++ b/pkg/serdes/google/descriptor.proto @@ -0,0 +1,911 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + +syntax = "proto2"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + + repeated OneofDescriptorProto oneof_decl = 8; + + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + } + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + // TODO(kenton): Base-64 encode? + optional string default_value = 7; + + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + + optional FieldOptions options = 8; + + // If true, this is a proto3 "optional". When a proto3 field is optional, it + // tracks presence regardless of field type. + // + // When proto3_optional is true, this field must be belong to a oneof to + // signal to old proto3 clients that presence is tracked for this field. This + // oneof is known as a "synthetic" oneof, and this field must be its sole + // member (each proto3 optional field gets its own synthetic oneof). Synthetic + // oneofs exist in the descriptor only, and do not generate any API. Synthetic + // oneofs must be ordered after all "real" oneofs. + // + // For message fields, proto3_optional doesn't create any semantic change, + // since non-repeated message fields always track presence. However it still + // indicates the semantic detail of whether the user wrote "optional" or not. + // This can be useful for round-tripping the .proto file. For consistency we + // give message fields a synthetic oneof also, even though it is not required + // to track presence. This is especially important because the parser can't + // tell if a field is a message or an enum, so it must always create a + // synthetic oneof. + // + // Proto2 optional fields do not set this flag, because they already indicate + // optional with `LABEL_OPTIONAL`. + optional bool proto3_optional = 17; +} + +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default = false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default = false]; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // Controls the name of the wrapper Java class generated for the .proto file. + // That class will always contain the .proto file's getDescriptor() method as + // well as any top-level extensions defined in the .proto file. + // If java_multiple_files is disabled, then all the other classes from the + // .proto file will be nested inside the single wrapper outer class. + optional string java_outer_classname = 8; + + // If enabled, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the wrapper class + // named by java_outer_classname. However, the wrapper class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default = false]; + + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default = false]; + + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default = SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + optional string go_package = 11; + + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool php_generic_services = 42 [default = false]; + + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default = false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default = true]; + + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // Use this option to change the namespace of php generated metadata classes. + // Default is empty. When this option is empty, the proto file name will be + // used for determining the namespace. + optional string php_metadata_namespace = 44; + + // Use this option to change the package of ruby generated classes. Default + // is empty. When this option is not set, the package name will be used for + // determining the ruby package. + optional string ruby_package = 45; + + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. + extensions 1000 to max; + + reserved 38; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default = false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default = false]; + + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default = false]; + + reserved 4, 5, 6; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementations still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. + optional bool packed = 2; + + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outer message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + optional bool lazy = 5 [default = false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default = false]; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default = false]; + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype +} + +message OneofOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to true to allow mapping different tag names to the same + // value. + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default = false]; + + reserved 5; // javanano_as_lite + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default = false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = 34 + [default = IDEMPOTENCY_UNKNOWN]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents + // "foo.(bar.baz).qux". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendant. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition occurs. For + // example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed = true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed = true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to qux. + // // + // // Another line attached to qux. + // optional double qux = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to qux or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed = true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; + } +} diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 9cb35e3e24..27b097e377 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -799,16 +799,14 @@ func TestProtobufSerdesValidWithRuleSet(t *testing.T) { schemaString := ` syntax = "proto3"; - import "meta.proto"; + import "confluent/meta.proto"; message Person { - string name = 1 [ - (confluent.field_meta) = { - tags: ["PII"] - } + string name = 1 [ + (confluent.field_meta).tags = "PII" ]; - int32 page = 2; - double result = 3; + int32 page = 2; + double result = 3; }` schemaPath := filepath.Join(dir, "person-schema-ruleset.proto") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) From f3a113f2357d425680189dbe913f27d63fd6c2b9 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Thu, 21 Nov 2024 09:22:46 -0800 Subject: [PATCH 06/24] Import all the built-in proto files used for CSFLE from SR team --- pkg/serdes/confluent/meta.proto | 2 +- pkg/serdes/confluent/{ => type}/decimal.proto | 2 +- pkg/serdes/google/protobuf/any.proto | 158 +++++++++++ pkg/serdes/google/protobuf/api.proto | 208 +++++++++++++++ .../google/{ => protobuf}/descriptor.proto | 0 pkg/serdes/google/protobuf/duration.proto | 116 +++++++++ pkg/serdes/google/protobuf/empty.proto | 52 ++++ pkg/serdes/google/protobuf/field_mask.proto | 245 ++++++++++++++++++ .../google/protobuf/source_context.proto | 48 ++++ pkg/serdes/google/protobuf/struct.proto | 95 +++++++ pkg/serdes/google/protobuf/timestamp.proto | 147 +++++++++++ pkg/serdes/google/protobuf/type.proto | 187 +++++++++++++ pkg/serdes/google/protobuf/wrappers.proto | 123 +++++++++ pkg/serdes/google/type/calendar_period.proto | 57 ++++ pkg/serdes/google/type/color.proto | 170 ++++++++++++ pkg/serdes/google/type/date.proto | 50 ++++ pkg/serdes/google/type/datetime.proto | 97 +++++++ pkg/serdes/google/type/dayofweek.proto | 51 ++++ pkg/serdes/google/type/expr.proto | 51 ++++ pkg/serdes/google/type/fraction.proto | 34 +++ pkg/serdes/google/type/latlng.proto | 38 +++ pkg/serdes/google/type/money.proto | 43 +++ pkg/serdes/google/type/month.proto | 66 +++++ pkg/serdes/google/type/postal_address.proto | 135 ++++++++++ pkg/serdes/google/type/quaternion.proto | 95 +++++++ pkg/serdes/google/type/timeofday.proto | 44 ++++ 26 files changed, 2312 insertions(+), 2 deletions(-) rename pkg/serdes/confluent/{ => type}/decimal.proto (78%) create mode 100644 pkg/serdes/google/protobuf/any.proto create mode 100644 pkg/serdes/google/protobuf/api.proto rename pkg/serdes/google/{ => protobuf}/descriptor.proto (100%) create mode 100644 pkg/serdes/google/protobuf/duration.proto create mode 100644 pkg/serdes/google/protobuf/empty.proto create mode 100644 pkg/serdes/google/protobuf/field_mask.proto create mode 100644 pkg/serdes/google/protobuf/source_context.proto create mode 100644 pkg/serdes/google/protobuf/struct.proto create mode 100644 pkg/serdes/google/protobuf/timestamp.proto create mode 100644 pkg/serdes/google/protobuf/type.proto create mode 100644 pkg/serdes/google/protobuf/wrappers.proto create mode 100644 pkg/serdes/google/type/calendar_period.proto create mode 100644 pkg/serdes/google/type/color.proto create mode 100644 pkg/serdes/google/type/date.proto create mode 100644 pkg/serdes/google/type/datetime.proto create mode 100644 pkg/serdes/google/type/dayofweek.proto create mode 100644 pkg/serdes/google/type/expr.proto create mode 100644 pkg/serdes/google/type/fraction.proto create mode 100644 pkg/serdes/google/type/latlng.proto create mode 100644 pkg/serdes/google/type/money.proto create mode 100644 pkg/serdes/google/type/month.proto create mode 100644 pkg/serdes/google/type/postal_address.proto create mode 100644 pkg/serdes/google/type/quaternion.proto create mode 100644 pkg/serdes/google/type/timeofday.proto diff --git a/pkg/serdes/confluent/meta.proto b/pkg/serdes/confluent/meta.proto index 6016459bea..c5152ea2e9 100644 --- a/pkg/serdes/confluent/meta.proto +++ b/pkg/serdes/confluent/meta.proto @@ -3,7 +3,7 @@ package confluent; import "google/protobuf/descriptor.proto"; -option go_package="../confluent"; +option csharp_namespace = "Confluent.SchemaRegistry.Serdes.Protobuf"; message Meta { string doc = 1; diff --git a/pkg/serdes/confluent/decimal.proto b/pkg/serdes/confluent/type/decimal.proto similarity index 78% rename from pkg/serdes/confluent/decimal.proto rename to pkg/serdes/confluent/type/decimal.proto index 75d8b9b46f..944dabe4f3 100644 --- a/pkg/serdes/confluent/decimal.proto +++ b/pkg/serdes/confluent/type/decimal.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package confluent.type; -option go_package="../types"; +option csharp_namespace = "Confluent.SchemaRegistry.Serdes.Protobuf"; message Decimal { diff --git a/pkg/serdes/google/protobuf/any.proto b/pkg/serdes/google/protobuf/any.proto new file mode 100644 index 0000000000..e2c2042fdc --- /dev/null +++ b/pkg/serdes/google/protobuf/any.proto @@ -0,0 +1,158 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/pkg/serdes/google/protobuf/api.proto b/pkg/serdes/google/protobuf/api.proto new file mode 100644 index 0000000000..3d598fc859 --- /dev/null +++ b/pkg/serdes/google/protobuf/api.proto @@ -0,0 +1,208 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/source_context.proto"; +import "google/protobuf/type.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "ApiProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/apipb"; + +// Api is a light-weight descriptor for an API Interface. +// +// Interfaces are also described as "protocol buffer services" in some contexts, +// such as by the "service" keyword in a .proto file, but they are different +// from API Services, which represent a concrete implementation of an interface +// as opposed to simply a description of methods and bindings. They are also +// sometimes simply referred to as "APIs" in other contexts, such as the name of +// this message itself. See https://cloud.google.com/apis/design/glossary for +// detailed terminology. +message Api { + // The fully qualified name of this interface, including package name + // followed by the interface's simple name. + string name = 1; + + // The methods of this interface, in unspecified order. + repeated Method methods = 2; + + // Any metadata attached to the interface. + repeated Option options = 3; + + // A version string for this interface. If specified, must have the form + // `major-version.minor-version`, as in `1.10`. If the minor version is + // omitted, it defaults to zero. If the entire version field is empty, the + // major version is derived from the package name, as outlined below. If the + // field is not empty, the version in the package name will be verified to be + // consistent with what is provided here. + // + // The versioning schema uses [semantic + // versioning](http://semver.org) where the major version number + // indicates a breaking change and the minor version an additive, + // non-breaking change. Both version numbers are signals to users + // what to expect from different versions, and should be carefully + // chosen based on the product plan. + // + // The major version is also reflected in the package name of the + // interface, which must end in `v`, as in + // `google.feature.v1`. For major versions 0 and 1, the suffix can + // be omitted. Zero major versions must only be used for + // experimental, non-GA interfaces. + // + // + string version = 4; + + // Source context for the protocol buffer service represented by this + // message. + SourceContext source_context = 5; + + // Included interfaces. See [Mixin][]. + repeated Mixin mixins = 6; + + // The source syntax of the service. + Syntax syntax = 7; +} + +// Method represents a method of an API interface. +message Method { + // The simple name of this method. + string name = 1; + + // A URL of the input message type. + string request_type_url = 2; + + // If true, the request is streamed. + bool request_streaming = 3; + + // The URL of the output message type. + string response_type_url = 4; + + // If true, the response is streamed. + bool response_streaming = 5; + + // Any metadata attached to the method. + repeated Option options = 6; + + // The source syntax of this method. + Syntax syntax = 7; +} + +// Declares an API Interface to be included in this interface. The including +// interface must redeclare all the methods from the included interface, but +// documentation and options are inherited as follows: +// +// - If after comment and whitespace stripping, the documentation +// string of the redeclared method is empty, it will be inherited +// from the original method. +// +// - Each annotation belonging to the service config (http, +// visibility) which is not set in the redeclared method will be +// inherited. +// +// - If an http annotation is inherited, the path pattern will be +// modified as follows. Any version prefix will be replaced by the +// version of the including interface plus the [root][] path if +// specified. +// +// Example of a simple mixin: +// +// package google.acl.v1; +// service AccessControl { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v1/{resource=**}:getAcl"; +// } +// } +// +// package google.storage.v2; +// service Storage { +// rpc GetAcl(GetAclRequest) returns (Acl); +// +// // Get a data record. +// rpc GetData(GetDataRequest) returns (Data) { +// option (google.api.http).get = "/v2/{resource=**}"; +// } +// } +// +// Example of a mixin configuration: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// +// The mixin construct implies that all methods in `AccessControl` are +// also declared with same name and request/response types in +// `Storage`. A documentation generator or annotation processor will +// see the effective `Storage.GetAcl` method after inheriting +// documentation and annotations as follows: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/{resource=**}:getAcl"; +// } +// ... +// } +// +// Note how the version in the path pattern changed from `v1` to `v2`. +// +// If the `root` field in the mixin is specified, it should be a +// relative path under which inherited HTTP paths are placed. Example: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// root: acls +// +// This implies the following inherited HTTP annotation: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; +// } +// ... +// } +message Mixin { + // The fully qualified name of the interface which is included. + string name = 1; + + // If non-empty specifies a path under which inherited HTTP paths + // are rooted. + string root = 2; +} diff --git a/pkg/serdes/google/descriptor.proto b/pkg/serdes/google/protobuf/descriptor.proto similarity index 100% rename from pkg/serdes/google/descriptor.proto rename to pkg/serdes/google/protobuf/descriptor.proto diff --git a/pkg/serdes/google/protobuf/duration.proto b/pkg/serdes/google/protobuf/duration.proto new file mode 100644 index 0000000000..81c3e369fd --- /dev/null +++ b/pkg/serdes/google/protobuf/duration.proto @@ -0,0 +1,116 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/durationpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (duration.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/pkg/serdes/google/protobuf/empty.proto b/pkg/serdes/google/protobuf/empty.proto new file mode 100644 index 0000000000..5f992de94a --- /dev/null +++ b/pkg/serdes/google/protobuf/empty.proto @@ -0,0 +1,52 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/emptypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "EmptyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +// The JSON representation for `Empty` is empty JSON object `{}`. +message Empty {} diff --git a/pkg/serdes/google/protobuf/field_mask.proto b/pkg/serdes/google/protobuf/field_mask.proto new file mode 100644 index 0000000000..6b5104f188 --- /dev/null +++ b/pkg/serdes/google/protobuf/field_mask.proto @@ -0,0 +1,245 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "FieldMaskProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/fieldmaskpb"; +option cc_enable_arenas = true; + +// `FieldMask` represents a set of symbolic field paths, for example: +// +// paths: "f.a" +// paths: "f.b.d" +// +// Here `f` represents a field in some root message, `a` and `b` +// fields in the message found in `f`, and `d` a field found in the +// message in `f.b`. +// +// Field masks are used to specify a subset of fields that should be +// returned by a get operation or modified by an update operation. +// Field masks also have a custom JSON encoding (see below). +// +// # Field Masks in Projections +// +// When used in the context of a projection, a response message or +// sub-message is filtered by the API to only contain those fields as +// specified in the mask. For example, if the mask in the previous +// example is applied to a response message as follows: +// +// f { +// a : 22 +// b { +// d : 1 +// x : 2 +// } +// y : 13 +// } +// z: 8 +// +// The result will not contain specific values for fields x,y and z +// (their value will be set to the default, and omitted in proto text +// output): +// +// +// f { +// a : 22 +// b { +// d : 1 +// } +// } +// +// A repeated field is not allowed except at the last position of a +// paths string. +// +// If a FieldMask object is not present in a get operation, the +// operation applies to all fields (as if a FieldMask of all fields +// had been specified). +// +// Note that a field mask does not necessarily apply to the +// top-level response message. In case of a REST get operation, the +// field mask applies directly to the response, but in case of a REST +// list operation, the mask instead applies to each individual message +// in the returned resource list. In case of a REST custom method, +// other definitions may be used. Where the mask applies will be +// clearly documented together with its declaration in the API. In +// any case, the effect on the returned resource/resources is required +// behavior for APIs. +// +// # Field Masks in Update Operations +// +// A field mask in update operations specifies which fields of the +// targeted resource are going to be updated. The API is required +// to only change the values of the fields as specified in the mask +// and leave the others untouched. If a resource is passed in to +// describe the updated values, the API ignores the values of all +// fields not covered by the mask. +// +// If a repeated field is specified for an update operation, new values will +// be appended to the existing repeated field in the target resource. Note that +// a repeated field is only allowed in the last position of a `paths` string. +// +// If a sub-message is specified in the last position of the field mask for an +// update operation, then new value will be merged into the existing sub-message +// in the target resource. +// +// For example, given the target message: +// +// f { +// b { +// d: 1 +// x: 2 +// } +// c: [1] +// } +// +// And an update message: +// +// f { +// b { +// d: 10 +// } +// c: [2] +// } +// +// then if the field mask is: +// +// paths: ["f.b", "f.c"] +// +// then the result will be: +// +// f { +// b { +// d: 10 +// x: 2 +// } +// c: [1, 2] +// } +// +// An implementation may provide options to override this default behavior for +// repeated and message fields. +// +// In order to reset a field's value to the default, the field must +// be in the mask and set to the default value in the provided resource. +// Hence, in order to reset all fields of a resource, provide a default +// instance of the resource and set all fields in the mask, or do +// not provide a mask as described below. +// +// If a field mask is not present on update, the operation applies to +// all fields (as if a field mask of all fields has been specified). +// Note that in the presence of schema evolution, this may mean that +// fields the client does not know and has therefore not filled into +// the request will be reset to their default. If this is unwanted +// behavior, a specific service may require a client to always specify +// a field mask, producing an error if not. +// +// As with get operations, the location of the resource which +// describes the updated values in the request message depends on the +// operation kind. In any case, the effect of the field mask is +// required to be honored by the API. +// +// ## Considerations for HTTP REST +// +// The HTTP kind of an update operation which uses a field mask must +// be set to PATCH instead of PUT in order to satisfy HTTP semantics +// (PUT must only be used for full updates). +// +// # JSON Encoding of Field Masks +// +// In JSON, a field mask is encoded as a single string where paths are +// separated by a comma. Fields name in each path are converted +// to/from lower-camel naming conventions. +// +// As an example, consider the following message declarations: +// +// message Profile { +// User user = 1; +// Photo photo = 2; +// } +// message User { +// string display_name = 1; +// string address = 2; +// } +// +// In proto a field mask for `Profile` may look as such: +// +// mask { +// paths: "user.display_name" +// paths: "photo" +// } +// +// In JSON, the same mask is represented as below: +// +// { +// mask: "user.displayName,photo" +// } +// +// # Field Masks and Oneof Fields +// +// Field masks treat fields in oneofs just as regular fields. Consider the +// following message: +// +// message SampleMessage { +// oneof test_oneof { +// string name = 4; +// SubMessage sub_message = 9; +// } +// } +// +// The field mask can be: +// +// mask { +// paths: "name" +// } +// +// Or: +// +// mask { +// paths: "sub_message" +// } +// +// Note that oneof type names ("test_oneof" in this case) cannot be used in +// paths. +// +// ## Field Mask Verification +// +// The implementation of any API method which has a FieldMask type field in the +// request should verify the included field paths, and return an +// `INVALID_ARGUMENT` error if any path is unmappable. +message FieldMask { + // The set of field mask paths. + repeated string paths = 1; +} diff --git a/pkg/serdes/google/protobuf/source_context.proto b/pkg/serdes/google/protobuf/source_context.proto new file mode 100644 index 0000000000..06bfc43a78 --- /dev/null +++ b/pkg/serdes/google/protobuf/source_context.proto @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "SourceContextProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/sourcecontextpb"; + +// `SourceContext` represents information about the source of a +// protobuf element, like the file in which it is defined. +message SourceContext { + // The path-qualified name of the .proto file that contained the associated + // protobuf element. For example: `"google/protobuf/source_context.proto"`. + string file_name = 1; +} diff --git a/pkg/serdes/google/protobuf/struct.proto b/pkg/serdes/google/protobuf/struct.proto new file mode 100644 index 0000000000..0ac843ca08 --- /dev/null +++ b/pkg/serdes/google/protobuf/struct.proto @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/structpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "StructProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Struct` represents a structured data value, consisting of fields +// which map to dynamically typed values. In some languages, `Struct` +// might be supported by a native representation. For example, in +// scripting languages like JS a struct is represented as an +// object. The details of that representation are described together +// with the proto support for the language. +// +// The JSON representation for `Struct` is JSON object. +message Struct { + // Unordered map of dynamically typed values. + map fields = 1; +} + +// `Value` represents a dynamically typed value which can be either +// null, a number, a string, a boolean, a recursive struct value, or a +// list of values. A producer of value is expected to set one of these +// variants. Absence of any variant indicates an error. +// +// The JSON representation for `Value` is JSON value. +message Value { + // The kind of value. + oneof kind { + // Represents a null value. + NullValue null_value = 1; + // Represents a double value. + double number_value = 2; + // Represents a string value. + string string_value = 3; + // Represents a boolean value. + bool bool_value = 4; + // Represents a structured value. + Struct struct_value = 5; + // Represents a repeated `Value`. + ListValue list_value = 6; + } +} + +// `NullValue` is a singleton enumeration to represent the null value for the +// `Value` type union. +// +// The JSON representation for `NullValue` is JSON `null`. +enum NullValue { + // Null value. + NULL_VALUE = 0; +} + +// `ListValue` is a wrapper around a repeated field of values. +// +// The JSON representation for `ListValue` is JSON array. +message ListValue { + // Repeated field of dynamically typed values. + repeated Value values = 1; +} diff --git a/pkg/serdes/google/protobuf/timestamp.proto b/pkg/serdes/google/protobuf/timestamp.proto new file mode 100644 index 0000000000..3b2df6d911 --- /dev/null +++ b/pkg/serdes/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/pkg/serdes/google/protobuf/type.proto b/pkg/serdes/google/protobuf/type.proto new file mode 100644 index 0000000000..d3f6a68b83 --- /dev/null +++ b/pkg/serdes/google/protobuf/type.proto @@ -0,0 +1,187 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/any.proto"; +import "google/protobuf/source_context.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TypeProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/typepb"; + +// A protocol buffer message type. +message Type { + // The fully qualified message name. + string name = 1; + // The list of fields. + repeated Field fields = 2; + // The list of types appearing in `oneof` definitions in this type. + repeated string oneofs = 3; + // The protocol buffer options. + repeated Option options = 4; + // The source context. + SourceContext source_context = 5; + // The source syntax. + Syntax syntax = 6; +} + +// A single field of a message type. +message Field { + // Basic field types. + enum Kind { + // Field type unknown. + TYPE_UNKNOWN = 0; + // Field type double. + TYPE_DOUBLE = 1; + // Field type float. + TYPE_FLOAT = 2; + // Field type int64. + TYPE_INT64 = 3; + // Field type uint64. + TYPE_UINT64 = 4; + // Field type int32. + TYPE_INT32 = 5; + // Field type fixed64. + TYPE_FIXED64 = 6; + // Field type fixed32. + TYPE_FIXED32 = 7; + // Field type bool. + TYPE_BOOL = 8; + // Field type string. + TYPE_STRING = 9; + // Field type group. Proto2 syntax only, and deprecated. + TYPE_GROUP = 10; + // Field type message. + TYPE_MESSAGE = 11; + // Field type bytes. + TYPE_BYTES = 12; + // Field type uint32. + TYPE_UINT32 = 13; + // Field type enum. + TYPE_ENUM = 14; + // Field type sfixed32. + TYPE_SFIXED32 = 15; + // Field type sfixed64. + TYPE_SFIXED64 = 16; + // Field type sint32. + TYPE_SINT32 = 17; + // Field type sint64. + TYPE_SINT64 = 18; + } + + // Whether a field is optional, required, or repeated. + enum Cardinality { + // For fields with unknown cardinality. + CARDINALITY_UNKNOWN = 0; + // For optional fields. + CARDINALITY_OPTIONAL = 1; + // For required fields. Proto2 syntax only. + CARDINALITY_REQUIRED = 2; + // For repeated fields. + CARDINALITY_REPEATED = 3; + } + + // The field type. + Kind kind = 1; + // The field cardinality. + Cardinality cardinality = 2; + // The field number. + int32 number = 3; + // The field name. + string name = 4; + // The field type URL, without the scheme, for message or enumeration + // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + string type_url = 6; + // The index of the field type in `Type.oneofs`, for message or enumeration + // types. The first type has index 1; zero means the type is not in the list. + int32 oneof_index = 7; + // Whether to use alternative packed wire representation. + bool packed = 8; + // The protocol buffer options. + repeated Option options = 9; + // The field JSON name. + string json_name = 10; + // The string value of the default value of this field. Proto2 syntax only. + string default_value = 11; +} + +// Enum type definition. +message Enum { + // Enum type name. + string name = 1; + // Enum value definitions. + repeated EnumValue enumvalue = 2; + // Protocol buffer options. + repeated Option options = 3; + // The source context. + SourceContext source_context = 4; + // The source syntax. + Syntax syntax = 5; +} + +// Enum value definition. +message EnumValue { + // Enum value name. + string name = 1; + // Enum value number. + int32 number = 2; + // Protocol buffer options. + repeated Option options = 3; +} + +// A protocol buffer option, which can be attached to a message, field, +// enumeration, etc. +message Option { + // The option's name. For protobuf built-in options (options defined in + // descriptor.proto), this is the short name. For example, `"map_entry"`. + // For custom options, it should be the fully-qualified name. For example, + // `"google.api.http"`. + string name = 1; + // The option's value packed in an Any message. If the value is a primitive, + // the corresponding wrapper type defined in google/protobuf/wrappers.proto + // should be used. If the value is an enum, it should be stored as an int32 + // value using the google.protobuf.Int32Value type. + Any value = 2; +} + +// The syntax in which a protocol buffer element is defined. +enum Syntax { + // Syntax `proto2`. + SYNTAX_PROTO2 = 0; + // Syntax `proto3`. + SYNTAX_PROTO3 = 1; +} diff --git a/pkg/serdes/google/protobuf/wrappers.proto b/pkg/serdes/google/protobuf/wrappers.proto new file mode 100644 index 0000000000..1959fa55a4 --- /dev/null +++ b/pkg/serdes/google/protobuf/wrappers.proto @@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Wrappers for primitive (non-message) types. These types are useful +// for embedding primitives in the `google.protobuf.Any` type and for places +// where we need to distinguish between the absence of a primitive +// typed field and its default value. +// +// These wrappers have no meaningful use within repeated fields as they lack +// the ability to detect presence on individual elements. +// These wrappers have no meaningful use within a map or a oneof since +// individual entries of a map or fields of a oneof can already detect presence. + +syntax = "proto3"; + +package google.protobuf; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "WrappersProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; + +// Wrapper message for `double`. +// +// The JSON representation for `DoubleValue` is JSON number. +message DoubleValue { + // The double value. + double value = 1; +} + +// Wrapper message for `float`. +// +// The JSON representation for `FloatValue` is JSON number. +message FloatValue { + // The float value. + float value = 1; +} + +// Wrapper message for `int64`. +// +// The JSON representation for `Int64Value` is JSON string. +message Int64Value { + // The int64 value. + int64 value = 1; +} + +// Wrapper message for `uint64`. +// +// The JSON representation for `UInt64Value` is JSON string. +message UInt64Value { + // The uint64 value. + uint64 value = 1; +} + +// Wrapper message for `int32`. +// +// The JSON representation for `Int32Value` is JSON number. +message Int32Value { + // The int32 value. + int32 value = 1; +} + +// Wrapper message for `uint32`. +// +// The JSON representation for `UInt32Value` is JSON number. +message UInt32Value { + // The uint32 value. + uint32 value = 1; +} + +// Wrapper message for `bool`. +// +// The JSON representation for `BoolValue` is JSON `true` and `false`. +message BoolValue { + // The bool value. + bool value = 1; +} + +// Wrapper message for `string`. +// +// The JSON representation for `StringValue` is JSON string. +message StringValue { + // The string value. + string value = 1; +} + +// Wrapper message for `bytes`. +// +// The JSON representation for `BytesValue` is JSON string. +message BytesValue { + // The bytes value. + bytes value = 1; +} diff --git a/pkg/serdes/google/type/calendar_period.proto b/pkg/serdes/google/type/calendar_period.proto new file mode 100644 index 0000000000..a91d0c35c8 --- /dev/null +++ b/pkg/serdes/google/type/calendar_period.proto @@ -0,0 +1,57 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/pkg/serdes/google/type/color.proto b/pkg/serdes/google/type/color.proto new file mode 100644 index 0000000000..417f1c4b19 --- /dev/null +++ b/pkg/serdes/google/type/color.proto @@ -0,0 +1,170 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness; for example, the fields of this representation +// can be trivially provided to the constructor of "java.awt.Color" in Java; it +// can also be trivially provided to UIColor's "+colorWithRed:green:blue:alpha" +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS "rgba()" string in JavaScript, as well. +// +// Note: this proto does not carry information about the absolute color space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications SHOULD assume the sRGB color +// space. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor_(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor_ = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // pixel color = alpha * (this color) + (1.0 - alpha) * (background color) + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is to be rendered as a solid color + // (as if the alpha value had been explicitly given with a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/pkg/serdes/google/type/date.proto b/pkg/serdes/google/type/date.proto new file mode 100644 index 0000000000..b958feeba5 --- /dev/null +++ b/pkg/serdes/google/type/date.proto @@ -0,0 +1,50 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, e.g. a birthday. The time of day +// and time zone are either specified elsewhere or are not significant. The date +// is relative to the Proleptic Gregorian Calendar. This can represent: +// +// * A full date, with non-zero year, month and day values +// * A month and day value, with a zero year, e.g. an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, e.g. a credit card expiration date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and `google.protobuf.Timestamp`. +message Date { + // Year of date. Must be from 1 to 9999, or 0 if specifying a date without + // a year. + int32 year = 1; + + // Month of year. Must be from 1 to 12, or 0 if specifying a year without a + // month and day. + int32 month = 2; + + // Day of month. Must be from 1 to 31 and valid for the year and month, or 0 + // if specifying a year by itself or a year and month where the day is not + // significant. + int32 day = 3; +} diff --git a/pkg/serdes/google/type/datetime.proto b/pkg/serdes/google/type/datetime.proto new file mode 100644 index 0000000000..5aebc4b97f --- /dev/null +++ b/pkg/serdes/google/type/datetime.proto @@ -0,0 +1,97 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/pkg/serdes/google/type/dayofweek.proto b/pkg/serdes/google/type/dayofweek.proto new file mode 100644 index 0000000000..7544e15161 --- /dev/null +++ b/pkg/serdes/google/type/dayofweek.proto @@ -0,0 +1,51 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of week. +enum DayOfWeek { + // The unspecified day-of-week. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // The day-of-week of Monday. + MONDAY = 1; + + // The day-of-week of Tuesday. + TUESDAY = 2; + + // The day-of-week of Wednesday. + WEDNESDAY = 3; + + // The day-of-week of Thursday. + THURSDAY = 4; + + // The day-of-week of Friday. + FRIDAY = 5; + + // The day-of-week of Saturday. + SATURDAY = 6; + + // The day-of-week of Sunday. + SUNDAY = 7; +} diff --git a/pkg/serdes/google/type/expr.proto b/pkg/serdes/google/type/expr.proto new file mode 100644 index 0000000000..5d4f2f71b2 --- /dev/null +++ b/pkg/serdes/google/type/expr.proto @@ -0,0 +1,51 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an expression text. Example: +// +// title: "User account presence" +// description: "Determines whether the request has a user account" +// expression: "size(request.user) > 0" +message Expr { + // Textual representation of an expression in + // Common Expression Language syntax. + // + // The application context of the containing message determines which + // well-known feature set of CEL is supported. + string expression = 1; + + // An optional title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // An optional description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // An optional string indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/pkg/serdes/google/type/fraction.proto b/pkg/serdes/google/type/fraction.proto new file mode 100644 index 0000000000..8ad008dda2 --- /dev/null +++ b/pkg/serdes/google/type/fraction.proto @@ -0,0 +1,34 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The portion of the denominator in the faction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/pkg/serdes/google/type/latlng.proto b/pkg/serdes/google/type/latlng.proto new file mode 100644 index 0000000000..473856f980 --- /dev/null +++ b/pkg/serdes/google/type/latlng.proto @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a latitude/longitude pair. This is expressed as a pair +// of doubles representing degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/pkg/serdes/google/type/money.proto b/pkg/serdes/google/type/money.proto new file mode 100644 index 0000000000..ef41f1089a --- /dev/null +++ b/pkg/serdes/google/type/money.proto @@ -0,0 +1,43 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The 3-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/pkg/serdes/google/type/month.proto b/pkg/serdes/google/type/month.proto new file mode 100644 index 0000000000..54b7865f4b --- /dev/null +++ b/pkg/serdes/google/type/month.proto @@ -0,0 +1,66 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecifed month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/pkg/serdes/google/type/postal_address.proto b/pkg/serdes/google/type/postal_address.proto new file mode 100644 index 0000000000..688af8a1bc --- /dev/null +++ b/pkg/serdes/google/type/postal_address.proto @@ -0,0 +1,135 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/pkg/serdes/google/type/quaternion.proto b/pkg/serdes/google/type/quaternion.proto new file mode 100644 index 0000000000..7ab5dc7283 --- /dev/null +++ b/pkg/serdes/google/type/quaternion.proto @@ -0,0 +1,95 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/pkg/serdes/google/type/timeofday.proto b/pkg/serdes/google/type/timeofday.proto new file mode 100644 index 0000000000..b609a48798 --- /dev/null +++ b/pkg/serdes/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} From 0427bd84684d29ad235f537bd394ea199a7422af Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:46:44 -0800 Subject: [PATCH 07/24] Copy the built-in schemas to the temp folder used for produce/consume --- go.mod | 5 ++- go.sum | 12 ++++-- pkg/serdes/confluent/meta.proto | 1 + pkg/serdes/protobuf_serialization_provider.go | 38 ++++++++++++++++++- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 5b6731eecc..c0639c3a06 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 github.com/olekukonko/tablewriter v0.0.5 github.com/opencontainers/image-spec v1.1.0 + github.com/otiai10/copy v1.14.0 github.com/panta/machineid v1.0.2 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c @@ -261,8 +262,8 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/mod v0.19.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sync v0.9.0 // indirect + golang.org/x/sys v0.27.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.23.0 // indirect google.golang.org/api v0.191.0 // indirect diff --git a/go.sum b/go.sum index ff97954e17..64a7f7e58c 100644 --- a/go.sum +++ b/go.sum @@ -736,6 +736,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= +github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panta/machineid v1.0.2 h1:LVYeEq1hZ+FwcM+/H6eB8KfXM2R5b2h1SWdnWwZ0OQw= github.com/panta/machineid v1.0.2/go.mod h1:AROj156fsca3R3rNw3q9h8xFkos25W9P0ZG9gu+3Uf0= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= @@ -1098,8 +1102,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1166,8 +1170,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= diff --git a/pkg/serdes/confluent/meta.proto b/pkg/serdes/confluent/meta.proto index c5152ea2e9..f81948018c 100644 --- a/pkg/serdes/confluent/meta.proto +++ b/pkg/serdes/confluent/meta.proto @@ -3,6 +3,7 @@ package confluent; import "google/protobuf/descriptor.proto"; +option go_package="../confluent"; option csharp_namespace = "Confluent.SchemaRegistry.Serdes.Protobuf"; message Meta { diff --git a/pkg/serdes/protobuf_serialization_provider.go b/pkg/serdes/protobuf_serialization_provider.go index a535017c96..4ea1bda8c2 100644 --- a/pkg/serdes/protobuf_serialization_provider.go +++ b/pkg/serdes/protobuf_serialization_provider.go @@ -3,10 +3,12 @@ package serdes import ( "context" "fmt" + "os" "path/filepath" "strings" "github.com/bufbuild/protocompile" + "github.com/otiai10/copy" "google.golang.org/protobuf/encoding/protojson" gproto "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/dynamicpb" @@ -26,6 +28,16 @@ import ( "github.com/confluentinc/cli/v4/pkg/errors" ) +const ( + confluentBuiltInSchemaFolder = "confluent" + googleBuiltInSchemaFolder = "google" +) + +var builtInSchemaFoldersToCopy = []string{ + confluentBuiltInSchemaFolder, + googleBuiltInSchemaFolder, +} + type ProtobufSerializationProvider struct { ser *protobuf.Serializer message gproto.Message @@ -120,7 +132,9 @@ func (p *ProtobufSerializationProvider) Serialize(topic, message string) ([]byte func parseMessage(schemaPath string, referencePathMap map[string]string) (gproto.Message, error) { // Collect import paths - importPaths := []string{filepath.Dir(schemaPath)} + importPath := filepath.Dir(schemaPath) + importPaths := []string{importPath} + for _, path := range referencePathMap { importPaths = append(importPaths, strings.SplitAfter(path, "ccloud-schema")[0]) } @@ -129,6 +143,26 @@ func parseMessage(schemaPath string, referencePathMap map[string]string) (gproto ImportPaths: importPaths, } + currDir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("Error getting current working directory: %v\n", err) + } + fmt.Printf("Current working directory is: %s\n", currDir) + + // Copy all the built-in proto schemas needed for CSFLE to where the main schema is stored + for _, folder := range builtInSchemaFoldersToCopy { + dst := importPath + "/" + folder + // Note: folder path should be set correctly based on current working directory + // Expected working directory in CLI test is: /Users/github.com/confluentinc/cli/pkg/serdes + // Expected working directory in CLI shell execution is: /Users/github.com/confluentinc/cli + if strings.HasSuffix(currDir, "confluentinc/cli") { + folder = "pkg/serdes/" + folder + } + if err := copy.Copy(folder, dst); err != nil { + return nil, fmt.Errorf("Error copying folder %s: %v\n", folder, err) + } + } + // Create the compiler compiler := protocompile.Compiler{ Resolver: resolver, @@ -137,7 +171,7 @@ func parseMessage(schemaPath string, referencePathMap map[string]string) (gproto // Parse and compile the .proto files compiledFiles, err := compiler.Compile(context.Background(), filepath.Base(schemaPath)) if err != nil { - return nil, fmt.Errorf("error compiling or finding .proto files") + return nil, fmt.Errorf("error compiling .proto files: %w\n", err) } if len(compiledFiles) == 0 { return nil, fmt.Errorf("error fetching valid compiled files") From f4dceb3133358fa7d194a765619f1b7776dfb159 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:18:50 -0800 Subject: [PATCH 08/24] Improve the built-in schema folder copying process --- pkg/serdes/protobuf_serialization_provider.go | 21 +++++++++++++------ pkg/serdes/serdes_test.go | 5 +++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/pkg/serdes/protobuf_serialization_provider.go b/pkg/serdes/protobuf_serialization_provider.go index 4ea1bda8c2..b7ce89312b 100644 --- a/pkg/serdes/protobuf_serialization_provider.go +++ b/pkg/serdes/protobuf_serialization_provider.go @@ -26,6 +26,7 @@ import ( "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/protobuf" "github.com/confluentinc/cli/v4/pkg/errors" + "github.com/confluentinc/cli/v4/pkg/log" ) const ( @@ -147,19 +148,27 @@ func parseMessage(schemaPath string, referencePathMap map[string]string) (gproto if err != nil { return nil, fmt.Errorf("Error getting current working directory: %v\n", err) } - fmt.Printf("Current working directory is: %s\n", currDir) + log.CliLogger.Debugf("Current working directory is: %s\n", currDir) // Copy all the built-in proto schemas needed for CSFLE to where the main schema is stored + // Note: folder path should be set correctly based on current working directory + // Expected working directory in CLI test is: /Users/github.com/confluentinc/cli/pkg/serdes + // Expected working directory in CLI shell execution is: /Users/github.com/confluentinc/cli for _, folder := range builtInSchemaFoldersToCopy { dst := importPath + "/" + folder - // Note: folder path should be set correctly based on current working directory - // Expected working directory in CLI test is: /Users/github.com/confluentinc/cli/pkg/serdes - // Expected working directory in CLI shell execution is: /Users/github.com/confluentinc/cli + + // Skip copying the built-in schema folders if they are present already in the temp folder + if _, err = os.Stat(dst); err == nil { + log.CliLogger.Debugf("Built-in schema folder already exists %s, skipping copy again:\n", dst) + continue + } + + // Locate the source of built-in schema folders if strings.HasSuffix(currDir, "confluentinc/cli") { folder = "pkg/serdes/" + folder } - if err := copy.Copy(folder, dst); err != nil { - return nil, fmt.Errorf("Error copying folder %s: %v\n", folder, err) + if err = copy.Copy(folder, dst); err != nil { + return nil, fmt.Errorf("Error copying built-in schemas folder %s: %w\n", folder, err) } } diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 27b097e377..4b4d354e92 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -603,7 +603,7 @@ func TestProtobufSerdesReference(t *testing.T) { referenceString := `syntax = "proto3"; -package io.confluent; +package test; message Address { string city = 1; @@ -617,7 +617,7 @@ message Address { schemaString := `syntax = "proto3"; -package io.confluent; +package test; import "address.proto"; @@ -800,6 +800,7 @@ func TestProtobufSerdesValidWithRuleSet(t *testing.T) { syntax = "proto3"; import "confluent/meta.proto"; + package test; message Person { string name = 1 [ From 7527302c08235c7aa0b0543dfe363ea9bd6a4a02 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:01:02 -0800 Subject: [PATCH 09/24] Update the protobuf deserializer process per Robert's suggestion, update test cases accordingly --- pkg/serdes/protobuf_deserialization_provider.go | 17 ++++++++++++++--- pkg/serdes/serdes_test.go | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/serdes/protobuf_deserialization_provider.go b/pkg/serdes/protobuf_deserialization_provider.go index 702d231e5e..b24c9b0a1e 100644 --- a/pkg/serdes/protobuf_deserialization_provider.go +++ b/pkg/serdes/protobuf_deserialization_provider.go @@ -2,6 +2,7 @@ package serdes import ( "fmt" + "regexp" "google.golang.org/protobuf/encoding/protojson" gproto "google.golang.org/protobuf/proto" @@ -81,19 +82,29 @@ func (p *ProtobufDeserializationProvider) LoadSchema(schemaPath string, referenc } func (p *ProtobufDeserializationProvider) Deserialize(topic string, payload []byte) (string, error) { - err := p.deser.DeserializeInto(topic, payload, p.message) + // Register the protobuf message + err := p.deser.ProtoRegistry.RegisterMessage(p.message.ProtoReflect().Type()) + re := regexp.MustCompile(`message .* is already registered`) + + // If the error is due to already registered message, we shouldn't early return + if err != nil && !re.MatchString(err.Error()) { + return "", fmt.Errorf("failed to deserialize payload: %w", err) + } + + // Deserialize the payload into the msgObj + msgObj, err := p.deser.Deserialize(topic, payload) if err != nil { return "", fmt.Errorf("failed to deserialize payload: %w", err) } - // Use protojson to marshal the message to JSON in a compact format + // Use protojson library to marshal the message to JSON in a compact format marshaler := protojson.MarshalOptions{ UseProtoNames: true, // Use original field names (snake_case) instead of camelCase EmitUnpopulated: false, // Emit unset fields, false here to omit Indent: "", // No indentation or additional formatting } - jsonBytes, err := marshaler.Marshal(p.message) + jsonBytes, err := marshaler.Marshal(msgObj.(gproto.Message)) if err != nil { return "", fmt.Errorf("failed to convert protobuf message into string after deserialization: %w", err) } diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 4b4d354e92..6ac312776c 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -623,7 +623,7 @@ import "address.proto"; message Person { string name = 1; - io.confluent.Address address = 2; + test.Address address = 2; int32 result = 3; } ` From f22d5fffdcc1884bc4b893be46ba714db0a61c45 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:11:03 -0800 Subject: [PATCH 10/24] Avoid harccoding the local KMS key/secret pair, and only apply for local testing --- pkg/serdes/avro_deserialization_provider.go | 9 +++++++-- pkg/serdes/avro_serialization_provider.go | 10 ++++++++-- pkg/serdes/json_deserialization_provider.go | 9 +++++++-- pkg/serdes/json_serialization_provider.go | 10 ++++++++-- pkg/serdes/protobuf_deserialization_provider.go | 9 +++++++-- pkg/serdes/protobuf_serialization_provider.go | 9 +++++++-- pkg/serdes/serdes.go | 17 ++++++++++------- pkg/serdes/serdes_test.go | 3 +++ 8 files changed, 57 insertions(+), 19 deletions(-) diff --git a/pkg/serdes/avro_deserialization_provider.go b/pkg/serdes/avro_deserialization_provider.go index a27f42b892..12137e5140 100644 --- a/pkg/serdes/avro_deserialization_provider.go +++ b/pkg/serdes/avro_deserialization_provider.go @@ -3,6 +3,7 @@ package serdes import ( "encoding/json" "fmt" + "os" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde" @@ -44,8 +45,12 @@ func (a *AvroDeserializationProvider) InitDeserializer(srClientUrl, srClusterId, } serdeConfig := avrov2.NewDeserializerConfig() - serdeConfig.RuleConfig = map[string]string{ - "secret": "avro_secret", + + // local KMS secret is only set and used during local testing with ruleSet + if localKmsSecretValue := os.Getenv(localKmsSecretMacro); srClientUrl == mockClientUrl && localKmsSecretValue != "" { + serdeConfig.RuleConfig = map[string]string{ + localKmsSecretKey: localKmsSecretValue, + } } var serdeType serde.Type diff --git a/pkg/serdes/avro_serialization_provider.go b/pkg/serdes/avro_serialization_provider.go index 6b977839c8..f21ef133f4 100644 --- a/pkg/serdes/avro_serialization_provider.go +++ b/pkg/serdes/avro_serialization_provider.go @@ -2,6 +2,7 @@ package serdes import ( "fmt" + "os" "github.com/linkedin/goavro/v2" @@ -58,9 +59,14 @@ func (a *AvroSerializationProvider) InitSerializer(srClientUrl, srClusterId, mod serdeConfig := avrov2.NewSerializerConfig() serdeConfig.AutoRegisterSchemas = false serdeConfig.UseLatestVersion = true - serdeConfig.RuleConfig = map[string]string{ - "secret": "avro_secret", + + // local KMS secret is only set and used during local testing with ruleSet + if localKmsSecretValue := os.Getenv(localKmsSecretMacro); srClientUrl == mockClientUrl && localKmsSecretValue != "" { + serdeConfig.RuleConfig = map[string]string{ + localKmsSecretKey: localKmsSecretValue, + } } + if schemaId > 0 { serdeConfig.UseSchemaID = schemaId serdeConfig.UseLatestVersion = false diff --git a/pkg/serdes/json_deserialization_provider.go b/pkg/serdes/json_deserialization_provider.go index 497d445c27..93e752b4c4 100644 --- a/pkg/serdes/json_deserialization_provider.go +++ b/pkg/serdes/json_deserialization_provider.go @@ -3,6 +3,7 @@ package serdes import ( "encoding/json" "fmt" + "os" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde" @@ -44,8 +45,12 @@ func (j *JsonDeserializationProvider) InitDeserializer(srClientUrl, srClusterId, serdeConfig := jsonschema.NewDeserializerConfig() serdeConfig.EnableValidation = true - serdeConfig.RuleConfig = map[string]string{ - "secret": "json_secret", + + // local KMS secret is only set and used during local testing with ruleSet + if localKmsSecretValue := os.Getenv(localKmsSecretMacro); srClientUrl == mockClientUrl && localKmsSecretValue != "" { + serdeConfig.RuleConfig = map[string]string{ + localKmsSecretKey: localKmsSecretValue, + } } var serdeType serde.Type diff --git a/pkg/serdes/json_serialization_provider.go b/pkg/serdes/json_serialization_provider.go index c5f75caa45..5e06b17130 100644 --- a/pkg/serdes/json_serialization_provider.go +++ b/pkg/serdes/json_serialization_provider.go @@ -3,6 +3,7 @@ package serdes import ( "encoding/json" "fmt" + "os" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/cel" @@ -55,9 +56,14 @@ func (j *JsonSerializationProvider) InitSerializer(srClientUrl, srClusterId, mod serdeConfig.AutoRegisterSchemas = false serdeConfig.UseLatestVersion = true serdeConfig.EnableValidation = true - serdeConfig.RuleConfig = map[string]string{ - "secret": "json_secret", + + // local KMS secret is only set and used during local testing with ruleSet + if localKmsSecretValue := os.Getenv(localKmsSecretMacro); srClientUrl == mockClientUrl && localKmsSecretValue != "" { + serdeConfig.RuleConfig = map[string]string{ + localKmsSecretKey: localKmsSecretValue, + } } + if schemaId > 0 { serdeConfig.UseSchemaID = schemaId serdeConfig.UseLatestVersion = false diff --git a/pkg/serdes/protobuf_deserialization_provider.go b/pkg/serdes/protobuf_deserialization_provider.go index b24c9b0a1e..2eb08a581d 100644 --- a/pkg/serdes/protobuf_deserialization_provider.go +++ b/pkg/serdes/protobuf_deserialization_provider.go @@ -2,6 +2,7 @@ package serdes import ( "fmt" + "os" "regexp" "google.golang.org/protobuf/encoding/protojson" @@ -48,8 +49,12 @@ func (p *ProtobufDeserializationProvider) InitDeserializer(srClientUrl, srCluste } serdeConfig := protobuf.NewDeserializerConfig() - serdeConfig.RuleConfig = map[string]string{ - "secret": "protobuf_secret", + + // local KMS secret is only set and used during local testing with ruleSet + if localKmsSecretValue := os.Getenv(localKmsSecretMacro); srClientUrl == mockClientUrl && localKmsSecretValue != "" { + serdeConfig.RuleConfig = map[string]string{ + localKmsSecretKey: localKmsSecretValue, + } } var serdeType serde.Type diff --git a/pkg/serdes/protobuf_serialization_provider.go b/pkg/serdes/protobuf_serialization_provider.go index b7ce89312b..213a859157 100644 --- a/pkg/serdes/protobuf_serialization_provider.go +++ b/pkg/serdes/protobuf_serialization_provider.go @@ -78,9 +78,14 @@ func (p *ProtobufSerializationProvider) InitSerializer(srClientUrl, srClusterId, serdeConfig := protobuf.NewSerializerConfig() serdeConfig.AutoRegisterSchemas = false serdeConfig.UseLatestVersion = true - serdeConfig.RuleConfig = map[string]string{ - "secret": "protobuf_secret", + + // local KMS secret is only set and used during local testing with ruleSet + if localKmsSecretValue := os.Getenv(localKmsSecretMacro); srClientUrl == mockClientUrl && localKmsSecretValue != "" { + serdeConfig.RuleConfig = map[string]string{ + localKmsSecretKey: localKmsSecretValue, + } } + if schemaId > 0 { serdeConfig.UseSchemaID = schemaId serdeConfig.UseLatestVersion = false diff --git a/pkg/serdes/serdes.go b/pkg/serdes/serdes.go index bfed0b3850..1c4f0e06f3 100644 --- a/pkg/serdes/serdes.go +++ b/pkg/serdes/serdes.go @@ -21,13 +21,16 @@ var KmsTypes = []string{ } const ( - avroSchemaName = "avro" - doubleSchemaName = "double" - integerSchemaName = "integer" - jsonSchemaName = "jsonschema" - protobufSchemaName = "protobuf" - stringSchemaName = "string" - mockClientUrl = "mock://" + avroSchemaName = "avro" + doubleSchemaName = "double" + integerSchemaName = "integer" + jsonSchemaName = "jsonschema" + protobufSchemaName = "protobuf" + stringSchemaName = "string" + mockClientUrl = "mock://" + localKmsSecretKey = "secret" + localKmsSecretValueDefault = "default_local_kms_secret_12345" + localKmsSecretMacro = "LOCAL_KMS_SECRET" ) var Formats = []string{ diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 6ac312776c..4d279227e4 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -208,6 +208,7 @@ func TestAvroSerdesValidWithRuleSet(t *testing.T) { dir, err := createTempDir() req.Nil(err) + t.Setenv(localKmsSecretMacro, localKmsSecretValueDefault) schemaString := `{"type":"record","name":"myRecord","fields":[{"name":"f1","type":"string","confluent:tags": ["PII"]}]}` schemaPath := filepath.Join(dir, "avro-schema.txt") @@ -490,6 +491,7 @@ func TestJsonSerdesValidWithRuleSet(t *testing.T) { dir, err := createTempDir() req.Nil(err) + t.Setenv(localKmsSecretMacro, localKmsSecretValueDefault) schemaString := `{"type":"object","properties":{"f1":{"type":"string","confluent:tags": ["PII"]}},"required":["f1"]}` schemaPath := filepath.Join(dir, "json-schema-ruleset.json") @@ -795,6 +797,7 @@ func TestProtobufSerdesValidWithRuleSet(t *testing.T) { dir, err := createTempDir() req.Nil(err) + t.Setenv(localKmsSecretMacro, localKmsSecretValueDefault) schemaString := ` syntax = "proto3"; From ec0b7ea925fdacd2780e293807b44494d6943d5c Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 27 Nov 2024 12:16:40 -0800 Subject: [PATCH 11/24] Experiment on CI variable --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 9189fc9738..6c538fa062 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -106,7 +106,7 @@ blocks: - $Env:GOCOVERDIR = "test/coverage" - New-Item $Env:GOCOVERDIR -ItemType Directory - go install gotest.tools/gotestsum@v1.8.2 - - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -v $(go list ./... | Select-String test -NotMatch) -ldflags "-buildmode=exe" + - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -race -v $(go list ./... | Select-String test -NotMatch) -ldflags "-buildmode=exe" - gotestsum --junitfile integration-test-report.xml -- -timeout 0 -v $(go list ./... | Select-String test) - go tool covdata textfmt -i $Env:GOCOVERDIR -o test/coverage.out epilogue: From 77e5bba989c856efee9561e902260cab5218e233 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 4 Dec 2024 11:08:42 -0800 Subject: [PATCH 12/24] Experinment 2 --- .semaphore/semaphore.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 6c538fa062..5daaf87681 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -16,6 +16,8 @@ blocks: - name: linux/amd64 dependencies: [] task: + when: + condition: false jobs: - name: "Test linux/amd64" commands: @@ -65,6 +67,8 @@ blocks: - name: darwin dependencies: [] task: + when: + condition: false agent: machine: type: s1-prod-macos-13-5-arm64 @@ -106,7 +110,7 @@ blocks: - $Env:GOCOVERDIR = "test/coverage" - New-Item $Env:GOCOVERDIR -ItemType Directory - go install gotest.tools/gotestsum@v1.8.2 - - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -race -v $(go list ./... | Select-String test -NotMatch) -ldflags "-buildmode=exe" + - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -v -parallel 1 $(go list ./... | Select-String test -NotMatch) -ldflags "-buildmode=exe" - gotestsum --junitfile integration-test-report.xml -- -timeout 0 -v $(go list ./... | Select-String test) - go tool covdata textfmt -i $Env:GOCOVERDIR -o test/coverage.out epilogue: From 5cbc67b36660754e5490cdd687293a950016c73b Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 4 Dec 2024 11:19:39 -0800 Subject: [PATCH 13/24] Experiment3 --- .semaphore/semaphore.yml | 77 ---------------------------------------- 1 file changed, 77 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5daaf87681..6a3cc58fa1 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -13,83 +13,6 @@ execution_time_limit: hours: 1 blocks: - - name: linux/amd64 - dependencies: [] - task: - when: - condition: false - jobs: - - name: "Test linux/amd64" - commands: - - checkout - - sem-version go $(cat .go-version) - - export PATH=$(go env GOPATH)/bin:$PATH - - make generate-packaging-patch - - diff -w -u <(git cat-file --filters HEAD:debian/patches/standard_build_layout.patch | awk "{if (NR>3) {print}}") <(cat debian/patches/standard_build_layout.patch | awk "{if (NR>3) {print}}") - - make lint - - make test - - make test-installer - - name: "Build linux/amd64 (GLIBC)" - commands: - - checkout - - docker build . --file docker/Dockerfile_build_linux_amd64 --tag test-build - - name: "Build linux/amd64 (Alpine)" - commands: - - checkout - - sem-version go $(cat .go-version) - - export PATH=$(go env GOPATH)/bin:$PATH - - sudo apt install musl-tools --yes - - CC=musl-gcc CGO_LDFLAGS="-static" CGO_ENABLED=1 GOEXPERIMENT=boringcrypto go build -tags musl -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" ./cmd/confluent - epilogue: - always: - commands: - - test-results publish . -N "linux/amd64" - - - name: linux/arm64 - dependencies: [] - task: - agent: - machine: - type: s1-prod-ubuntu20-04-arm64-1 - jobs: - - name: "Build linux/arm64 (GLIBC)" - commands: - - checkout - - docker build . --file docker/Dockerfile_build_linux_arm64 --tag test-build - - name: "Build linux/arm64 (Alpine)" - commands: - - checkout - - sem-version go $(cat .go-version) - - export PATH=$(go env GOPATH)/bin:$PATH - - sudo apt install musl-tools --yes - - CC=musl-gcc CGO_LDFLAGS="-static" CGO_ENABLED=1 GOEXPERIMENT=boringcrypto go build -tags musl -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" ./cmd/confluent - - - name: darwin - dependencies: [] - task: - when: - condition: false - agent: - machine: - type: s1-prod-macos-13-5-arm64 - jobs: - - name: "Build & Test darwin/arm64" - commands: - - checkout - - sem-version go $(cat .go-version) - - export PATH=$(go env GOPATH)/bin:$PATH - - make test - - name: "Build darwin/amd64" - commands: - - checkout - - sem-version go $(cat .go-version) - - export PATH=$(go env GOPATH)/bin:$PATH - - GOARCH=amd64 CGO_ENABLED=1 go build -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" ./cmd/confluent - epilogue: - always: - commands: - - test-results publish . -N "darwin/arm64" - - name: windows/amd64 dependencies: [] task: From 175f3141cd626fa55e4528d0f43f8f6024c80ad6 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:08:07 -0800 Subject: [PATCH 14/24] Revert "Experiment3" This reverts commit 5cbc67b36660754e5490cdd687293a950016c73b. --- .semaphore/semaphore.yml | 77 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 6a3cc58fa1..5daaf87681 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -13,6 +13,83 @@ execution_time_limit: hours: 1 blocks: + - name: linux/amd64 + dependencies: [] + task: + when: + condition: false + jobs: + - name: "Test linux/amd64" + commands: + - checkout + - sem-version go $(cat .go-version) + - export PATH=$(go env GOPATH)/bin:$PATH + - make generate-packaging-patch + - diff -w -u <(git cat-file --filters HEAD:debian/patches/standard_build_layout.patch | awk "{if (NR>3) {print}}") <(cat debian/patches/standard_build_layout.patch | awk "{if (NR>3) {print}}") + - make lint + - make test + - make test-installer + - name: "Build linux/amd64 (GLIBC)" + commands: + - checkout + - docker build . --file docker/Dockerfile_build_linux_amd64 --tag test-build + - name: "Build linux/amd64 (Alpine)" + commands: + - checkout + - sem-version go $(cat .go-version) + - export PATH=$(go env GOPATH)/bin:$PATH + - sudo apt install musl-tools --yes + - CC=musl-gcc CGO_LDFLAGS="-static" CGO_ENABLED=1 GOEXPERIMENT=boringcrypto go build -tags musl -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" ./cmd/confluent + epilogue: + always: + commands: + - test-results publish . -N "linux/amd64" + + - name: linux/arm64 + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu20-04-arm64-1 + jobs: + - name: "Build linux/arm64 (GLIBC)" + commands: + - checkout + - docker build . --file docker/Dockerfile_build_linux_arm64 --tag test-build + - name: "Build linux/arm64 (Alpine)" + commands: + - checkout + - sem-version go $(cat .go-version) + - export PATH=$(go env GOPATH)/bin:$PATH + - sudo apt install musl-tools --yes + - CC=musl-gcc CGO_LDFLAGS="-static" CGO_ENABLED=1 GOEXPERIMENT=boringcrypto go build -tags musl -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" ./cmd/confluent + + - name: darwin + dependencies: [] + task: + when: + condition: false + agent: + machine: + type: s1-prod-macos-13-5-arm64 + jobs: + - name: "Build & Test darwin/arm64" + commands: + - checkout + - sem-version go $(cat .go-version) + - export PATH=$(go env GOPATH)/bin:$PATH + - make test + - name: "Build darwin/amd64" + commands: + - checkout + - sem-version go $(cat .go-version) + - export PATH=$(go env GOPATH)/bin:$PATH + - GOARCH=amd64 CGO_ENABLED=1 go build -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" ./cmd/confluent + epilogue: + always: + commands: + - test-results publish . -N "darwin/arm64" + - name: windows/amd64 dependencies: [] task: From 987f5e486bbe2b36ac12697479699c2bcbef1829 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 4 Dec 2024 15:21:26 -0800 Subject: [PATCH 15/24] Experiment4 --- go.mod | 2 +- pkg/serdes/serdes_test.go | 159 ++++++++++++++------------------------ 2 files changed, 58 insertions(+), 103 deletions(-) diff --git a/go.mod b/go.mod index c0639c3a06..e59398d445 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,6 @@ require ( github.com/go-jose/go-jose/v3 v3.0.3 github.com/gobuffalo/flect v1.0.2 github.com/gogo/protobuf v1.3.2 - github.com/golang/mock v1.6.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 @@ -183,6 +182,7 @@ require ( github.com/gogo/googleapis v1.4.1 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/cel-go v0.20.1 // indirect diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 4d279227e4..066f60b790 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -3,6 +3,7 @@ package serdes import ( "bytes" "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -12,6 +13,24 @@ import ( "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" ) +var tempDir string + +func TestMain(m *testing.M) { + // Create the temporary directory used for placing schemas + tempDir, _ = createTempDir() + + // Run the required test(s) + code := m.Run() + + // Cleanup directory here will always run after all test(s) have been executed. + if err := os.RemoveAll(tempDir); err != nil { + fmt.Printf("failed to remove temporary directory: %s", err) + code = 1 + } + + os.Exit(code) +} + func TestGetSerializationProvider(t *testing.T) { req := require.New(t) valueFormats := []string{avroSchemaName, jsonSchemaName, protobufSchemaName} @@ -61,11 +80,8 @@ func TestStringSerdes(t *testing.T) { func TestAvroSerdesValid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := `{"type":"record","name":"myRecord","fields":[{"name":"f1","type":"int"}]}` - schemaPath := filepath.Join(dir, "avro-schema.txt") + schemaPath := filepath.Join(tempDir, "avro-schema.txt") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) // expectedBytes[0] is the magic byte, expectedBytes[1:5] is the schemaId in BigIndian @@ -74,7 +90,7 @@ func TestAvroSerdesValid(t *testing.T) { // Initialize the mock serializer and use latest schemaId serializationProvider, _ := GetSerializationProvider(avroSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -103,22 +119,18 @@ func TestAvroSerdesValid(t *testing.T) { req.Nil(err) req.Equal(expectedString, actualString) - req.NoError(os.RemoveAll(dir)) } func TestAvroSerdesInvalid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := `{"type":"record","name":"myRecord","fields":[{"name":"f1","type":"string"}]}` - schemaPath := filepath.Join(dir, "avro-schema-invalid.txt") + schemaPath := filepath.Join(tempDir, "avro-schema-invalid.txt") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) // Initialize the mock serializer and use latest schemaId serializationProvider, _ := GetSerializationProvider(avroSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -151,18 +163,13 @@ func TestAvroSerdesInvalid(t *testing.T) { invalidString := `{"f2": "abc"}` _, err = serializationProvider.Serialize("topic1", invalidString) req.Regexp(`cannot decode textual map: cannot determine codec: "f2"$`, err) - - req.NoError(os.RemoveAll(dir)) } func TestAvroSerdesNestedValid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := `{"type":"record","name":"myRecord","fields":[{"name":"f1","type":"string"},{"name":"nestedField","type":{"type":"record","name":"AnotherNestedRecord","fields":[{"name":"nested_field1","type":"float"},{"name":"nested_field2","type":"string"}]}}]}` - schemaPath := filepath.Join(dir, "avro-schema-nested.txt") + schemaPath := filepath.Join(tempDir, "avro-schema-nested.txt") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) // expectedBytes[0] is the magic byte, expectedBytes[1:5] is the schemaId in BigIndian @@ -171,7 +178,7 @@ func TestAvroSerdesNestedValid(t *testing.T) { // Initialize the mock serializer and use latest schemaId serializationProvider, _ := GetSerializationProvider(avroSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -200,25 +207,22 @@ func TestAvroSerdesNestedValid(t *testing.T) { req.Nil(err) req.Equal(expectedString, actualString) - req.NoError(os.RemoveAll(dir)) } func TestAvroSerdesValidWithRuleSet(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) t.Setenv(localKmsSecretMacro, localKmsSecretValueDefault) schemaString := `{"type":"record","name":"myRecord","fields":[{"name":"f1","type":"string","confluent:tags": ["PII"]}]}` - schemaPath := filepath.Join(dir, "avro-schema.txt") + schemaPath := filepath.Join(tempDir, "avro-schema.txt") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) expectedString := `{"f1":"this is a confidential message in AVRO schema"}` // Initialize the mock serializer and use latest schemaId serializationProvider, _ := GetSerializationProvider(avroSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -263,17 +267,13 @@ func TestAvroSerdesValidWithRuleSet(t *testing.T) { req.Nil(err) req.Equal(expectedString, actualString) - req.NoError(os.RemoveAll(dir)) } func TestJsonSerdesValid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := `{"type":"object","properties":{"f1":{"type":"string"}},"required":["f1"]}` - schemaPath := filepath.Join(dir, "json-schema.json") + schemaPath := filepath.Join(tempDir, "json-schema.json") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) expectedString := `{"f1":"asd"}` @@ -281,7 +281,7 @@ func TestJsonSerdesValid(t *testing.T) { // Initialize the mock serializer and use latest schemaId serializationProvider, _ := GetSerializationProvider(jsonSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -311,26 +311,21 @@ func TestJsonSerdesValid(t *testing.T) { actualString, err := deserializationProvider.Deserialize("topic1", expectedBytes) req.Nil(err) req.Equal(expectedString, actualString) - - req.NoError(os.RemoveAll(dir)) } func TestJsonSerdesReference(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - // Reference schema should be registered from user side prior to be used as reference // So subject and schema version will be known value at this time referenceContent := `[{"name":"RefSchema","subject":"topic2-value","version":1}]` referenceString := `{"type": "string"}` - referencePath := filepath.Join(dir, "json-reference.json") + referencePath := filepath.Join(tempDir, "json-reference.json") req.NoError(os.WriteFile(referencePath, []byte(referenceContent), 0644)) // Prepare main schema information schemaString := `{"type":"object","properties":{"f1":{"$ref":"RefSchema"}},"required":["f1"]}` - schemaPath := filepath.Join(dir, "json-schema.json") + schemaPath := filepath.Join(tempDir, "json-schema.json") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) // Read references from local reference schema file @@ -380,23 +375,18 @@ func TestJsonSerdesReference(t *testing.T) { actualString, err := deserializationProvider.Deserialize("topic1", expectedBytes) req.Nil(err) req.Equal(expectedString, actualString) - - req.NoError(os.RemoveAll(dir)) } func TestJsonSerdesInvalid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := `{"type":"object","properties":{"f1":{"type":"string"}},"required":["f1"]}` - schemaPath := filepath.Join(dir, "json-schema-invalid.json") + schemaPath := filepath.Join(tempDir, "json-schema-invalid.json") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) // Initialize the mock serializer and use latest schemaId serializationProvider, _ := GetSerializationProvider(jsonSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -432,18 +422,13 @@ func TestJsonSerdesInvalid(t *testing.T) { _, err = deserializationProvider.Deserialize("topic1", invalidBytes) req.Regexp("unknown magic byte$", err) - - req.NoError(os.RemoveAll(dir)) } func TestJsonSerdesNestedValid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := `{"type":"object","name":"myRecord","properties":{"f1":{"type":"string"},"nestedField":{"type":"object","name":"AnotherNestedRecord","properties":{"nested_field1":{"type":"number"},"nested_field2":{"type":"string"}}}},"required":["f1","nestedField"]}` - schemaPath := filepath.Join(dir, "json-schema-nested.txt") + schemaPath := filepath.Join(tempDir, "json-schema-nested.txt") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) // expectedBytes[0] is the magic byte, expectedBytes[1:5] is the schemaId in BigIndian @@ -454,7 +439,7 @@ func TestJsonSerdesNestedValid(t *testing.T) { // Initialize the mock serializer and use latest schemaId serializationProvider, _ := GetSerializationProvider(jsonSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -483,25 +468,21 @@ func TestJsonSerdesNestedValid(t *testing.T) { req.Nil(err) req.Equal(expectedString, actualString) - req.NoError(os.RemoveAll(dir)) } func TestJsonSerdesValidWithRuleSet(t *testing.T) { req := require.New(t) - - dir, err := createTempDir() - req.Nil(err) t.Setenv(localKmsSecretMacro, localKmsSecretValueDefault) schemaString := `{"type":"object","properties":{"f1":{"type":"string","confluent:tags": ["PII"]}},"required":["f1"]}` - schemaPath := filepath.Join(dir, "json-schema-ruleset.json") + schemaPath := filepath.Join(tempDir, "json-schema-ruleset.json") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) expectedString := `{"f1":"this is a confidential message in JSON schema"}` // Initialize the mock serializer and use latest schemaId serializationProvider, _ := GetSerializationProvider(jsonSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -546,15 +527,11 @@ func TestJsonSerdesValidWithRuleSet(t *testing.T) { req.Nil(err) req.Equal(expectedString, actualString) - req.NoError(os.RemoveAll(dir)) } func TestProtobufSerdesValid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := ` syntax = "proto3"; message Person { @@ -562,13 +539,13 @@ func TestProtobufSerdesValid(t *testing.T) { int32 page = 2; double result = 3; }` - schemaPath := filepath.Join(dir, "person-schema.proto") + schemaPath := filepath.Join(tempDir, "person-schema.proto") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) expectedString := `{"name":"abc","page":1,"result":2.5}` serializationProvider, _ := GetSerializationProvider(protobufSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -593,28 +570,23 @@ func TestProtobufSerdesValid(t *testing.T) { actualString, err := deserializationProvider.Deserialize("topic1", data) req.Nil(err) req.JSONEq(expectedString, actualString) - - req.NoError(os.RemoveAll(dir)) } func TestProtobufSerdesReference(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - referenceString := `syntax = "proto3"; package test; message Address { - string city = 1; + string city = 1; } ` // Reference schema should be registered from user side prior to be used as reference // So subject and schema version will be known value at this time - referencePath := filepath.Join(dir, "address.proto") + referencePath := filepath.Join(tempDir, "address.proto") req.NoError(os.WriteFile(referencePath, []byte(referenceString), 0644)) schemaString := `syntax = "proto3"; @@ -624,18 +596,18 @@ package test; import "address.proto"; message Person { - string name = 1; - test.Address address = 2; - int32 result = 3; + string name = 1; + test.Address address = 2; + int32 result = 3; } ` - schemaPath := filepath.Join(dir, "person.proto") + schemaPath := filepath.Join(tempDir, "person.proto") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) expectedString := `{"name":"abc","address":{"city":"LA"},"result":2}` serializationProvider, _ := GetSerializationProvider(protobufSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{"address.proto": referencePath}) req.Nil(err) @@ -674,16 +646,11 @@ message Person { str, err := deserializationProvider.Deserialize("topic1", data) req.Nil(err) req.JSONEq(str, expectedString) - - req.NoError(os.RemoveAll(dir)) } func TestProtobufSerdesInvalid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := ` syntax = "proto3"; message Person { @@ -691,11 +658,11 @@ func TestProtobufSerdesInvalid(t *testing.T) { int32 page = 2; int32 result = 3; }` - schemaPath := filepath.Join(dir, "person-invalid.proto") + schemaPath := filepath.Join(tempDir, "person-invalid.proto") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) serializationProvider, _ := GetSerializationProvider(protobufSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -732,16 +699,11 @@ func TestProtobufSerdesInvalid(t *testing.T) { _, err = deserializationProvider.Deserialize("topic1", invalidBytes) req.Regexp("^failed to deserialize payload:.*Subject Not Found$", err) - - req.NoError(os.RemoveAll(dir)) } func TestProtobufSerdesNestedValid(t *testing.T) { req := require.New(t) - dir, err := createTempDir() - req.Nil(err) - schemaString := ` syntax = "proto3"; message Input { @@ -757,13 +719,13 @@ func TestProtobufSerdesNestedValid(t *testing.T) { string street = 2; } }` - schemaPath := filepath.Join(dir, "person-nested.proto") + schemaPath := filepath.Join(tempDir, "person-nested.proto") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) expectedString := `{"name":"abc","id":2,"add":{"zip":"123","street":"def"},"phones":{"number":"234"}}` serializationProvider, _ := GetSerializationProvider(protobufSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -788,37 +750,32 @@ func TestProtobufSerdesNestedValid(t *testing.T) { actualString, err := deserializationProvider.Deserialize("topic1", data) req.Nil(err) req.JSONEq(expectedString, actualString) - - req.NoError(os.RemoveAll(dir)) } func TestProtobufSerdesValidWithRuleSet(t *testing.T) { req := require.New(t) - - dir, err := createTempDir() - req.Nil(err) t.Setenv(localKmsSecretMacro, localKmsSecretValueDefault) schemaString := ` syntax = "proto3"; - import "confluent/meta.proto"; + import "confluent/meta.proto"; package test; message Person { - string name = 1 [ + string name = 1 [ (confluent.field_meta).tags = "PII" - ]; - int32 page = 2; - double result = 3; + ]; + int32 page = 2; + double result = 3; }` - schemaPath := filepath.Join(dir, "person-schema-ruleset.proto") + schemaPath := filepath.Join(tempDir, "person-schema-ruleset.proto") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) expectedString := `{"name":"abc","page":1,"result":2.5}` serializationProvider, _ := GetSerializationProvider(protobufSchemaName) - err = serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) + err := serializationProvider.InitSerializer(mockClientUrl, "", "value", "", "", "", -1) req.Nil(err) err = serializationProvider.LoadSchema(schemaPath, map[string]string{}) req.Nil(err) @@ -862,8 +819,6 @@ func TestProtobufSerdesValidWithRuleSet(t *testing.T) { actualString, err := deserializationProvider.Deserialize("topic1", data) req.Nil(err) req.JSONEq(expectedString, actualString) - - req.NoError(os.RemoveAll(dir)) } func createTempDir() (string, error) { From d808c337936dbdbd722c90314988ca9ded2eabff Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 4 Dec 2024 15:26:14 -0800 Subject: [PATCH 16/24] Experiment5 --- .semaphore/semaphore.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5daaf87681..98c8074baa 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -16,8 +16,6 @@ blocks: - name: linux/amd64 dependencies: [] task: - when: - condition: false jobs: - name: "Test linux/amd64" commands: @@ -67,8 +65,6 @@ blocks: - name: darwin dependencies: [] task: - when: - condition: false agent: machine: type: s1-prod-macos-13-5-arm64 From e795022bed4a0f341469f04cbe0f4b6466d09b14 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 4 Dec 2024 15:27:30 -0800 Subject: [PATCH 17/24] Experiment6 --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 98c8074baa..9189fc9738 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -106,7 +106,7 @@ blocks: - $Env:GOCOVERDIR = "test/coverage" - New-Item $Env:GOCOVERDIR -ItemType Directory - go install gotest.tools/gotestsum@v1.8.2 - - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -v -parallel 1 $(go list ./... | Select-String test -NotMatch) -ldflags "-buildmode=exe" + - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -v $(go list ./... | Select-String test -NotMatch) -ldflags "-buildmode=exe" - gotestsum --junitfile integration-test-report.xml -- -timeout 0 -v $(go list ./... | Select-String test) - go tool covdata textfmt -i $Env:GOCOVERDIR -o test/coverage.out epilogue: From d45e7a279c7151022559fb9e24f559092df585fa Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:16:24 -0800 Subject: [PATCH 18/24] Update schema names with uniqueness and correct a typo --- pkg/serdes/avro_serialization_provider.go | 2 +- pkg/serdes/serdes_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/serdes/avro_serialization_provider.go b/pkg/serdes/avro_serialization_provider.go index f21ef133f4..0b93ef95b9 100644 --- a/pkg/serdes/avro_serialization_provider.go +++ b/pkg/serdes/avro_serialization_provider.go @@ -132,7 +132,7 @@ func (a *AvroSerializationProvider) Serialize(topic, message string) ([]byte, er // Passing the Go native object directly could cause issues during ruleSet execution v, ok := object.(map[string]interface{}) if !ok { - return nil, fmt.Errorf("failed to serialize message: unexpected message type asserion result") + return nil, fmt.Errorf("failed to serialize message: unexpected message type assertion result") } payload, err := a.ser.Serialize(topic, &v) if err != nil { diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 066f60b790..613210ff87 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -215,7 +215,7 @@ func TestAvroSerdesValidWithRuleSet(t *testing.T) { t.Setenv(localKmsSecretMacro, localKmsSecretValueDefault) schemaString := `{"type":"record","name":"myRecord","fields":[{"name":"f1","type":"string","confluent:tags": ["PII"]}]}` - schemaPath := filepath.Join(tempDir, "avro-schema.txt") + schemaPath := filepath.Join(tempDir, "avro-schema-ruleset.txt") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) expectedString := `{"f1":"this is a confidential message in AVRO schema"}` @@ -325,7 +325,7 @@ func TestJsonSerdesReference(t *testing.T) { // Prepare main schema information schemaString := `{"type":"object","properties":{"f1":{"$ref":"RefSchema"}},"required":["f1"]}` - schemaPath := filepath.Join(tempDir, "json-schema.json") + schemaPath := filepath.Join(tempDir, "json-schema-main.json") req.NoError(os.WriteFile(schemaPath, []byte(schemaString), 0644)) // Read references from local reference schema file From 973c6b8a44fb1e575f8b4866390c3a4567e60497 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Sun, 8 Dec 2024 23:59:57 -0800 Subject: [PATCH 19/24] Use go embed package to save all required built-in schemas during compile time for Protobuf --- pkg/serdes/protobuf_serialization_provider.go | 84 +++++++++++-------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/pkg/serdes/protobuf_serialization_provider.go b/pkg/serdes/protobuf_serialization_provider.go index 213a859157..0fbf41cb00 100644 --- a/pkg/serdes/protobuf_serialization_provider.go +++ b/pkg/serdes/protobuf_serialization_provider.go @@ -2,13 +2,14 @@ package serdes import ( "context" + "embed" "fmt" + "io/fs" "os" "path/filepath" "strings" "github.com/bufbuild/protocompile" - "github.com/otiai10/copy" "google.golang.org/protobuf/encoding/protojson" gproto "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/dynamicpb" @@ -26,18 +27,15 @@ import ( "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/protobuf" "github.com/confluentinc/cli/v4/pkg/errors" - "github.com/confluentinc/cli/v4/pkg/log" ) -const ( - confluentBuiltInSchemaFolder = "confluent" - googleBuiltInSchemaFolder = "google" -) - -var builtInSchemaFoldersToCopy = []string{ - confluentBuiltInSchemaFolder, - googleBuiltInSchemaFolder, -} +// Embed all .proto built-in schema files in both folders +// +//go:embed google/protobuf/*.proto +//go:embed google/type/*.proto +//go:embed confluent/*.proto +//go:embed confluent/type/*.proto +var builtInSchemas embed.FS type ProtobufSerializationProvider struct { ser *protobuf.Serializer @@ -149,32 +147,10 @@ func parseMessage(schemaPath string, referencePathMap map[string]string) (gproto ImportPaths: importPaths, } - currDir, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("Error getting current working directory: %v\n", err) - } - log.CliLogger.Debugf("Current working directory is: %s\n", currDir) - - // Copy all the built-in proto schemas needed for CSFLE to where the main schema is stored - // Note: folder path should be set correctly based on current working directory - // Expected working directory in CLI test is: /Users/github.com/confluentinc/cli/pkg/serdes - // Expected working directory in CLI shell execution is: /Users/github.com/confluentinc/cli - for _, folder := range builtInSchemaFoldersToCopy { - dst := importPath + "/" + folder - - // Skip copying the built-in schema folders if they are present already in the temp folder - if _, err = os.Stat(dst); err == nil { - log.CliLogger.Debugf("Built-in schema folder already exists %s, skipping copy again:\n", dst) - continue - } - - // Locate the source of built-in schema folders - if strings.HasSuffix(currDir, "confluentinc/cli") { - folder = "pkg/serdes/" + folder - } - if err = copy.Copy(folder, dst); err != nil { - return nil, fmt.Errorf("Error copying built-in schemas folder %s: %w\n", folder, err) - } + // Extract and copy embedded builtin proto files schemas needed for CSFLE to a temp destination directory + if err := copyBuiltInProtoFiles(importPaths[0]); err != nil { + fmt.Printf("Error copying proto files to the temp folder: %v\n", err) + return nil, err } // Create the compiler @@ -213,3 +189,37 @@ func parseMessage(schemaPath string, referencePathMap map[string]string) (gproto func (p *ProtobufSerializationProvider) GetSchemaRegistryClient() schemaregistry.Client { return p.ser.Client } + +func copyBuiltInProtoFiles(destinationDir string) error { + return fs.WalkDir(builtInSchemas, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("error accessing path %s: %w", path, err) + } + + // Skip directories + if d.IsDir() { + return nil + } + + // Read file content from the embedded filesystem + content, err := builtInSchemas.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read embedded file %s: %w", path, err) + } + + // Determine the destination path + destPath := filepath.Join(destinationDir, path) + + // Ensure the destination directory exists + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return fmt.Errorf("failed to create directory for %s: %w", destPath, err) + } + + // Write the built-in schema files to the destination + if err := os.WriteFile(destPath, content, 0644); err != nil { + return fmt.Errorf("failed to write file %s: %w", destPath, err) + } + + return nil + }) +} From e02c415136b985df3eda9b27975cea0ac89bf1f1 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:32:04 -0800 Subject: [PATCH 20/24] Add retry mechanism --- pkg/serdes/protobuf_serialization_provider.go | 3 +-- pkg/serdes/serdes_test.go | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/serdes/protobuf_serialization_provider.go b/pkg/serdes/protobuf_serialization_provider.go index 0fbf41cb00..35a3964b5b 100644 --- a/pkg/serdes/protobuf_serialization_provider.go +++ b/pkg/serdes/protobuf_serialization_provider.go @@ -149,8 +149,7 @@ func parseMessage(schemaPath string, referencePathMap map[string]string) (gproto // Extract and copy embedded builtin proto files schemas needed for CSFLE to a temp destination directory if err := copyBuiltInProtoFiles(importPaths[0]); err != nil { - fmt.Printf("Error copying proto files to the temp folder: %v\n", err) - return nil, err + return nil, fmt.Errorf("failed to copy built-in proto files to the temp folder: %w", err) } // Create the compiler diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 613210ff87..12c59aff35 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" @@ -22,9 +23,18 @@ func TestMain(m *testing.M) { // Run the required test(s) code := m.Run() - // Cleanup directory here will always run after all test(s) have been executed. - if err := os.RemoveAll(tempDir); err != nil { - fmt.Printf("failed to remove temporary directory: %s", err) + // Sleep for additional 200ms to avoid certain OS still holding the files lock + time.Sleep(200 * time.Millisecond) + + // Cleanup directory (with retry) here will always run after all test(s) have been executed. + for i := 0; i < 3; i++ { + err := os.RemoveAll(tempDir) + if err != nil { + fmt.Printf("successfully removed temporary schema directory: %s", err) + code = 0 + break + } + fmt.Printf("failed to remove temporary schema directory: %s", err) code = 1 } From aa82fe24984b7d4477d53dce6cd40a4e950a9362 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:47:36 -0800 Subject: [PATCH 21/24] Fix bug --- pkg/serdes/serdes_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 12c59aff35..6d037936a7 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -29,7 +29,7 @@ func TestMain(m *testing.M) { // Cleanup directory (with retry) here will always run after all test(s) have been executed. for i := 0; i < 3; i++ { err := os.RemoveAll(tempDir) - if err != nil { + if err == nil { fmt.Printf("successfully removed temporary schema directory: %s", err) code = 0 break From 96c164df5ae3c44189fd5954ae77d00e5350ebee Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Mon, 9 Dec 2024 10:40:07 -0800 Subject: [PATCH 22/24] Try with explicit files handle open/close before removing the directory --- pkg/serdes/serdes_test.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 6d037936a7..52152e60c8 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -23,19 +23,40 @@ func TestMain(m *testing.M) { // Run the required test(s) code := m.Run() - // Sleep for additional 200ms to avoid certain OS still holding the files lock - time.Sleep(200 * time.Millisecond) + // Walk through the directory + err := filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Skip directories + if info.IsDir() { + return nil + } + // Try opening and closing the file to release any handle + file, err := os.Open(path) + if err != nil { + fmt.Printf("Failed to open %s: %v", path, err) + return nil // Skip if file cannot be opened + } + defer file.Close() + return nil + }) + + if err != nil { + fmt.Printf("failed to release all file handles for directory: %s", err) + os.Exit(1) + } // Cleanup directory (with retry) here will always run after all test(s) have been executed. for i := 0; i < 3; i++ { err := os.RemoveAll(tempDir) if err == nil { fmt.Printf("successfully removed temporary schema directory: %s", err) - code = 0 break } fmt.Printf("failed to remove temporary schema directory: %s", err) code = 1 + time.Sleep(500 * time.Millisecond) } os.Exit(code) From 92944c45945ccb7265e5d2abdf977f539d07d010 Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:59:06 -0800 Subject: [PATCH 23/24] Additional sleep before remove all temp directories --- pkg/serdes/serdes_test.go | 38 +++++--------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/pkg/serdes/serdes_test.go b/pkg/serdes/serdes_test.go index 52152e60c8..d5fc8d98ad 100644 --- a/pkg/serdes/serdes_test.go +++ b/pkg/serdes/serdes_test.go @@ -22,41 +22,13 @@ func TestMain(m *testing.M) { // Run the required test(s) code := m.Run() + // Sleep for 2s as the Windows usually holds the file(s) access longer + time.Sleep(2 * time.Second) - // Walk through the directory - err := filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - // Skip directories - if info.IsDir() { - return nil - } - // Try opening and closing the file to release any handle - file, err := os.Open(path) - if err != nil { - fmt.Printf("Failed to open %s: %v", path, err) - return nil // Skip if file cannot be opened - } - defer file.Close() - return nil - }) - - if err != nil { - fmt.Printf("failed to release all file handles for directory: %s", err) - os.Exit(1) - } - - // Cleanup directory (with retry) here will always run after all test(s) have been executed. - for i := 0; i < 3; i++ { - err := os.RemoveAll(tempDir) - if err == nil { - fmt.Printf("successfully removed temporary schema directory: %s", err) - break - } - fmt.Printf("failed to remove temporary schema directory: %s", err) + // Cleanup directory here will always run after all test(s) have been executed. + if err := os.RemoveAll(tempDir); err != nil { + fmt.Printf("failed to remove temporary directory: %s", err) code = 1 - time.Sleep(500 * time.Millisecond) } os.Exit(code) From 47c80f1f5b26cd48e2b46ca98a659c7f49ddb74f Mon Sep 17 00:00:00 2001 From: Channing Dong <165202380+channingdong@users.noreply.github.com> Date: Fri, 13 Dec 2024 21:38:00 -0800 Subject: [PATCH 24/24] Update the go.mod files to remove redundant dependency --- go.mod | 1 - go.sum | 4 ---- 2 files changed, 5 deletions(-) diff --git a/go.mod b/go.mod index e59398d445..989aa31641 100644 --- a/go.mod +++ b/go.mod @@ -89,7 +89,6 @@ require ( github.com/mattn/go-runewidth v0.0.15 github.com/olekukonko/tablewriter v0.0.5 github.com/opencontainers/image-spec v1.1.0 - github.com/otiai10/copy v1.14.0 github.com/panta/machineid v1.0.2 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c diff --git a/go.sum b/go.sum index 64a7f7e58c..d118f308fa 100644 --- a/go.sum +++ b/go.sum @@ -736,10 +736,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= -github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= -github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= -github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panta/machineid v1.0.2 h1:LVYeEq1hZ+FwcM+/H6eB8KfXM2R5b2h1SWdnWwZ0OQw= github.com/panta/machineid v1.0.2/go.mod h1:AROj156fsca3R3rNw3q9h8xFkos25W9P0ZG9gu+3Uf0= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=