From db0296f402e7b6ac17999174126121520293ac7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lovro=20Ma=C5=BEgon?= Date: Wed, 28 Feb 2024 19:34:52 +0100 Subject: [PATCH 1/2] display raw data in processor examples as string --- go.mod | 2 +- go.sum | 4 +-- pkg/plugin/processor/builtin/examples_test.go | 29 +++++++++++++++---- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index f48c816e0..5c6ccfd1a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 github.com/NYTimes/gziphandler v1.1.1 github.com/bufbuild/buf v1.29.0 - github.com/conduitio/conduit-commons v0.0.0-20240216175021-58737e44eb62 + github.com/conduitio/conduit-commons v0.0.0-20240228183204-44c456787a25 github.com/conduitio/conduit-connector-file v0.6.0 github.com/conduitio/conduit-connector-generator v0.5.0 github.com/conduitio/conduit-connector-kafka v0.7.1 diff --git a/go.sum b/go.sum index 39c3fde22..cb26914fb 100644 --- a/go.sum +++ b/go.sum @@ -1077,8 +1077,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= -github.com/conduitio/conduit-commons v0.0.0-20240216175021-58737e44eb62 h1:h1AcneviaadJqw1KAUfzIDBVrODma4rneg13q726KQU= -github.com/conduitio/conduit-commons v0.0.0-20240216175021-58737e44eb62/go.mod h1:Hhr5nik71/Wz3yrLjaKSZ2HRQp1wNSOPY2JGGKz7fsw= +github.com/conduitio/conduit-commons v0.0.0-20240228183204-44c456787a25 h1:AEifS3EV44zg7w3D4ynf9y1iDJTHASTnQdmrpfgb4Co= +github.com/conduitio/conduit-commons v0.0.0-20240228183204-44c456787a25/go.mod h1:Hhr5nik71/Wz3yrLjaKSZ2HRQp1wNSOPY2JGGKz7fsw= github.com/conduitio/conduit-connector-file v0.6.0 h1:8tsGeGhKvFwYQZztOOL5/tmOhVShsfo9lQ3b/0fX8kQ= github.com/conduitio/conduit-connector-file v0.6.0/go.mod h1:ju7PiB4kTJgqng4KVXDt/Gvw/53kFwSzi5Ez9EDXxNI= github.com/conduitio/conduit-connector-generator v0.5.0 h1:zpXHif89DCJ13nftKLv31uI2AJGicpY5H1V7SwldRNo= diff --git a/pkg/plugin/processor/builtin/examples_test.go b/pkg/plugin/processor/builtin/examples_test.go index e77bdfa12..870d8b127 100644 --- a/pkg/plugin/processor/builtin/examples_test.go +++ b/pkg/plugin/processor/builtin/examples_test.go @@ -17,14 +17,15 @@ package builtin import ( + "bytes" "context" - "encoding/json" "fmt" "log" "github.com/conduitio/conduit-commons/opencdc" sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal/diff" + "github.com/goccy/go-json" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" ) @@ -86,15 +87,15 @@ func RunExample(p sdk.Processor, e example) { switch rec := got[0].(type) { case sdk.SingleRecord: - // produce JSON diff - havePrettyJSON, err := json.MarshalIndent(e.Have, "", " ") + // Serialize records to pretty JSON for comparison. + havePrettyJSON, err := recordToPrettyJSON(e.Have) if err != nil { - log.Fatalf("failed to marshal test record to JSON: %v", err) + log.Fatalf("failed to marshal test record to pretty JSON: %v", err) } - gotPrettyJSON, err := json.MarshalIndent(rec, "", " ") + gotPrettyJSON, err := recordToPrettyJSON(opencdc.Record(rec)) if err != nil { - log.Fatalf("failed to marshal processed record to JSON: %v", err) + log.Fatalf("failed to marshal processed record to pretty JSON: %v", err) } edits := diff.Strings(string(havePrettyJSON), string(gotPrettyJSON)) @@ -113,3 +114,19 @@ func RunExample(p sdk.Processor, e example) { // append example to processor pi.Examples = append(pi.Examples, e) } + +func recordToPrettyJSON(r opencdc.Record) ([]byte, error) { + serializer := opencdc.JSONSerializer{RawDataAsString: true} + + // Serialize records to pretty JSON for comparison. + haveJSON, err := serializer.Serialize(r) + if err != nil { + return nil, fmt.Errorf("failed to marshal test record to JSON: %w", err) + } + var buf bytes.Buffer + err = json.Indent(&buf, haveJSON, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to indent test record JSON: %w", err) + } + return buf.Bytes(), nil +} From 4fa0b499175a1b7d1a70f058678b209b7c46dcb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lovro=20Ma=C5=BEgon?= Date: Wed, 28 Feb 2024 20:03:28 +0100 Subject: [PATCH 2/2] ensure examples are formatted the same way in processors.json --- go.mod | 2 +- go.sum | 4 ++-- .../processor/builtin/examples_exporter_test.go | 11 ++++++----- pkg/plugin/processor/builtin/processors.json | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 5c6ccfd1a..7c48ea919 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 github.com/NYTimes/gziphandler v1.1.1 github.com/bufbuild/buf v1.29.0 - github.com/conduitio/conduit-commons v0.0.0-20240228183204-44c456787a25 + github.com/conduitio/conduit-commons v0.0.0-20240228185905-49ce3f31fbcd github.com/conduitio/conduit-connector-file v0.6.0 github.com/conduitio/conduit-connector-generator v0.5.0 github.com/conduitio/conduit-connector-kafka v0.7.1 diff --git a/go.sum b/go.sum index cb26914fb..64306ea9d 100644 --- a/go.sum +++ b/go.sum @@ -1077,8 +1077,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= -github.com/conduitio/conduit-commons v0.0.0-20240228183204-44c456787a25 h1:AEifS3EV44zg7w3D4ynf9y1iDJTHASTnQdmrpfgb4Co= -github.com/conduitio/conduit-commons v0.0.0-20240228183204-44c456787a25/go.mod h1:Hhr5nik71/Wz3yrLjaKSZ2HRQp1wNSOPY2JGGKz7fsw= +github.com/conduitio/conduit-commons v0.0.0-20240228185905-49ce3f31fbcd h1:BuKtwSyIMzhGWkU7diay5vimV4qJfKQZxMhwNjPaygc= +github.com/conduitio/conduit-commons v0.0.0-20240228185905-49ce3f31fbcd/go.mod h1:Hhr5nik71/Wz3yrLjaKSZ2HRQp1wNSOPY2JGGKz7fsw= github.com/conduitio/conduit-connector-file v0.6.0 h1:8tsGeGhKvFwYQZztOOL5/tmOhVShsfo9lQ3b/0fX8kQ= github.com/conduitio/conduit-connector-file v0.6.0/go.mod h1:ju7PiB4kTJgqng4KVXDt/Gvw/53kFwSzi5Ez9EDXxNI= github.com/conduitio/conduit-connector-generator v0.5.0 h1:zpXHif89DCJ13nftKLv31uI2AJGicpY5H1V7SwldRNo= diff --git a/pkg/plugin/processor/builtin/examples_exporter_test.go b/pkg/plugin/processor/builtin/examples_exporter_test.go index 9307e6957..8afe61a1d 100644 --- a/pkg/plugin/processor/builtin/examples_exporter_test.go +++ b/pkg/plugin/processor/builtin/examples_exporter_test.go @@ -17,6 +17,7 @@ package builtin import ( + "context" "io" "log" "os" @@ -24,6 +25,7 @@ import ( "strings" "testing" + "github.com/conduitio/conduit-commons/opencdc" "github.com/goccy/go-json" ) @@ -48,12 +50,11 @@ func TestMain(m *testing.M) { func exportProcessors(output io.Writer) { sorted := sortProcessors(processors) - bytes, err := json.MarshalIndent(sorted, "", " ") - if err != nil { - log.Fatalf("failed to marshal processors to JSON: %v", err) - } + ctx := opencdc.WithJSONMarshalOptions(context.Background(), &opencdc.JSONMarshalOptions{RawDataAsString: true}) + enc := json.NewEncoder(output) + enc.SetIndent("", " ") - _, err = output.Write(bytes) + err := enc.EncodeContext(ctx, sorted) if err != nil { log.Fatalf("failed to write processors to output: %v", err) } diff --git a/pkg/plugin/processor/builtin/processors.json b/pkg/plugin/processor/builtin/processors.json index 0637a088a..fe51488c7 100644 --- a/pkg/plugin/processor/builtin/processors.json +++ b/pkg/plugin/processor/builtin/processors.json @@ -1 +1 @@ -[] \ No newline at end of file +[]