From 81445a7b0707b5ecd9378d0a8dc0b22431f085a4 Mon Sep 17 00:00:00 2001 From: mao3267 Date: Tue, 19 Nov 2024 11:34:40 +0800 Subject: [PATCH] fix: support strict subset match with ambiguity Signed-off-by: mao3267 --- .run/single-binary.run.xml | 4 +- flyteadmin/go.mod | 2 + flyteadmin/go.sum | 3 ++ flytepropeller/go.mod | 2 + flytepropeller/go.sum | 4 ++ .../pkg/compiler/validators/typing.go | 46 ++++++++++--------- go.mod | 2 + go.sum | 3 ++ 8 files changed, 41 insertions(+), 25 deletions(-) diff --git a/.run/single-binary.run.xml b/.run/single-binary.run.xml index 77de6fe10a..4b5de3705a 100644 --- a/.run/single-binary.run.xml +++ b/.run/single-binary.run.xml @@ -8,8 +8,6 @@ - - + \ No newline at end of file diff --git a/flyteadmin/go.mod b/flyteadmin/go.mod index b8507cc0e5..82e2189f34 100644 --- a/flyteadmin/go.mod +++ b/flyteadmin/go.mod @@ -167,6 +167,7 @@ require ( github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/shamaton/msgpack/v2 v2.2.2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -180,6 +181,7 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect diff --git a/flyteadmin/go.sum b/flyteadmin/go.sum index a7d12f01fd..afd775c3ba 100644 --- a/flyteadmin/go.sum +++ b/flyteadmin/go.sum @@ -1159,6 +1159,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/santhosh-tekuri/jsonschema/v2 v2.1.0/go.mod h1:yzJzKUGV4RbWqWIBBP4wSOBqavX5saE02yirLS0OTyg= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -1320,6 +1321,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.elastic.co/apm v1.8.0/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= go.elastic.co/apm/module/apmhttp v1.8.0/go.mod h1:9LPFlEON51/lRbnWDfqAWErihIiAFDUMfMV27YjoWQ8= go.elastic.co/apm/module/apmot v1.8.0/go.mod h1:Q5Xzabte8G/fkvDjr1jlDuOSUt9hkVWNZEHh6ZNaTjI= diff --git a/flytepropeller/go.mod b/flytepropeller/go.mod index 3e83107905..c129312e44 100644 --- a/flytepropeller/go.mod +++ b/flytepropeller/go.mod @@ -22,12 +22,14 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 + github.com/santhosh-tekuri/jsonschema v1.2.4 github.com/shamaton/msgpack/v2 v2.2.2 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/wI2L/jsondiff v0.6.0 + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 diff --git a/flytepropeller/go.sum b/flytepropeller/go.sum index 3ab11fff27..63c498dc77 100644 --- a/flytepropeller/go.sum +++ b/flytepropeller/go.sum @@ -373,6 +373,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/shamaton/msgpack/v2 v2.2.2 h1:GOIg0c9LV04VwzOOqZSrmsv/JzjNOOMxnS/HvOHGdgs= github.com/shamaton/msgpack/v2 v2.2.2/go.mod h1:6khjYnkx73f7VQU7wjcFS9DFjs+59naVWJv1TB7qdOI= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -427,6 +429,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/flytepropeller/pkg/compiler/validators/typing.go b/flytepropeller/pkg/compiler/validators/typing.go index 988308b15b..7f7aadf699 100644 --- a/flytepropeller/pkg/compiler/validators/typing.go +++ b/flytepropeller/pkg/compiler/validators/typing.go @@ -1,11 +1,14 @@ package validators import ( + "bytes" "encoding/json" "strings" structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/santhosh-tekuri/jsonschema" "github.com/wI2L/jsondiff" + jscmp "gitlab.com/yvesf/json-schema-compare" flyte "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) @@ -26,32 +29,31 @@ func isSuperTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { srcSchemaBytes, _ := json.Marshal(sourceMetaData.GetFields()) tgtSchemaBytes, _ := json.Marshal(targetMetaData.GetFields()) - patch, _ := jsondiff.CompareJSON(srcSchemaBytes, tgtSchemaBytes) - for _, p := range patch { - if p.Type == jsondiff.OperationReplace { - if strings.Contains(p.Path, "title") { - // Ignore title changes to support inheritance - continue - } else if strings.Contains(p.Path, "max") { - // If the value of maxItems, maxProperties, maxLength is changed, the target schema should have a smaller range. - if p.OldValue.(int) == -1 || p.OldValue.(int) < p.Value.(int) { - return false - } - continue - } else if strings.Contains(p.Path, "min") { - // If the value of minItems, minProperties, minLength is changed, the target schema should have a larger range. - if p.OldValue.(int) == -1 || p.OldValue.(int) > p.Value.(int) { - return false - } - continue - } - return false - } else if p.Type == jsondiff.OperationAdd { + compiler := jsonschema.NewCompiler() + + err := compiler.AddResource("src", bytes.NewReader(srcSchemaBytes)) + if err != nil { + return false + } + err = compiler.AddResource("tgt", bytes.NewReader(tgtSchemaBytes)) + if err != nil { + return false + } + + srcSchema, _ := compiler.Compile("src") + tgtSchema, _ := compiler.Compile("tgt") + + // Compare the two schemas + errs := jscmp.Compare(tgtSchema, srcSchema) + + // Ignore not implemented errors in json-schema-compare (additionalProperties) + for _, err := range errs { + if !strings.Contains(err.Error(), "not implemented") { return false } } - return true + return true } func isSameTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { diff --git a/go.mod b/go.mod index aaeda42cf8..48cf680956 100644 --- a/go.mod +++ b/go.mod @@ -164,6 +164,7 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/robfig/cron/v3 v3.0.0 // indirect github.com/samber/lo v1.47.0 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.10.0+incompatible // indirect github.com/shamaton/msgpack/v2 v2.2.2 // indirect @@ -180,6 +181,7 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/wI2L/jsondiff v0.6.0 // indirect + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/go.sum b/go.sum index fb4d1bc05b..d61dd5d2d9 100644 --- a/go.sum +++ b/go.sum @@ -1198,6 +1198,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/santhosh-tekuri/jsonschema/v2 v2.1.0/go.mod h1:yzJzKUGV4RbWqWIBBP4wSOBqavX5saE02yirLS0OTyg= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -1359,6 +1360,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.elastic.co/apm v1.8.0/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= go.elastic.co/apm/module/apmhttp v1.8.0/go.mod h1:9LPFlEON51/lRbnWDfqAWErihIiAFDUMfMV27YjoWQ8= go.elastic.co/apm/module/apmot v1.8.0/go.mod h1:Q5Xzabte8G/fkvDjr1jlDuOSUt9hkVWNZEHh6ZNaTjI=