From 48245860d6535c49c0ded5289ae2fed62ae6971b Mon Sep 17 00:00:00 2001 From: Miguel Mendoza Date: Mon, 6 May 2024 19:09:14 -0700 Subject: [PATCH] feat: implement remove_oas_extensions template command Also, re-implement the cycle detection, and $refs resolution. The logic was too convoluted, because a single function tried to do too many things. --- cmd/apigee-go-gen/main.go | 13 +- examples/templates/oas3/apiproxy.yaml | 1 + go.mod | 2 +- go.sum | 4 +- pkg/apigee/v1/apiproxymodel.go | 2 +- pkg/apigee/v1/model.go | 12 +- pkg/apigee/v1/sharedflowbundlemodel.go | 2 +- pkg/apigee/v1/unknown.go | 8 - pkg/common/resources/helper_functions.txt | 9 + pkg/render/helper_funcs.go | 37 ++ pkg/render/model.go | 12 +- pkg/render/render.go | 1 + pkg/utils/extension_remover.go | 137 +++++++ pkg/utils/io.go | 16 - pkg/utils/openapi2.go | 201 ++++------ pkg/utils/openapi2_test.go | 8 +- pkg/utils/openapi3.go | 104 ----- pkg/utils/resolver.go | 2 +- pkg/utils/resolver_test.go | 2 +- .../oas2-to-oas3/petstore-cycle/exp-oas3.json | 28 +- pkg/utils/utils.go | 26 ++ pkg/utils/yaml.go | 369 +----------------- pkg/utils/yaml_cycle_detector.go | 157 ++++++++ pkg/utils/yaml_nodes.go | 69 ++++ pkg/utils/yaml_ref_resolver.go | 258 ++++++++++++ 25 files changed, 823 insertions(+), 657 deletions(-) create mode 100644 pkg/utils/extension_remover.go create mode 100644 pkg/utils/yaml_cycle_detector.go create mode 100644 pkg/utils/yaml_nodes.go create mode 100644 pkg/utils/yaml_ref_resolver.go diff --git a/cmd/apigee-go-gen/main.go b/cmd/apigee-go-gen/main.go index 0f47cf8..0e67715 100644 --- a/cmd/apigee-go-gen/main.go +++ b/cmd/apigee-go-gen/main.go @@ -16,7 +16,7 @@ package main import ( "fmt" - v1 "github.com/apigee/apigee-go-gen/pkg/apigee/v1" + "github.com/apigee/apigee-go-gen/pkg/utils" "github.com/go-errors/errors" "os" ) @@ -25,11 +25,12 @@ func main() { err := RootCmd.Execute() if err != nil { - var validationErrors v1.ValidationErrors - isValidationErrors := errors.As(err, &validationErrors) - if isValidationErrors { - for i := 0; i < len(validationErrors.Errors) && i < 10; i++ { - _, _ = fmt.Fprintf(os.Stderr, "Error: %s\n", validationErrors.Errors[i].Error()) + var multiErrors utils.MultiError + isMultiErrors := errors.As(err, &multiErrors) + + if isMultiErrors { + for i := 0; i < len(multiErrors.Errors) && i < 10; i++ { + _, _ = fmt.Fprintf(os.Stderr, "Error: %s\n", multiErrors.Errors[i].Error()) } } else { _, _ = fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) diff --git a/examples/templates/oas3/apiproxy.yaml b/examples/templates/oas3/apiproxy.yaml index 1d89492..2d4b2c8 100755 --- a/examples/templates/oas3/apiproxy.yaml +++ b/examples/templates/oas3/apiproxy.yaml @@ -65,6 +65,7 @@ Resources: - Resource: Type: oas #{{ os_writefile "./spec.yaml" $.Values.spec_string }} + #{{ remove_oas_extensions "./spec.yaml" }} Path: ./spec.yaml - Resource: Type: properties diff --git a/go.mod b/go.mod index 62e5ce0..4b719d6 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/getkin/kin-openapi v0.124.0 github.com/go-errors/errors v1.5.1 github.com/gosimple/slug v1.14.0 - github.com/pb33f/libopenapi v0.16.2 + github.com/pb33f/libopenapi v0.16.5 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 github.com/vektah/gqlparser/v2 v2.5.11 diff --git a/go.sum b/go.sum index 4c69b06..df2cdd9 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/pb33f/libopenapi v0.16.2 h1:vAZisFmtMLScN68Hu26/T7ClEt5tWg+2HZd1Omw3abA= -github.com/pb33f/libopenapi v0.16.2/go.mod h1:PEXNwvtT4KNdjrwudp5OYnD1ryqK6uJ68aMNyWvoMuc= +github.com/pb33f/libopenapi v0.16.5 h1:jqb/N5nc2zuSUSWDgCXi2vKGxMQfTsWHgtmPWSKQGqc= +github.com/pb33f/libopenapi v0.16.5/go.mod h1:PEXNwvtT4KNdjrwudp5OYnD1ryqK6uJ68aMNyWvoMuc= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/apigee/v1/apiproxymodel.go b/pkg/apigee/v1/apiproxymodel.go index 5d871e5..2e12ec8 100644 --- a/pkg/apigee/v1/apiproxymodel.go +++ b/pkg/apigee/v1/apiproxymodel.go @@ -64,7 +64,7 @@ func (a *APIProxyModel) Validate() error { return nil } - err := ValidationErrors{Errors: []error{}} + err := utils.MultiError{Errors: []error{}} path := "Root" if len(a.UnknownNode) > 0 { err.Errors = append(err.Errors, &UnknownNodeError{path, a.UnknownNode[0]}) diff --git a/pkg/apigee/v1/model.go b/pkg/apigee/v1/model.go index 178ce2a..bb28e31 100644 --- a/pkg/apigee/v1/model.go +++ b/pkg/apigee/v1/model.go @@ -109,16 +109,8 @@ func Model2BundleZip(model Model, outputZip string) error { func HydrateResources(model Model, fromDir string) error { // switch to directory relative to the YAML file so that resource paths are valid - wd, err := os.Getwd() - if err != nil { - return errors.New(err) - } - defer func() { utils.Must(os.Chdir(wd)) }() - - err = os.Chdir(fromDir) - if err != nil { - return errors.New(err) - } + popd := utils.PushDir(fromDir) + defer popd() for _, resource := range model.GetResources().List { parsedUrl, err := url.Parse(resource.Path) diff --git a/pkg/apigee/v1/sharedflowbundlemodel.go b/pkg/apigee/v1/sharedflowbundlemodel.go index 352d694..48f3f5e 100644 --- a/pkg/apigee/v1/sharedflowbundlemodel.go +++ b/pkg/apigee/v1/sharedflowbundlemodel.go @@ -124,7 +124,7 @@ func (a *SharedFlowBundleModel) Validate() error { return nil } - err := ValidationErrors{Errors: []error{}} + err := utils.MultiError{Errors: []error{}} path := "Root" if len(a.UnknownNode) > 0 { err.Errors = append(err.Errors, &UnknownNodeError{path, a.UnknownNode[0]}) diff --git a/pkg/apigee/v1/unknown.go b/pkg/apigee/v1/unknown.go index 04e0378..69bf64e 100644 --- a/pkg/apigee/v1/unknown.go +++ b/pkg/apigee/v1/unknown.go @@ -38,11 +38,3 @@ type UnknownNodeError struct { func (e *UnknownNodeError) Error() string { return fmt.Sprintf(`unknown node "%s" found at "%s"`, e.Node.XMLName.Local, e.Location) } - -type ValidationErrors struct { - Errors []error -} - -func (e ValidationErrors) Error() string { - return e.Errors[0].Error() -} diff --git a/pkg/common/resources/helper_functions.txt b/pkg/common/resources/helper_functions.txt index bfb098d..b07bf6b 100644 --- a/pkg/common/resources/helper_functions.txt +++ b/pkg/common/resources/helper_functions.txt @@ -59,6 +59,15 @@ You can also use it to dump values to stdout in order to see the contents. e.g. {{ fmt_printf "url: %%v\n" $url }} + remove_oas_extensions(src string) string + Removes the OpenAPI spec extensions from the file specified by src + The file must already exist in the output directory + + This is useful if to make the spec files small within the generated bundles + e.g. + {{ os_writefile "./spec.yaml" $.Values.spec_string }} + {{ remove_oas_extensions "./spec.yaml" }} + Sprig Template functions from Sprig library e.g. {{ "Hello World" | upper }} diff --git a/pkg/render/helper_funcs.go b/pkg/render/helper_funcs.go index 2229990..707b5f3 100644 --- a/pkg/render/helper_funcs.go +++ b/pkg/render/helper_funcs.go @@ -15,6 +15,7 @@ package render import ( + "github.com/apigee/apigee-go-gen/pkg/utils" "os" "path/filepath" "strings" @@ -76,3 +77,39 @@ func getOSCopyFileFunc(templateFile string, outputFile string, dryRun bool) Help return _osCopyFileFunc } + +func getRemoveOASExtensions(templateFile string, outputFile string, dryRun bool) HelperFunc { + _removeOASExtensionsFunc := func(args ...any) string { + if len(args) < 1 { + panic("remove_oas_extensions function requires one argument") + } + + //both destination and source are the same + dst := args[0].(string) + src := args[0].(string) + + if filepath.IsAbs(src) { + panic("remove_oas_extensions src must not be absolute") + } + if strings.Index(src, "..") >= 0 { + panic("remove_oas_extensions src must not use ..") + } + + //both destination and source are relative to the output file + dstPath := filepath.Join(filepath.Dir(outputFile), dst) + srcPath := filepath.Join(filepath.Dir(outputFile), src) + + if !dryRun { + err := utils.RemoveExtensions(srcPath, dstPath) + if err != nil { + panic(err) + } + } else { + //fmt.Printf(`remove_oas_extensions("%s", "%s")\n`, dstPath, srcPath) + } + + return dst + } + + return _removeOASExtensionsFunc +} diff --git a/pkg/render/model.go b/pkg/render/model.go index 8d0eff9..a067f66 100644 --- a/pkg/render/model.go +++ b/pkg/render/model.go @@ -108,16 +108,8 @@ func CreateBundle(model v1.Model, output string, validate bool, dryRun string) ( } func ResolveYAML(text []byte, filePath string) ([]byte, error) { - wd, err := os.Getwd() - if err != nil { - return nil, errors.New(err) - } - defer func() { utils.Must(os.Chdir(wd)) }() - - err = os.Chdir(filepath.Dir(filePath)) - if err != nil { - return nil, errors.New(err) - } + popd := utils.PushDir(filepath.Dir(filePath)) + defer popd() yaml, err := utils.Text2YAML(bytes.NewReader(text)) if err != nil { diff --git a/pkg/render/render.go b/pkg/render/render.go index 44bb0ee..d28568c 100644 --- a/pkg/render/render.go +++ b/pkg/render/render.go @@ -246,6 +246,7 @@ func CreateTemplate(templateFile string, includeList []string, outputFile string helperFuncs["os_getenv"] = osGetEnvFunc helperFuncs["os_getenvs"] = osGetEnvs helperFuncs["os_copyfile"] = getOSCopyFileFunc(templateFile, outputFile, dryRun) + helperFuncs["remove_oas_extensions"] = getRemoveOASExtensions(templateFile, outputFile, dryRun) helperFuncs["blank"] = blankFunc helperFuncs["deref"] = derefFunc helperFuncs["slug_make"] = slugMakeFunc diff --git a/pkg/utils/extension_remover.go b/pkg/utils/extension_remover.go new file mode 100644 index 0000000..979db6f --- /dev/null +++ b/pkg/utils/extension_remover.go @@ -0,0 +1,137 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "github.com/go-errors/errors" + libopenapijson "github.com/pb33f/libopenapi/json" + "gopkg.in/yaml.v3" + "path/filepath" + "strings" +) + +func RemoveExtensions(input string, output string) error { + text, err := ReadInputText(input) + if err != nil { + return err + } + + //unmarshall input as YAML + var yamlNode *yaml.Node + yamlNode = &yaml.Node{} + err = yaml.Unmarshal(text, yamlNode) + if err != nil { + return errors.New(err) + } + + yamlNode, err = RemoveOASExtensions(yamlNode) + if err != nil { + return err + } + + //convert back to text + ext := filepath.Ext(output) + if ext == "" { + ext = filepath.Ext(input) + } + + //depending on the file extension write output as either JSON or YAML + var outputText []byte + if ext == ".json" { + outputText, err = libopenapijson.YAMLNodeToJSON(yamlNode, " ") + if err != nil { + return errors.New(err) + } + } else { + outputText, err = YAML2Text(UnFlowYAMLNode(yamlNode), 2) + if err != nil { + return err + } + } + + return WriteOutputText(output, outputText) +} + +func RemoveOASExtensions(root *yaml.Node) (*yaml.Node, error) { + modified, err := RemoveOASExtensionsRecursive(root, "") + if err != nil { + return nil, err + } + return modified, nil +} + +func RemoveOASExtensionsRecursive(node *yaml.Node, parentField string) (*yaml.Node, error) { + var err error + + if node == nil { + return nil, errors.Errorf("nil node detected") + } + + var modifiedNode *yaml.Node + + if node.Kind == yaml.MappingNode && isYAMLRef(node) { + if err != nil { + return nil, err + } + return node, nil + } else if node.Kind == yaml.MappingNode { + var newContent []*yaml.Node + for i := 0; i+1 < len(node.Content); i += 2 { + fieldName := node.Content[i].Value + if strings.Index(fieldName, "x-") == 0 && + !(parentField == "headers" || + parentField == "properties" || + parentField == "responses" || + parentField == "schemas" || + parentField == "paths" || + parentField == "variables" || + parentField == "securitySchemes" || + parentField == "examples" || + parentField == "links" || + parentField == "callbacks" || + parentField == "requestBodies" || + parentField == "mapping" || + parentField == "scopes" || + parentField == "encoding" || + parentField == "definitions" || + parentField == "securityDefinitions" || + parentField == "parameters") { + continue + } + if modifiedNode, err = RemoveOASExtensionsRecursive(node.Content[i+1], fieldName); err != nil { + return nil, err + } + newContent = append(newContent, node.Content[i], modifiedNode) + } + node.Content = newContent + return node, nil + } else if node.Kind == yaml.DocumentNode { + if modifiedNode, err = RemoveOASExtensionsRecursive(node.Content[0], ""); err != nil { + return nil, err + } + node.Content[0] = modifiedNode + return node, nil + } else if node.Kind == yaml.SequenceNode { + for i := 0; i < len(node.Content); i += 1 { + if modifiedNode, err = RemoveOASExtensionsRecursive(node.Content[i], ""); err != nil { + return nil, err + } + node.Content[i] = modifiedNode + } + return node, nil + } + + return node, nil +} diff --git a/pkg/utils/io.go b/pkg/utils/io.go index 218e800..d1bf703 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -17,7 +17,6 @@ package utils import ( "fmt" "github.com/go-errors/errors" - "gopkg.in/yaml.v3" "io" "os" "path/filepath" @@ -104,18 +103,3 @@ func ReadInputText(input string) ([]byte, error) { } return text, nil } - -func RunWithinDirectory[ResultType *yaml.Node](dir string, operation func() (ResultType, error)) (ResultType, error) { - wd, err := os.Getwd() - if err != nil { - return nil, errors.New(err) - } - defer func() { Must(os.Chdir(wd)) }() - - err = os.Chdir(dir) - if err != nil { - return nil, errors.New(err) - } - - return operation() -} diff --git a/pkg/utils/openapi2.go b/pkg/utils/openapi2.go index 28d8261..6ff35f1 100644 --- a/pkg/utils/openapi2.go +++ b/pkg/utils/openapi2.go @@ -16,14 +16,17 @@ package utils import ( "encoding/json" + "fmt" "github.com/getkin/kin-openapi/openapi2" "github.com/getkin/kin-openapi/openapi2conv" "github.com/getkin/kin-openapi/openapi3" "github.com/go-errors/errors" libopenapijson "github.com/pb33f/libopenapi/json" "gopkg.in/yaml.v3" + "net/url" "path/filepath" "slices" + "strings" ) func OAS2YAMLtoOAS3YAML(oasNode *yaml.Node) (*yaml.Node, error) { @@ -50,9 +53,7 @@ func OAS2YAMLtoOAS3YAML(oasNode *yaml.Node) (*yaml.Node, error) { openapi3.DisableReadOnlyValidation() openapi3.DisableWriteOnlyValidation() - loader := openapi3.NewLoader() - loader.IsExternalRefsAllowed = true - oas3doc, err := openapi2conv.ToV3WithLoader(&oas2doc, loader, nil) + oas3doc, err := ToV3(&oas2doc) if err != nil { return nil, errors.New(err) } @@ -82,22 +83,20 @@ func OAS2FileToOAS3File(input string, output string, allowCycles bool) error { return errors.Errorf("input %s is not an OpenAPI 2.0 spec", input) } - //detect JSONRef cycles - _, err = DetectCycle(oas2node, input) + cycles, err := YAMLDetectRefCycles(oas2node, input) if err != nil { - var cyclicError CyclicJSONRefError - isCyclicError := errors.As(err, &cyclicError) + return err + } - if isCyclicError && allowCycles == true { - oas2node, err = ResolveCycles(oas2node, input) - } else { - return err + if len(cycles) > 0 && allowCycles == false { + var multiError MultiError + for _, cycle := range cycles { + multiError.Errors = append(multiError.Errors, errors.Errorf("cyclic ref at %s", strings.Join(cycle, ":"))) } + return errors.New(multiError) } - oas3node, err := RunWithinDirectory(filepath.Dir(input), func() (*yaml.Node, error) { - return OAS2YAMLtoOAS3YAML(oas2node) - }) + oas3node, err := OAS2YAMLtoOAS3YAML(oas2node) if err != nil { return err } @@ -124,130 +123,98 @@ func OAS2FileToOAS3File(input string, output string, allowCycles bool) error { return WriteOutputText(output, outputText) } -func OAS2ToYAML(doc *openapi2.T) (*yaml.Node, error) { - var err error - oas := &yaml.Node{Kind: yaml.MappingNode} - - //required - _, err = AddEntryToOASYAML(oas, "swagger", doc.Swagger, nil) - if err != nil { - return nil, err - } - - //required - _, err = AddEntryToOASYAML(oas, "info", doc.Info, &yaml.Node{Kind: yaml.MappingNode}) - if err != nil { - return nil, err +func ToV3(doc2 *openapi2.T) (*openapi3.T, error) { + doc3 := &openapi3.T{ + OpenAPI: "3.0.3", + Info: &doc2.Info, + Components: &openapi3.Components{}, + Tags: doc2.Tags, + Extensions: doc2.Extensions, + ExternalDocs: doc2.ExternalDocs, } - //optional - for k, v := range doc.Extensions { - _, err = AddEntryToOASYAML(oas, k, v, &yaml.Node{Kind: yaml.MappingNode}) - if err != nil { + if host := doc2.Host; host != "" { + if strings.Contains(host, "://") { + err := fmt.Errorf("%s host is not valid", host) return nil, err } - } - - //optional - if doc.BasePath != "" { - _, err = AddEntryToOASYAML(oas, "basePath", doc.BasePath, nil) - if err != nil { - return nil, err + schemes := doc2.Schemes + if len(schemes) == 0 { + schemes = []string{"https"} } - } - - //optional - if doc.Host != "" { - _, err = AddEntryToOASYAML(oas, "host", doc.Host, nil) - if err != nil { - return nil, err + basePath := doc2.BasePath + if basePath == "" { + basePath = "/" } - } - - //optional - if len(doc.Schemes) > 0 { - _, err = AddEntryToOASYAML(oas, "schemes", doc.Schemes, &yaml.Node{Kind: yaml.SequenceNode}) - if err != nil { - return nil, err + for _, scheme := range schemes { + u := url.URL{ + Scheme: scheme, + Host: host, + Path: basePath, + } + doc3.AddServer(&openapi3.Server{URL: u.String()}) } } - //optional - if doc.ExternalDocs != nil { - _, err = AddEntryToOASYAML(oas, "externalDocs", doc.ExternalDocs, &yaml.Node{Kind: yaml.MappingNode}) - if err != nil { - return nil, err + doc3.Components.Schemas = make(map[string]*openapi3.SchemaRef) + if parameters := doc2.Parameters; len(parameters) != 0 { + doc3.Components.Parameters = make(map[string]*openapi3.ParameterRef) + doc3.Components.RequestBodies = make(map[string]*openapi3.RequestBodyRef) + for k, parameter := range parameters { + v3Parameter, v3RequestBody, v3SchemaMap, err := openapi2conv.ToV3Parameter(doc3.Components, parameter, doc2.Consumes) + switch { + case err != nil: + return nil, err + case v3RequestBody != nil: + doc3.Components.RequestBodies[k] = v3RequestBody + case v3SchemaMap != nil: + for _, v3Schema := range v3SchemaMap { + doc3.Components.Schemas[k] = v3Schema + } + default: + doc3.Components.Parameters[k] = v3Parameter + } } } - //optional - if len(doc.Tags) > 0 { - _, err = AddEntryToOASYAML(oas, "tags", doc.Tags, &yaml.Node{Kind: yaml.SequenceNode}) - if err != nil { - return nil, err + if paths := doc2.Paths; len(paths) != 0 { + doc3.Paths = openapi3.NewPathsWithCapacity(len(paths)) + for path, pathItem := range paths { + r, err := openapi2conv.ToV3PathItem(doc2, doc3.Components, pathItem, doc2.Consumes) + if err != nil { + return nil, err + } + doc3.Paths.Set(path, r) } } - //optional - if len(doc.Security) > 0 { - _, err = AddEntryToOASYAML(oas, "security", doc.Security, &yaml.Node{Kind: yaml.SequenceNode}) - if err != nil { - return nil, err + if responses := doc2.Responses; len(responses) != 0 { + doc3.Components.Responses = make(openapi3.ResponseBodies, len(responses)) + for k, response := range responses { + r, err := openapi2conv.ToV3Response(response, doc2.Produces) + if err != nil { + return nil, err + } + doc3.Components.Responses[k] = r } } - _, err = AddEntryToOASYAML(oas, "paths", doc.Paths, &yaml.Node{Kind: yaml.MappingNode}) - if err != nil { - return nil, err + for key, schema := range openapi2conv.ToV3Schemas(doc2.Definitions) { + doc3.Components.Schemas[key] = schema } - //optional - if len(doc.Consumes) > 0 { - _, err = AddEntryToOASYAML(oas, "consumes", doc.Consumes, &yaml.Node{Kind: yaml.SequenceNode}) - if err != nil { - return nil, err - } - } - - //optional - if len(doc.Produces) > 0 { - _, err = AddEntryToOASYAML(oas, "produces", doc.Produces, &yaml.Node{Kind: yaml.SequenceNode}) - if err != nil { - return nil, err - } - } - - //optional - if len(doc.SecurityDefinitions) > 0 { - _, err = AddEntryToOASYAML(oas, "securityDefinitions", doc.SecurityDefinitions, &yaml.Node{Kind: yaml.MappingNode}) - if err != nil { - return nil, err - } - } - - //optional - if len(doc.Parameters) > 0 { - _, err = AddEntryToOASYAML(oas, "parameters", doc.Parameters, &yaml.Node{Kind: yaml.MappingNode}) - if err != nil { - return nil, err - } - } - - //optional - if len(doc.Responses) > 0 { - _, err = AddEntryToOASYAML(oas, "responses", doc.Responses, &yaml.Node{Kind: yaml.MappingNode}) - if err != nil { - return nil, err - } - } - - //optional - if len(doc.Definitions) > 0 { - _, err = AddEntryToOASYAML(oas, "definitions", doc.Definitions, &yaml.Node{Kind: yaml.MappingNode}) - if err != nil { - return nil, err + if m := doc2.SecurityDefinitions; len(m) != 0 { + doc3SecuritySchemes := make(map[string]*openapi3.SecuritySchemeRef) + for k, v := range m { + r, err := openapi2conv.ToV3SecurityScheme(v) + if err != nil { + return nil, err + } + doc3SecuritySchemes[k] = r } + doc3.Components.SecuritySchemes = doc3SecuritySchemes } - return oas, nil + doc3.Security = openapi2conv.ToV3SecurityRequirements(doc2.Security) + return doc3, nil } diff --git a/pkg/utils/openapi2_test.go b/pkg/utils/openapi2_test.go index ed0376a..1b2fca3 100644 --- a/pkg/utils/openapi2_test.go +++ b/pkg/utils/openapi2_test.go @@ -89,7 +89,11 @@ func TestOpenAPI2FileToOpenAPI3File(t *testing.T) { "oas2.json", "oas3.json", false, - errors.New("cyclic JSONRef at $.definitions.Widgets.properties.widgets.items.properties.subWidgets"), + MultiError{Errors: []error{ + errors.New("cyclic ref at schemas/widget.json:$.properties.subWidgets"), + errors.New("cyclic ref at oas2.json:$.definitions.Error.properties.errors"), + errors.New("cyclic ref at oas2.json:$.definitions.Errors.items"), + }}, }, } for _, tt := range tests { @@ -107,7 +111,7 @@ func TestOpenAPI2FileToOpenAPI3File(t *testing.T) { err = OAS2FileToOAS3File(inputFile, outputFile, tt.allowCycles) if tt.wantErr != nil { - require.EqualError(t, tt.wantErr, err.Error()) + require.EqualError(t, err, tt.wantErr.Error()) return } diff --git a/pkg/utils/openapi3.go b/pkg/utils/openapi3.go index 5d111d1..4dcd4fb 100644 --- a/pkg/utils/openapi3.go +++ b/pkg/utils/openapi3.go @@ -15,114 +15,10 @@ package utils import ( - "encoding/json" - "github.com/getkin/kin-openapi/openapi2conv" "github.com/getkin/kin-openapi/openapi3" - "github.com/go-errors/errors" - libopenapijson "github.com/pb33f/libopenapi/json" "gopkg.in/yaml.v3" - "path/filepath" - "slices" ) -func OAS3YAMLtoOAS2YAML(oasNode *yaml.Node) (*yaml.Node, error) { - //convert it to JSON, since the converter library depends on JSON text - jsonText, err := libopenapijson.YAMLNodeToJSON(oasNode, " ") - if err != nil { - return nil, errors.New(err) - } - - //then, convert it to the OAS2 data model - var oas3doc openapi3.T - err = json.Unmarshal(jsonText, &oas3doc) - if err != nil { - return nil, errors.New(err) - } - - //finally, convert it to the OAS3 data model - openapi3.CircularReferenceCounter = 5 - openapi3.DisableSchemaDefaultsValidation() - openapi3.DisablePatternValidation() - openapi3.DisableExamplesValidation() - openapi3.DisableSchemaPatternValidation() - openapi3.DisableSchemaFormatValidation() - openapi3.DisableReadOnlyValidation() - openapi3.DisableWriteOnlyValidation() - - //loader := openapi3.NewLoader() - //loader.IsExternalRefsAllowed = true - oas2doc, err := openapi2conv.FromV3(&oas3doc) - if err != nil { - return nil, errors.New(err) - } - - //and back to YAML node - return OAS2ToYAML(oas2doc) -} - -func OAS3FileToOAS2File(input string, output string, allowCycles bool) error { - text, err := ReadInputText(input) - if err != nil { - return err - } - - //first, use the YAML library to parse it (regardless if it's JSON or YAML) - var oas3node *yaml.Node - oas3node = &yaml.Node{} - err = yaml.Unmarshal(text, oas3node) - if err != nil { - return errors.New(err) - } - - //verify we are actually working with OAS2 - if slices.IndexFunc(oas3node.Content[0].Content, func(n *yaml.Node) bool { - return n.Value == "openapi" - }) < 0 { - return errors.Errorf("input %s is not an OpenAPI 3.0 spec", input) - } - - //detect JSONRef cycles - _, err = DetectCycle(oas3node, input) - if err != nil { - var cyclicError CyclicJSONRefError - isCyclicError := errors.As(err, &cyclicError) - - if isCyclicError && allowCycles == true { - oas3node, err = ResolveCycles(oas3node, input) - } else { - return err - } - } - - oas2node, err := RunWithinDirectory(filepath.Dir(input), func() (*yaml.Node, error) { - return OAS3YAMLtoOAS2YAML(oas3node) - }) - if err != nil { - return err - } - - ext := filepath.Ext(output) - if ext == "" { - ext = filepath.Ext(input) - } - - //depending on the file extension write the output as either JSON or YAML - var outputText []byte - if ext == ".json" { - outputText, err = libopenapijson.YAMLNodeToJSON(oas2node, " ") - if err != nil { - return errors.New(err) - } - } else { - outputText, err = YAML2Text(UnFlowYAMLNode(oas2node), 2) - if err != nil { - return err - } - } - - return WriteOutputText(output, outputText) -} - func OAS3ToYAML(doc *openapi3.T) (*yaml.Node, error) { var err error oas := &yaml.Node{Kind: yaml.MappingNode} diff --git a/pkg/utils/resolver.go b/pkg/utils/resolver.go index 08fb42e..0bf107b 100644 --- a/pkg/utils/resolver.go +++ b/pkg/utils/resolver.go @@ -36,7 +36,7 @@ func ResolveDollarRefs(input string, output string, allowCycles bool) error { } //resolve references - yamlNode, err = ResolveYAMLRefs(yamlNode, input, allowCycles) + yamlNode, err = YAMLResolveRefs(yamlNode, input, allowCycles) if err != nil { return err } diff --git a/pkg/utils/resolver_test.go b/pkg/utils/resolver_test.go index f8a2438..146c1d7 100644 --- a/pkg/utils/resolver_test.go +++ b/pkg/utils/resolver_test.go @@ -56,7 +56,7 @@ func TestResolveDollarRefs(t *testing.T) { "oas2.json", "oas2.json", false, - errors.New("cyclic JSONRef at $.definitions.Widgets.properties.widgets.items.properties.subWidgets"), + errors.New("cyclic ref at schemas/widget.json:$.properties.subWidgets"), }, } for _, tt := range tests { diff --git a/pkg/utils/testdata/oas2-to-oas3/petstore-cycle/exp-oas3.json b/pkg/utils/testdata/oas2-to-oas3/petstore-cycle/exp-oas3.json index bdaae30..5360bbf 100755 --- a/pkg/utils/testdata/oas2-to-oas3/petstore-cycle/exp-oas3.json +++ b/pkg/utils/testdata/oas2-to-oas3/petstore-cycle/exp-oas3.json @@ -1,8 +1,8 @@ { "openapi": "3.0.3", "info": { - "title": "Cycle OAS2", "description": "This is a sample OAS2 that contains a cycle", + "title": "Cycle OAS2", "version": "1.0.7" }, "servers": [ @@ -18,7 +18,6 @@ "components": { "schemas": { "Error": { - "type": "object", "properties": { "errors": { "$ref": "#/components/schemas/Errors" @@ -26,32 +25,17 @@ "message": { "type": "string" } - } + }, + "type": "object" }, "Errors": { - "type": "array", "items": { "$ref": "#/components/schemas/Error" - } + }, + "type": "array" }, "Widgets": { - "type": "object", - "properties": { - "widgets": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "subWidgets": { - "description": "cyclic JSONRef to $.definitions.Widgets" - } - } - } - } - } + "$ref": "./schemas/definitions.json#/definitions/Widgets" } } } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a678d5f..2ac732e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -58,3 +58,29 @@ func AddEntryToOASYAML(oas *yaml.Node, key string, value any, defaultVal *yaml.N oas.Content = append(oas.Content, content) return &yamlNode, nil } + +type MultiError struct { + Errors []error +} + +func (e MultiError) Error() string { + return errors.Join(e.Errors...).Error() +} + +func PushDir(dir string) func() { + wd, err := os.Getwd() + if err != nil { + panic(err) + } + + err = os.Chdir(dir) + if err != nil { + panic(err) + } + + popDir := func() { + Must(os.Chdir(wd)) + } + + return popDir +} diff --git a/pkg/utils/yaml.go b/pkg/utils/yaml.go index 66592ee..e76d2d8 100644 --- a/pkg/utils/yaml.go +++ b/pkg/utils/yaml.go @@ -20,13 +20,10 @@ import ( "github.com/beevik/etree" "github.com/go-errors/errors" libopenapijson "github.com/pb33f/libopenapi/json" - "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" "gopkg.in/yaml.v3" "io" - "net/url" "os" "path/filepath" - "slices" "strings" ) @@ -78,7 +75,7 @@ func Text2YAML(reader io.Reader) (*yaml.Node, error) { } filePath := "./input.yaml" - resultNode, err := ResolveYAMLRefs(&yamlNode, filePath, false) + resultNode, err := YAMLResolveRefs(&yamlNode, filePath, false) if err != nil { return nil, err } @@ -163,333 +160,6 @@ func YAML2XMLRecursive(node *yaml.Node, parent *etree.Element) (*etree.Element, return nil, fmt.Errorf("unknown yaml node kind %v", node.Kind) } -func isYAMLRef(node *yaml.Node) bool { - if node == nil { - return false - } - - return node.Kind == yaml.MappingNode && slices.IndexFunc(node.Content, func(n *yaml.Node) bool { - return n.Value == "$ref" - }) >= 0 - -} - -func getYAMLRefString(node *yaml.Node) string { - index := slices.IndexFunc(node.Content, func(n *yaml.Node) bool { - return n.Value == "$ref" - }) - if index < 0 { - return "" - } - return node.Content[index+1].Value -} - -func ParseYAMLFile(filePath string) (*yaml.Node, error) { - var err error - - absFilePath, err := filepath.Abs(filePath) - if err != nil { - return nil, errors.New(err) - } - - //check if the file has already been parsed - rootNode, ok := ParsedYAMLFiles[absFilePath] - if ok { - return rootNode, nil - } - - file, err := os.Open(filePath) - if err != nil { - return nil, errors.New(err) - } - - decoder := yaml.NewDecoder(file) - yamlNode := yaml.Node{} - if err = decoder.Decode(&yamlNode); err != nil { - return nil, errors.Errorf("could not parse %s. %s", filePath, err.Error()) - } - - ParsedYAMLFiles[filePath] = &yamlNode - return &yamlNode, nil -} - -func JSONPointer2JSONPath(jsonPointer string) (jsonPath string, err error) { - - pointer := strings.TrimSpace(jsonPointer) - if pointer == "" || - pointer == "#" || - pointer == "#/" { - return "$", nil - } - - if strings.Index(pointer, "#/") != 0 { - return "", errors.Errorf("relative JSONPointer %s is not supported", jsonPointer) - } - - pointer = "$" + strings.ReplaceAll(pointer[1:], "/", ".") - return pointer, nil -} - -func SplitJSONRef(refStr string) (location string, jsonPath string, err error) { - parsedUrl, err := url.Parse(refStr) - if err != nil { - return "", "", errors.New(err) - } - - if parsedUrl.Scheme != "" { - return "", "", errors.Errorf("JSONRef %s is not supported", refStr) - } - - jsonPath, err = JSONPointer2JSONPath("#" + parsedUrl.Fragment) - if err != nil { - return "", "", err - } - - return parsedUrl.Path, jsonPath, nil -} - -func ResolveYAMLRef(root *yaml.Node, - node *yaml.Node, nodePath string, nodePaths []string, - filePath string, filePaths []string, - allowCycles bool, resolveCyclesOnly bool) (*yaml.Node, bool, error) { - var err error - - jsonRef := getYAMLRefString(node) - if jsonRef == "" { - return nil, false, errors.Errorf("JSONRef at %s is not valid", nodePath) - } - - refFilePath, refJSONPath, err := SplitJSONRef(jsonRef) - if err != nil { - return nil, false, err - } - - absFilePath, err := filepath.Abs(filePath) - if err != nil { - return nil, false, errors.New(err) - } - - if refFilePath == "" { - refFilePath = filepath.Base(filePath) - } - - var absRefFilePath string - if filepath.IsAbs(refFilePath) { - absRefFilePath = refFilePath - } else { - absRefFilePath, err = filepath.Abs(filepath.Join(filepath.Dir(absFilePath), refFilePath)) - } - - absJSONRef := fmt.Sprintf("%s%s", absRefFilePath, refJSONPath) - if _, ok := ResolvedRefs[absJSONRef]; ok { - //return cached result - return ResolvedRefs[absJSONRef], false, nil - } - - //detect cycles - if index := slices.Index(nodePaths, absJSONRef); index >= 0 { - if !allowCycles { - return nil, false, errors.New(NewCyclicJSONRefError(nodePaths)) - } - ResolvedRefs[absJSONRef] = MakeCyclicRefPlaceholder(refJSONPath) - return ResolvedRefs[absJSONRef], true, nil - } - - isSelfRef := absRefFilePath == absFilePath - - if isSelfRef && len(filePaths) == 1 { - //self ref at the first level, no need to de-reference it - ResolvedRefs[absJSONRef] = node - return ResolvedRefs[absJSONRef], false, nil - } else if isSelfRef && len(filePaths) > 1 { - //self ref below first level, need to de-reference it - yamlNode, err := LocateRef(root, refJSONPath, jsonRef) - - newNodePaths := append([]string{}, nodePaths...) - newNodePaths = append(newNodePaths, absJSONRef) - - resolvedNode, cycleDetected, err := ResolveYAMLRefsRecursive(root, yamlNode, nodePath, newNodePaths, absRefFilePath, filePaths, allowCycles, resolveCyclesOnly) - if err != nil { - return nil, cycleDetected, err - } - - if resolveCyclesOnly && !cycleDetected { - resolvedNode = node - } - - ResolvedRefs[absJSONRef] = resolvedNode - return ResolvedRefs[absJSONRef], cycleDetected, nil - } - - //ref to a different file, need to de-reference it - var refFileNode *yaml.Node - refFileNode, err = ParseYAMLFile(absRefFilePath) - if err != nil { - return nil, false, err - } - - yamlNode, err := LocateRef(refFileNode, refJSONPath, jsonRef) - if err != nil { - return nil, false, err - } - - newFilePaths := append([]string{}, filePaths...) - newFilePaths = append(newFilePaths, absRefFilePath) - - newNodePaths := append([]string{}, nodePaths...) - newNodePaths = append(newNodePaths, absJSONRef) - - resolvedNode, cycleDetected, err := ResolveYAMLRefsRecursive(refFileNode, yamlNode, nodePath, newNodePaths, absRefFilePath, newFilePaths, allowCycles, resolveCyclesOnly) - if err != nil { - return nil, cycleDetected, err - } - - if resolveCyclesOnly && !cycleDetected { - resolvedNode = node - } - - ResolvedRefs[absJSONRef] = resolvedNode - return ResolvedRefs[absJSONRef], cycleDetected, nil -} - -func MakeCyclicRefPlaceholder(refJSONPath string) *yaml.Node { - cyclicRef := &yaml.Node{Kind: yaml.MappingNode} - key := yaml.Node{Kind: yaml.ScalarNode, Style: yaml.DoubleQuotedStyle} - key.SetString("description") - - value := yaml.Node{Kind: yaml.ScalarNode, Style: yaml.DoubleQuotedStyle} - value.SetString(fmt.Sprintf("cyclic JSONRef to %s", refJSONPath)) - - cyclicRef.Content = append(cyclicRef.Content, &key, &value) - return cyclicRef -} - -func LocateRef(refFileNode *yaml.Node, refJSONPath string, jsonRef string) (*yaml.Node, error) { - var err error - var yamlPath *yamlpath.Path - - yamlPath, err = yamlpath.NewPath(refJSONPath) - if err != nil { - return nil, errors.New(err) - } - - var yamlNodes []*yaml.Node - yamlNodes, err = yamlPath.Find(refFileNode) - if err != nil { - return nil, errors.New(err) - } - - if len(yamlNodes) == 0 { - return nil, errors.Errorf("no node found at JSONRef '%s'", jsonRef) - } - - if len(yamlNodes) > 1 { - return nil, errors.Errorf("more than one node found at JSONRef '%s'", jsonRef) - } - return yamlNodes[0], nil -} - -func ResolveYAMLRefs(node *yaml.Node, filePath string, allowCycles bool) (*yaml.Node, error) { - ResetYAMLRefs() - node, _, err := ResolveYAMLRefsRecursive(node, node, "$", nil, filePath, nil, allowCycles, false) - return node, err -} - -func DetectCycle(node *yaml.Node, filePath string) (bool, error) { - ResetYAMLRefs() - _, cycleDetected, err := ResolveYAMLRefsRecursive(node, node, "$", nil, filePath, nil, false, true) - return cycleDetected, err -} - -func ResolveCycles(node *yaml.Node, filePath string) (*yaml.Node, error) { - ResetYAMLRefs() - node, _, err := ResolveYAMLRefsRecursive(node, node, "$", nil, filePath, nil, true, true) - return node, err -} - -func ResolveYAMLRefsRecursive(root *yaml.Node, node *yaml.Node, - nodePath string, nodePaths []string, - filePath string, filePaths []string, - allowCycles bool, - resolveCyclesOnly bool) (*yaml.Node, bool, error) { - var err error - - if !filepath.IsAbs(filePath) { - filePath, err = filepath.Abs(filePath) - if err != nil { - return nil, false, errors.New(err) - } - } - - if len(filePaths) == 0 { - filePaths = append(filePaths, filePath) - } - - if len(nodePaths) == 0 { - nodePaths = append(nodePaths, nodePath) - } - - if node == nil { - return nil, false, nil - } - - var resolvedNode *yaml.Node - var cycleDetected bool - - if node.Kind == yaml.MappingNode && isYAMLRef(node) { - if resolvedNode, cycleDetected, err = ResolveYAMLRef(root, node, nodePath, nodePaths, filePath, filePaths, allowCycles, resolveCyclesOnly); err != nil { - return nil, cycleDetected, err - } - return resolvedNode, cycleDetected, nil - } else if node.Kind == yaml.MappingNode { - for i := 0; i+1 < len(node.Content); i += 2 { - subPath := fmt.Sprintf("%s.%s", nodePath, node.Content[i].Value) - subContent := node.Content[i+1] - - if slices.Index(nodePaths, subPath) >= 0 { - return nil, true, errors.Errorf("cycle detected at %v", nodePaths) - } - - newNodePaths := append([]string{}, nodePaths...) - newNodePaths = append(newNodePaths, subPath) - if resolvedNode, cycleDetected, err = ResolveYAMLRefsRecursive(root, subContent, subPath, newNodePaths, filePath, filePaths, allowCycles, resolveCyclesOnly); err != nil { - return nil, cycleDetected, err - } - node.Content[i+1] = resolvedNode - } - return node, cycleDetected, nil - } else if node.Kind == yaml.DocumentNode { - subPath := nodePath - subContent := node.Content[0] - newNodePaths := nodePaths - if resolvedNode, cycleDetected, err = ResolveYAMLRefsRecursive(root, subContent, subPath, newNodePaths, filePath, filePaths, allowCycles, resolveCyclesOnly); err != nil { - return nil, cycleDetected, err - } - node.Content[0] = resolvedNode - return node, cycleDetected, nil - - } else if node.Kind == yaml.SequenceNode { - for i := 0; i < len(node.Content); i += 1 { - subPath := fmt.Sprintf("%s.%v", nodePath, i) - subContent := node.Content[i] - - if slices.Index(nodePaths, subPath) >= 0 { - return nil, true, errors.Errorf("cycle detected at %v", nodePaths) - } - - newNodePaths := append([]string{}, nodePaths...) - newNodePaths = append(newNodePaths, subPath) - if resolvedNode, cycleDetected, err = ResolveYAMLRefsRecursive(root, subContent, subPath, newNodePaths, filePath, filePaths, allowCycles, resolveCyclesOnly); err != nil { - return nil, cycleDetected, err - } - node.Content[i] = resolvedNode - } - return node, cycleDetected, nil - } - - return node, cycleDetected, nil -} - func YAMLDoc2File(docNode *yaml.Node, outputFile string) error { var err error var docBytes []byte @@ -510,18 +180,6 @@ func YAMLDoc2File(docNode *yaml.Node, outputFile string) error { return nil } -var ResolvedRefs map[string]*yaml.Node -var ParsedYAMLFiles map[string]*yaml.Node - -func init() { - ResetYAMLRefs() -} - -func ResetYAMLRefs() { - ParsedYAMLFiles = make(map[string]*yaml.Node) - ResolvedRefs = make(map[string]*yaml.Node) -} - func YAMLFile2YAML(filePath string) (*yaml.Node, error) { var file *os.File var err error @@ -531,16 +189,8 @@ func YAMLFile2YAML(filePath string) (*yaml.Node, error) { defer func() { MustClose(file) }() //switch to directory relative to the YAML file so that JSON $refs are valid - wd, err := os.Getwd() - if err != nil { - return nil, errors.New(err) - } - defer func() { Must(os.Chdir(wd)) }() - - err = os.Chdir(filepath.Dir(filePath)) - if err != nil { - return nil, errors.New(err) - } + popd := PushDir(filepath.Dir(filePath)) + defer popd() dataNode, err := Text2YAML(file) if err != nil { @@ -555,13 +205,22 @@ func UnFlowYAMLNode(node *yaml.Node) *yaml.Node { case yaml.DocumentNode: fallthrough case yaml.SequenceNode: - fallthrough - case yaml.MappingNode: node.Style = 0 for _, v := range node.Content { v.Style = 0 UnFlowYAMLNode(v) } + case yaml.MappingNode: + node.Style = 0 + for i := 0; i+1 < len(node.Content); i += 2 { + key := node.Content[i] + val := node.Content[i+1] + key.Style = 0 + if key.Value == "$ref" { + continue + } + UnFlowYAMLNode(val) + } case yaml.ScalarNode: node.Style = 0 } diff --git a/pkg/utils/yaml_cycle_detector.go b/pkg/utils/yaml_cycle_detector.go new file mode 100644 index 0000000..457d625 --- /dev/null +++ b/pkg/utils/yaml_cycle_detector.go @@ -0,0 +1,157 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "github.com/go-errors/errors" + "gopkg.in/yaml.v3" + "os" + "path/filepath" + "slices" + "strings" +) + +func YAMLDetectRefCycles(root *yaml.Node, filePath string) (cycles [][]string, err error) { + fileDir := filepath.Dir(filePath) + filePath = filepath.Base(filePath) + + PopD := PushDir(fileDir) + defer PopD() + + err = YAMLDetectRefCyclesRecursive(root, "", filePath, []string{}, &map[string]*yaml.Node{}, &cycles) + + return cycles, err +} + +func YAMLDetectRefCyclesRecursive(node *yaml.Node, relParentPath string, parentFile string, activePaths []string, loaded *map[string]*yaml.Node, cycles *[][]string) error { + var err error + + if node == nil { + return nil + } + + absFilePath, err := filepath.Abs(parentFile) + if err != nil { + return errors.Errorf("could not process %s:%s. %s", parentFile, relParentPath, err.Error()) + } + + activePath := fmt.Sprintf("%s:%s", absFilePath, relParentPath) + if slices.Contains(activePaths, activePath) { + rootPath, _, _ := strings.Cut(activePaths[0], ":") + + lastPath := activePaths[len(activePaths)-1] + lastFilePath, lastNodePath, _ := strings.Cut(lastPath, ":") + + rel, _ := filepath.Rel(filepath.Dir(rootPath), lastFilePath) + *cycles = append(*cycles, []string{rel, lastNodePath}) + return nil + } + + activePaths = append(activePaths, activePath) + + if node.Kind == yaml.MappingNode && isYAMLRef(node) { + jsonRef := getYAMLRefString(node) + + if jsonRef == "" { + return errors.Errorf("JSONRef %s at %s is not valid", jsonRef, parentFile) + } + + refFilePath, refJSONPath, err := SplitJSONRef(jsonRef) + if err != nil { + return errors.Errorf("could not process JSONRef %s at %s. %s", jsonRef, parentFile, err.Error()) + } + + if refFilePath == "" { + refFilePath = parentFile + } + + refFileNode, err := loadYAMLFile(refFilePath, loaded) + + if err != nil { + return errors.Errorf("could not process JSONRef %s at %s. %s", jsonRef, parentFile, err.Error()) + } + + yamlNode, err := LocateRef(refFileNode, refJSONPath, jsonRef) + if err != nil { + return errors.Errorf("could not process JSONRef %s at %s, %s", jsonRef, parentFile, err.Error()) + } + + //switch-dir + PopD := PushDir(filepath.Dir(refFilePath)) + defer PopD() + + err = YAMLDetectRefCyclesRecursive(yamlNode, refJSONPath, filepath.Base(refFilePath), activePaths, loaded, cycles) + if err != nil { + return err + } + + return nil + } else if node.Kind == yaml.MappingNode { + for i := 0; i+1 < len(node.Content); i += 2 { + curPath := fmt.Sprintf("%s.%s", relParentPath, node.Content[i].Value) + + err = YAMLDetectRefCyclesRecursive(node.Content[i+1], curPath, parentFile, activePaths, loaded, cycles) + if err != nil { + return err + } + } + } else if node.Kind == yaml.DocumentNode { + err = YAMLDetectRefCyclesRecursive(node.Content[0], "$", parentFile, activePaths, loaded, cycles) + if err != nil { + return err + } + } else if node.Kind == yaml.SequenceNode { + for i := 0; i < len(node.Content); i += 1 { + curPath := fmt.Sprintf("%s.%d", relParentPath, i) + + err = YAMLDetectRefCyclesRecursive(node.Content[i], curPath, parentFile, activePaths, loaded, cycles) + if err != nil { + return err + } + } + } + + return nil +} + +func loadYAMLFile(filePath string, loaded *map[string]*yaml.Node) (*yaml.Node, error) { + var err error + + absFilePath, err := filepath.Abs(filePath) + if err != nil { + return nil, errors.New(err) + } + + //check if the file has already been parsed + cachedNode, ok := (*loaded)[absFilePath] + if ok { + return cachedNode, nil + } + + file, err := os.Open(absFilePath) + if err != nil { + return nil, errors.New(err) + } + + decoder := yaml.NewDecoder(file) + yamlNode := yaml.Node{} + if err = decoder.Decode(&yamlNode); err != nil { + return nil, errors.Errorf("could not load %s. %s", filePath, err.Error()) + } + + (*loaded)[absFilePath] = &yamlNode + return &yamlNode, nil +} diff --git a/pkg/utils/yaml_nodes.go b/pkg/utils/yaml_nodes.go new file mode 100644 index 0000000..a5a9b8b --- /dev/null +++ b/pkg/utils/yaml_nodes.go @@ -0,0 +1,69 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import "gopkg.in/yaml.v3" + +func NewRefNode(refJSONPath string) *yaml.Node { + mapNode := NewMapNode() + mapNode.Content = append(mapNode.Content, NewStringNode("$ref", yaml.SingleQuotedStyle)) + mapNode.Content = append(mapNode.Content, NewStringNode(refJSONPath, yaml.SingleQuotedStyle)) + return mapNode +} + +func NewStringNode(value string, style yaml.Style) *yaml.Node { + strNode := yaml.Node{Kind: yaml.ScalarNode, Style: style} + strNode.SetString(value) + return &strNode +} + +func NewMapNode() *yaml.Node { + mapNode := yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + return &mapNode +} + +func GetFieldOrCreateNew(node *yaml.Node, key string, value *yaml.Node) *yaml.Node { + + for i := 0; i < len(node.Content); i += 2 { + curKey := node.Content[i] + curVal := node.Content[i+1] + + if curKey.Value == key { + return curVal + } + } + + node.Content = append(node.Content, NewStringNode(key, 0)) + + node.Content = append(node.Content, value) + return value +} + +func GetDocMapRoot(yamlNode *yaml.Node) *yaml.Node { + if yamlNode.Kind != yaml.DocumentNode { + return nil + } + + if len(yamlNode.Content) == 0 { + return nil + } + + root := yamlNode.Content[0] + if root.Kind != yaml.MappingNode { + return nil + } + + return root +} diff --git a/pkg/utils/yaml_ref_resolver.go b/pkg/utils/yaml_ref_resolver.go new file mode 100644 index 0000000..5a3402d --- /dev/null +++ b/pkg/utils/yaml_ref_resolver.go @@ -0,0 +1,258 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "github.com/go-errors/errors" + "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" + "gopkg.in/yaml.v3" + "net/url" + "path/filepath" + "slices" + "strings" +) + +func YAMLResolveRefs(root *yaml.Node, filePath string, allowCycles bool) (*yaml.Node, error) { + fileDir := filepath.Dir(filePath) + filePath = filepath.Base(filePath) + + PopD := PushDir(fileDir) + defer PopD() + + cycles := [][]string{} + resolved, err := YAMLResolveRefsRecursive(root, "", filePath, []string{}, &map[string]*yaml.Node{}, &cycles) + if err != nil { + return nil, err + } + + if len(cycles) > 0 && allowCycles == false { + var multiError MultiError + for _, cycle := range cycles { + multiError.Errors = append(multiError.Errors, errors.Errorf("cyclic ref at %s", strings.Join(cycle, ":"))) + } + return nil, errors.New(multiError) + } + + return resolved, nil +} + +func YAMLResolveRefsRecursive(node *yaml.Node, relParentPath string, parentFile string, activePaths []string, loaded *map[string]*yaml.Node, cycles *[][]string) (*yaml.Node, error) { + var err error + + if node == nil { + return nil, nil + } + + absFilePath, err := filepath.Abs(parentFile) + if err != nil { + return nil, errors.Errorf("could not process %s:%s. %s", parentFile, relParentPath, err.Error()) + } + + activePath := fmt.Sprintf("%s:%s", absFilePath, relParentPath) + if slices.Contains(activePaths, activePath) { + rootPath, _, _ := strings.Cut(activePaths[0], ":") + + lastPath := activePaths[len(activePaths)-1] + lastFilePath, lastNodePath, _ := strings.Cut(lastPath, ":") + + rel, _ := filepath.Rel(filepath.Dir(rootPath), lastFilePath) + *cycles = append(*cycles, []string{rel, lastNodePath}) + + return MakeCyclicRefPlaceholder(relParentPath), nil + } + + activePaths = append(activePaths, activePath) + + if node.Kind == yaml.MappingNode && isYAMLRef(node) { + jsonRef := getYAMLRefString(node) + + if jsonRef == "" { + return nil, errors.Errorf("JSONRef %s at %s is not valid", jsonRef, parentFile) + } + + refFilePath, refJSONPath, err := SplitJSONRef(jsonRef) + if err != nil { + return nil, errors.Errorf("could not process JSONRef %s at %s. %s", jsonRef, parentFile, err.Error()) + } + + if refFilePath == "" { + refFilePath = parentFile + } + + //do not resolve refs that point back to the main file back + absRefFilePath, _ := filepath.Abs(refFilePath) + rootPath, _, _ := strings.Cut(activePaths[0], ":") + if absRefFilePath == rootPath { + return node, nil + } + + refFileNode, err := loadYAMLFile(refFilePath, loaded) + + if err != nil { + return nil, errors.Errorf("could not process JSONRef %s at %s. %s", jsonRef, parentFile, err.Error()) + } + + yamlNode, err := LocateRef(refFileNode, refJSONPath, jsonRef) + if err != nil { + return nil, errors.Errorf("could not process JSONRef %s at %s, %s", jsonRef, parentFile, err.Error()) + } + + //switch-dir + popd := PushDir(filepath.Dir(refFilePath)) + defer popd() + + resolved, err := YAMLResolveRefsRecursive(yamlNode, refJSONPath, filepath.Base(refFilePath), activePaths, loaded, cycles) + if err != nil { + return nil, err + } + return resolved, nil + + } else if node.Kind == yaml.MappingNode { + for i := 0; i+1 < len(node.Content); i += 2 { + curPath := fmt.Sprintf("%s.%s", relParentPath, node.Content[i].Value) + + resolved, err := YAMLResolveRefsRecursive(node.Content[i+1], curPath, parentFile, activePaths, loaded, cycles) + if err != nil { + return nil, err + } + node.Content[i+1] = resolved + } + return node, nil + } else if node.Kind == yaml.DocumentNode { + resolved, err := YAMLResolveRefsRecursive(node.Content[0], "$", parentFile, activePaths, loaded, cycles) + if err != nil { + return nil, err + } + node.Content[0] = resolved + return node, nil + } else if node.Kind == yaml.SequenceNode { + for i := 0; i < len(node.Content); i += 1 { + curPath := fmt.Sprintf("%s.%d", relParentPath, i) + + resolved, err := YAMLResolveRefsRecursive(node.Content[i], curPath, parentFile, activePaths, loaded, cycles) + if err != nil { + return nil, err + } + node.Content[i] = resolved + } + return node, nil + } + + return node, nil +} + +func isYAMLRef(node *yaml.Node) bool { + if node == nil { + return false + } + + return node.Kind == yaml.MappingNode && slices.IndexFunc(node.Content, func(n *yaml.Node) bool { + return n.Value == "$ref" + }) >= 0 + +} + +func getYAMLRefString(node *yaml.Node) string { + index := slices.IndexFunc(node.Content, func(n *yaml.Node) bool { + return n.Value == "$ref" + }) + if index < 0 { + return "" + } + return node.Content[index+1].Value +} + +func setYAMLRefString(node *yaml.Node, value string) { + index := slices.IndexFunc(node.Content, func(n *yaml.Node) bool { + return n.Value == "$ref" + }) + if index < 0 { + return + } + node.Content[index+1].Value = value +} + +func MakeCyclicRefPlaceholder(refJSONPath string) *yaml.Node { + cyclicRef := &yaml.Node{Kind: yaml.MappingNode} + key := yaml.Node{Kind: yaml.ScalarNode, Style: yaml.DoubleQuotedStyle} + key.SetString("description") + + value := yaml.Node{Kind: yaml.ScalarNode, Style: yaml.DoubleQuotedStyle} + value.SetString(fmt.Sprintf("cyclic JSONRef to %s", refJSONPath)) + + cyclicRef.Content = append(cyclicRef.Content, &key, &value) + return cyclicRef +} + +func LocateRef(refFileNode *yaml.Node, refJSONPath string, jsonRef string) (*yaml.Node, error) { + var err error + var yamlPath *yamlpath.Path + + yamlPath, err = yamlpath.NewPath(refJSONPath) + if err != nil { + return nil, errors.New(err) + } + + var yamlNodes []*yaml.Node + yamlNodes, err = yamlPath.Find(refFileNode) + if err != nil { + return nil, errors.New(err) + } + + if len(yamlNodes) == 0 { + return nil, errors.Errorf("no node found at JSONRef '%s'", jsonRef) + } + + if len(yamlNodes) > 1 { + return nil, errors.Errorf("more than one node found at JSONRef '%s'", jsonRef) + } + return yamlNodes[0], nil +} + +func JSONPointer2JSONPath(jsonPointer string) (jsonPath string, err error) { + + pointer := strings.TrimSpace(jsonPointer) + if pointer == "" || + pointer == "#" || + pointer == "#/" { + return "$", nil + } + + if strings.Index(pointer, "#/") != 0 { + return "", errors.Errorf("relative JSONPointer %s is not supported", jsonPointer) + } + + pointer = "$" + strings.ReplaceAll(pointer[1:], "/", ".") + return pointer, nil +} + +func SplitJSONRef(refStr string) (location string, jsonPath string, err error) { + parsedUrl, err := url.Parse(refStr) + if err != nil { + return "", "", errors.New(err) + } + + if parsedUrl.Scheme != "" { + return "", "", errors.Errorf("JSONRef %s is not supported", refStr) + } + + jsonPath, err = JSONPointer2JSONPath("#" + parsedUrl.Fragment) + if err != nil { + return "", "", err + } + + return parsedUrl.Path, jsonPath, nil +}