Skip to content

Commit

Permalink
feat: implement remove_oas_extensions template command
Browse files Browse the repository at this point in the history
Also, re-implement the cycle detection, and $refs resolution.
The logic was too convoluted, because a single function tried to
do too many things.
  • Loading branch information
micovery committed May 7, 2024
1 parent 8d08625 commit 4824586
Show file tree
Hide file tree
Showing 25 changed files with 823 additions and 657 deletions.
13 changes: 7 additions & 6 deletions cmd/apigee-go-gen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ package main

import (
"fmt"
v1 "github.com/apigee/apigee-go-gen/pkg/apigee/v1"
"github.com/apigee/apigee-go-gen/pkg/utils"
"github.com/go-errors/errors"
"os"
)
Expand All @@ -25,11 +25,12 @@ func main() {
err := RootCmd.Execute()

if err != nil {
var validationErrors v1.ValidationErrors
isValidationErrors := errors.As(err, &validationErrors)
if isValidationErrors {
for i := 0; i < len(validationErrors.Errors) && i < 10; i++ {
_, _ = fmt.Fprintf(os.Stderr, "Error: %s\n", validationErrors.Errors[i].Error())
var multiErrors utils.MultiError
isMultiErrors := errors.As(err, &multiErrors)

if isMultiErrors {
for i := 0; i < len(multiErrors.Errors) && i < 10; i++ {
_, _ = fmt.Fprintf(os.Stderr, "Error: %s\n", multiErrors.Errors[i].Error())
}
} else {
_, _ = fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error())
Expand Down
1 change: 1 addition & 0 deletions examples/templates/oas3/apiproxy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ Resources:
- Resource:
Type: oas
#{{ os_writefile "./spec.yaml" $.Values.spec_string }}
#{{ remove_oas_extensions "./spec.yaml" }}
Path: ./spec.yaml
- Resource:
Type: properties
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ require (
github.com/getkin/kin-openapi v0.124.0
github.com/go-errors/errors v1.5.1
github.com/gosimple/slug v1.14.0
github.com/pb33f/libopenapi v0.16.2
github.com/pb33f/libopenapi v0.16.5
github.com/spf13/cobra v1.8.0
github.com/stretchr/testify v1.9.0
github.com/vektah/gqlparser/v2 v2.5.11
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
github.com/pb33f/libopenapi v0.16.2 h1:vAZisFmtMLScN68Hu26/T7ClEt5tWg+2HZd1Omw3abA=
github.com/pb33f/libopenapi v0.16.2/go.mod h1:PEXNwvtT4KNdjrwudp5OYnD1ryqK6uJ68aMNyWvoMuc=
github.com/pb33f/libopenapi v0.16.5 h1:jqb/N5nc2zuSUSWDgCXi2vKGxMQfTsWHgtmPWSKQGqc=
github.com/pb33f/libopenapi v0.16.5/go.mod h1:PEXNwvtT4KNdjrwudp5OYnD1ryqK6uJ68aMNyWvoMuc=
github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s=
github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down
2 changes: 1 addition & 1 deletion pkg/apigee/v1/apiproxymodel.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func (a *APIProxyModel) Validate() error {
return nil
}

err := ValidationErrors{Errors: []error{}}
err := utils.MultiError{Errors: []error{}}
path := "Root"
if len(a.UnknownNode) > 0 {
err.Errors = append(err.Errors, &UnknownNodeError{path, a.UnknownNode[0]})
Expand Down
12 changes: 2 additions & 10 deletions pkg/apigee/v1/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,16 +109,8 @@ func Model2BundleZip(model Model, outputZip string) error {

func HydrateResources(model Model, fromDir string) error {
// switch to directory relative to the YAML file so that resource paths are valid
wd, err := os.Getwd()
if err != nil {
return errors.New(err)
}
defer func() { utils.Must(os.Chdir(wd)) }()

err = os.Chdir(fromDir)
if err != nil {
return errors.New(err)
}
popd := utils.PushDir(fromDir)
defer popd()

for _, resource := range model.GetResources().List {
parsedUrl, err := url.Parse(resource.Path)
Expand Down
2 changes: 1 addition & 1 deletion pkg/apigee/v1/sharedflowbundlemodel.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func (a *SharedFlowBundleModel) Validate() error {
return nil
}

err := ValidationErrors{Errors: []error{}}
err := utils.MultiError{Errors: []error{}}
path := "Root"
if len(a.UnknownNode) > 0 {
err.Errors = append(err.Errors, &UnknownNodeError{path, a.UnknownNode[0]})
Expand Down
8 changes: 0 additions & 8 deletions pkg/apigee/v1/unknown.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,3 @@ type UnknownNodeError struct {
func (e *UnknownNodeError) Error() string {
return fmt.Sprintf(`unknown node "%s" found at "%s"`, e.Node.XMLName.Local, e.Location)
}

type ValidationErrors struct {
Errors []error
}

func (e ValidationErrors) Error() string {
return e.Errors[0].Error()
}
9 changes: 9 additions & 0 deletions pkg/common/resources/helper_functions.txt
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,15 @@
You can also use it to dump values to stdout in order to see the contents.
e.g. {{ fmt_printf "url: %%v\n" $url }}

remove_oas_extensions(src string) string
Removes the OpenAPI spec extensions from the file specified by src
The file must already exist in the output directory

This is useful if to make the spec files small within the generated bundles
e.g.
{{ os_writefile "./spec.yaml" $.Values.spec_string }}
{{ remove_oas_extensions "./spec.yaml" }}

Sprig
Template functions from Sprig library
e.g. {{ "Hello World" | upper }}
Expand Down
37 changes: 37 additions & 0 deletions pkg/render/helper_funcs.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package render

import (
"github.com/apigee/apigee-go-gen/pkg/utils"
"os"
"path/filepath"
"strings"
Expand Down Expand Up @@ -76,3 +77,39 @@ func getOSCopyFileFunc(templateFile string, outputFile string, dryRun bool) Help

return _osCopyFileFunc
}

func getRemoveOASExtensions(templateFile string, outputFile string, dryRun bool) HelperFunc {
_removeOASExtensionsFunc := func(args ...any) string {
if len(args) < 1 {
panic("remove_oas_extensions function requires one argument")
}

//both destination and source are the same
dst := args[0].(string)
src := args[0].(string)

if filepath.IsAbs(src) {
panic("remove_oas_extensions src must not be absolute")
}
if strings.Index(src, "..") >= 0 {
panic("remove_oas_extensions src must not use ..")
}

//both destination and source are relative to the output file
dstPath := filepath.Join(filepath.Dir(outputFile), dst)
srcPath := filepath.Join(filepath.Dir(outputFile), src)

if !dryRun {
err := utils.RemoveExtensions(srcPath, dstPath)
if err != nil {
panic(err)
}
} else {
//fmt.Printf(`remove_oas_extensions("%s", "%s")\n`, dstPath, srcPath)
}

return dst
}

return _removeOASExtensionsFunc
}
12 changes: 2 additions & 10 deletions pkg/render/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,16 +108,8 @@ func CreateBundle(model v1.Model, output string, validate bool, dryRun string) (
}

func ResolveYAML(text []byte, filePath string) ([]byte, error) {
wd, err := os.Getwd()
if err != nil {
return nil, errors.New(err)
}
defer func() { utils.Must(os.Chdir(wd)) }()

err = os.Chdir(filepath.Dir(filePath))
if err != nil {
return nil, errors.New(err)
}
popd := utils.PushDir(filepath.Dir(filePath))
defer popd()

yaml, err := utils.Text2YAML(bytes.NewReader(text))
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions pkg/render/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ func CreateTemplate(templateFile string, includeList []string, outputFile string
helperFuncs["os_getenv"] = osGetEnvFunc
helperFuncs["os_getenvs"] = osGetEnvs
helperFuncs["os_copyfile"] = getOSCopyFileFunc(templateFile, outputFile, dryRun)
helperFuncs["remove_oas_extensions"] = getRemoveOASExtensions(templateFile, outputFile, dryRun)
helperFuncs["blank"] = blankFunc
helperFuncs["deref"] = derefFunc
helperFuncs["slug_make"] = slugMakeFunc
Expand Down
137 changes: 137 additions & 0 deletions pkg/utils/extension_remover.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
"github.com/go-errors/errors"
libopenapijson "github.com/pb33f/libopenapi/json"
"gopkg.in/yaml.v3"
"path/filepath"
"strings"
)

func RemoveExtensions(input string, output string) error {
text, err := ReadInputText(input)
if err != nil {
return err
}

//unmarshall input as YAML
var yamlNode *yaml.Node
yamlNode = &yaml.Node{}
err = yaml.Unmarshal(text, yamlNode)
if err != nil {
return errors.New(err)
}

yamlNode, err = RemoveOASExtensions(yamlNode)
if err != nil {
return err
}

//convert back to text
ext := filepath.Ext(output)
if ext == "" {
ext = filepath.Ext(input)
}

//depending on the file extension write output as either JSON or YAML
var outputText []byte
if ext == ".json" {
outputText, err = libopenapijson.YAMLNodeToJSON(yamlNode, " ")
if err != nil {
return errors.New(err)
}
} else {
outputText, err = YAML2Text(UnFlowYAMLNode(yamlNode), 2)
if err != nil {
return err
}
}

return WriteOutputText(output, outputText)
}

func RemoveOASExtensions(root *yaml.Node) (*yaml.Node, error) {
modified, err := RemoveOASExtensionsRecursive(root, "")
if err != nil {
return nil, err
}
return modified, nil
}

func RemoveOASExtensionsRecursive(node *yaml.Node, parentField string) (*yaml.Node, error) {
var err error

if node == nil {
return nil, errors.Errorf("nil node detected")
}

var modifiedNode *yaml.Node

if node.Kind == yaml.MappingNode && isYAMLRef(node) {
if err != nil {
return nil, err
}
return node, nil
} else if node.Kind == yaml.MappingNode {
var newContent []*yaml.Node
for i := 0; i+1 < len(node.Content); i += 2 {
fieldName := node.Content[i].Value
if strings.Index(fieldName, "x-") == 0 &&
!(parentField == "headers" ||
parentField == "properties" ||
parentField == "responses" ||
parentField == "schemas" ||
parentField == "paths" ||
parentField == "variables" ||
parentField == "securitySchemes" ||
parentField == "examples" ||
parentField == "links" ||
parentField == "callbacks" ||
parentField == "requestBodies" ||
parentField == "mapping" ||
parentField == "scopes" ||
parentField == "encoding" ||
parentField == "definitions" ||
parentField == "securityDefinitions" ||
parentField == "parameters") {
continue
}
if modifiedNode, err = RemoveOASExtensionsRecursive(node.Content[i+1], fieldName); err != nil {
return nil, err
}
newContent = append(newContent, node.Content[i], modifiedNode)
}
node.Content = newContent
return node, nil
} else if node.Kind == yaml.DocumentNode {
if modifiedNode, err = RemoveOASExtensionsRecursive(node.Content[0], ""); err != nil {
return nil, err
}
node.Content[0] = modifiedNode
return node, nil
} else if node.Kind == yaml.SequenceNode {
for i := 0; i < len(node.Content); i += 1 {
if modifiedNode, err = RemoveOASExtensionsRecursive(node.Content[i], ""); err != nil {
return nil, err
}
node.Content[i] = modifiedNode
}
return node, nil
}

return node, nil
}
16 changes: 0 additions & 16 deletions pkg/utils/io.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package utils
import (
"fmt"
"github.com/go-errors/errors"
"gopkg.in/yaml.v3"
"io"
"os"
"path/filepath"
Expand Down Expand Up @@ -104,18 +103,3 @@ func ReadInputText(input string) ([]byte, error) {
}
return text, nil
}

func RunWithinDirectory[ResultType *yaml.Node](dir string, operation func() (ResultType, error)) (ResultType, error) {
wd, err := os.Getwd()
if err != nil {
return nil, errors.New(err)
}
defer func() { Must(os.Chdir(wd)) }()

err = os.Chdir(dir)
if err != nil {
return nil, errors.New(err)
}

return operation()
}
Loading

0 comments on commit 4824586

Please sign in to comment.