Skip to content

Commit

Permalink
Merge pull request #189 from meshery/nithish/refactor/compose_validation
Browse files Browse the repository at this point in the history
[Refactor] Use CUE to validate docker compose files
  • Loading branch information
humblenginr authored Jul 1, 2022
2 parents b21e073 + 203b23c commit b8ea136
Show file tree
Hide file tree
Showing 9 changed files with 176 additions and 88 deletions.
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ require (
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/docker/libcompose v0.4.0 // indirect
github.com/emicklei/proto v1.6.15 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
Expand All @@ -86,6 +87,7 @@ require (
github.com/go-openapi/swag v0.19.14 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/btree v1.0.1 // indirect
Expand Down Expand Up @@ -151,6 +153,7 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.30.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc // indirect
github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
github.com/russross/blackfriday v1.5.2 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
Expand All @@ -169,6 +172,7 @@ require (
go.opencensus.io v0.23.0 // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
golang.org/x/mod v0.5.0 // indirect
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
Expand All @@ -177,6 +181,7 @@ require (
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
google.golang.org/grpc v1.46.0 // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -804,6 +804,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubernetes/kompose v1.26.1 h1:jUpXFobChy2U2uqEuk//l7Ur/HzXi0FTAQKRtHCBLbI=
github.com/kubernetes/kompose v1.26.1/go.mod h1:IJd7R1JvhxPSLPAM3MNPGGuHGprVft//t6OTSt1vY5I=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
Expand Down Expand Up @@ -1038,6 +1039,7 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -1369,6 +1371,7 @@ golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hM
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -1699,6 +1702,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
Expand Down Expand Up @@ -1895,6 +1899,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
Expand Down
2 changes: 1 addition & 1 deletion helpers/component_info.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"name": "meshkit",
"type": "library",
"next_error_code": 11083
"next_error_code": 11085
}
6 changes: 6 additions & 0 deletions models/interfaces.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package models

// anything that can be validated is a Validator
type Validator interface {
Validate([]byte) error
}
74 changes: 74 additions & 0 deletions utils/kubernetes/kompose/composefile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package kompose

import (
"context"
"io"
"os"
"path/filepath"

"cuelang.org/go/cmd/cue/cmd"
"cuelang.org/go/cue/cuecontext"
"cuelang.org/go/encoding/yaml"
"github.com/layer5io/meshkit/utils"
)

type DockerComposeFile []byte

// TODO: This should be dynamic
// once we move to using encoding/jsonschema, this will be removed
var (
DefaultSpecDirectory = filepath.Join(utils.GetHome(), ".meshery", "cue")
DefaultSpecPath = filepath.Join(DefaultSpecDirectory, "compose-spec.cue")
)

func generateCueSchema(jsonSchema []byte, dir string) error {
err := os.MkdirAll(dir, os.ModePerm)
if err != nil {
return ErrValidateDockerComposeFile(err)
}
err = utils.CreateFile(jsonSchema, "compose-spec.json", dir)
if err != nil {
return ErrValidateDockerComposeFile(err)
}
defer os.Remove(filepath.Join(DefaultSpecDirectory, "compose-spec.json"))
//load and parse the schema
// NOTE: this approach makes the process slow and is not recommended
// TODO: use encoding/jsonschema.
importCmd, err := cmd.New([]string{"import", "-f", "--files", filepath.Join(DefaultSpecDirectory, "compose-spec.json")})
if err != nil {
return ErrValidateDockerComposeFile(err)
}
// no need for logs
importCmd.SetOut(io.Discard)
err = importCmd.Run(context.TODO())
if err != nil {
return ErrValidateDockerComposeFile(err)
}
return nil
}

func (dc *DockerComposeFile) Validate(schema []byte) error {
cueCtx := cuecontext.New()
// check if the spec is already present
if _, err := os.Stat(DefaultSpecPath); err != nil {
// if not, create the spec
err = generateCueSchema(schema, DefaultSpecDirectory)
if err != nil {
return ErrValidateDockerComposeFile(err)
}
}

cueSchema, err := utils.ReadLocalFile(DefaultSpecPath)
if err != nil {
return ErrValidateDockerComposeFile(err)
}
sv := cueCtx.CompileString(cueSchema)
if sv.Err() != nil {
return ErrValidateDockerComposeFile(sv.Err())
}
err = yaml.Validate(*dc, sv)
if err != nil {
return ErrValidateDockerComposeFile(err)
}
return nil
}
77 changes: 76 additions & 1 deletion utils/kubernetes/kompose/convert.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package kompose

import (
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"

"github.com/kubernetes/kompose/pkg/app"
Expand All @@ -15,8 +17,28 @@ var (
list = "List"
)

const DefaultDockerComposeSchemaURL = "https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json"

// Checks whether the given manifest is a valid docker-compose file.
// schemaURL is assigned a default url if not specified
// error will be 'nil' if it is a valid docker compose file
func IsManifestADockerCompose(manifest []byte, schemaURL string) error {
if schemaURL == "" {
schemaURL = DefaultDockerComposeSchemaURL
}
schema, err := utils.ReadRemoteFile(schemaURL)
if err != nil {
return err
}
var dockerComposeFile DockerComposeFile
dockerComposeFile = manifest
err = dockerComposeFile.Validate([]byte(schema))
return err
}

// converts a given docker-compose file into kubernetes manifests
func Convert(dockerCompose []byte) (string, error) {
// expects a validated docker-compose file
func Convert(dockerCompose DockerComposeFile) (string, error) {
err := utils.CreateFile(dockerCompose, "temp.data", "./")
if err != nil {
return "", ErrCvrtKompose(err)
Expand All @@ -27,6 +49,12 @@ func Convert(dockerCompose []byte) (string, error) {
os.Remove("result.yaml")
}()

err = formatComposeFile(&dockerCompose)
err = versionCheck(dockerCompose)
if err != nil {
return "", ErrCvrtKompose(err)
}

ConvertOpt := kobject.ConvertOptions{
ToStdout: false,
CreateChart: false, // for helm charts
Expand Down Expand Up @@ -95,3 +123,50 @@ func formatConvertedManifest(k8sMan string) (string, error) {
}
return formattedManifest, nil
}

type composeFile struct {
Version string `yaml:"version,omitempty"`
}

// checks if the version is compatible with `kompose`
// expects a valid docker compose yaml
// error = nil means it is compatible
func versionCheck(dc DockerComposeFile) error {
cf := composeFile{}
err := yaml.Unmarshal(dc, &cf)
if err != nil {
return utils.ErrUnmarshal(err)
}
if cf.Version == "" {
return ErrNoVersion()
}
versionFloatVal, err := strconv.ParseFloat(cf.Version, 64)
if err != nil {
return utils.ErrExpectedTypeMismatch(err, "float")
} else {
if versionFloatVal > 3.3 {
// kompose throws a fatal error when version exceeds 3.3
// need this till this PR gets merged https://github.com/kubernetes/kompose/pull/1440(move away from libcompose to compose-go)
return ErrIncompatibleVersion()
}
}
return nil
}

// formatComposeFile takes in a pointer to the compose file byte array and formats it so that it is compatible with `Kompose`
// it expects a validated docker compose file and does not validate
func formatComposeFile(yamlManifest *DockerComposeFile) error {
data := composeFile{}
err := yaml.Unmarshal(*yamlManifest, &data)
if err != nil {
return utils.ErrUnmarshal(err)
}
// so that "3.3" and 3.3 are treated differently by `Kompose`
data.Version = fmt.Sprintf("%s", data.Version)
out, err := yaml.Marshal(data)
if err != nil {
return utils.ErrMarshal(err)
}
*yamlManifest = out
return nil
}
15 changes: 9 additions & 6 deletions utils/kubernetes/kompose/error.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,22 @@ package kompose
import "github.com/layer5io/meshkit/errors"

const (
ErrCvrtKomposeCode = "11075"
ErrNoVersionCode = "11077"
ErrIncompatibleVersionCode = "11078"
ErrCvrtKomposeCode = "11075"
ErrValidateDockerComposeFileCode = "11084"
ErrIncompatibleVersionCode = "11083"
ErrNoVersionCode = "11077"
)

func ErrCvrtKompose(err error) error {
return errors.New(ErrCvrtKomposeCode, errors.Alert, []string{"Error converting the docker compose file into kubernetes manifests"}, []string{err.Error()}, []string{"Could not convert docker-compose file into kubernetes manifests"}, []string{"Make sure the docker-compose file is valid", ""})
}

func ErrNoVersion() error {
return errors.New(ErrNoVersionCode, errors.Alert, []string{"version not found in the docker compose file"}, []string{"The docker compose file does not have version field in it. The underlying tool that is used for conversion mandates the presence of version field."}, []string{"Since the Docker Compose specification does not mandate the version field from version 3 onwards, most sources do not provide them."}, []string{"Make sure that the compose file has version specified,", "Add any version less than or equal to 3.3 if you cannot get the exact version from the source"})
func ErrValidateDockerComposeFile(err error) error {
return errors.New(ErrValidateDockerComposeFileCode, errors.Alert, []string{"Invalid docker compose file"}, []string{err.Error()}, []string{""}, []string{"Make sure that the compose file is valid,", "Make sure that the schema is valid"})
}

func ErrIncompatibleVersion() error {
return errors.New(ErrIncompatibleVersionCode, errors.Alert, []string{"This version of docker compose file is not compatible."}, []string{"This docker compose file is invalid since it's version is incompatible."}, []string{"docker compose file with version greater than 3.3 is probably being used"}, []string{"Make sure that the compose file has version less than or equal to 3.3,", ""})
}
func ErrNoVersion() error {
return errors.New(ErrNoVersionCode, errors.Alert, []string{"version not found in the docker compose file"}, []string{"The docker compose file does not have version field in it. The underlying tool that is used for conversion mandates the presence of version field."}, []string{"Since the Docker Compose specification does not mandate the version field from version 3 onwards, most sources do not provide them."}, []string{"Make sure that the compose file has version specified,", "Add any version less than or equal to 3.3 if you cannot get the exact version from the source"})
}
10 changes: 0 additions & 10 deletions utils/kubernetes/kompose/models/composefile.go

This file was deleted.

70 changes: 0 additions & 70 deletions utils/kubernetes/kompose/utils.go

This file was deleted.

0 comments on commit b8ea136

Please sign in to comment.