diff --git a/api/pipeline/template.go b/api/pipeline/template.go index aef79d165..0ff2a54bc 100644 --- a/api/pipeline/template.go +++ b/api/pipeline/template.go @@ -13,7 +13,7 @@ import ( "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry/github" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/internal" "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" diff --git a/api/types/string.go b/api/types/string.go index 33ddacb59..a7637d714 100644 --- a/api/types/string.go +++ b/api/types/string.go @@ -7,8 +7,8 @@ import ( "strconv" "strings" - "github.com/buildkite/yaml" json "github.com/ghodss/yaml" + "gopkg.in/yaml.v3" ) // ToString is a helper function to convert diff --git a/compiler/engine.go b/compiler/engine.go index 31f85852e..9c7ef3496 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -9,7 +9,7 @@ import ( "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/internal" ) diff --git a/compiler/native/clone.go b/compiler/native/clone.go index f6f4ffdbc..179e80db5 100644 --- a/compiler/native/clone.go +++ b/compiler/native/clone.go @@ -3,7 +3,7 @@ package native import ( - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/clone_test.go b/compiler/native/clone_test.go index 6bd92697c..ffee2ceaf 100644 --- a/compiler/native/clone_test.go +++ b/compiler/native/clone_test.go @@ -9,7 +9,7 @@ import ( "github.com/urfave/cli/v2" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) const defaultCloneImage = "target/vela-git-slim:latest" diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 13ad8eba7..a18a76d2b 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -12,14 +12,14 @@ import ( "strings" "time" - yml "github.com/buildkite/yaml" "github.com/hashicorp/go-cleanhttp" "github.com/hashicorp/go-retryablehttp" + yml "gopkg.in/yaml.v3" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index e98b13e2b..35e3d2a17 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -13,16 +13,16 @@ import ( "testing" "time" - yml "github.com/buildkite/yaml" "github.com/gin-gonic/gin" "github.com/google/go-cmp/cmp" "github.com/google/go-github/v65/github" "github.com/urfave/cli/v2" + yml "gopkg.in/yaml.v3" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" "github.com/go-vela/server/internal" ) @@ -1981,6 +1981,203 @@ func TestNative_Compile_StepsandStages(t *testing.T) { } } +func TestNative_Compile_LegacyMergeAnchor(t *testing.T) { + // setup types + set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") + c := cli.NewContext(nil, set, nil) + name := "foo" + author := "author" + event := "push" + number := 1 + + m := &internal.Metadata{ + Database: &internal.Database{ + Driver: "foo", + Host: "foo", + }, + Queue: &internal.Queue{ + Channel: "foo", + Driver: "foo", + Host: "foo", + }, + Source: &internal.Source{ + Driver: "foo", + Host: "foo", + }, + Vela: &internal.Vela{ + Address: "foo", + WebAddress: "foo", + }, + } + + compiler, err := FromCLIContext(c) + if err != nil { + t.Errorf("Creating compiler returned err: %v", err) + } + + compiler.repo = &api.Repo{Name: &author} + compiler.build = &api.Build{Author: &name, Number: &number, Event: &event} + compiler.WithMetadata(m) + + testEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil) + + serviceEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil) + serviceEnv["REGION"] = "dev" + + alphaEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil) + alphaEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo alpha"}) + alphaEnv["HOME"] = "/root" + alphaEnv["SHELL"] = "/bin/sh" + + betaEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil) + betaEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo beta"}) + betaEnv["HOME"] = "/root" + betaEnv["SHELL"] = "/bin/sh" + + gammaEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil) + gammaEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo gamma"}) + gammaEnv["HOME"] = "/root" + gammaEnv["SHELL"] = "/bin/sh" + gammaEnv["REGION"] = "dev" + + want := &pipeline.Build{ + Version: "legacy", + ID: "_author_1", + Metadata: pipeline.Metadata{ + Clone: true, + Template: false, + Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{ + Running: false, + Pending: false, + DefaultBranch: false, + }, + }, + Worker: pipeline.Worker{ + Flavor: "", + Platform: "", + }, + Services: pipeline.ContainerSlice{ + &pipeline.Container{ + ID: "service__author_1_service-a", + Detach: true, + Directory: "", + Environment: serviceEnv, + Image: "postgres", + Name: "service-a", + Number: 1, + Pull: "not_present", + Ports: []string{"5432:5432"}, + }, + }, + Steps: pipeline.ContainerSlice{ + &pipeline.Container{ + ID: "step__author_1_init", + Directory: "/vela/src/foo//author", + Environment: testEnv, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + &pipeline.Container{ + ID: "step__author_1_clone", + Directory: "/vela/src/foo//author", + Environment: testEnv, + Image: defaultCloneImage, + Name: "clone", + Number: 2, + Pull: "not_present", + }, + &pipeline.Container{ + ID: "step__author_1_alpha", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//author", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: alphaEnv, + Image: "alpine:latest", + Name: "alpha", + Number: 3, + Pull: "not_present", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + &pipeline.Container{ + ID: "step__author_1_beta", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//author", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: betaEnv, + Image: "alpine:latest", + Name: "beta", + Number: 4, + Pull: "not_present", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + &pipeline.Container{ + ID: "step__author_1_gamma", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//author", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: gammaEnv, + Image: "alpine:latest", + Name: "gamma", + Number: 5, + Pull: "not_present", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + }, + } + + // run test on legacy version + yaml, err := os.ReadFile("testdata/steps_merge_anchor.yml") + if err != nil { + t.Errorf("Reading yaml file return err: %v", err) + } + + got, _, err := compiler.Compile(context.Background(), yaml) + if err != nil { + t.Errorf("Compile returned err: %v", err) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Compile() mismatch (-want +got):\n%s", diff) + } + + // run test on current version (should fail) + yaml, err = os.ReadFile("../types/yaml/buildkite/testdata/merge_anchor.yml") // has `version: "1"` instead of `version: "legacy"` + if err != nil { + t.Errorf("Reading yaml file return err: %v", err) + } + + got, _, err = compiler.Compile(context.Background(), yaml) + if err == nil { + t.Errorf("Compile should have returned err") + } + + if got != nil { + t.Errorf("Compile is %v, want %v", got, nil) + } +} + // convertResponse converts the build to the ModifyResponse. func convertResponse(build *yaml.Build) (*ModifyResponse, error) { data, err := yml.Marshal(build) @@ -2056,7 +2253,7 @@ func Test_client_modifyConfig(t *testing.T) { Name: "docker", Pull: "always", Parameters: map[string]interface{}{ - "init_options": map[interface{}]interface{}{ + "init_options": map[string]interface{}{ "get_plugins": "true", }, }, @@ -2089,7 +2286,7 @@ func Test_client_modifyConfig(t *testing.T) { Name: "docker", Pull: "always", Parameters: map[string]interface{}{ - "init_options": map[interface{}]interface{}{ + "init_options": map[string]interface{}{ "get_plugins": "true", }, }, diff --git a/compiler/native/environment.go b/compiler/native/environment.go index fae7557d1..7a3643581 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -9,7 +9,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" "github.com/go-vela/server/internal" ) diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index fae1f4552..648ba2f17 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -13,7 +13,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/internal" ) diff --git a/compiler/native/expand.go b/compiler/native/expand.go index 0420338c0..5f89d667c 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -15,7 +15,7 @@ import ( "github.com/go-vela/server/compiler/template/starlark" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index 8702f9c8b..6671d3390 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -17,7 +17,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_ExpandStages(t *testing.T) { @@ -589,7 +589,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { "auth_method": "token", "username": "octocat", "items": []interface{}{ - map[interface{}]interface{}{"path": "docker", "source": "secret/docker"}, + map[string]interface{}{"path": "docker", "source": "secret/docker"}, }, }, }, @@ -610,7 +610,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { "auth_method": "token", "username": "octocat", "items": []interface{}{ - map[interface{}]interface{}{"path": "docker", "source": "secret/docker"}, + map[string]interface{}{"path": "docker", "source": "secret/docker"}, }, }, }, diff --git a/compiler/native/initialize.go b/compiler/native/initialize.go index bbfdb071c..6380278e4 100644 --- a/compiler/native/initialize.go +++ b/compiler/native/initialize.go @@ -3,7 +3,7 @@ package native import ( - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/initialize_test.go b/compiler/native/initialize_test.go index 26c170581..738d388d7 100644 --- a/compiler/native/initialize_test.go +++ b/compiler/native/initialize_test.go @@ -9,7 +9,7 @@ import ( "github.com/urfave/cli/v2" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_InitStage(t *testing.T) { diff --git a/compiler/native/parse.go b/compiler/native/parse.go index 7afeefef3..5d1d58cd6 100644 --- a/compiler/native/parse.go +++ b/compiler/native/parse.go @@ -7,13 +7,12 @@ import ( "io" "os" - "github.com/buildkite/yaml" - "github.com/go-vela/server/compiler/template/native" "github.com/go-vela/server/compiler/template/starlark" typesRaw "github.com/go-vela/server/compiler/types/raw" - types "github.com/go-vela/server/compiler/types/yaml" + types "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" + "github.com/go-vela/server/internal" ) // ParseRaw converts an object to a string. @@ -113,12 +112,9 @@ func (c *client) Parse(v interface{}, pipelineType string, template *types.Templ // ParseBytes converts a byte slice to a yaml configuration. func ParseBytes(data []byte) (*types.Build, []byte, error) { - config := new(types.Build) - - // unmarshal the bytes into the yaml configuration - err := yaml.Unmarshal(data, config) + config, err := internal.ParseYAML(data) if err != nil { - return nil, data, fmt.Errorf("unable to unmarshal yaml: %w", err) + return nil, nil, err } // initializing Environment to prevent nil error diff --git a/compiler/native/parse_test.go b/compiler/native/parse_test.go index fa2635a48..1ac5953b9 100644 --- a/compiler/native/parse_test.go +++ b/compiler/native/parse_test.go @@ -15,7 +15,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) @@ -604,6 +604,84 @@ func TestNative_Parse_Stages(t *testing.T) { } } +func TestNative_Parse_StagesLegacyMergeAnchor(t *testing.T) { + // setup types + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + want := &yaml.Build{ + Version: "legacy", + Metadata: yaml.Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + Environment: raw.StringSliceMap{}, + Stages: yaml.StageSlice{ + &yaml.Stage{ + Name: "install", + Needs: raw.StringSlice{"clone"}, + Steps: yaml.StepSlice{ + &yaml.Step{ + Commands: []string{"./gradlew downloadDependencies"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + }, + }, + }, + &yaml.Stage{ + Name: "test", + Needs: []string{"install", "clone"}, + Steps: yaml.StepSlice{ + &yaml.Step{ + Commands: []string{"./gradlew check"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "test", + Pull: "always", + }, + }, + }, + &yaml.Stage{ + Name: "build", + Needs: []string{"install", "clone"}, + Steps: yaml.StepSlice{ + &yaml.Step{ + Commands: []string{"./gradlew build"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "build", + Pull: "always", + }, + }, + }, + }, + } + + // run test + b, err := os.ReadFile("testdata/stages_merged.yml") + if err != nil { + t.Errorf("Reading file returned err: %v", err) + } + + got, _, err := client.Parse(b, "", new(yaml.Template)) + + if err != nil { + t.Errorf("Parse returned err: %v", err) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Parse() mismatch (-want +got):\n%s", diff) + } +} + func TestNative_Parse_Steps(t *testing.T) { // setup types client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) diff --git a/compiler/native/script.go b/compiler/native/script.go index 03aa093bb..049b01f91 100644 --- a/compiler/native/script.go +++ b/compiler/native/script.go @@ -8,7 +8,7 @@ import ( "fmt" "strings" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) // ScriptStages injects the script for each step in every stage in a yaml configuration. diff --git a/compiler/native/script_test.go b/compiler/native/script_test.go index e12e6a4b8..48f328682 100644 --- a/compiler/native/script_test.go +++ b/compiler/native/script_test.go @@ -10,7 +10,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/urfave/cli/v2" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_ScriptStages(t *testing.T) { diff --git a/compiler/native/substitute.go b/compiler/native/substitute.go index a675e92d5..8e3f44b53 100644 --- a/compiler/native/substitute.go +++ b/compiler/native/substitute.go @@ -6,10 +6,10 @@ import ( "fmt" "strings" - "github.com/buildkite/yaml" "github.com/drone/envsubst" + "gopkg.in/yaml.v3" - types "github.com/go-vela/server/compiler/types/yaml" + types "github.com/go-vela/server/compiler/types/yaml/yaml" ) // SubstituteStages replaces every declared environment diff --git a/compiler/native/substitute_test.go b/compiler/native/substitute_test.go index e8db1565c..df7a2230c 100644 --- a/compiler/native/substitute_test.go +++ b/compiler/native/substitute_test.go @@ -9,7 +9,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/urfave/cli/v2" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func Test_client_SubstituteStages(t *testing.T) { diff --git a/compiler/native/testdata/stages_merged.yml b/compiler/native/testdata/stages_merged.yml new file mode 100644 index 000000000..6bb17c276 --- /dev/null +++ b/compiler/native/testdata/stages_merged.yml @@ -0,0 +1,38 @@ +version: "legacy" + +stages: + install: + steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + + test: + needs: [ install ] + steps: + - name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +stages: + build: + needs: [ install ] + steps: + - name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true \ No newline at end of file diff --git a/compiler/native/testdata/steps_merge_anchor.yml b/compiler/native/testdata/steps_merge_anchor.yml new file mode 100644 index 000000000..1d488edd2 --- /dev/null +++ b/compiler/native/testdata/steps_merge_anchor.yml @@ -0,0 +1,46 @@ +# test file that uses the non-standard multiple anchor keys in one step to test custom step unmarshaler + +version: "legacy" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + postgres: &pg-image + image: postgres + + events: + push: &event-push + ruleset: + event: + - push + env: + dev-env: &dev-environment + environment: + REGION: dev + +services: + - name: service-a + <<: *pg-image + <<: *dev-environment + ports: + - "5432:5432" + +steps: + - name: alpha + <<: *alpine-image + <<: *event-push + commands: + - echo alpha + + - name: beta + <<: [ *alpine-image, *event-push ] + commands: + - echo beta + + - name: gamma + <<: *alpine-image + <<: *event-push + <<: *dev-environment + commands: + - echo gamma \ No newline at end of file diff --git a/compiler/native/transform.go b/compiler/native/transform.go index 2f54c41a3..f9b24bfe1 100644 --- a/compiler/native/transform.go +++ b/compiler/native/transform.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) const ( diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index 14da2b332..df95bcfcf 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -10,7 +10,7 @@ import ( "github.com/urfave/cli/v2" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/internal" ) diff --git a/compiler/native/validate.go b/compiler/native/validate.go index 0c0775019..77a8578f2 100644 --- a/compiler/native/validate.go +++ b/compiler/native/validate.go @@ -7,7 +7,7 @@ import ( "github.com/hashicorp/go-multierror" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/validate_test.go b/compiler/native/validate_test.go index 4d3876f0a..a5771e953 100644 --- a/compiler/native/validate_test.go +++ b/compiler/native/validate_test.go @@ -10,7 +10,7 @@ import ( "github.com/urfave/cli/v2" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_Validate_NoVersion(t *testing.T) { diff --git a/compiler/template/native/convert.go b/compiler/template/native/convert.go index 4a75a5793..5715cecd5 100644 --- a/compiler/template/native/convert.go +++ b/compiler/template/native/convert.go @@ -5,7 +5,7 @@ package native import ( "strings" - "github.com/buildkite/yaml" + "gopkg.in/yaml.v3" "github.com/go-vela/server/compiler/types/raw" ) diff --git a/compiler/template/native/render.go b/compiler/template/native/render.go index 528188e0e..a9313ebdf 100644 --- a/compiler/template/native/render.go +++ b/compiler/template/native/render.go @@ -8,16 +8,15 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" - "github.com/buildkite/yaml" "github.com/go-vela/server/compiler/types/raw" - types "github.com/go-vela/server/compiler/types/yaml" + types "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/internal" ) // Render combines the template with the step in the yaml pipeline. func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}) (*types.Build, error) { buffer := new(bytes.Buffer) - config := new(types.Build) velaFuncs := funcHandler{envs: convertPlatformVars(environment, name)} templateFuncMap := map[string]interface{}{ @@ -47,7 +46,7 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM } // unmarshal the template to the pipeline - err = yaml.Unmarshal(buffer.Bytes(), config) + config, err := internal.ParseYAML(buffer.Bytes()) if err != nil { return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } @@ -63,7 +62,6 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM // RenderBuild renders the templated build. func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}) (*types.Build, error) { buffer := new(bytes.Buffer) - config := new(types.Build) velaFuncs := funcHandler{envs: convertPlatformVars(envs, tmpl)} templateFuncMap := map[string]interface{}{ @@ -93,7 +91,7 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st } // unmarshal the template to the pipeline - err = yaml.Unmarshal(buffer.Bytes(), config) + config, err := internal.ParseYAML(buffer.Bytes()) if err != nil { return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } diff --git a/compiler/template/native/render_test.go b/compiler/template/native/render_test.go index 3b4daacb8..d204f7572 100644 --- a/compiler/template/native/render_test.go +++ b/compiler/template/native/render_test.go @@ -6,11 +6,11 @@ import ( "os" "testing" - goyaml "github.com/buildkite/yaml" "github.com/google/go-cmp/cmp" + goyaml "gopkg.in/yaml.v3" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_Render(t *testing.T) { diff --git a/compiler/template/starlark/render.go b/compiler/template/starlark/render.go index edd822463..1abc52319 100644 --- a/compiler/template/starlark/render.go +++ b/compiler/template/starlark/render.go @@ -7,13 +7,13 @@ import ( "errors" "fmt" - yaml "github.com/buildkite/yaml" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" "go.starlark.net/syntax" "github.com/go-vela/server/compiler/types/raw" - types "github.com/go-vela/server/compiler/types/yaml" + types "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/internal" ) var ( @@ -32,8 +32,6 @@ var ( // Render combines the template with the step in the yaml pipeline. func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}, limit int64) (*types.Build, error) { - config := new(types.Build) - thread := &starlark.Thread{Name: name} if limit < 0 { @@ -124,7 +122,7 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM } // unmarshal the template to the pipeline - err = yaml.Unmarshal(buf.Bytes(), config) + config, err := internal.ParseYAML(buf.Bytes()) if err != nil { return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } @@ -141,8 +139,6 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM // //nolint:lll // ignore function length due to input args func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}, limit int64) (*types.Build, error) { - config := new(types.Build) - thread := &starlark.Thread{Name: "templated-base"} if limit < 0 { @@ -233,7 +229,7 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st } // unmarshal the template to the pipeline - err = yaml.Unmarshal(buf.Bytes(), config) + config, err := internal.ParseYAML(buf.Bytes()) if err != nil { return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } diff --git a/compiler/template/starlark/render_test.go b/compiler/template/starlark/render_test.go index 33db5a7f0..b9d184dae 100644 --- a/compiler/template/starlark/render_test.go +++ b/compiler/template/starlark/render_test.go @@ -6,11 +6,11 @@ import ( "os" "testing" - goyaml "github.com/buildkite/yaml" "github.com/google/go-cmp/cmp" + goyaml "gopkg.in/yaml.v3" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestStarlark_Render(t *testing.T) { diff --git a/compiler/template/template.go b/compiler/template/template.go index a16cc4057..ee503e980 100644 --- a/compiler/template/template.go +++ b/compiler/template/template.go @@ -2,7 +2,7 @@ package template -import "github.com/go-vela/server/compiler/types/yaml" +import "github.com/go-vela/server/compiler/types/yaml/yaml" // Engine represents the interface for Vela integrating // with the different supported template engines. diff --git a/compiler/types/raw/map_test.go b/compiler/types/raw/map_test.go index 0494c8ed4..918664a65 100644 --- a/compiler/types/raw/map_test.go +++ b/compiler/types/raw/map_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/buildkite/yaml" + "gopkg.in/yaml.v3" ) func TestRaw_StringSliceMap_UnmarshalJSON(t *testing.T) { diff --git a/compiler/types/raw/slice_test.go b/compiler/types/raw/slice_test.go index 7a32bd3d7..0ae76f9dc 100644 --- a/compiler/types/raw/slice_test.go +++ b/compiler/types/raw/slice_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "github.com/buildkite/yaml" + "gopkg.in/yaml.v3" ) func TestRaw_StringSlice_UnmarshalJSON(t *testing.T) { diff --git a/compiler/types/yaml/buildkite/build.go b/compiler/types/yaml/buildkite/build.go new file mode 100644 index 000000000..c63c129b4 --- /dev/null +++ b/compiler/types/yaml/buildkite/build.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +// Build is the yaml representation of a build for a pipeline. +type Build struct { + Version string `yaml:"version,omitempty" json:"version,omitempty" jsonschema:"required,minLength=1,description=Provide syntax version used to evaluate the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/version/"` + Metadata Metadata `yaml:"metadata,omitempty" json:"metadata,omitempty" jsonschema:"description=Pass extra information.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/"` + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Provide global environment variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-environment-key"` + Worker Worker `yaml:"worker,omitempty" json:"worker,omitempty" jsonschema:"description=Limit the pipeline to certain types of workers.\nReference: https://go-vela.github.io/docs/reference/yaml/worker/"` + Secrets SecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Provide sensitive information.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/"` + Services ServiceSlice `yaml:"services,omitempty" json:"services,omitempty" jsonschema:"description=Provide detached (headless) execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/services/"` + Stages StageSlice `yaml:"stages,omitempty" json:"stages,omitempty" jsonschema:"oneof_required=stages,description=Provide parallel execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/"` + Steps StepSlice `yaml:"steps,omitempty" json:"steps,omitempty" jsonschema:"oneof_required=steps,description=Provide sequential execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/"` + Templates TemplateSlice `yaml:"templates,omitempty" json:"templates,omitempty" jsonschema:"description=Provide the name of templates to expand.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/"` +} + +// ToPipelineAPI converts the Build type to an API Pipeline type. +func (b *Build) ToPipelineAPI() *api.Pipeline { + pipeline := new(api.Pipeline) + + pipeline.SetFlavor(b.Worker.Flavor) + pipeline.SetPlatform(b.Worker.Platform) + pipeline.SetVersion(b.Version) + pipeline.SetServices(len(b.Services) > 0) + pipeline.SetStages(len(b.Stages) > 0) + pipeline.SetSteps(len(b.Steps) > 0) + pipeline.SetTemplates(len(b.Templates) > 0) + + // set default for external and internal secrets + external := false + internal := false + + // iterate through all secrets in the build + for _, secret := range b.Secrets { + // check if external and internal secrets have been found + if external && internal { + // exit the loop since both secrets have been found + break + } + + // check if the secret origin is empty + if secret.Origin.Empty() { + // origin was empty so an internal secret was found + internal = true + } else { + // origin was not empty so an external secret was found + external = true + } + } + + pipeline.SetExternalSecrets(external) + pipeline.SetInternalSecrets(internal) + + return pipeline +} + +// UnmarshalYAML implements the Unmarshaler interface for the Build type. +func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { + // build we try unmarshalling to + build := new(struct { + Version string + Metadata Metadata + Environment raw.StringSliceMap + Worker Worker + Secrets SecretSlice + Services ServiceSlice + Stages StageSlice + Steps StepSlice + Templates TemplateSlice + }) + + // attempt to unmarshal as a build type + err := unmarshal(build) + if err != nil { + return err + } + + // give the documented default value to metadata environment + if build.Metadata.Environment == nil { + build.Metadata.Environment = []string{"steps", "services", "secrets"} + } + + // override the values + b.Version = build.Version + b.Metadata = build.Metadata + b.Environment = build.Environment + b.Worker = build.Worker + b.Secrets = build.Secrets + b.Services = build.Services + b.Stages = build.Stages + b.Steps = build.Steps + b.Templates = build.Templates + + return nil +} + +func (b *Build) ToYAML() *yaml.Build { + return &yaml.Build{ + Version: b.Version, + Metadata: *b.Metadata.ToYAML(), + Environment: b.Environment, + Worker: *b.Worker.ToYAML(), + Secrets: *b.Secrets.ToYAML(), + Services: *b.Services.ToYAML(), + Stages: *b.Stages.ToYAML(), + Steps: *b.Steps.ToYAML(), + Templates: *b.Templates.ToYAML(), + } +} diff --git a/compiler/types/yaml/build_test.go b/compiler/types/yaml/buildkite/build_test.go similarity index 99% rename from compiler/types/yaml/build_test.go rename to compiler/types/yaml/buildkite/build_test.go index fc72abc85..1dd1d2a05 100644 --- a/compiler/types/yaml/build_test.go +++ b/compiler/types/yaml/buildkite/build_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/doc.go b/compiler/types/yaml/buildkite/doc.go new file mode 100644 index 000000000..07e33c5a4 --- /dev/null +++ b/compiler/types/yaml/buildkite/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +// package buildkite provides the defined yaml types for Vela. +// +// Usage: +// +// import "github.com/go-vela/server/compiler/types/yaml/yaml" +package buildkite diff --git a/compiler/types/yaml/buildkite/metadata.go b/compiler/types/yaml/buildkite/metadata.go new file mode 100644 index 000000000..a57d83273 --- /dev/null +++ b/compiler/types/yaml/buildkite/metadata.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // Metadata is the yaml representation of + // the metadata block for a pipeline. + Metadata struct { + Template bool `yaml:"template,omitempty" json:"template,omitempty" jsonschema:"description=Enables compiling the pipeline as a template.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-template-key"` + RenderInline bool `yaml:"render_inline,omitempty" json:"render_inline,omitempty" jsonschema:"description=Enables inline compiling for the pipeline templates.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-render-inline-key"` + Clone *bool `yaml:"clone,omitempty" json:"clone,omitempty" jsonschema:"default=true,description=Enables injecting the default clone process.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-clone-key"` + Environment []string `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Controls which containers processes can have global env injected.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-environment-key"` + AutoCancel *CancelOptions `yaml:"auto_cancel,omitempty" json:"auto_cancel,omitempty" jsonschema:"description=Enables auto canceling of queued or running pipelines that become stale due to new push.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-auto-cancel-key"` + } + + // CancelOptions is the yaml representation of + // the auto_cancel block for a pipeline. + CancelOptions struct { + Running *bool `yaml:"running,omitempty" json:"running,omitempty" jsonschema:"description=Enables auto canceling of running pipelines that become stale due to new push.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-auto-cancel-key"` + Pending *bool `yaml:"pending,omitempty" json:"pending,omitempty" jsonschema:"description=Enables auto canceling of queued pipelines that become stale due to new push.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-auto-cancel-key"` + DefaultBranch *bool `yaml:"default_branch,omitempty" json:"default_branch,omitempty" jsonschema:"description=Enables auto canceling of queued or running pipelines that become stale due to new push to default branch.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-auto-cancel-key"` + } +) + +// ToPipeline converts the Metadata type +// to a pipeline Metadata type. +func (m *Metadata) ToPipeline() *pipeline.Metadata { + var clone bool + if m.Clone == nil { + clone = true + } else { + clone = *m.Clone + } + + autoCancel := new(pipeline.CancelOptions) + + // default to false for all fields if block isn't found + if m.AutoCancel == nil { + autoCancel.Pending = false + autoCancel.Running = false + autoCancel.DefaultBranch = false + } else { + // if block is found but pending field isn't, default to true + if m.AutoCancel.Pending != nil { + autoCancel.Pending = *m.AutoCancel.Pending + } else { + autoCancel.Pending = true + } + + if m.AutoCancel.Running != nil { + autoCancel.Running = *m.AutoCancel.Running + } + + if m.AutoCancel.DefaultBranch != nil { + autoCancel.DefaultBranch = *m.AutoCancel.DefaultBranch + } + } + + return &pipeline.Metadata{ + Template: m.Template, + Clone: clone, + Environment: m.Environment, + AutoCancel: autoCancel, + } +} + +// HasEnvironment checks if the container type +// is contained within the environment list. +func (m *Metadata) HasEnvironment(container string) bool { + for _, e := range m.Environment { + if e == container { + return true + } + } + + return false +} + +func (m *Metadata) ToYAML() *yaml.Metadata { + if m == nil { + return nil + } + + return &yaml.Metadata{ + Template: m.Template, + RenderInline: m.RenderInline, + Clone: m.Clone, + Environment: m.Environment, + AutoCancel: m.AutoCancel.ToYAML(), + } +} + +func (ac *CancelOptions) ToYAML() *yaml.CancelOptions { + if ac == nil { + return nil + } + + return &yaml.CancelOptions{ + Pending: ac.Pending, + Running: ac.Running, + DefaultBranch: ac.DefaultBranch, + } +} diff --git a/compiler/types/yaml/buildkite/metadata_test.go b/compiler/types/yaml/buildkite/metadata_test.go new file mode 100644 index 000000000..a3c92bec8 --- /dev/null +++ b/compiler/types/yaml/buildkite/metadata_test.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "reflect" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Metadata_ToPipeline(t *testing.T) { + tBool := true + fBool := false + // setup tests + tests := []struct { + metadata *Metadata + want *pipeline.Metadata + }{ + { + metadata: &Metadata{ + Template: false, + Clone: &fBool, + Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &CancelOptions{ + Pending: &tBool, + Running: &tBool, + DefaultBranch: &fBool, + }, + }, + want: &pipeline.Metadata{ + Template: false, + Clone: false, + Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{ + Pending: true, + Running: true, + DefaultBranch: false, + }, + }, + }, + { + metadata: &Metadata{ + Template: false, + Clone: &tBool, + Environment: []string{"steps", "services"}, + }, + want: &pipeline.Metadata{ + Template: false, + Clone: true, + Environment: []string{"steps", "services"}, + AutoCancel: &pipeline.CancelOptions{ + Pending: false, + Running: false, + DefaultBranch: false, + }, + }, + }, + { + metadata: &Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps"}, + AutoCancel: &CancelOptions{ + Running: &tBool, + DefaultBranch: &tBool, + }, + }, + want: &pipeline.Metadata{ + Template: false, + Clone: true, + Environment: []string{"steps"}, + AutoCancel: &pipeline.CancelOptions{ + Pending: true, + Running: true, + DefaultBranch: true, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.metadata.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Metadata_HasEnvironment(t *testing.T) { + // setup tests + tests := []struct { + metadata *Metadata + container string + want bool + }{ + { + metadata: &Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + container: "steps", + want: true, + }, + { + metadata: &Metadata{ + Environment: []string{"services", "secrets"}, + }, + container: "services", + want: true, + }, + { + metadata: &Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + container: "notacontainer", + want: false, + }, + } + + // run tests + for _, test := range tests { + got := test.metadata.HasEnvironment(test.container) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/buildkite/ruleset.go b/compiler/types/yaml/buildkite/ruleset.go new file mode 100644 index 000000000..1b56e3062 --- /dev/null +++ b/compiler/types/yaml/buildkite/ruleset.go @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/constants" +) + +type ( + // Ruleset is the yaml representation of a + // ruleset block for a step in a pipeline. + Ruleset struct { + If Rules `yaml:"if,omitempty" json:"if,omitempty" jsonschema:"description=Limit execution to when all rules match.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Unless Rules `yaml:"unless,omitempty" json:"unless,omitempty" jsonschema:"description=Limit execution to when all rules do not match.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Matcher string `yaml:"matcher,omitempty" json:"matcher,omitempty" jsonschema:"enum=filepath,enum=regexp,default=filepath,description=Use the defined matching method.\nReference: coming soon"` + Operator string `yaml:"operator,omitempty" json:"operator,omitempty" jsonschema:"enum=or,enum=and,default=and,description=Whether all rule conditions must be met or just any one of them.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Continue bool `yaml:"continue,omitempty" json:"continue,omitempty" jsonschema:"default=false,description=Limits the execution of a step to continuing on any failure.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + } + + // Rules is the yaml representation of the ruletypes + // from a ruleset block for a step in a pipeline. + Rules struct { + Branch []string `yaml:"branch,omitempty,flow" json:"branch,omitempty" jsonschema:"description=Limits the execution of a step to matching build branches.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Comment []string `yaml:"comment,omitempty,flow" json:"comment,omitempty" jsonschema:"description=Limits the execution of a step to matching a pull request comment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Event []string `yaml:"event,omitempty,flow" json:"event,omitempty" jsonschema:"description=Limits the execution of a step to matching build events.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Path []string `yaml:"path,omitempty,flow" json:"path,omitempty" jsonschema:"description=Limits the execution of a step to matching files changed in a repository.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Repo []string `yaml:"repo,omitempty,flow" json:"repo,omitempty" jsonschema:"description=Limits the execution of a step to matching repos.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Sender []string `yaml:"sender,omitempty,flow" json:"sender,omitempty" jsonschema:"description=Limits the execution of a step to matching build senders.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Status []string `yaml:"status,omitempty,flow" json:"status,omitempty" jsonschema:"enum=[failure],enum=[success],description=Limits the execution of a step to matching build statuses.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Tag []string `yaml:"tag,omitempty,flow" json:"tag,omitempty" jsonschema:"description=Limits the execution of a step to matching build tag references.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Target []string `yaml:"target,omitempty,flow" json:"target,omitempty" jsonschema:"description=Limits the execution of a step to matching build deployment targets.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Label []string `yaml:"label,omitempty,flow" json:"label,omitempty" jsonschema:"description=Limits step execution to match on pull requests labels.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Instance []string `yaml:"instance,omitempty,flow" json:"instance,omitempty" jsonschema:"description=Limits step execution to match on certain instances.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + } +) + +// ToPipeline converts the Ruleset type +// to a pipeline Ruleset type. +func (r *Ruleset) ToPipeline() *pipeline.Ruleset { + return &pipeline.Ruleset{ + If: *r.If.ToPipeline(), + Unless: *r.Unless.ToPipeline(), + Matcher: r.Matcher, + Operator: r.Operator, + Continue: r.Continue, + } +} + +// UnmarshalYAML implements the Unmarshaler interface for the Ruleset type. +func (r *Ruleset) UnmarshalYAML(unmarshal func(interface{}) error) error { + // simple struct we try unmarshalling to + simple := new(Rules) + + // advanced struct we try unmarshalling to + advanced := new(struct { + If Rules + Unless Rules + Matcher string + Operator string + Continue bool + }) + + // attempt to unmarshal simple ruleset + //nolint:errcheck // intentionally not handling error + unmarshal(simple) + // attempt to unmarshal advanced ruleset + //nolint:errcheck // intentionally not handling error + unmarshal(advanced) + + // set ruleset `unless` to advanced `unless` rules + r.Unless = advanced.Unless + // set ruleset `matcher` to advanced `matcher` + r.Matcher = advanced.Matcher + // set ruleset `operator` to advanced `operator` + r.Operator = advanced.Operator + // set ruleset `continue` to advanced `continue` + r.Continue = advanced.Continue + + // implicitly add simple ruleset to the advanced ruleset for each rule type + advanced.If.Branch = append(advanced.If.Branch, simple.Branch...) + advanced.If.Comment = append(advanced.If.Comment, simple.Comment...) + advanced.If.Event = append(advanced.If.Event, simple.Event...) + advanced.If.Path = append(advanced.If.Path, simple.Path...) + advanced.If.Repo = append(advanced.If.Repo, simple.Repo...) + advanced.If.Sender = append(advanced.If.Sender, simple.Sender...) + advanced.If.Status = append(advanced.If.Status, simple.Status...) + advanced.If.Tag = append(advanced.If.Tag, simple.Tag...) + advanced.If.Target = append(advanced.If.Target, simple.Target...) + advanced.If.Label = append(advanced.If.Label, simple.Label...) + advanced.If.Instance = append(advanced.If.Instance, simple.Instance...) + + // set ruleset `if` to advanced `if` rules + r.If = advanced.If + + // implicitly set `matcher` field if empty for ruleset + if len(r.Matcher) == 0 { + r.Matcher = constants.MatcherFilepath + } + + // implicitly set `operator` field if empty for ruleset + if len(r.Operator) == 0 { + r.Operator = constants.OperatorAnd + } + + return nil +} + +// ToPipeline converts the Rules +// type to a pipeline Rules type. +func (r *Rules) ToPipeline() *pipeline.Rules { + return &pipeline.Rules{ + Branch: r.Branch, + Comment: r.Comment, + Event: r.Event, + Path: r.Path, + Repo: r.Repo, + Sender: r.Sender, + Status: r.Status, + Tag: r.Tag, + Target: r.Target, + Label: r.Label, + Instance: r.Instance, + } +} + +// UnmarshalYAML implements the Unmarshaler interface for the Rules type. +func (r *Rules) UnmarshalYAML(unmarshal func(interface{}) error) error { + // rules struct we try unmarshalling to + rules := new(struct { + Branch raw.StringSlice + Comment raw.StringSlice + Event raw.StringSlice + Path raw.StringSlice + Repo raw.StringSlice + Sender raw.StringSlice + Status raw.StringSlice + Tag raw.StringSlice + Target raw.StringSlice + Label raw.StringSlice + Instance raw.StringSlice + }) + + // attempt to unmarshal rules + err := unmarshal(rules) + if err == nil { + r.Branch = rules.Branch + r.Comment = rules.Comment + r.Path = rules.Path + r.Repo = rules.Repo + r.Sender = rules.Sender + r.Status = rules.Status + r.Tag = rules.Tag + r.Target = rules.Target + r.Label = rules.Label + r.Instance = rules.Instance + + // account for users who use non-scoped pull_request event + events := []string{} + + for _, e := range rules.Event { + switch e { + // backwards compatibility + // pull_request = pull_request:opened + pull_request:synchronize + pull_request:reopened + // comment = comment:created + comment:edited + case constants.EventPull: + events = append(events, + constants.EventPull+":"+constants.ActionOpened, + constants.EventPull+":"+constants.ActionSynchronize, + constants.EventPull+":"+constants.ActionReopened) + case constants.EventDeploy: + events = append(events, + constants.EventDeploy+":"+constants.ActionCreated) + case constants.EventComment: + events = append(events, + constants.EventComment+":"+constants.ActionCreated, + constants.EventComment+":"+constants.ActionEdited) + default: + events = append(events, e) + } + } + + r.Event = events + } + + return err +} + +func (r *Ruleset) ToYAML() *yaml.Ruleset { + if r == nil { + return nil + } + + return &yaml.Ruleset{ + If: *r.If.ToYAML(), + Unless: *r.Unless.ToYAML(), + Matcher: r.Matcher, + Operator: r.Operator, + Continue: r.Continue, + } +} + +func (r *Rules) ToYAML() *yaml.Rules { + if r == nil { + return nil + } + + return &yaml.Rules{ + Branch: r.Branch, + Comment: r.Comment, + Event: r.Event, + Path: r.Path, + Repo: r.Repo, + Sender: r.Sender, + Status: r.Status, + Tag: r.Tag, + Target: r.Target, + Label: r.Label, + Instance: r.Instance, + } +} diff --git a/compiler/types/yaml/ruleset_test.go b/compiler/types/yaml/buildkite/ruleset_test.go similarity index 99% rename from compiler/types/yaml/ruleset_test.go rename to compiler/types/yaml/buildkite/ruleset_test.go index 6275046b0..adc926c9a 100644 --- a/compiler/types/yaml/ruleset_test.go +++ b/compiler/types/yaml/buildkite/ruleset_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/secret.go b/compiler/types/yaml/buildkite/secret.go new file mode 100644 index 000000000..b6ea8d71a --- /dev/null +++ b/compiler/types/yaml/buildkite/secret.go @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "errors" + "fmt" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/constants" +) + +type ( + // SecretSlice is the yaml representation + // of the secrets block for a pipeline. + SecretSlice []*Secret + + // Secret is the yaml representation of a secret + // from the secrets block for a pipeline. + Secret struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Name of secret to reference in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-name-key"` + Key string `yaml:"key,omitempty" json:"key,omitempty" jsonschema:"minLength=1,description=Path to secret to fetch from storage backend.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-key-key"` + Engine string `yaml:"engine,omitempty" json:"engine,omitempty" jsonschema:"enum=native,enum=vault,default=native,description=Name of storage backend to fetch secret from.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-engine-key"` + Type string `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"enum=repo,enum=org,enum=shared,default=repo,description=Type of secret to fetch from storage backend.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-type-key"` + Origin Origin `yaml:"origin,omitempty" json:"origin,omitempty" jsonschema:"description=Declaration to pull secrets from non-internal secret providers.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-origin-key"` + Pull string `yaml:"pull,omitempty" json:"pull,omitempty" jsonschema:"enum=step_start,enum=build_start,default=build_start,description=When to pull in secrets from storage backend.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-pull-key"` + } + + // Origin is the yaml representation of a method + // for looking up secrets with a secret plugin. + Origin struct { + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Variables to inject into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-environment-key"` + Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"required,minLength=1,description=Docker image to use to create the ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-image-key"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique name for the secret origin.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-name-key"` + Parameters map[string]interface{} `yaml:"parameters,omitempty" json:"parameters,omitempty" jsonschema:"description=Extra configuration variables for the secret plugin.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-parameters-key"` + Secrets StepSecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Secrets to inject that are necessary to retrieve the secrets.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-secrets-key"` + Pull string `yaml:"pull,omitempty" json:"pull,omitempty" jsonschema:"enum=always,enum=not_present,enum=on_start,enum=never,default=not_present,description=Declaration to configure if and when the Docker image is pulled.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-pull-key"` + Ruleset Ruleset `yaml:"ruleset,omitempty" json:"ruleset,omitempty" jsonschema:"description=Conditions to limit the execution of the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + } +) + +// ToPipeline converts the SecretSlice type +// to a pipeline SecretSlice type. +func (s *SecretSlice) ToPipeline() *pipeline.SecretSlice { + // secret slice we want to return + secretSlice := new(pipeline.SecretSlice) + + // iterate through each element in the secret slice + for _, secret := range *s { + // append the element to the pipeline secret slice + *secretSlice = append(*secretSlice, &pipeline.Secret{ + Name: secret.Name, + Key: secret.Key, + Engine: secret.Engine, + Type: secret.Type, + Origin: secret.Origin.ToPipeline(), + Pull: secret.Pull, + }) + } + + return secretSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the SecretSlice type. +func (s *SecretSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // secret slice we try unmarshalling to + secretSlice := new([]*Secret) + + // attempt to unmarshal as a secret slice type + err := unmarshal(secretSlice) + if err != nil { + return err + } + + tmp := SecretSlice{} + + // iterate through each element in the secret slice + for _, secret := range *secretSlice { + if secret.Origin.Empty() && len(secret.Name) == 0 { + continue + } + + if secret.Origin.Empty() && len(secret.Key) == 0 { + secret.Key = secret.Name + } + + // implicitly set `engine` field if empty + if secret.Origin.Empty() && len(secret.Engine) == 0 { + secret.Engine = constants.DriverNative + } + + // implicitly set `type` field if empty + if secret.Origin.Empty() && len(secret.Type) == 0 { + secret.Type = constants.SecretRepo + } + + // implicitly set `type` field if empty + if secret.Origin.Empty() && len(secret.Pull) == 0 { + secret.Pull = constants.SecretPullBuild + } + + // implicitly set `pull` field if empty + if !secret.Origin.Empty() && len(secret.Origin.Pull) == 0 { + secret.Origin.Pull = constants.PullNotPresent + } + + // TODO: remove this in a future release + // + // handle true deprecated pull policy + // + // a `true` pull policy equates to `always` + if !secret.Origin.Empty() && strings.EqualFold(secret.Origin.Pull, "true") { + secret.Origin.Pull = constants.PullAlways + } + + // TODO: remove this in a future release + // + // handle false deprecated pull policy + // + // a `false` pull policy equates to `not_present` + if !secret.Origin.Empty() && strings.EqualFold(secret.Origin.Pull, "false") { + secret.Origin.Pull = constants.PullNotPresent + } + + tmp = append(tmp, secret) + } + + // overwrite existing SecretSlice + *s = tmp + + return nil +} + +// Empty returns true if the provided origin is empty. +func (o *Origin) Empty() bool { + // return true if the origin is nil + if o == nil { + return true + } + + // return true if every origin field is empty + if o.Environment == nil && + len(o.Image) == 0 && + len(o.Name) == 0 && + o.Parameters == nil && + len(o.Secrets) == 0 && + len(o.Pull) == 0 { + return true + } + + return false +} + +// MergeEnv takes a list of environment variables and attempts +// to set them in the secret environment. If the environment +// variable already exists in the secret, than this will +// overwrite the existing environment variable. +func (o *Origin) MergeEnv(environment map[string]string) error { + // check if the secret container is empty + if o.Empty() { + // TODO: evaluate if we should error here + // + // immediately return and do nothing + // + // treated as a no-op + return nil + } + + // check if the environment provided is empty + if environment == nil { + return fmt.Errorf("empty environment provided for secret %s", o.Name) + } + + // iterate through all environment variables provided + for key, value := range environment { + // set or update the secret environment variable + o.Environment[key] = value + } + + return nil +} + +// ToPipeline converts the Origin type +// to a pipeline Container type. +func (o *Origin) ToPipeline() *pipeline.Container { + return &pipeline.Container{ + Environment: o.Environment, + Image: o.Image, + Name: o.Name, + Pull: o.Pull, + Ruleset: *o.Ruleset.ToPipeline(), + Secrets: *o.Secrets.ToPipeline(), + } +} + +type ( + // StepSecretSlice is the yaml representation of + // the secrets block for a step in a pipeline. + StepSecretSlice []*StepSecret + + // StepSecret is the yaml representation of a secret + // from a secrets block for a step in a pipeline. + StepSecret struct { + Source string `yaml:"source,omitempty"` + Target string `yaml:"target,omitempty"` + } +) + +// ToPipeline converts the StepSecretSlice type +// to a pipeline StepSecretSlice type. +func (s *StepSecretSlice) ToPipeline() *pipeline.StepSecretSlice { + // step secret slice we want to return + secretSlice := new(pipeline.StepSecretSlice) + + // iterate through each element in the step secret slice + for _, secret := range *s { + // append the element to the pipeline step secret slice + *secretSlice = append(*secretSlice, &pipeline.StepSecret{ + Source: secret.Source, + Target: secret.Target, + }) + } + + return secretSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the StepSecretSlice type. +func (s *StepSecretSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // string slice we try unmarshalling to + stringSlice := new(raw.StringSlice) + + // attempt to unmarshal as a string slice type + err := unmarshal(stringSlice) + if err == nil { + // iterate through each element in the string slice + for _, secret := range *stringSlice { + // append the element to the step secret slice + *s = append(*s, &StepSecret{ + Source: secret, + Target: strings.ToUpper(secret), + }) + } + + return nil + } + + // step secret slice we try unmarshalling to + secrets := new([]*StepSecret) + + // attempt to unmarshal as a step secret slice type + err = unmarshal(secrets) + if err == nil { + // check for secret source and target + for _, secret := range *secrets { + if len(secret.Source) == 0 || len(secret.Target) == 0 { + return fmt.Errorf("no secret source or target found") + } + + secret.Target = strings.ToUpper(secret.Target) + } + + // overwrite existing StepSecretSlice + *s = StepSecretSlice(*secrets) + + return nil + } + + return errors.New("failed to unmarshal StepSecretSlice") +} + +func (s *StepSecretSlice) ToYAML() *yaml.StepSecretSlice { + // step secret slice we want to return + secretSlice := new(yaml.StepSecretSlice) + + // iterate through each element in the step secret slice + for _, secret := range *s { + // append the element to the yaml step secret slice + *secretSlice = append(*secretSlice, &yaml.StepSecret{ + Source: secret.Source, + Target: secret.Target, + }) + } + + return secretSlice +} + +func (o *Origin) ToYAML() yaml.Origin { + return yaml.Origin{ + Environment: o.Environment, + Image: o.Image, + Name: o.Name, + Parameters: o.Parameters, + Secrets: *o.Secrets.ToYAML(), + Pull: o.Pull, + Ruleset: *o.Ruleset.ToYAML(), + } +} + +func (s *Secret) ToYAML() *yaml.Secret { + if s == nil { + return nil + } + + return &yaml.Secret{ + Name: s.Name, + Key: s.Key, + Engine: s.Engine, + Type: s.Type, + Origin: s.Origin.ToYAML(), + Pull: s.Pull, + } +} + +func (s *SecretSlice) ToYAML() *yaml.SecretSlice { + // secret slice we want to return + secretSlice := new(yaml.SecretSlice) + + // iterate through each element in the secret slice + for _, secret := range *s { + // append the element to the yaml secret slice + *secretSlice = append(*secretSlice, secret.ToYAML()) + } + + return secretSlice +} diff --git a/compiler/types/yaml/secret_test.go b/compiler/types/yaml/buildkite/secret_test.go similarity index 99% rename from compiler/types/yaml/secret_test.go rename to compiler/types/yaml/buildkite/secret_test.go index 68a9ed4d6..1d731e477 100644 --- a/compiler/types/yaml/secret_test.go +++ b/compiler/types/yaml/buildkite/secret_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/service.go b/compiler/types/yaml/buildkite/service.go new file mode 100644 index 000000000..9fb9a0ed1 --- /dev/null +++ b/compiler/types/yaml/buildkite/service.go @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "fmt" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/constants" +) + +type ( + // ServiceSlice is the yaml representation + // of the Services block for a pipeline. + ServiceSlice []*Service + + // Service is the yaml representation + // of a Service in a pipeline. + Service struct { + Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"required,minLength=1,description=Docker image used to create ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-image-key"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique identifier for the container in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-name-key"` + Entrypoint raw.StringSlice `yaml:"entrypoint,omitempty" json:"entrypoint,omitempty" jsonschema:"description=Commands to execute inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-entrypoint-key"` + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Variables to inject into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-environment-key"` + Ports raw.StringSlice `yaml:"ports,omitempty" json:"ports,omitempty" jsonschema:"description=List of ports to map for the container in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-ports-key"` + Pull string `yaml:"pull,omitempty" json:"pull,omitempty" jsonschema:"enum=always,enum=not_present,enum=on_start,enum=never,default=not_present,description=Declaration to configure if and when the Docker image is pulled.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-pul-key"` + Ulimits UlimitSlice `yaml:"ulimits,omitempty" json:"ulimits,omitempty" jsonschema:"description=Set the user limits for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-ulimits-key"` + User string `yaml:"user,omitempty" json:"user,omitempty" jsonschema:"description=Set the user for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-user-key"` + } +) + +// ToPipeline converts the ServiceSlice type +// to a pipeline ContainerSlice type. +func (s *ServiceSlice) ToPipeline() *pipeline.ContainerSlice { + // service slice we want to return + serviceSlice := new(pipeline.ContainerSlice) + + // iterate through each element in the service slice + for _, service := range *s { + // append the element to the pipeline container slice + *serviceSlice = append(*serviceSlice, &pipeline.Container{ + Detach: true, + Image: service.Image, + Name: service.Name, + Entrypoint: service.Entrypoint, + Environment: service.Environment, + Ports: service.Ports, + Pull: service.Pull, + Ulimits: *service.Ulimits.ToPipeline(), + User: service.User, + }) + } + + return serviceSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the ServiceSlice type. +func (s *ServiceSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // service slice we try unmarshalling to + serviceSlice := new([]*Service) + + // attempt to unmarshal as a service slice type + err := unmarshal(serviceSlice) + if err != nil { + return err + } + + // iterate through each element in the service slice + for _, service := range *serviceSlice { + // handle nil service to avoid panic + if service == nil { + return fmt.Errorf("invalid service with nil content found") + } + + // implicitly set `pull` field if empty + if len(service.Pull) == 0 { + service.Pull = constants.PullNotPresent + } + + // TODO: remove this in a future release + // + // handle true deprecated pull policy + // + // a `true` pull policy equates to `always` + if strings.EqualFold(service.Pull, "true") { + service.Pull = constants.PullAlways + } + + // TODO: remove this in a future release + // + // handle false deprecated pull policy + // + // a `false` pull policy equates to `not_present` + if strings.EqualFold(service.Pull, "false") { + service.Pull = constants.PullNotPresent + } + } + + // overwrite existing ServiceSlice + *s = ServiceSlice(*serviceSlice) + + return nil +} + +// MergeEnv takes a list of environment variables and attempts +// to set them in the service environment. If the environment +// variable already exists in the service, than this will +// overwrite the existing environment variable. +func (s *Service) MergeEnv(environment map[string]string) error { + // check if the service container is empty + if s == nil || s.Environment == nil { + // TODO: evaluate if we should error here + // + // immediately return and do nothing + // + // treated as a no-op + return nil + } + + // check if the environment provided is empty + if environment == nil { + return fmt.Errorf("empty environment provided for service %s", s.Name) + } + + // iterate through all environment variables provided + for key, value := range environment { + // set or update the service environment variable + s.Environment[key] = value + } + + return nil +} + +func (s *Service) ToYAML() *yaml.Service { + if s == nil { + return nil + } + + return &yaml.Service{ + Image: s.Image, + Name: s.Name, + Entrypoint: s.Entrypoint, + Environment: s.Environment, + Ports: s.Ports, + Pull: s.Pull, + Ulimits: *s.Ulimits.ToYAML(), + User: s.User, + } +} + +func (s *ServiceSlice) ToYAML() *yaml.ServiceSlice { + // service slice we want to return + serviceSlice := new(yaml.ServiceSlice) + + // iterate through each element in the service slice + for _, service := range *s { + // append the element to the yaml service slice + *serviceSlice = append(*serviceSlice, service.ToYAML()) + } + + return serviceSlice +} diff --git a/compiler/types/yaml/service_test.go b/compiler/types/yaml/buildkite/service_test.go similarity index 99% rename from compiler/types/yaml/service_test.go rename to compiler/types/yaml/buildkite/service_test.go index 09f4fbc16..bc97c258e 100644 --- a/compiler/types/yaml/service_test.go +++ b/compiler/types/yaml/buildkite/service_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/stage.go b/compiler/types/yaml/buildkite/stage.go similarity index 84% rename from compiler/types/yaml/stage.go rename to compiler/types/yaml/buildkite/stage.go index 93bc8c8ca..313ec18f7 100644 --- a/compiler/types/yaml/stage.go +++ b/compiler/types/yaml/buildkite/stage.go @@ -1,14 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "fmt" - "github.com/buildkite/yaml" + bkYaml "github.com/buildkite/yaml" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) type ( @@ -52,7 +53,7 @@ func (s *StageSlice) ToPipeline() *pipeline.StageSlice { // UnmarshalYAML implements the Unmarshaler interface for the StageSlice type. func (s *StageSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { // map slice we try unmarshalling to - mapSlice := new(yaml.MapSlice) + mapSlice := new(bkYaml.MapSlice) // attempt to unmarshal as a map slice type err := unmarshal(mapSlice) @@ -66,10 +67,10 @@ func (s *StageSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { stage := new(Stage) // marshal interface value from ordered map - out, _ := yaml.Marshal(v.Value) + out, _ := bkYaml.Marshal(v.Value) // unmarshal interface value as stage - err = yaml.Unmarshal(out, stage) + err = bkYaml.Unmarshal(out, stage) if err != nil { return err } @@ -102,7 +103,7 @@ func (s *StageSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { // MarshalYAML implements the marshaler interface for the StageSlice type. func (s StageSlice) MarshalYAML() (interface{}, error) { // map slice to return as marshaled output - var output yaml.MapSlice + var output bkYaml.MapSlice // loop over the input stages for _, inputStage := range s { @@ -119,7 +120,7 @@ func (s StageSlice) MarshalYAML() (interface{}, error) { outputStage.Steps = inputStage.Steps // append stage to MapSlice - output = append(output, yaml.MapItem{Key: inputStage.Name, Value: outputStage}) + output = append(output, bkYaml.MapItem{Key: inputStage.Name, Value: outputStage}) } return output, nil @@ -153,3 +154,30 @@ func (s *Stage) MergeEnv(environment map[string]string) error { return nil } + +func (s *Stage) ToYAML() *yaml.Stage { + if s == nil { + return nil + } + + return &yaml.Stage{ + Environment: s.Environment, + Name: s.Name, + Needs: s.Needs, + Independent: s.Independent, + Steps: *s.Steps.ToYAML(), + } +} + +func (s *StageSlice) ToYAML() *yaml.StageSlice { + // stage slice we want to return + stageSlice := new(yaml.StageSlice) + + // iterate through each element in the stage slice + for _, stage := range *s { + // append the element to the yaml stage slice + *stageSlice = append(*stageSlice, stage.ToYAML()) + } + + return stageSlice +} diff --git a/compiler/types/yaml/stage_test.go b/compiler/types/yaml/buildkite/stage_test.go similarity index 99% rename from compiler/types/yaml/stage_test.go rename to compiler/types/yaml/buildkite/stage_test.go index 7c63253b8..3c87a3859 100644 --- a/compiler/types/yaml/stage_test.go +++ b/compiler/types/yaml/buildkite/stage_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/step.go b/compiler/types/yaml/buildkite/step.go new file mode 100644 index 000000000..06a9eadb7 --- /dev/null +++ b/compiler/types/yaml/buildkite/step.go @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "fmt" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/constants" +) + +type ( + // StepSlice is the yaml representation + // of the steps block for a pipeline. + StepSlice []*Step + + // Step is the yaml representation of a step + // from the steps block for a pipeline. + Step struct { + Ruleset Ruleset `yaml:"ruleset,omitempty" json:"ruleset,omitempty" jsonschema:"description=Conditions to limit the execution of the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Commands raw.StringSlice `yaml:"commands,omitempty" json:"commands,omitempty" jsonschema:"description=Execution instructions to run inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-commands-key"` + Entrypoint raw.StringSlice `yaml:"entrypoint,omitempty" json:"entrypoint,omitempty" jsonschema:"description=Command to execute inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-entrypoint-key"` + Secrets StepSecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Sensitive variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-secrets-key"` + Template StepTemplate `yaml:"template,omitempty" json:"template,omitempty" jsonschema:"oneof_required=template,description=Name of template to expand in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` + Ulimits UlimitSlice `yaml:"ulimits,omitempty" json:"ulimits,omitempty" jsonschema:"description=Set the user limits for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` + Volumes VolumeSlice `yaml:"volumes,omitempty" json:"volumes,omitempty" jsonschema:"description=Mount volumes for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` + Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"oneof_required=image,minLength=1,description=Docker image to use to create the ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-image-key"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique name for the step.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-name-key"` + Pull string `yaml:"pull,omitempty" json:"pull,omitempty" jsonschema:"enum=always,enum=not_present,enum=on_start,enum=never,default=not_present,description=Declaration to configure if and when the Docker image is pulled.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-pull-key"` + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Provide environment variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-environment-key"` + Parameters map[string]interface{} `yaml:"parameters,omitempty" json:"parameters,omitempty" jsonschema:"description=Extra configuration variables for a plugin.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-parameters-key"` + Detach bool `yaml:"detach,omitempty" json:"detach,omitempty" jsonschema:"description=Run the container in a detached (headless) state.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-detach-key"` + Privileged bool `yaml:"privileged,omitempty" json:"privileged,omitempty" jsonschema:"description=Run the container with extra privileges.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-privileged-key"` + User string `yaml:"user,omitempty" json:"user,omitempty" jsonschema:"description=Set the user for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-user-key"` + ReportAs string `yaml:"report_as,omitempty" json:"report_as,omitempty" jsonschema:"description=Set the name of the step to report as.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-report_as-key"` + IDRequest string `yaml:"id_request,omitempty" json:"id_request,omitempty" jsonschema:"description=Request ID Request Token for the step.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-id_request-key"` + } +) + +// ToPipeline converts the StepSlice type +// to a pipeline ContainerSlice type. +func (s *StepSlice) ToPipeline() *pipeline.ContainerSlice { + // step slice we want to return + stepSlice := new(pipeline.ContainerSlice) + + // iterate through each element in the step slice + for _, step := range *s { + // append the element to the pipeline container slice + *stepSlice = append(*stepSlice, &pipeline.Container{ + Commands: step.Commands, + Detach: step.Detach, + Entrypoint: step.Entrypoint, + Environment: step.Environment, + Image: step.Image, + Name: step.Name, + Privileged: step.Privileged, + Pull: step.Pull, + Ruleset: *step.Ruleset.ToPipeline(), + Secrets: *step.Secrets.ToPipeline(), + Ulimits: *step.Ulimits.ToPipeline(), + Volumes: *step.Volumes.ToPipeline(), + User: step.User, + ReportAs: step.ReportAs, + IDRequest: step.IDRequest, + }) + } + + return stepSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the StepSlice type. +func (s *StepSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // step slice we try unmarshalling to + stepSlice := new([]*Step) + + // attempt to unmarshal as a step slice type + err := unmarshal(stepSlice) + if err != nil { + return err + } + + // iterate through each element in the step slice + for _, step := range *stepSlice { + // handle nil step to avoid panic + if step == nil { + return fmt.Errorf("invalid step with nil content found") + } + + // implicitly set `pull` field if empty + if len(step.Pull) == 0 { + step.Pull = constants.PullNotPresent + } + + // TODO: remove this in a future release + // + // handle true deprecated pull policy + // + // a `true` pull policy equates to `always` + if strings.EqualFold(step.Pull, "true") { + step.Pull = constants.PullAlways + } + + // TODO: remove this in a future release + // + // handle false deprecated pull policy + // + // a `false` pull policy equates to `not_present` + if strings.EqualFold(step.Pull, "false") { + step.Pull = constants.PullNotPresent + } + } + + // overwrite existing StepSlice + *s = StepSlice(*stepSlice) + + return nil +} + +// MergeEnv takes a list of environment variables and attempts +// to set them in the step environment. If the environment +// variable already exists in the step, than this will +// overwrite the existing environment variable. +func (s *Step) MergeEnv(environment map[string]string) error { + // check if the step container is empty + if s == nil || s.Environment == nil { + // TODO: evaluate if we should error here + // + // immediately return and do nothing + // + // treated as a no-op + return nil + } + + // check if the environment provided is empty + if environment == nil { + return fmt.Errorf("empty environment provided for step %s", s.Name) + } + + // iterate through all environment variables provided + for key, value := range environment { + // set or update the step environment variable + s.Environment[key] = value + } + + return nil +} + +func (s *Step) ToYAML() *yaml.Step { + if s == nil { + return nil + } + + return &yaml.Step{ + Commands: s.Commands, + Detach: s.Detach, + Entrypoint: s.Entrypoint, + Environment: s.Environment, + Image: s.Image, + Name: s.Name, + Privileged: s.Privileged, + Pull: s.Pull, + Ruleset: *s.Ruleset.ToYAML(), + Secrets: *s.Secrets.ToYAML(), + Template: s.Template.ToYAML(), + Ulimits: *s.Ulimits.ToYAML(), + Volumes: *s.Volumes.ToYAML(), + User: s.User, + ReportAs: s.ReportAs, + IDRequest: s.IDRequest, + } +} + +func (s *StepSlice) ToYAML() *yaml.StepSlice { + // step slice we want to return + stepSlice := new(yaml.StepSlice) + + // iterate through each element in the step slice + for _, step := range *s { + // append the element to the yaml step slice + *stepSlice = append(*stepSlice, step.ToYAML()) + } + + return stepSlice +} diff --git a/compiler/types/yaml/step_test.go b/compiler/types/yaml/buildkite/step_test.go similarity index 99% rename from compiler/types/yaml/step_test.go rename to compiler/types/yaml/buildkite/step_test.go index 2f5336be4..da4df26fe 100644 --- a/compiler/types/yaml/step_test.go +++ b/compiler/types/yaml/buildkite/step_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/template.go b/compiler/types/yaml/buildkite/template.go new file mode 100644 index 000000000..f9380dab3 --- /dev/null +++ b/compiler/types/yaml/buildkite/template.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // TemplateSlice is the yaml representation + // of the templates block for a pipeline. + TemplateSlice []*Template + + // Template is the yaml representation of a template + // from the templates block for a pipeline. + Template struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique identifier for the template.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-name-key"` + Source string `yaml:"source,omitempty" json:"source,omitempty" jsonschema:"required,minLength=1,description=Path to template in remote system.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-source-key"` + Format string `yaml:"format,omitempty" json:"format,omitempty" jsonschema:"enum=starlark,enum=golang,enum=go,default=go,minLength=1,description=language used within the template file \nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-format-key"` + Type string `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"minLength=1,example=github,description=Type of template provided from the remote system.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-type-key"` + Variables map[string]interface{} `yaml:"vars,omitempty" json:"vars,omitempty" jsonschema:"description=Variables injected into the template.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-variables-key"` + } + + // StepTemplate is the yaml representation of the + // template block for a step in a pipeline. + StepTemplate struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique identifier for the template.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` + Variables map[string]interface{} `yaml:"vars,omitempty" json:"vars,omitempty" jsonschema:"description=Variables injected into the template.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` + } +) + +// UnmarshalYAML implements the Unmarshaler interface for the TemplateSlice type. +func (t *TemplateSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // template slice we try unmarshalling to + templateSlice := new([]*Template) + + // attempt to unmarshal as a template slice type + err := unmarshal(templateSlice) + if err != nil { + return err + } + + // overwrite existing TemplateSlice + *t = TemplateSlice(*templateSlice) + + return nil +} + +// ToAPI converts the Template type +// to an API Template type. +func (t *Template) ToAPI() *api.Template { + template := new(api.Template) + + template.SetName(t.Name) + template.SetSource(t.Source) + template.SetType(t.Type) + + return template +} + +// TemplateFromAPI converts the API Template type +// to a yaml Template type. +func TemplateFromAPI(t *api.Template) *Template { + template := &Template{ + Name: t.GetName(), + Source: t.GetSource(), + Type: t.GetType(), + } + + return template +} + +// Map helper function that creates a map of templates from a slice of templates. +func (t *TemplateSlice) Map() map[string]*Template { + m := make(map[string]*Template) + + if t == nil { + return m + } + + for _, tmpl := range *t { + m[tmpl.Name] = tmpl + } + + return m +} + +func (t *Template) ToYAML() *yaml.Template { + if t == nil { + return nil + } + + return &yaml.Template{ + Name: t.Name, + Source: t.Source, + Format: t.Format, + Type: t.Type, + Variables: t.Variables, + } +} + +func (t *TemplateSlice) ToYAML() *yaml.TemplateSlice { + // template slice we want to return + templateSlice := new(yaml.TemplateSlice) + + // iterate through each element in the template slice + for _, template := range *t { + // append the element to the yaml template slice + *templateSlice = append(*templateSlice, template.ToYAML()) + } + + return templateSlice +} + +func (t *StepTemplate) ToYAML() yaml.StepTemplate { + return yaml.StepTemplate{ + Name: t.Name, + Variables: t.Variables, + } +} diff --git a/compiler/types/yaml/template_test.go b/compiler/types/yaml/buildkite/template_test.go similarity index 99% rename from compiler/types/yaml/template_test.go rename to compiler/types/yaml/buildkite/template_test.go index b000cd70b..f81481005 100644 --- a/compiler/types/yaml/template_test.go +++ b/compiler/types/yaml/buildkite/template_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/testdata/build.yml b/compiler/types/yaml/buildkite/testdata/build.yml similarity index 100% rename from compiler/types/yaml/testdata/build.yml rename to compiler/types/yaml/buildkite/testdata/build.yml diff --git a/compiler/types/yaml/testdata/build/validate/bad_pipeline0.yml b/compiler/types/yaml/buildkite/testdata/build/validate/bad_pipeline0.yml similarity index 100% rename from compiler/types/yaml/testdata/build/validate/bad_pipeline0.yml rename to compiler/types/yaml/buildkite/testdata/build/validate/bad_pipeline0.yml diff --git a/compiler/types/yaml/testdata/build/validate/bad_pipeline1.yml b/compiler/types/yaml/buildkite/testdata/build/validate/bad_pipeline1.yml similarity index 100% rename from compiler/types/yaml/testdata/build/validate/bad_pipeline1.yml rename to compiler/types/yaml/buildkite/testdata/build/validate/bad_pipeline1.yml diff --git a/compiler/types/yaml/testdata/build/validate/bad_version.yml b/compiler/types/yaml/buildkite/testdata/build/validate/bad_version.yml similarity index 100% rename from compiler/types/yaml/testdata/build/validate/bad_version.yml rename to compiler/types/yaml/buildkite/testdata/build/validate/bad_version.yml diff --git a/compiler/types/yaml/testdata/build/validate/step.yml b/compiler/types/yaml/buildkite/testdata/build/validate/step.yml similarity index 100% rename from compiler/types/yaml/testdata/build/validate/step.yml rename to compiler/types/yaml/buildkite/testdata/build/validate/step.yml diff --git a/compiler/types/yaml/testdata/build_anchor_stage.yml b/compiler/types/yaml/buildkite/testdata/build_anchor_stage.yml similarity index 100% rename from compiler/types/yaml/testdata/build_anchor_stage.yml rename to compiler/types/yaml/buildkite/testdata/build_anchor_stage.yml diff --git a/compiler/types/yaml/testdata/build_anchor_step.yml b/compiler/types/yaml/buildkite/testdata/build_anchor_step.yml similarity index 100% rename from compiler/types/yaml/testdata/build_anchor_step.yml rename to compiler/types/yaml/buildkite/testdata/build_anchor_step.yml diff --git a/compiler/types/yaml/testdata/build_empty_env.yml b/compiler/types/yaml/buildkite/testdata/build_empty_env.yml similarity index 100% rename from compiler/types/yaml/testdata/build_empty_env.yml rename to compiler/types/yaml/buildkite/testdata/build_empty_env.yml diff --git a/compiler/types/yaml/testdata/invalid.yml b/compiler/types/yaml/buildkite/testdata/invalid.yml similarity index 100% rename from compiler/types/yaml/testdata/invalid.yml rename to compiler/types/yaml/buildkite/testdata/invalid.yml diff --git a/compiler/types/yaml/testdata/merge_anchor.yml b/compiler/types/yaml/buildkite/testdata/merge_anchor.yml similarity index 100% rename from compiler/types/yaml/testdata/merge_anchor.yml rename to compiler/types/yaml/buildkite/testdata/merge_anchor.yml diff --git a/compiler/types/yaml/testdata/metadata.yml b/compiler/types/yaml/buildkite/testdata/metadata.yml similarity index 100% rename from compiler/types/yaml/testdata/metadata.yml rename to compiler/types/yaml/buildkite/testdata/metadata.yml diff --git a/compiler/types/yaml/testdata/metadata_env.yml b/compiler/types/yaml/buildkite/testdata/metadata_env.yml similarity index 100% rename from compiler/types/yaml/testdata/metadata_env.yml rename to compiler/types/yaml/buildkite/testdata/metadata_env.yml diff --git a/compiler/types/yaml/testdata/ruleset_advanced.yml b/compiler/types/yaml/buildkite/testdata/ruleset_advanced.yml similarity index 100% rename from compiler/types/yaml/testdata/ruleset_advanced.yml rename to compiler/types/yaml/buildkite/testdata/ruleset_advanced.yml diff --git a/compiler/types/yaml/testdata/ruleset_regex.yml b/compiler/types/yaml/buildkite/testdata/ruleset_regex.yml similarity index 100% rename from compiler/types/yaml/testdata/ruleset_regex.yml rename to compiler/types/yaml/buildkite/testdata/ruleset_regex.yml diff --git a/compiler/types/yaml/testdata/ruleset_simple.yml b/compiler/types/yaml/buildkite/testdata/ruleset_simple.yml similarity index 100% rename from compiler/types/yaml/testdata/ruleset_simple.yml rename to compiler/types/yaml/buildkite/testdata/ruleset_simple.yml diff --git a/compiler/types/yaml/testdata/secret.yml b/compiler/types/yaml/buildkite/testdata/secret.yml similarity index 100% rename from compiler/types/yaml/testdata/secret.yml rename to compiler/types/yaml/buildkite/testdata/secret.yml diff --git a/compiler/types/yaml/testdata/secret/validate/no_name.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/no_name.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/no_name.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/no_name.yml diff --git a/compiler/types/yaml/testdata/secret/validate/org.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/org.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/org.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/org.yml diff --git a/compiler/types/yaml/testdata/secret/validate/org_bad_engine.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/org_bad_engine.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/org_bad_engine.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/org_bad_engine.yml diff --git a/compiler/types/yaml/testdata/secret/validate/org_bad_key.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/org_bad_key.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/org_bad_key.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/org_bad_key.yml diff --git a/compiler/types/yaml/testdata/secret/validate/plugin.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/plugin.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/plugin.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/plugin.yml diff --git a/compiler/types/yaml/testdata/secret/validate/plugin_bad_image.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/plugin_bad_image.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/plugin_bad_image.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/plugin_bad_image.yml diff --git a/compiler/types/yaml/testdata/secret/validate/plugin_bad_name.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/plugin_bad_name.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/plugin_bad_name.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/plugin_bad_name.yml diff --git a/compiler/types/yaml/testdata/secret/validate/repo.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/repo.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/repo.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/repo.yml diff --git a/compiler/types/yaml/testdata/secret/validate/repo_bad_engine.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/repo_bad_engine.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/repo_bad_engine.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/repo_bad_engine.yml diff --git a/compiler/types/yaml/testdata/secret/validate/repo_bad_key.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/repo_bad_key.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/repo_bad_key.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/repo_bad_key.yml diff --git a/compiler/types/yaml/testdata/secret/validate/shared.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/shared.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/shared.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/shared.yml diff --git a/compiler/types/yaml/testdata/secret/validate/shared_bad_engine.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/shared_bad_engine.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/shared_bad_engine.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/shared_bad_engine.yml diff --git a/compiler/types/yaml/testdata/secret/validate/shared_bad_key.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/shared_bad_key.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/shared_bad_key.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/shared_bad_key.yml diff --git a/compiler/types/yaml/testdata/service.yml b/compiler/types/yaml/buildkite/testdata/service.yml similarity index 100% rename from compiler/types/yaml/testdata/service.yml rename to compiler/types/yaml/buildkite/testdata/service.yml diff --git a/compiler/types/yaml/testdata/service/validate/bad_image.yml b/compiler/types/yaml/buildkite/testdata/service/validate/bad_image.yml similarity index 100% rename from compiler/types/yaml/testdata/service/validate/bad_image.yml rename to compiler/types/yaml/buildkite/testdata/service/validate/bad_image.yml diff --git a/compiler/types/yaml/testdata/service/validate/minimal.yml b/compiler/types/yaml/buildkite/testdata/service/validate/minimal.yml similarity index 100% rename from compiler/types/yaml/testdata/service/validate/minimal.yml rename to compiler/types/yaml/buildkite/testdata/service/validate/minimal.yml diff --git a/compiler/types/yaml/testdata/service/validate/missing_image.yml b/compiler/types/yaml/buildkite/testdata/service/validate/missing_image.yml similarity index 100% rename from compiler/types/yaml/testdata/service/validate/missing_image.yml rename to compiler/types/yaml/buildkite/testdata/service/validate/missing_image.yml diff --git a/compiler/types/yaml/testdata/service/validate/missing_name.yml b/compiler/types/yaml/buildkite/testdata/service/validate/missing_name.yml similarity index 100% rename from compiler/types/yaml/testdata/service/validate/missing_name.yml rename to compiler/types/yaml/buildkite/testdata/service/validate/missing_name.yml diff --git a/compiler/types/yaml/testdata/service_nil.yml b/compiler/types/yaml/buildkite/testdata/service_nil.yml similarity index 100% rename from compiler/types/yaml/testdata/service_nil.yml rename to compiler/types/yaml/buildkite/testdata/service_nil.yml diff --git a/compiler/types/yaml/testdata/stage.yml b/compiler/types/yaml/buildkite/testdata/stage.yml similarity index 100% rename from compiler/types/yaml/testdata/stage.yml rename to compiler/types/yaml/buildkite/testdata/stage.yml diff --git a/compiler/types/yaml/testdata/stage/validate/bad_image.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/bad_image.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/bad_image.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/bad_image.yml diff --git a/compiler/types/yaml/testdata/stage/validate/minimal.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/minimal.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/minimal.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/minimal.yml diff --git a/compiler/types/yaml/testdata/stage/validate/missing.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/missing.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/missing.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/missing.yml diff --git a/compiler/types/yaml/testdata/stage/validate/missing_image.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/missing_image.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/missing_image.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/missing_image.yml diff --git a/compiler/types/yaml/testdata/stage/validate/missing_name.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/missing_name.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/missing_name.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/missing_name.yml diff --git a/compiler/types/yaml/testdata/step.yml b/compiler/types/yaml/buildkite/testdata/step.yml similarity index 100% rename from compiler/types/yaml/testdata/step.yml rename to compiler/types/yaml/buildkite/testdata/step.yml diff --git a/compiler/types/yaml/testdata/step/validate/bad_image.yml b/compiler/types/yaml/buildkite/testdata/step/validate/bad_image.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/bad_image.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/bad_image.yml diff --git a/compiler/types/yaml/testdata/step/validate/minimal.yml b/compiler/types/yaml/buildkite/testdata/step/validate/minimal.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/minimal.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/minimal.yml diff --git a/compiler/types/yaml/testdata/step/validate/missing.yml b/compiler/types/yaml/buildkite/testdata/step/validate/missing.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/missing.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/missing.yml diff --git a/compiler/types/yaml/testdata/step/validate/missing_image.yml b/compiler/types/yaml/buildkite/testdata/step/validate/missing_image.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/missing_image.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/missing_image.yml diff --git a/compiler/types/yaml/testdata/step/validate/missing_name.yml b/compiler/types/yaml/buildkite/testdata/step/validate/missing_name.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/missing_name.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/missing_name.yml diff --git a/compiler/types/yaml/testdata/step_malformed.yml b/compiler/types/yaml/buildkite/testdata/step_malformed.yml similarity index 100% rename from compiler/types/yaml/testdata/step_malformed.yml rename to compiler/types/yaml/buildkite/testdata/step_malformed.yml diff --git a/compiler/types/yaml/testdata/step_nil.yml b/compiler/types/yaml/buildkite/testdata/step_nil.yml similarity index 100% rename from compiler/types/yaml/testdata/step_nil.yml rename to compiler/types/yaml/buildkite/testdata/step_nil.yml diff --git a/compiler/types/yaml/testdata/step_secret_slice.yml b/compiler/types/yaml/buildkite/testdata/step_secret_slice.yml similarity index 100% rename from compiler/types/yaml/testdata/step_secret_slice.yml rename to compiler/types/yaml/buildkite/testdata/step_secret_slice.yml diff --git a/compiler/types/yaml/testdata/step_secret_slice_invalid_no_source.yml b/compiler/types/yaml/buildkite/testdata/step_secret_slice_invalid_no_source.yml similarity index 100% rename from compiler/types/yaml/testdata/step_secret_slice_invalid_no_source.yml rename to compiler/types/yaml/buildkite/testdata/step_secret_slice_invalid_no_source.yml diff --git a/compiler/types/yaml/testdata/step_secret_slice_invalid_no_target.yml b/compiler/types/yaml/buildkite/testdata/step_secret_slice_invalid_no_target.yml similarity index 100% rename from compiler/types/yaml/testdata/step_secret_slice_invalid_no_target.yml rename to compiler/types/yaml/buildkite/testdata/step_secret_slice_invalid_no_target.yml diff --git a/compiler/types/yaml/testdata/step_secret_string.yml b/compiler/types/yaml/buildkite/testdata/step_secret_string.yml similarity index 100% rename from compiler/types/yaml/testdata/step_secret_string.yml rename to compiler/types/yaml/buildkite/testdata/step_secret_string.yml diff --git a/compiler/types/yaml/testdata/template.yml b/compiler/types/yaml/buildkite/testdata/template.yml similarity index 100% rename from compiler/types/yaml/testdata/template.yml rename to compiler/types/yaml/buildkite/testdata/template.yml diff --git a/compiler/types/yaml/testdata/ulimit_colon_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_colon_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_colon_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_colon_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_equal_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_equal_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_equal_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_equal_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_hardlimit1_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_hardlimit1_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_hardlimit1_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_hardlimit1_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_hardlimit2_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_hardlimit2_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_hardlimit2_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_hardlimit2_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_slice.yml b/compiler/types/yaml/buildkite/testdata/ulimit_slice.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_slice.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_slice.yml diff --git a/compiler/types/yaml/testdata/ulimit_softlimit_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_softlimit_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_softlimit_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_softlimit_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_string.yml b/compiler/types/yaml/buildkite/testdata/ulimit_string.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_string.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_string.yml diff --git a/compiler/types/yaml/testdata/volume_error.yml b/compiler/types/yaml/buildkite/testdata/volume_error.yml similarity index 100% rename from compiler/types/yaml/testdata/volume_error.yml rename to compiler/types/yaml/buildkite/testdata/volume_error.yml diff --git a/compiler/types/yaml/testdata/volume_slice.yml b/compiler/types/yaml/buildkite/testdata/volume_slice.yml similarity index 100% rename from compiler/types/yaml/testdata/volume_slice.yml rename to compiler/types/yaml/buildkite/testdata/volume_slice.yml diff --git a/compiler/types/yaml/testdata/volume_string.yml b/compiler/types/yaml/buildkite/testdata/volume_string.yml similarity index 100% rename from compiler/types/yaml/testdata/volume_string.yml rename to compiler/types/yaml/buildkite/testdata/volume_string.yml diff --git a/compiler/types/yaml/buildkite/ulimit.go b/compiler/types/yaml/buildkite/ulimit.go new file mode 100644 index 000000000..75271f293 --- /dev/null +++ b/compiler/types/yaml/buildkite/ulimit.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "fmt" + "strconv" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // UlimitSlice is the yaml representation of + // the ulimits block for a step in a pipeline. + UlimitSlice []*Ulimit + + // Ulimit is the yaml representation of a ulimit + // from the ulimits block for a step in a pipeline. + Ulimit struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique name of the user limit.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` + Soft int64 `yaml:"soft,omitempty" json:"soft,omitempty" jsonschema:"description=Set the soft limit.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` + Hard int64 `yaml:"hard,omitempty" json:"hard,omitempty" jsonschema:"description=Set the hard limit.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` + } +) + +// ToPipeline converts the UlimitSlice type +// to a pipeline UlimitSlice type. +func (u *UlimitSlice) ToPipeline() *pipeline.UlimitSlice { + // ulimit slice we want to return + ulimitSlice := new(pipeline.UlimitSlice) + + // iterate through each element in the ulimit slice + for _, ulimit := range *u { + // append the element to the pipeline ulimit slice + *ulimitSlice = append(*ulimitSlice, &pipeline.Ulimit{ + Name: ulimit.Name, + Soft: ulimit.Soft, + Hard: ulimit.Hard, + }) + } + + return ulimitSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the UlimitSlice type. +func (u *UlimitSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // string slice we try unmarshalling to + stringSlice := new(raw.StringSlice) + + // attempt to unmarshal as a string slice type + err := unmarshal(stringSlice) + if err == nil { + // iterate through each element in the string slice + for _, ulimit := range *stringSlice { + // split each slice element into key/value pairs + parts := strings.Split(ulimit, "=") + if len(parts) != 2 { + return fmt.Errorf("ulimit %s must contain 1 `=` (equal)", ulimit) + } + + // split each value into soft and hard limits + limitParts := strings.Split(parts[1], ":") + + switch { + case len(limitParts) == 1: + // capture value for soft and hard limit + value, err := strconv.ParseInt(limitParts[0], 10, 64) + if err != nil { + return err + } + + // append the element to the ulimit slice + *u = append(*u, &Ulimit{ + Name: parts[0], + Soft: value, + Hard: value, + }) + + continue + case len(limitParts) == 2: + // capture value for soft limit + firstValue, err := strconv.ParseInt(limitParts[0], 10, 64) + if err != nil { + return err + } + + // capture value for hard limit + secondValue, err := strconv.ParseInt(limitParts[1], 10, 64) + if err != nil { + return err + } + + // append the element to the ulimit slice + *u = append(*u, &Ulimit{ + Name: parts[0], + Soft: firstValue, + Hard: secondValue, + }) + + continue + default: + return fmt.Errorf("ulimit %s can only contain 1 `:` (colon)", ulimit) + } + } + + return nil + } + + // ulimit slice we try unmarshalling to + ulimits := new([]*Ulimit) + + // attempt to unmarshal as a ulimit slice type + err = unmarshal(ulimits) + if err != nil { + return err + } + + // iterate through each element in the volume slice + for _, ulimit := range *ulimits { + // implicitly set `hard` field if empty + if ulimit.Hard == 0 { + ulimit.Hard = ulimit.Soft + } + } + + // overwrite existing UlimitSlice + *u = UlimitSlice(*ulimits) + + return nil +} + +func (u *Ulimit) ToYAML() *yaml.Ulimit { + if u == nil { + return nil + } + + return &yaml.Ulimit{ + Name: u.Name, + Soft: u.Soft, + Hard: u.Hard, + } +} + +func (u *UlimitSlice) ToYAML() *yaml.UlimitSlice { + // ulimit slice we want to return + ulimitSlice := new(yaml.UlimitSlice) + + // iterate through each element in the ulimit slice + for _, ulimit := range *u { + // append the element to the yaml ulimit slice + *ulimitSlice = append(*ulimitSlice, ulimit.ToYAML()) + } + + return ulimitSlice +} diff --git a/compiler/types/yaml/ulimit_test.go b/compiler/types/yaml/buildkite/ulimit_test.go similarity index 99% rename from compiler/types/yaml/ulimit_test.go rename to compiler/types/yaml/buildkite/ulimit_test.go index 3a2d9fcfb..5337007fa 100644 --- a/compiler/types/yaml/ulimit_test.go +++ b/compiler/types/yaml/buildkite/ulimit_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/volume.go b/compiler/types/yaml/buildkite/volume.go new file mode 100644 index 000000000..ee7328075 --- /dev/null +++ b/compiler/types/yaml/buildkite/volume.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "fmt" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // VolumeSlice is the yaml representation of + // the volumes block for a step in a pipeline. + VolumeSlice []*Volume + + // Volume is the yaml representation of a volume + // from a volumes block for a step in a pipeline. + Volume struct { + Source string `yaml:"source,omitempty" json:"source,omitempty" jsonschema:"required,minLength=1,description=Set the source directory to be mounted.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` + Destination string `yaml:"destination,omitempty" json:"destination,omitempty" jsonschema:"required,minLength=1,description=Set the destination directory for the mount in the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` + AccessMode string `yaml:"access_mode,omitempty" json:"access_mode,omitempty" jsonschema:"default=ro,description=Set the access mode for the mounted volume.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` + } +) + +// ToPipeline converts the VolumeSlice type +// to a pipeline VolumeSlice type. +func (v *VolumeSlice) ToPipeline() *pipeline.VolumeSlice { + // volume slice we want to return + volumes := new(pipeline.VolumeSlice) + + // iterate through each element in the volume slice + for _, volume := range *v { + // append the element to the pipeline volume slice + *volumes = append(*volumes, &pipeline.Volume{ + Source: volume.Source, + Destination: volume.Destination, + AccessMode: volume.AccessMode, + }) + } + + return volumes +} + +// UnmarshalYAML implements the Unmarshaler interface for the VolumeSlice type. +func (v *VolumeSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // string slice we try unmarshalling to + stringSlice := new(raw.StringSlice) + + // attempt to unmarshal as a string slice type + err := unmarshal(stringSlice) + if err == nil { + // iterate through each element in the string slice + for _, volume := range *stringSlice { + // split each slice element into source, destination and access mode + parts := strings.Split(volume, ":") + + switch { + case len(parts) == 1: + // append the element to the volume slice + *v = append(*v, &Volume{ + Source: parts[0], + Destination: parts[0], + AccessMode: "ro", + }) + + continue + case len(parts) == 2: + // append the element to the volume slice + *v = append(*v, &Volume{ + Source: parts[0], + Destination: parts[1], + AccessMode: "ro", + }) + + continue + case len(parts) == 3: + // append the element to the volume slice + *v = append(*v, &Volume{ + Source: parts[0], + Destination: parts[1], + AccessMode: parts[2], + }) + + continue + default: + return fmt.Errorf("volume %s must contain at least 1 but no more than 2 `:`(colons)", volume) + } + } + + return nil + } + + // volume slice we try unmarshalling to + volumes := new([]*Volume) + + // attempt to unmarshal as a volume slice type + err = unmarshal(volumes) + if err != nil { + return err + } + + // iterate through each element in the volume slice + for _, volume := range *volumes { + // implicitly set `destination` field if empty + if len(volume.Destination) == 0 { + volume.Destination = volume.Source + } + + // implicitly set `access_mode` field if empty + if len(volume.AccessMode) == 0 { + volume.AccessMode = "ro" + } + } + + // overwrite existing VolumeSlice + *v = VolumeSlice(*volumes) + + return nil +} + +func (v *Volume) ToYAML() *yaml.Volume { + if v == nil { + return nil + } + + return &yaml.Volume{ + Source: v.Source, + Destination: v.Destination, + AccessMode: v.AccessMode, + } +} + +func (v *VolumeSlice) ToYAML() *yaml.VolumeSlice { + // volume slice we want to return + volumeSlice := new(yaml.VolumeSlice) + + // iterate through each element in the volume slice + for _, volume := range *v { + // append the element to the yaml volume slice + *volumeSlice = append(*volumeSlice, volume.ToYAML()) + } + + return volumeSlice +} diff --git a/compiler/types/yaml/buildkite/volume_test.go b/compiler/types/yaml/buildkite/volume_test.go new file mode 100644 index 000000000..c2192f944 --- /dev/null +++ b/compiler/types/yaml/buildkite/volume_test.go @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "os" + "reflect" + "testing" + + "github.com/buildkite/yaml" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_VolumeSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + volumes *VolumeSlice + want *pipeline.VolumeSlice + }{ + { + volumes: &VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + want: &pipeline.VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.volumes.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_VolumeSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *VolumeSlice + }{ + { + failure: false, + file: "testdata/volume_slice.yml", + want: &VolumeSlice{ + { + Source: "/foo", + Destination: "/foo", + AccessMode: "ro", + }, + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + { + Source: "/foo", + Destination: "/foobar", + AccessMode: "ro", + }, + }, + }, + { + failure: false, + file: "testdata/volume_string.yml", + want: &VolumeSlice{ + { + Source: "/foo", + Destination: "/foo", + AccessMode: "ro", + }, + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + { + Source: "/foo", + Destination: "/foobar", + AccessMode: "ro", + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "testdata/volume_error.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(VolumeSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/buildkite/worker.go b/compiler/types/yaml/buildkite/worker.go new file mode 100644 index 000000000..d7fa9e891 --- /dev/null +++ b/compiler/types/yaml/buildkite/worker.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +// Worker is the yaml representation of a worker +// from a worker block in a pipeline. +type Worker struct { + Flavor string `yaml:"flavor,omitempty" json:"flavor,omitempty" jsonschema:"minLength=1,description=Flavor identifier for worker.\nReference: https://go-vela.github.io/docs/reference/yaml/worker/#the-flavor-key,example=large"` + Platform string `yaml:"platform,omitempty" json:"platform,omitempty" jsonschema:"minLength=1,description=Platform identifier for the worker.\nReference: https://go-vela.github.io/docs/reference/yaml/worker/#the-platform-key,example=kubernetes"` +} + +// ToPipeline converts the Worker type +// to a pipeline Worker type. +func (w *Worker) ToPipeline() *pipeline.Worker { + return &pipeline.Worker{ + Flavor: w.Flavor, + Platform: w.Platform, + } +} + +func (w *Worker) ToYAML() *yaml.Worker { + return &yaml.Worker{ + Flavor: w.Flavor, + Platform: w.Platform, + } +} diff --git a/compiler/types/yaml/buildkite/worker_test.go b/compiler/types/yaml/buildkite/worker_test.go new file mode 100644 index 000000000..049fd090c --- /dev/null +++ b/compiler/types/yaml/buildkite/worker_test.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "reflect" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Worker_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + worker *Worker + want *pipeline.Worker + }{ + { + worker: &Worker{ + Flavor: "8cpu16gb", + Platform: "gcp", + }, + want: &pipeline.Worker{ + Flavor: "8cpu16gb", + Platform: "gcp", + }, + }, + } + + // run tests + for _, test := range tests { + got := test.worker.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/doc.go b/compiler/types/yaml/doc.go deleted file mode 100644 index 2a2e39cd3..000000000 --- a/compiler/types/yaml/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -// Package yaml provides the defined yaml types for Vela. -// -// Usage: -// -// import "github.com/go-vela/server/compiler/types/yaml" -package yaml diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/yaml/build.go similarity index 100% rename from compiler/types/yaml/build.go rename to compiler/types/yaml/yaml/build.go diff --git a/compiler/types/yaml/yaml/build_test.go b/compiler/types/yaml/yaml/build_test.go new file mode 100644 index 000000000..f0c284f75 --- /dev/null +++ b/compiler/types/yaml/yaml/build_test.go @@ -0,0 +1,686 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/raw" +) + +func TestYaml_Build_ToAPI(t *testing.T) { + build := new(api.Pipeline) + build.SetFlavor("16cpu8gb") + build.SetPlatform("gcp") + build.SetVersion("1") + build.SetExternalSecrets(true) + build.SetInternalSecrets(true) + build.SetServices(true) + build.SetStages(false) + build.SetSteps(true) + build.SetTemplates(true) + + stages := new(api.Pipeline) + stages.SetFlavor("") + stages.SetPlatform("") + stages.SetVersion("1") + stages.SetExternalSecrets(false) + stages.SetInternalSecrets(false) + stages.SetServices(false) + stages.SetStages(true) + stages.SetSteps(false) + stages.SetTemplates(false) + + steps := new(api.Pipeline) + steps.SetFlavor("") + steps.SetPlatform("") + steps.SetVersion("1") + steps.SetExternalSecrets(false) + steps.SetInternalSecrets(false) + steps.SetServices(false) + steps.SetStages(false) + steps.SetSteps(true) + steps.SetTemplates(false) + + // setup tests + tests := []struct { + name string + file string + want *api.Pipeline + }{ + { + name: "build", + file: "testdata/build.yml", + want: build, + }, + { + name: "stages", + file: "testdata/build_anchor_stage.yml", + want: stages, + }, + { + name: "steps", + file: "testdata/build_anchor_step.yml", + want: steps, + }, + } + + // run tests + for _, test := range tests { + b := new(Build) + + data, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file %s for %s: %v", test.file, test.name, err) + } + + err = yaml.Unmarshal(data, b) + if err != nil { + t.Errorf("unable to unmarshal YAML for %s: %v", test.name, err) + } + + got := b.ToPipelineAPI() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipelineAPI for %s is %v, want %v", test.name, got, test.want) + } + } +} + +func TestYaml_Build_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + file string + want *Build + }{ + { + file: "testdata/build.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Environment: raw.StringSliceMap{ + "HELLO": "Hello, Global Message", + }, + Worker: Worker{ + Flavor: "16cpu8gb", + Platform: "gcp", + }, + Services: ServiceSlice{ + { + Ports: []string{"5432:5432"}, + Environment: raw.StringSliceMap{ + "POSTGRES_DB": "foo", + }, + Name: "postgres", + Image: "postgres:latest", + Pull: "not_present", + }, + }, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:edited"}}, + Matcher: "filepath", + Operator: "and", + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + { + Commands: raw.StringSlice{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + Commands: raw.StringSlice{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + Name: "docker_build", + Parameters: map[string]interface{}{ + "dry_run": true, + "registry": "index.docker.io", + "repo": "github/octocat", + "tags": []interface{}{"latest", "dev"}, + }, + Image: "plugins/docker:18.09", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Name: "docker_publish", + Parameters: map[string]interface{}{ + "registry": "index.docker.io", + "repo": "github/octocat", + "tags": []interface{}{"latest", "dev"}, + }, + Image: "plugins/docker:18.09", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Branch: []string{"main"}, Event: []string{"push"}}, + Matcher: "filepath", + Operator: "and", + }, + Secrets: StepSecretSlice{ + { + Source: "docker_username", + Target: "PLUGIN_USERNAME", + }, + { + Source: "docker_password", + Target: "PLUGIN_PASSWORD", + }, + }, + }, + }, + Secrets: SecretSlice{ + { + Name: "docker_username", + Key: "org/repo/docker/username", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "docker_password", + Key: "org/repo/docker/password", + Engine: "vault", + Type: "repo", + Pull: "build_start", + }, + { + Name: "docker_username", + Key: "org/docker/username", + Engine: "native", + Type: "org", + Pull: "build_start", + }, + { + Name: "docker_password", + Key: "org/docker/password", + Engine: "vault", + Type: "org", + Pull: "build_start", + }, + { + Name: "docker_username", + Key: "org/team/docker/username", + Engine: "native", + Type: "shared", + Pull: "build_start", + }, + { + Name: "docker_password", + Key: "org/team/docker/password", + Engine: "vault", + Type: "shared", + Pull: "build_start", + }, + { + Origin: Origin{ + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.example.com", + }, + Pull: "always", + Secrets: StepSecretSlice{ + { + Source: "docker_username", + Target: "DOCKER_USERNAME", + }, + { + Source: "docker_password", + Target: "DOCKER_PASSWORD", + }, + }, + }, + }, + }, + Templates: TemplateSlice{ + { + Name: "docker_publish", + Source: "github.com/go-vela/atlas/stable/docker_publish", + Type: "github", + }, + }, + }, + }, + { + file: "testdata/build_anchor_stage.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Stages: StageSlice{ + { + Name: "dependencies", + Needs: []string{"clone"}, + Independent: false, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + }, + }, + { + Name: "test", + Needs: []string{"dependencies", "clone"}, + Independent: false, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + }, + }, + { + Name: "build", + Needs: []string{"dependencies", "clone"}, + Independent: true, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + }, + }, + }, + }, + }, + { + file: "testdata/build_anchor_step.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + Commands: raw.StringSlice{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + Commands: raw.StringSlice{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + }, + }, + }, + { + file: "testdata/build_empty_env.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{}, + }, + Environment: raw.StringSliceMap{ + "HELLO": "Hello, Global Message", + }, + Worker: Worker{ + Flavor: "16cpu8gb", + Platform: "gcp"}, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + }, + }, + { + file: "testdata/merge_anchor.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Services: ServiceSlice{ + { + Name: "service-a", + Ports: []string{"5432:5432"}, + Environment: raw.StringSliceMap{ + "REGION": "dev", + }, + Image: "postgres", + Pull: "not_present", + }, + }, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"echo alpha"}, + Name: "alpha", + Image: "alpine:latest", + Pull: "not_present", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Commands: raw.StringSlice{"echo beta"}, + Name: "beta", + Image: "alpine:latest", + Pull: "not_present", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Commands: raw.StringSlice{"echo gamma"}, + Name: "gamma", + Image: "alpine:latest", + Pull: "not_present", + Environment: raw.StringSliceMap{ + "REGION": "dev", + }, + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := new(Build) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("Reading file for UnmarshalYAML returned err: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/yaml/doc.go b/compiler/types/yaml/yaml/doc.go new file mode 100644 index 000000000..30870a01c --- /dev/null +++ b/compiler/types/yaml/yaml/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +// package yaml provides the defined yaml types for Vela. +// +// Usage: +// +// import "github.com/go-vela/server/compiler/types/yaml/yaml" +package yaml diff --git a/compiler/types/yaml/metadata.go b/compiler/types/yaml/yaml/metadata.go similarity index 100% rename from compiler/types/yaml/metadata.go rename to compiler/types/yaml/yaml/metadata.go diff --git a/compiler/types/yaml/metadata_test.go b/compiler/types/yaml/yaml/metadata_test.go similarity index 100% rename from compiler/types/yaml/metadata_test.go rename to compiler/types/yaml/yaml/metadata_test.go diff --git a/compiler/types/yaml/ruleset.go b/compiler/types/yaml/yaml/ruleset.go similarity index 100% rename from compiler/types/yaml/ruleset.go rename to compiler/types/yaml/yaml/ruleset.go diff --git a/compiler/types/yaml/yaml/ruleset_test.go b/compiler/types/yaml/yaml/ruleset_test.go new file mode 100644 index 000000000..cb4e0466f --- /dev/null +++ b/compiler/types/yaml/yaml/ruleset_test.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Ruleset_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + ruleset *Ruleset + want *pipeline.Ruleset + }{ + { + ruleset: &Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push", "pull_request:labeled"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + Label: []string{"enhancement"}, + Instance: []string{"http://localhost:8080"}, + }, + Unless: Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octokitty"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + Instance: []string{"http://localhost:8080"}, + }, + Matcher: "filepath", + Operator: "and", + Continue: false, + }, + want: &pipeline.Ruleset{ + If: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push", "pull_request:labeled"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + Label: []string{"enhancement"}, + Instance: []string{"http://localhost:8080"}, + }, + Unless: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octokitty"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + Instance: []string{"http://localhost:8080"}, + }, + Matcher: "filepath", + Operator: "and", + Continue: false, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.ruleset.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Ruleset_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + file string + want *Ruleset + }{ + { + file: "testdata/ruleset_simple.yml", + want: &Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Instance: []string{"vela-server"}, + Label: []string{"bug"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Matcher: "filepath", + Operator: "and", + Continue: true, + }, + }, + { + file: "testdata/ruleset_advanced.yml", + want: &Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Event: []string{"push"}, + Tag: []string{"^refs/tags/(\\d+\\.)+\\d+$"}, + }, + Unless: Rules{ + Event: []string{"deployment:created", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened", "comment:created", "comment:edited", "schedule"}, + Path: []string{"foo.txt", "/foo/bar.txt"}, + }, + Matcher: "regexp", + Operator: "or", + Continue: true, + }, + }, + { + file: "testdata/ruleset_regex.yml", + want: &Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Event: []string{"tag"}, + Tag: []string{"^refs/tags/(\\d+\\.)+\\d+$"}, + }, + Operator: "and", + Matcher: "regex", + }, + }, + } + + // run tests + for _, test := range tests { + got := new(Ruleset) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Rules_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + rules *Rules + want *pipeline.Rules + }{ + { + rules: &Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push", "pull_request:labeled"}, + Instance: []string{"vela-server"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + Label: []string{"enhancement"}, + }, + want: &pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push", "pull_request:labeled"}, + Instance: []string{"vela-server"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + Label: []string{"enhancement"}, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.rules.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Rules_UnmarshalYAML(t *testing.T) { + // setup types + var ( + b []byte + err error + ) + + // setup tests + tests := []struct { + failure bool + file string + want *Rules + }{ + { + failure: false, + file: "testdata/ruleset_simple.yml", + want: &Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Instance: []string{"vela-server"}, + Label: []string{"bug"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + }, + { + failure: true, + file: "", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(Rules) + + if len(test.file) > 0 { + b, err = os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + } else { + b = []byte("``") + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/secret.go b/compiler/types/yaml/yaml/secret.go similarity index 100% rename from compiler/types/yaml/secret.go rename to compiler/types/yaml/yaml/secret.go diff --git a/compiler/types/yaml/yaml/secret_test.go b/compiler/types/yaml/yaml/secret_test.go new file mode 100644 index 000000000..d35b91171 --- /dev/null +++ b/compiler/types/yaml/yaml/secret_test.go @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Origin_MergeEnv(t *testing.T) { + // setup tests + tests := []struct { + origin *Origin + environment map[string]string + failure bool + }{ + { + origin: &Origin{ + Name: "vault", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.example.com", + "auth_method": "token", + "items": []interface{}{ + map[string]string{"source": "secret/docker", "path": "docker"}, + }, + }, + Pull: "always", + Secrets: StepSecretSlice{ + { + Source: "vault_token", + Target: "vault_token", + }, + }, + }, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + origin: &Origin{}, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + origin: nil, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + origin: &Origin{ + Name: "vault", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.example.com", + "auth_method": "token", + "items": []interface{}{ + map[string]string{"source": "secret/docker", "path": "docker"}, + }, + }, + Pull: "always", + Secrets: StepSecretSlice{ + { + Source: "vault_token", + Target: "vault_token", + }, + }, + }, + environment: nil, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err := test.origin.MergeEnv(test.environment) + + if test.failure { + if err == nil { + t.Errorf("MergeEnv should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("MergeEnv returned err: %v", err) + } + } +} + +func TestYaml_SecretSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + secrets *SecretSlice + want *pipeline.SecretSlice + }{ + { + secrets: &SecretSlice{ + { + Name: "docker_username", + Key: "github/octocat/docker/username", + Engine: "native", + Type: "repo", + Origin: Origin{}, + Pull: "build_start", + }, + { + Name: "docker_username", + Key: "", + Engine: "", + Type: "", + Origin: Origin{ + Name: "vault", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.company.com", + }, + Pull: "always", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Operator: "and", + }, + Secrets: StepSecretSlice{ + { + Source: "foo", + Target: "foo", + }, + { + Source: "foobar", + Target: "foobar", + }, + }, + }, + Pull: "build_start", + }, + }, + want: &pipeline.SecretSlice{ + { + Name: "docker_username", + Key: "github/octocat/docker/username", + Engine: "native", + Type: "repo", + Origin: &pipeline.Container{}, + Pull: "build_start", + }, + { + Name: "docker_username", + Key: "", + Engine: "", + Type: "", + Origin: &pipeline.Container{ + Name: "vault", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Pull: "always", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Event: []string{"push"}, + }, + Operator: "and", + }, + Secrets: pipeline.StepSecretSlice{ + { + Source: "foo", + Target: "foo", + }, + { + Source: "foobar", + Target: "foobar", + }, + }, + }, + Pull: "build_start", + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.secrets.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_SecretSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *SecretSlice + }{ + { + failure: false, + file: "testdata/secret.yml", + want: &SecretSlice{ + { + Name: "foo", + Key: "bar", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "noKey", + Key: "noKey", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "noType", + Key: "bar", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "noEngine", + Key: "bar", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "noKeyEngineAndType", + Key: "noKeyEngineAndType", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "externalSecret", + Key: "", + Engine: "", + Type: "", + Origin: Origin{ + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.company.com", + }, + Pull: "always", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Operator: "and", + Matcher: "filepath", + }, + Secrets: StepSecretSlice{ + { + Source: "foo", + Target: "FOO", + }, + { + Source: "foobar", + Target: "FOOBAR", + }, + }, + }, + Pull: "", + }, + { + Name: "", + Key: "", + Engine: "", + Type: "", + Origin: Origin{ + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.company.com", + }, + Pull: "always", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Operator: "and", + Matcher: "filepath", + }, + Secrets: StepSecretSlice{ + { + Source: "foo", + Target: "FOO", + }, + { + Source: "foobar", + Target: "FOOBAR", + }, + }, + }, + Pull: "", + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(SecretSlice) + + // run test + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYaml_StepSecretSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + secrets *StepSecretSlice + want *pipeline.StepSecretSlice + }{ + { + secrets: &StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + want: &pipeline.StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.secrets.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_StepSecretSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *StepSecretSlice + }{ + { + failure: false, + file: "testdata/step_secret_slice.yml", + want: &StepSecretSlice{ + { + Source: "foo", + Target: "BAR", + }, + { + Source: "hello", + Target: "WORLD", + }, + }, + }, + { + failure: false, + file: "testdata/step_secret_string.yml", + want: &StepSecretSlice{ + { + Source: "foo", + Target: "FOO", + }, + { + Source: "hello", + Target: "HELLO", + }, + }, + }, + { + failure: true, + file: "testdata/step_secret_slice_invalid_no_source.yml", + want: nil, + }, + { + failure: true, + file: "testdata/step_secret_slice_invalid_no_target.yml", + want: nil, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(StepSecretSlice) + + // run test + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/service.go b/compiler/types/yaml/yaml/service.go similarity index 100% rename from compiler/types/yaml/service.go rename to compiler/types/yaml/yaml/service.go diff --git a/compiler/types/yaml/yaml/service_test.go b/compiler/types/yaml/yaml/service_test.go new file mode 100644 index 000000000..85c432d8b --- /dev/null +++ b/compiler/types/yaml/yaml/service_test.go @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" +) + +func TestYaml_ServiceSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + services *ServiceSlice + want *pipeline.ContainerSlice + }{ + { + services: &ServiceSlice{ + { + Entrypoint: []string{"/usr/local/bin/docker-entrypoint.sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + want: &pipeline.ContainerSlice{ + { + Detach: true, + Entrypoint: []string{"/usr/local/bin/docker-entrypoint.sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.services.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_ServiceSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *ServiceSlice + }{ + { + failure: false, + file: "testdata/service.yml", + want: &ServiceSlice{ + { + Environment: raw.StringSliceMap{ + "POSTGRES_DB": "foo", + }, + Image: "postgres:latest", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + { + Environment: raw.StringSliceMap{ + "MYSQL_DATABASE": "foo", + }, + Image: "mysql:latest", + Name: "mysql", + Ports: []string{"3061:3061"}, + Pull: "not_present", + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "testdata/service_nil.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(ServiceSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Service_MergeEnv(t *testing.T) { + // setup tests + tests := []struct { + service *Service + environment map[string]string + failure bool + }{ + { + service: &Service{ + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:latest", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + service: &Service{}, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + service: nil, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + service: &Service{ + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:latest", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + environment: nil, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err := test.service.MergeEnv(test.environment) + + if test.failure { + if err == nil { + t.Errorf("MergeEnv should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("MergeEnv returned err: %v", err) + } + } +} diff --git a/compiler/types/yaml/yaml/stage.go b/compiler/types/yaml/yaml/stage.go new file mode 100644 index 000000000..5f9ba37ae --- /dev/null +++ b/compiler/types/yaml/yaml/stage.go @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "fmt" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" +) + +type ( + // StageSlice is the yaml representation + // of the stages block for a pipeline. + StageSlice []*Stage + + // Stage is the yaml representation + // of a stage in a pipeline. + Stage struct { + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Provide environment variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/#the-environment-key"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"minLength=1,description=Unique identifier for the stage in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/#the-name-key"` + Needs raw.StringSlice `yaml:"needs,omitempty,flow" json:"needs,omitempty" jsonschema:"description=Stages that must complete before starting the current one.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/#the-needs-key"` + Independent bool `yaml:"independent,omitempty" json:"independent,omitempty" jsonschema:"description=Stage will continue executing if other stage fails"` + Steps StepSlice `yaml:"steps,omitempty" json:"steps,omitempty" jsonschema:"required,description=Sequential execution instructions for the stage.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/#the-steps-key"` + } +) + +// ToPipeline converts the StageSlice type +// to a pipeline StageSlice type. +func (s *StageSlice) ToPipeline() *pipeline.StageSlice { + // stage slice we want to return + stageSlice := new(pipeline.StageSlice) + + // iterate through each element in the stage slice + for _, stage := range *s { + // append the element to the pipeline stage slice + *stageSlice = append(*stageSlice, &pipeline.Stage{ + Done: make(chan error, 1), + Environment: stage.Environment, + Name: stage.Name, + Needs: stage.Needs, + Independent: stage.Independent, + Steps: *stage.Steps.ToPipeline(), + }) + } + + return stageSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the StageSlice type. +func (s *StageSlice) UnmarshalYAML(v *yaml.Node) error { + if v.Kind != yaml.MappingNode { + return fmt.Errorf("invalid yaml: expected map node for stage") + } + + // iterate through each element in the map slice + for i := 0; i < len(v.Content); i += 2 { + key := v.Content[i] + value := v.Content[i+1] + + stage := new(Stage) + + // unmarshal value into stage + err := value.Decode(stage) + if err != nil { + return err + } + + // implicitly set stage `name` if empty + if len(stage.Name) == 0 { + stage.Name = fmt.Sprintf("%v", key.Value) + } + + // implicitly set the stage `needs` + if stage.Name != "clone" && stage.Name != "init" { + // add clone if not present + stage.Needs = func(needs []string) []string { + for _, s := range needs { + if s == "clone" { + return needs + } + } + + return append(needs, "clone") + }(stage.Needs) + } + // append stage to stage slice + *s = append(*s, stage) + } + + return nil +} + +// MarshalYAML implements the marshaler interface for the StageSlice type. +func (s StageSlice) MarshalYAML() (interface{}, error) { + output := new(yaml.Node) + output.Kind = yaml.MappingNode + + for _, inputStage := range s { + n := new(yaml.Node) + + // create new stage with existing properties + outputStage := &Stage{ + Name: inputStage.Name, + Needs: inputStage.Needs, + Independent: inputStage.Independent, + Steps: inputStage.Steps, + } + + err := n.Encode(outputStage) + if err != nil { + return nil, err + } + + // append stage to map output + output.Content = append(output.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: inputStage.Name}) + output.Content = append(output.Content, n) + } + + return output, nil +} + +// MergeEnv takes a list of environment variables and attempts +// to set them in the stage environment. If the environment +// variable already exists in the stage, than this will +// overwrite the existing environment variable. +func (s *Stage) MergeEnv(environment map[string]string) error { + // check if the stage is empty + if s == nil || s.Environment == nil { + // TODO: evaluate if we should error here + // + // immediately return and do nothing + // + // treated as a no-op + return nil + } + + // check if the environment provided is empty + if environment == nil { + return fmt.Errorf("empty environment provided for stage %s", s.Name) + } + + // iterate through all environment variables provided + for key, value := range environment { + // set or update the stage environment variable + s.Environment[key] = value + } + + return nil +} diff --git a/compiler/types/yaml/yaml/stage_test.go b/compiler/types/yaml/yaml/stage_test.go new file mode 100644 index 000000000..86169f373 --- /dev/null +++ b/compiler/types/yaml/yaml/stage_test.go @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_StageSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + stages *StageSlice + want *pipeline.StageSlice + }{ + { + stages: &StageSlice{ + { + Name: "echo", + Needs: []string{"clone"}, + Steps: StepSlice{ + { + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + Ruleset: Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Unless: Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + }, + Operator: "and", + Continue: false, + }, + Secrets: StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + }, + }, + want: &pipeline.StageSlice{ + { + Name: "echo", + Needs: []string{"clone"}, + Steps: pipeline.ContainerSlice{ + { + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Unless: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + }, + Operator: "and", + Continue: false, + }, + Secrets: pipeline.StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + Ulimits: pipeline.UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: pipeline.VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.stages.ToPipeline() + + // WARNING: hack to compare stages + // + // Channel values can only be compared for equality. + // Two channel values are considered equal if they + // originated from the same make call meaning they + // refer to the same channel value in memory. + for i, stage := range *got { + tmp := *test.want + + tmp[i].Done = stage.Done + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_StageSlice_UnmarshalYAML(t *testing.T) { + // setup types + var ( + b []byte + err error + ) + + // setup tests + tests := []struct { + failure bool + file string + want *StageSlice + }{ + { + failure: false, + file: "testdata/stage.yml", + want: &StageSlice{ + { + Name: "dependencies", + Needs: []string{"clone"}, + Environment: map[string]string{ + "STAGE_ENV_VAR": "stage", + }, + Independent: true, + Steps: StepSlice{ + { + Commands: []string{"./gradlew downloadDependencies"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + }, + }, + }, + { + Name: "test", + Needs: []string{"dependencies", "clone"}, + Environment: map[string]string{ + "STAGE_ENV_VAR": "stage", + "SECOND_STAGE_ENV": "stage2", + }, + Independent: false, + Steps: StepSlice{ + { + Commands: []string{"./gradlew check"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + }, + }, + }, + { + Name: "build", + Needs: []string{"dependencies", "clone"}, + Environment: map[string]string{ + "STAGE_ENV_VAR": "stage", + }, + Independent: false, + Steps: StepSlice{ + { + Commands: []string{"./gradlew build"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + }, + }, + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(StageSlice) + + if len(test.file) > 0 { + b, err = os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + } else { + b = []byte("- foo") + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("(Unmarshal mismatch: -want +got):\n%s", diff) + } + } +} + +func TestYaml_StageSlice_MarshalYAML(t *testing.T) { + // setup types + var ( + b []byte + err error + ) + + // setup tests + tests := []struct { + failure bool + file string + want *StageSlice + }{ + { + failure: false, + file: "testdata/stage.yml", + want: &StageSlice{ + { + Name: "dependencies", + Needs: []string{"clone"}, + Independent: true, + Steps: StepSlice{ + { + Commands: []string{"./gradlew downloadDependencies"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + }, + }, + }, + { + Name: "test", + Needs: []string{"dependencies", "clone"}, + Independent: false, + Steps: StepSlice{ + { + Commands: []string{"./gradlew check"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + }, + }, + }, + { + Name: "build", + Needs: []string{"dependencies", "clone"}, + Independent: false, + Steps: StepSlice{ + { + Commands: []string{"./gradlew build"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + }, + }, + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(StageSlice) + got2 := new(StageSlice) + + if len(test.file) > 0 { + b, err = os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + } else { + b = []byte("- foo") + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + out, err := yaml.Marshal(got) + if err != nil { + t.Errorf("MarshalYAML returned err: %v", err) + } + + err = yaml.Unmarshal(out, got2) + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if diff := cmp.Diff(got2, test.want); diff != "" { + t.Errorf("(Marshal mismatch: -got +want):\n%s", diff) + } + } +} + +func TestYaml_Stage_MergeEnv(t *testing.T) { + // setup tests + tests := []struct { + stage *Stage + environment map[string]string + failure bool + }{ + { + stage: &Stage{ + Environment: map[string]string{"FOO": "bar"}, + Name: "testStage", + }, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + stage: &Stage{}, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + stage: nil, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + stage: &Stage{ + Environment: map[string]string{"FOO": "bar"}, + Name: "testStage", + }, + environment: nil, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err := test.stage.MergeEnv(test.environment) + + if test.failure { + if err == nil { + t.Errorf("MergeEnv should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("MergeEnv returned err: %v", err) + } + } +} diff --git a/compiler/types/yaml/step.go b/compiler/types/yaml/yaml/step.go similarity index 100% rename from compiler/types/yaml/step.go rename to compiler/types/yaml/yaml/step.go diff --git a/compiler/types/yaml/yaml/step_test.go b/compiler/types/yaml/yaml/step_test.go new file mode 100644 index 000000000..343daf230 --- /dev/null +++ b/compiler/types/yaml/yaml/step_test.go @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" +) + +func TestYaml_StepSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + steps *StepSlice + want *pipeline.ContainerSlice + }{ + { + steps: &StepSlice{ + { + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + ReportAs: "my-step", + IDRequest: "yes", + Ruleset: Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Unless: Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + }, + Operator: "and", + Continue: false, + }, + Secrets: StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + want: &pipeline.ContainerSlice{ + { + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + ReportAs: "my-step", + IDRequest: "yes", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Unless: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + }, + Operator: "and", + Continue: false, + }, + Secrets: pipeline.StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + Ulimits: pipeline.UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: pipeline.VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.steps.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_StepSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *StepSlice + }{ + { + failure: false, + file: "testdata/step.yml", + want: &StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "install", + Image: "openjdk:latest", + Pull: "always", + }, + { + Commands: raw.StringSlice{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + }, + { + Commands: raw.StringSlice{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + }, + { + Name: "docker_build", + Image: "plugins/docker:18.09", + Pull: "always", + ReportAs: "docker", + Parameters: map[string]interface{}{ + "registry": "index.docker.io", + "repo": "github/octocat", + "tags": []interface{}{"latest", "dev"}, + }, + }, + { + Name: "templated_publish", + Pull: "not_present", + Template: StepTemplate{ + Name: "docker_publish", + Variables: map[string]interface{}{ + "registry": "index.docker.io", + "repo": "github/octocat", + "tags": []interface{}{"latest", "dev"}, + }, + }, + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "testdata/step_malformed.yml", + want: nil, + }, + { + failure: true, + file: "testdata/step_nil.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(StepSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Step_MergeEnv(t *testing.T) { + // setup tests + tests := []struct { + step *Step + environment map[string]string + failure bool + }{ + { + step: &Step{ + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + }, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + step: &Step{}, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + step: nil, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + step: &Step{ + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + }, + environment: nil, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err := test.step.MergeEnv(test.environment) + + if test.failure { + if err == nil { + t.Errorf("MergeEnv should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("MergeEnv returned err: %v", err) + } + } +} diff --git a/compiler/types/yaml/template.go b/compiler/types/yaml/yaml/template.go similarity index 100% rename from compiler/types/yaml/template.go rename to compiler/types/yaml/yaml/template.go diff --git a/compiler/types/yaml/yaml/template_test.go b/compiler/types/yaml/yaml/template_test.go new file mode 100644 index 000000000..128c998a0 --- /dev/null +++ b/compiler/types/yaml/yaml/template_test.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + api "github.com/go-vela/server/api/types" +) + +func TestBuild_TemplateSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *TemplateSlice + }{ + { + failure: false, + file: "testdata/template.yml", + want: &TemplateSlice{ + { + Name: "docker_build", + Source: "github.com/go-vela/atlas/stable/docker_create", + Type: "github", + }, + { + Name: "docker_build", + Source: "github.com/go-vela/atlas/stable/docker_build", + Format: "go", + Type: "github", + }, + { + Name: "docker_publish", + Source: "github.com/go-vela/atlas/stable/docker_publish", + Format: "starlark", + Type: "github", + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(TemplateSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYAML_Template_ToAPI(t *testing.T) { + // setup types + want := new(api.Template) + want.SetName("docker_build") + want.SetSource("github.com/go-vela/atlas/stable/docker_build") + want.SetType("github") + + tmpl := &Template{ + Name: "docker_build", + Source: "github.com/go-vela/atlas/stable/docker_build", + Type: "github", + } + + // run test + got := tmpl.ToAPI() + + if !reflect.DeepEqual(got, want) { + t.Errorf("ToAPI is %v, want %v", got, want) + } +} + +func TestYAML_TemplateFromAPI(t *testing.T) { + // setup types + want := &Template{ + Name: "docker_build", + Source: "github.com/go-vela/atlas/stable/docker_build", + Type: "github", + } + + tmpl := new(api.Template) + tmpl.SetName("docker_build") + tmpl.SetSource("github.com/go-vela/atlas/stable/docker_build") + tmpl.SetType("github") + + // run test + got := TemplateFromAPI(tmpl) + + if !reflect.DeepEqual(got, want) { + t.Errorf("TemplateFromAPI is %v, want %v", got, want) + } +} diff --git a/compiler/types/yaml/yaml/testdata/build.yml b/compiler/types/yaml/yaml/testdata/build.yml new file mode 100644 index 000000000..e1a7fbc9f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build.yml @@ -0,0 +1,144 @@ +--- +version: "1" + +environment: + HELLO: "Hello, Global Message" + +templates: + - name: docker_publish + source: github.com/go-vela/atlas/stable/docker_publish + type: github + +worker: + flavor: 16cpu8gb + platform: gcp + +services: + - name: postgres + image: postgres:latest + environment: + POSTGRES_DB: foo + ports: + - "5432:5432" + +steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + ruleset: + event: [ push, pull_request:opened, pull_request:synchronize, pull_request:edited ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + - name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + - name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true + ruleset: + event: [ push, pull_request ] + volumes: + - source: /foo + destination: /bar + access_mode: ro + ulimits: + - name: foo + soft: 1024 + hard: 2048 + + - name: docker_build + image: plugins/docker:18.09 + parameters: + dry_run: true + registry: index.docker.io + repo: github/octocat + tags: + - latest + - dev + pull: true + ruleset: + if: + event: [ push, pull_request ] + operator: and + + - name: docker_publish + image: plugins/docker:18.09 + parameters: + registry: index.docker.io + repo: github/octocat + tags: + - latest + - dev + pull: true + ruleset: + if: + branch: main + event: push + operator: and + secrets: + - source: docker_username + target: plugin_username + - source: docker_password + target: plugin_password + +secrets: + # Repo secrets + - name: docker_username + key: org/repo/docker/username + engine: native + type: repo + + - name: docker_password + key: org/repo/docker/password + engine: vault + type: repo + + # Org secrets + - name: docker_username + key: org/docker/username + engine: native + type: org + + - name: docker_password + key: org/docker/password + engine: vault + type: org + + # Shared secrets + - name: docker_username + key: org/team/docker/username + engine: native + type: shared + + - name: docker_password + key: org/team/docker/password + engine: vault + type: shared + + - origin: + image: target/vela-vault:latest + pull: always + parameters: + addr: vault.example.com + secrets: [ docker_username, docker_password ] diff --git a/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline0.yml b/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline0.yml new file mode 100644 index 000000000..8cbe12806 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline0.yml @@ -0,0 +1 @@ +version: 1 \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline1.yml b/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline1.yml new file mode 100644 index 000000000..b1db03665 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline1.yml @@ -0,0 +1,3 @@ +version: "1" +steps: +stages: \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/build/validate/bad_version.yml b/compiler/types/yaml/yaml/testdata/build/validate/bad_version.yml new file mode 100644 index 000000000..e2489f424 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build/validate/bad_version.yml @@ -0,0 +1,2 @@ +--- +steps: \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/build/validate/step.yml b/compiler/types/yaml/yaml/testdata/build/validate/step.yml new file mode 100644 index 000000000..a70942591 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build/validate/step.yml @@ -0,0 +1,47 @@ +--- +version: 1 +steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + + - name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + + - name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true + + - name: docker_build + image: plugins/docker:18.09 + parameters: + registry: index.docker.io + repo: github/octocat + tags: + - latest + - dev + pull: true + + - name: templated_publish + template: + name: docker_publish + vars: + registry: index.docker.io + repo: github/octocat + tags: [ latest, dev ] diff --git a/compiler/types/yaml/yaml/testdata/build_anchor_stage.yml b/compiler/types/yaml/yaml/testdata/build_anchor_stage.yml new file mode 100644 index 000000000..2fc87932b --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build_anchor_stage.yml @@ -0,0 +1,57 @@ +--- +version: "1" + +metadata: + template: false + +stage-anchor: &stage-anchor + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + +stages: + dependencies: + steps: + - name: install + commands: + - ./gradlew downloadDependencies + <<: *stage-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + test: + needs: [ dependencies ] + steps: + - name: test + commands: + - ./gradlew check + <<: *stage-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + build: + needs: [ dependencies ] + independent: true + steps: + - name: build + commands: + - ./gradlew build + <<: *stage-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: + - source: /foo + destination: /bar + access_mode: ro + ulimits: + - name: foo + soft: 1024 + hard: 2048 diff --git a/compiler/types/yaml/yaml/testdata/build_anchor_step.yml b/compiler/types/yaml/yaml/testdata/build_anchor_step.yml new file mode 100644 index 000000000..ffc7abf50 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build_anchor_step.yml @@ -0,0 +1,48 @@ +--- +version: "1" + +metadata: + template: false + +step-anchor: &step-anchor + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + +steps: + - name: install + commands: + - ./gradlew downloadDependencies + <<: *step-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + - name: test + commands: + - ./gradlew check + <<: *step-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + - name: build + commands: + - ./gradlew build + <<: *step-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: + - source: /foo + destination: /bar + access_mode: ro + ulimits: + - name: foo + soft: 1024 + hard: 2048 diff --git a/compiler/types/yaml/yaml/testdata/build_empty_env.yml b/compiler/types/yaml/yaml/testdata/build_empty_env.yml new file mode 100644 index 000000000..6b4f7d063 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build_empty_env.yml @@ -0,0 +1,27 @@ +--- +version: "1" + +metadata: + template: false + environment: [] + +environment: + HELLO: "Hello, Global Message" + +worker: + flavor: 16cpu8gb + platform: gcp + +steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/invalid.yml b/compiler/types/yaml/yaml/testdata/invalid.yml new file mode 100644 index 000000000..23809fe06 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/invalid.yml @@ -0,0 +1,2 @@ +--- +foo: bar diff --git a/compiler/types/yaml/yaml/testdata/merge_anchor.yml b/compiler/types/yaml/yaml/testdata/merge_anchor.yml new file mode 100644 index 000000000..626b695f5 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/merge_anchor.yml @@ -0,0 +1,45 @@ +# test file that uses the non-standard multiple anchor keys in one step to test custom step unmarshaler + +version: "1" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + postgres: &pg-image + image: postgres + + events: + push: &event-push + ruleset: + event: + - push + env: + dev-env: &dev-environment + environment: + REGION: dev + +services: + - name: service-a + <<: [ *pg-image, *dev-environment ] + ports: + - "5432:5432" + +steps: + - name: alpha + <<: [ *alpine-image, *event-push ] + commands: + - echo alpha + + - name: beta + <<: [ *alpine-image, *event-push ] + commands: + - echo beta + + - name: gamma + <<: + - *alpine-image + - *event-push + - *dev-environment + commands: + - echo gamma \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/metadata.yml b/compiler/types/yaml/yaml/testdata/metadata.yml new file mode 100644 index 000000000..6946050b0 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/metadata.yml @@ -0,0 +1,2 @@ +--- +template: false diff --git a/compiler/types/yaml/yaml/testdata/metadata_env.yml b/compiler/types/yaml/yaml/testdata/metadata_env.yml new file mode 100644 index 000000000..0b7932a30 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/metadata_env.yml @@ -0,0 +1,3 @@ +--- +template: false +environment: [ steps ] diff --git a/compiler/types/yaml/yaml/testdata/ruleset_advanced.yml b/compiler/types/yaml/yaml/testdata/ruleset_advanced.yml new file mode 100644 index 000000000..24039e2a6 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ruleset_advanced.yml @@ -0,0 +1,15 @@ +--- +if: + branch: [ main ] + event: push + tag: "^refs/tags/(\\d+\\.)+\\d+$" +unless: + event: + - deployment + - pull_request + - comment + - schedule + path: [ foo.txt, /foo/bar.txt ] +matcher: regexp +operator: or +continue: true \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/ruleset_regex.yml b/compiler/types/yaml/yaml/testdata/ruleset_regex.yml new file mode 100644 index 000000000..eb6b1fd31 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ruleset_regex.yml @@ -0,0 +1,7 @@ +--- +if: + branch: main + event: tag + tag: [ "^refs/tags/(\\d+\\.)+\\d+$" ] + operator: and +matcher: regex diff --git a/compiler/types/yaml/yaml/testdata/ruleset_simple.yml b/compiler/types/yaml/yaml/testdata/ruleset_simple.yml new file mode 100644 index 000000000..acbe4e198 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ruleset_simple.yml @@ -0,0 +1,13 @@ +--- +branch: main +comment: "test comment" +continue: true +event: push +instance: vela-server +label: bug +path: foo.txt +repo: github/octocat +sender: octocat +status: success +tag: v0.1.0 +target: production diff --git a/compiler/types/yaml/yaml/testdata/secret.yml b/compiler/types/yaml/yaml/testdata/secret.yml new file mode 100644 index 000000000..c432eb291 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret.yml @@ -0,0 +1,39 @@ +--- +- source: foo + target: bar +- name: foo + key: bar + engine: native + type: repo + pull: build_start +- name: noKey + engine: native + type: repo +- name: noType + key: bar + engine: native +- name: noEngine + key: bar + type: repo +- name: noKeyEngineAndType +- name: externalSecret + origin: + environment: + FOO: bar + image: target/vela-vault:latest + pull: true + parameters: + addr: vault.company.com + ruleset: + event: [ push ] + secrets: [ foo, foobar ] +- origin: + environment: + FOO: bar + image: target/vela-vault:latest + pull: true + parameters: + addr: vault.company.com + ruleset: + event: [ push ] + secrets: [ foo, foobar ] diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/no_name.yml b/compiler/types/yaml/yaml/testdata/secret/validate/no_name.yml new file mode 100644 index 000000000..d674817c5 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/no_name.yml @@ -0,0 +1,11 @@ +secrets: +# Declarative repository secret definition. + - key: github/ocotocat/foob + engine: native + type: repo + - key: github/ocotocat + engine: native + type: org + - key: github/octokitties/foobar + engine: native + type: org \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/org.yml b/compiler/types/yaml/yaml/testdata/secret/validate/org.yml new file mode 100644 index 000000000..a5aad5e0d --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/org.yml @@ -0,0 +1,5 @@ +secrets: + - name: foobar + key: github/foobar + engine: native + type: org \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_engine.yml b/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_engine.yml new file mode 100644 index 000000000..18f9f8c6f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_engine.yml @@ -0,0 +1,9 @@ +secrets: + - name: foo + key: github/foobar + type: org + + - name: foobar + key: github/foobar + engine: badengine + type: org \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_key.yml b/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_key.yml new file mode 100644 index 000000000..bae2fcc9a --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_key.yml @@ -0,0 +1,9 @@ +secrets: + - name: foo + engine: native + type: org + + - name: foobar + key: github + engine: native + type: org \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/plugin.yml b/compiler/types/yaml/yaml/testdata/secret/validate/plugin.yml new file mode 100644 index 000000000..180ab5da4 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/plugin.yml @@ -0,0 +1,8 @@ +secrets: + - origin: + name: vault secrets + image: target/vela/secret-vault:latest + parameters: + items: + - source: secret/vela/dev/docker + path: docker \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_image.yml b/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_image.yml new file mode 100644 index 000000000..c21be424e --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_image.yml @@ -0,0 +1,15 @@ +secrets: + - origin: + name: vault secrets + parameters: + items: + - source: secret/vela/dev/docker + path: docker + + - origin: + name: vault secrets + image: bazel/:java:3240943c9ea3f72db51bea0a2428e83f3c5fa1312e19af017d026f9bcf70f84b + parameters: + items: + - source: secret/vela/dev/docker + path: docker \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_name.yml b/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_name.yml new file mode 100644 index 000000000..6ebb1505f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_name.yml @@ -0,0 +1,7 @@ +secrets: + - origin: + image: target/vela/secret-vault:latest + parameters: + items: + - source: secret/vela/dev/docker + path: docker \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/repo.yml b/compiler/types/yaml/yaml/testdata/secret/validate/repo.yml new file mode 100644 index 000000000..fcad31edb --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/repo.yml @@ -0,0 +1,13 @@ +secrets: + # Implicit native secret definition. + - name: foo + + # Declarative repository secret definition. + - name: foob + key: github/ocotocat/foob + engine: native + type: repo + - name: foo_bar + key: github/ocotocat/foo/bar + engine: native + type: repo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_engine.yml b/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_engine.yml new file mode 100644 index 000000000..3eac1d359 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_engine.yml @@ -0,0 +1,5 @@ +secrets: + - name: foobar + key: github/ocotocat/foobar + engine: badengine + type: repo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_key.yml b/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_key.yml new file mode 100644 index 000000000..cf031b30c --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_key.yml @@ -0,0 +1,14 @@ +secrets: + - name: foo + engine: native + type: repo + + - name: bar + key: github/ocotocat + engine: native + type: repo + + - name: foobar + key: github + engine: native + type: repo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/shared.yml b/compiler/types/yaml/yaml/testdata/secret/validate/shared.yml new file mode 100644 index 000000000..4037a97a3 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/shared.yml @@ -0,0 +1,5 @@ +secrets: + - name: foobar + key: github/ocotokitties/foo + engine: native + type: shared \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_engine.yml b/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_engine.yml new file mode 100644 index 000000000..ca22067dd --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_engine.yml @@ -0,0 +1,9 @@ +secrets: + - name: foo + key: github/ocotokitties/foo + type: shared + + - name: foobar + key: github/ocotokitties/foo + engine: badengine + type: shared \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_key.yml b/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_key.yml new file mode 100644 index 000000000..b80945618 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_key.yml @@ -0,0 +1,9 @@ +secrets: + - name: foo + engine: native + type: shared + + - name: foobar + key: github/ocotokitties + engine: native + type: shared \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service.yml b/compiler/types/yaml/yaml/testdata/service.yml new file mode 100644 index 000000000..b6bc0456f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service.yml @@ -0,0 +1,14 @@ +--- +- name: postgres + image: postgres:latest + environment: + POSTGRES_DB: foo + ports: + - "5432:5432" + +- name: mysql + image: mysql:latest + environment: + MYSQL_DATABASE: foo + ports: + - "3061:3061" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service/validate/bad_image.yml b/compiler/types/yaml/yaml/testdata/service/validate/bad_image.yml new file mode 100644 index 000000000..47a875e67 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service/validate/bad_image.yml @@ -0,0 +1,3 @@ +services: + - name: badimage + image: bazel/:java:3240943c9ea3f72db51bea0a2428e83f3c5fa1312e19af017d026f9bcf70f84b \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service/validate/minimal.yml b/compiler/types/yaml/yaml/testdata/service/validate/minimal.yml new file mode 100644 index 000000000..469d2dddf --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service/validate/minimal.yml @@ -0,0 +1,3 @@ +services: + - name: postgres + image: postgres:latest diff --git a/compiler/types/yaml/yaml/testdata/service/validate/missing_image.yml b/compiler/types/yaml/yaml/testdata/service/validate/missing_image.yml new file mode 100644 index 000000000..3c2b76f1f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service/validate/missing_image.yml @@ -0,0 +1,2 @@ +services: + - name: postgres \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service/validate/missing_name.yml b/compiler/types/yaml/yaml/testdata/service/validate/missing_name.yml new file mode 100644 index 000000000..0c1034e5d --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service/validate/missing_name.yml @@ -0,0 +1,2 @@ +services: + - image: postgres:latest \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service_nil.yml b/compiler/types/yaml/yaml/testdata/service_nil.yml new file mode 100644 index 000000000..41cd65e60 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service_nil.yml @@ -0,0 +1,2 @@ +--- +- diff --git a/compiler/types/yaml/yaml/testdata/stage.yml b/compiler/types/yaml/yaml/testdata/stage.yml new file mode 100644 index 000000000..543ffdf83 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage.yml @@ -0,0 +1,44 @@ +--- +dependencies: + environment: + STAGE_ENV_VAR: stage + independent: true + steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +test: + needs: [ dependencies ] + environment: + STAGE_ENV_VAR: stage + SECOND_STAGE_ENV: stage2 + independent: false + steps: + - name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +build: + needs: [ dependencies ] + environment: + STAGE_ENV_VAR: stage + steps: + - name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/bad_image.yml b/compiler/types/yaml/yaml/testdata/stage/validate/bad_image.yml new file mode 100644 index 000000000..3dbed3107 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/bad_image.yml @@ -0,0 +1,7 @@ +stages: + badimage: + steps: + - name: badimage + image: bazel/:java:3240943c9ea3f72db51bea0a2428e83f3c5fa1312e19af017d026f9bcf70f84b + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/minimal.yml b/compiler/types/yaml/yaml/testdata/stage/validate/minimal.yml new file mode 100644 index 000000000..665887c87 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/minimal.yml @@ -0,0 +1,7 @@ +stages: + hello: + steps: + - name: hello + image: alpine:latest + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/missing.yml b/compiler/types/yaml/yaml/testdata/stage/validate/missing.yml new file mode 100644 index 000000000..954fdeabe --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/missing.yml @@ -0,0 +1,5 @@ +stages: + hello: + steps: + - name: hello + image: alpine:latest \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/missing_image.yml b/compiler/types/yaml/yaml/testdata/stage/validate/missing_image.yml new file mode 100644 index 000000000..90b361748 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/missing_image.yml @@ -0,0 +1,6 @@ +stages: + hello: + steps: + - name: hello + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/missing_name.yml b/compiler/types/yaml/yaml/testdata/stage/validate/missing_name.yml new file mode 100644 index 000000000..1394939a3 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/missing_name.yml @@ -0,0 +1,6 @@ +stages: + hello: + steps: + - image: alpine:latest + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step.yml b/compiler/types/yaml/yaml/testdata/step.yml new file mode 100644 index 000000000..1d6d9cc93 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step.yml @@ -0,0 +1,46 @@ +--- +- name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +- name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +- name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true + +- name: docker_build + image: plugins/docker:18.09 + report_as: docker + parameters: + registry: index.docker.io + repo: github/octocat + tags: + - latest + - dev + pull: true + +- name: templated_publish + template: + name: docker_publish + vars: + registry: index.docker.io + repo: github/octocat + tags: [ latest, dev ] diff --git a/compiler/types/yaml/yaml/testdata/step/validate/bad_image.yml b/compiler/types/yaml/yaml/testdata/step/validate/bad_image.yml new file mode 100644 index 000000000..a97e8f12c --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/bad_image.yml @@ -0,0 +1,5 @@ +steps: + - name: badimage + image: bazel/:java:3240943c9ea3f72db51bea0a2428e83f3c5fa1312e19af017d026f9bcf70f84b + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step/validate/minimal.yml b/compiler/types/yaml/yaml/testdata/step/validate/minimal.yml new file mode 100644 index 000000000..da2283d4b --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/minimal.yml @@ -0,0 +1,5 @@ +steps: + - name: hello + image: alpine:latest + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step/validate/missing.yml b/compiler/types/yaml/yaml/testdata/step/validate/missing.yml new file mode 100644 index 000000000..0aa30db52 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/missing.yml @@ -0,0 +1,3 @@ +steps: + - name: hello + image: alpine:latest \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step/validate/missing_image.yml b/compiler/types/yaml/yaml/testdata/step/validate/missing_image.yml new file mode 100644 index 000000000..b36acf70d --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/missing_image.yml @@ -0,0 +1,4 @@ +steps: + - name: hello + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step/validate/missing_name.yml b/compiler/types/yaml/yaml/testdata/step/validate/missing_name.yml new file mode 100644 index 000000000..228ac30ec --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/missing_name.yml @@ -0,0 +1,4 @@ +steps: + - image: alpine:latest + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step_malformed.yml b/compiler/types/yaml/yaml/testdata/step_malformed.yml new file mode 100644 index 000000000..5d70e5a4f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_malformed.yml @@ -0,0 +1,4 @@ +--- +- name: Testing + environment: + - 'This: Shouldnt Panic' diff --git a/compiler/types/yaml/yaml/testdata/step_nil.yml b/compiler/types/yaml/yaml/testdata/step_nil.yml new file mode 100644 index 000000000..41cd65e60 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_nil.yml @@ -0,0 +1,2 @@ +--- +- diff --git a/compiler/types/yaml/yaml/testdata/step_secret_slice.yml b/compiler/types/yaml/yaml/testdata/step_secret_slice.yml new file mode 100644 index 000000000..64bc68b7f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_secret_slice.yml @@ -0,0 +1,5 @@ +--- +- source: foo + target: bar +- source: hello + target: world diff --git a/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_source.yml b/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_source.yml new file mode 100644 index 000000000..1f7e6fc3e --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_source.yml @@ -0,0 +1,2 @@ +--- +- target: foo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_target.yml b/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_target.yml new file mode 100644 index 000000000..3e0e29b1c --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_target.yml @@ -0,0 +1,2 @@ +--- +- source: foo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step_secret_string.yml b/compiler/types/yaml/yaml/testdata/step_secret_string.yml new file mode 100644 index 000000000..930977980 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_secret_string.yml @@ -0,0 +1,2 @@ +--- +[ foo, hello ] diff --git a/compiler/types/yaml/yaml/testdata/template.yml b/compiler/types/yaml/yaml/testdata/template.yml new file mode 100644 index 000000000..6d5615e01 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/template.yml @@ -0,0 +1,12 @@ +--- +- name: docker_build + source: github.com/go-vela/atlas/stable/docker_create + type: github +- name: docker_build + source: github.com/go-vela/atlas/stable/docker_build + format: go + type: github +- name: docker_publish + source: github.com/go-vela/atlas/stable/docker_publish + format: starlark + type: github diff --git a/compiler/types/yaml/yaml/testdata/ulimit_colon_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_colon_error.yml new file mode 100644 index 000000000..3e948cb08 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_colon_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=bar:1024:2048 ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_equal_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_equal_error.yml new file mode 100644 index 000000000..f72b3b461 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_equal_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=1024=2048 ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_hardlimit1_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_hardlimit1_error.yml new file mode 100644 index 000000000..1472c22b7 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_hardlimit1_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=bar:1024 ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_hardlimit2_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_hardlimit2_error.yml new file mode 100644 index 000000000..4569bc3ad --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_hardlimit2_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=1024:bar ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_slice.yml b/compiler/types/yaml/yaml/testdata/ulimit_slice.yml new file mode 100644 index 000000000..9ee862c06 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_slice.yml @@ -0,0 +1,6 @@ +--- +- name: foo + soft: 1024 +- name: bar + soft: 1024 + hard: 2048 diff --git a/compiler/types/yaml/yaml/testdata/ulimit_softlimit_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_softlimit_error.yml new file mode 100644 index 000000000..63f68f1c4 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_softlimit_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=bar ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_string.yml b/compiler/types/yaml/yaml/testdata/ulimit_string.yml new file mode 100644 index 000000000..59669af36 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_string.yml @@ -0,0 +1,2 @@ +--- +[ foo=1024, bar=1024:2048 ] diff --git a/compiler/types/yaml/yaml/testdata/volume_error.yml b/compiler/types/yaml/yaml/testdata/volume_error.yml new file mode 100644 index 000000000..8c36e5057 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/volume_error.yml @@ -0,0 +1,2 @@ +--- +[ /foo:/bar:/foo:bar ] diff --git a/compiler/types/yaml/yaml/testdata/volume_slice.yml b/compiler/types/yaml/yaml/testdata/volume_slice.yml new file mode 100644 index 000000000..fbad0133b --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/volume_slice.yml @@ -0,0 +1,7 @@ +--- +- source: /foo +- source: /foo + destination: /bar +- source: /foo + destination: /foobar + access_mode: ro diff --git a/compiler/types/yaml/yaml/testdata/volume_string.yml b/compiler/types/yaml/yaml/testdata/volume_string.yml new file mode 100644 index 000000000..a596a9116 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/volume_string.yml @@ -0,0 +1,2 @@ +--- +[ /foo, /foo:/bar, /foo:/foobar:ro ] diff --git a/compiler/types/yaml/ulimit.go b/compiler/types/yaml/yaml/ulimit.go similarity index 100% rename from compiler/types/yaml/ulimit.go rename to compiler/types/yaml/yaml/ulimit.go diff --git a/compiler/types/yaml/yaml/ulimit_test.go b/compiler/types/yaml/yaml/ulimit_test.go new file mode 100644 index 000000000..a8da414ee --- /dev/null +++ b/compiler/types/yaml/yaml/ulimit_test.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_UlimitSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + ulimits *UlimitSlice + want *pipeline.UlimitSlice + }{ + { + ulimits: &UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + want: &pipeline.UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.ulimits.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_UlimitSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *UlimitSlice + }{ + { + failure: false, + file: "testdata/ulimit_slice.yml", + want: &UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 1024, + }, + { + Name: "bar", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + failure: false, + file: "testdata/ulimit_string.yml", + want: &UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 1024, + }, + { + Name: "bar", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_equal_error.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_colon_error.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_softlimit_error.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_hardlimit1_error.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_hardlimit2_error.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(UlimitSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/volume.go b/compiler/types/yaml/yaml/volume.go similarity index 100% rename from compiler/types/yaml/volume.go rename to compiler/types/yaml/yaml/volume.go diff --git a/compiler/types/yaml/volume_test.go b/compiler/types/yaml/yaml/volume_test.go similarity index 100% rename from compiler/types/yaml/volume_test.go rename to compiler/types/yaml/yaml/volume_test.go diff --git a/compiler/types/yaml/worker.go b/compiler/types/yaml/yaml/worker.go similarity index 100% rename from compiler/types/yaml/worker.go rename to compiler/types/yaml/yaml/worker.go diff --git a/compiler/types/yaml/worker_test.go b/compiler/types/yaml/yaml/worker_test.go similarity index 100% rename from compiler/types/yaml/worker_test.go rename to compiler/types/yaml/yaml/worker_test.go diff --git a/go.mod b/go.mod index 290078d95..02d450edb 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/time v0.6.0 + gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.9 gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.12 @@ -148,7 +149,6 @@ require ( google.golang.org/grpc v1.66.1 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect ) diff --git a/internal/testdata/buildkite.yml b/internal/testdata/buildkite.yml new file mode 100644 index 000000000..1a74a344b --- /dev/null +++ b/internal/testdata/buildkite.yml @@ -0,0 +1,18 @@ +version: "legacy" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + + env: + dev-env: &dev-environment + environment: + REGION: dev + +steps: + - name: example + <<: *alpine-image + <<: *dev-environment + commands: + - echo $REGION \ No newline at end of file diff --git a/internal/testdata/go-yaml.yml b/internal/testdata/go-yaml.yml new file mode 100644 index 000000000..7a9d284f5 --- /dev/null +++ b/internal/testdata/go-yaml.yml @@ -0,0 +1,19 @@ +version: "1" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + + env: + dev-env: &dev-environment + environment: + REGION: dev + +steps: + - name: example + <<: + - *alpine-image + - *dev-environment + commands: + - echo $REGION diff --git a/internal/testdata/invalid.yml b/internal/testdata/invalid.yml new file mode 100644 index 000000000..a1cde504a --- /dev/null +++ b/internal/testdata/invalid.yml @@ -0,0 +1,2 @@ +- sliceNodeA +- sliceNodeB \ No newline at end of file diff --git a/internal/testdata/no_version.yml b/internal/testdata/no_version.yml new file mode 100644 index 000000000..5b59dea2a --- /dev/null +++ b/internal/testdata/no_version.yml @@ -0,0 +1,17 @@ +aliases: + images: + alpine: &alpine-image + image: alpine:latest + + env: + dev-env: &dev-environment + environment: + REGION: dev + +steps: + - name: example + <<: + - *alpine-image + - *dev-environment + commands: + - echo $REGION \ No newline at end of file diff --git a/internal/yaml.go b/internal/yaml.go new file mode 100644 index 000000000..fab806aaa --- /dev/null +++ b/internal/yaml.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +package internal + +import ( + "fmt" + + bkYaml "github.com/buildkite/yaml" + yaml "gopkg.in/yaml.v3" + + legacyTypes "github.com/go-vela/server/compiler/types/yaml/buildkite" + types "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +// parseYAML is a helper function for transitioning teams away from legacy buildkite YAML parser. +func ParseYAML(data []byte) (*types.Build, error) { + var ( + rootNode yaml.Node + version string + ) + + err := yaml.Unmarshal(data, &rootNode) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal pipeline version yaml: %w", err) + } + + if len(rootNode.Content) == 0 || rootNode.Content[0].Kind != yaml.MappingNode { + return nil, fmt.Errorf("unable to find pipeline version in yaml") + } + + for i, subNode := range rootNode.Content[0].Content { + if subNode.Kind == yaml.ScalarNode && subNode.Value == "version" { + if len(rootNode.Content[0].Content) > i { + version = rootNode.Content[0].Content[i+1].Value + + break + } + } + } + + config := new(types.Build) + + switch version { + case "legacy": + legacyConfig := new(legacyTypes.Build) + + err := bkYaml.Unmarshal(data, legacyConfig) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal legacy yaml: %w", err) + } + + config = legacyConfig.ToYAML() + + default: + // unmarshal the bytes into the yaml configuration + err := yaml.Unmarshal(data, config) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) + } + } + + return config, nil +} diff --git a/internal/yaml_test.go b/internal/yaml_test.go new file mode 100644 index 000000000..627139229 --- /dev/null +++ b/internal/yaml_test.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 + +package internal + +import ( + "os" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +func TestInternal_ParseYAML(t *testing.T) { + // wantBuild + wantBuild := &yaml.Build{ + Version: "1", + Metadata: yaml.Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + Steps: yaml.StepSlice{ + &yaml.Step{ + Name: "example", + Image: "alpine:latest", + Environment: map[string]string{ + "REGION": "dev", + }, + Pull: "not_present", + Commands: []string{ + "echo $REGION", + }, + }, + }, + } + + // set up tests + tests := []struct { + file string + want *yaml.Build + wantErr bool + }{ + { + file: "testdata/go-yaml.yml", + want: wantBuild, + }, + { + file: "testdata/buildkite.yml", + want: wantBuild, + }, + { + file: "testdata/no_version.yml", + want: wantBuild, + }, + { + file: "testdata/invalid.yml", + want: nil, + wantErr: true, + }, + } + + // run tests + for _, test := range tests { + bytes, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + gotBuild, err := ParseYAML(bytes) + if err != nil && !test.wantErr { + t.Errorf("ParseYAML returned err: %v", err) + } + + if err == nil && test.wantErr { + t.Errorf("ParseYAML returned nil error") + } + + if err != nil && test.wantErr { + continue + } + + // different versions expected + wantBuild.Version = gotBuild.Version + + if diff := cmp.Diff(gotBuild, test.want); diff != "" { + t.Errorf("ParseYAML returned diff (-got +want):\n%s", diff) + } + } +} diff --git a/mock/server/pipeline.go b/mock/server/pipeline.go index f7eabde04..f4d65c596 100644 --- a/mock/server/pipeline.go +++ b/mock/server/pipeline.go @@ -8,11 +8,11 @@ import ( "net/http" "strings" - yml "github.com/buildkite/yaml" "github.com/gin-gonic/gin" + yml "gopkg.in/yaml.v3" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) const (