From e3b1f6916fe348ae40300582dd98acd3587d872d Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 5 Mar 2024 12:22:44 -0600 Subject: [PATCH 1/3] enhance(compiler): add ruledata option to CompileLite --- api/pipeline/compile.go | 58 +- api/pipeline/expand.go | 4 +- api/pipeline/validate.go | 4 +- compiler/engine.go | 2 +- compiler/native/compile.go | 43 +- compiler/native/compile_test.go | 685 +++++++++++++----- .../native/testdata/golang_inline_stages.yml | 3 + .../native/testdata/inline_with_stages.yml | 9 +- .../native/testdata/inline_with_steps.yml | 15 +- .../testdata/steps_pipeline_template.yml | 11 + 10 files changed, 654 insertions(+), 180 deletions(-) diff --git a/api/pipeline/compile.go b/api/pipeline/compile.go index d829ab755..b47d243e8 100644 --- a/api/pipeline/compile.go +++ b/api/pipeline/compile.go @@ -6,15 +6,17 @@ package pipeline import ( "fmt" "net/http" + "strings" "github.com/gin-gonic/gin" "github.com/go-vela/server/compiler" "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/router/middleware/pipeline" + pMiddleware "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types" + "github.com/go-vela/types/pipeline" "github.com/sirupsen/logrus" ) @@ -72,7 +74,7 @@ func CompilePipeline(c *gin.Context) { // capture middleware values m := c.MustGet("metadata").(*types.Metadata) o := org.Retrieve(c) - p := pipeline.Retrieve(c) + p := pMiddleware.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) @@ -94,8 +96,10 @@ func CompilePipeline(c *gin.Context) { // create the compiler object compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) + ruleData := prepareRuleData(c) + // compile the pipeline - pipeline, _, err := compiler.CompileLite(p.GetData(), true) + pipeline, _, err := compiler.CompileLite(p.GetData(), ruleData, true) if err != nil { retErr := fmt.Errorf("unable to compile pipeline %s: %w", entry, err) @@ -106,3 +110,51 @@ func CompilePipeline(c *gin.Context) { writeOutput(c, pipeline) } + +// prepareRuleData is a helper function to prepare the rule data from the query parameters. +func prepareRuleData(c *gin.Context) *pipeline.RuleData { + // capture the branch name parameter + branch := c.Query("branch") + // capture the comment parameter + comment := c.Query("comment") + // capture the event type parameter + event := c.Query("event") + // capture the repo parameter + ruleDataRepo := c.Query("repo") + // capture the status type parameter + status := c.Query("status") + // capture the tag parameter + tag := c.Query("tag") + // capture the target parameter + target := c.Query("target") + + var pathSet []string + // capture the path parameter + path := c.Query("path") + if len(path) > 0 { + pathSet = strings.Split(path, ",") + } + + // if any ruledata query params were provided, create ruledata struct + if len(branch) > 0 || + len(comment) > 0 || + len(event) > 0 || + len(path) > 0 || + len(ruleDataRepo) > 0 || + len(status) > 0 || + len(tag) > 0 || + len(target) > 0 { + return &pipeline.RuleData{ + Branch: branch, + Comment: comment, + Event: event, + Path: pathSet, + Repo: ruleDataRepo, + Status: status, + Tag: tag, + Target: target, + } + } + + return nil +} diff --git a/api/pipeline/expand.go b/api/pipeline/expand.go index 8fb19303f..948dcf1fd 100644 --- a/api/pipeline/expand.go +++ b/api/pipeline/expand.go @@ -95,8 +95,10 @@ func ExpandPipeline(c *gin.Context) { // create the compiler object compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) + ruleData := prepareRuleData(c) + // expand the templates in the pipeline - pipeline, _, err := compiler.CompileLite(p.GetData(), false) + pipeline, _, err := compiler.CompileLite(p.GetData(), ruleData, false) if err != nil { retErr := fmt.Errorf("unable to expand pipeline %s: %w", entry, err) diff --git a/api/pipeline/validate.go b/api/pipeline/validate.go index 333e7f0d6..fbaf1116b 100644 --- a/api/pipeline/validate.go +++ b/api/pipeline/validate.go @@ -93,8 +93,10 @@ func ValidatePipeline(c *gin.Context) { // create the compiler object compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) + ruleData := prepareRuleData(c) + // validate the pipeline - pipeline, _, err := compiler.CompileLite(p.GetData(), false) + pipeline, _, err := compiler.CompileLite(p.GetData(), ruleData, false) if err != nil { retErr := fmt.Errorf("unable to validate pipeline %s: %w", entry, err) diff --git a/compiler/engine.go b/compiler/engine.go index 69812a519..b0a5c032f 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -23,7 +23,7 @@ type Engine interface { // CompileLite defines a function that produces an light executable // representation of a pipeline from an object. This calls // Parse internally to convert the object to a yaml configuration. - CompileLite(interface{}, bool) (*yaml.Build, *library.Pipeline, error) + CompileLite(interface{}, *pipeline.RuleData, bool) (*yaml.Build, *library.Pipeline, error) // Duplicate defines a function that // creates a clone of the Engine. diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 9768d90bb..b2390b16b 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -97,7 +97,7 @@ func (c *client) Compile(v interface{}) (*pipeline.Build, *library.Pipeline, err } // CompileLite produces a partial of an executable pipeline from a yaml configuration. -func (c *client) CompileLite(v interface{}, substitute bool) (*yaml.Build, *library.Pipeline, error) { +func (c *client) CompileLite(v interface{}, ruleData *pipeline.RuleData, substitute bool) (*yaml.Build, *library.Pipeline, error) { p, data, err := c.Parse(v, c.repo.GetPipelineType(), new(yaml.Template)) if err != nil { return nil, nil, err @@ -129,7 +129,7 @@ func (c *client) CompileLite(v interface{}, substitute bool) (*yaml.Build, *libr switch { case len(p.Stages) > 0: // inject the templates into the steps - p, err = c.ExpandStages(p, templates, nil) + p, err = c.ExpandStages(p, templates, ruleData) if err != nil { return nil, _pipeline, err } @@ -141,9 +141,33 @@ func (c *client) CompileLite(v interface{}, substitute bool) (*yaml.Build, *libr return nil, _pipeline, err } } + + if ruleData != nil { + purgedStages := new(yaml.StageSlice) + + for _, stg := range p.Stages { + purgedSteps := new(yaml.StepSlice) + + for _, s := range stg.Steps { + cRuleset := s.Ruleset.ToPipeline() + if match, err := cRuleset.Match(ruleData); err == nil && match { + *purgedSteps = append(*purgedSteps, s) + } + } + + stg.Steps = *purgedSteps + + if len(stg.Steps) > 0 { + *purgedStages = append(*purgedStages, stg) + } + } + + p.Stages = *purgedStages + } + case len(p.Steps) > 0: // inject the templates into the steps - p, err = c.ExpandSteps(p, templates, nil, c.TemplateDepth) + p, err = c.ExpandSteps(p, templates, ruleData, c.TemplateDepth) if err != nil { return nil, _pipeline, err } @@ -155,6 +179,19 @@ func (c *client) CompileLite(v interface{}, substitute bool) (*yaml.Build, *libr return nil, _pipeline, err } } + + if ruleData != nil { + purgedSteps := new(yaml.StepSlice) + + for _, s := range p.Steps { + cRuleset := s.Ruleset.ToPipeline() + if match, err := cRuleset.Match(ruleData); err == nil && match { + *purgedSteps = append(*purgedSteps, s) + } + } + + p.Steps = *purgedSteps + } } } diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index e71956306..81df616ea 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -2308,60 +2308,6 @@ func Test_Compile_Inline(t *testing.T) { }, }, }, - { - Name: "golang_foo", - Needs: []string{"clone"}, - Environment: initEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_golang_foo_golang_foo", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from foo", m, ""), - Image: "golang:latest", - Name: "golang_foo", - Pull: "not_present", - Number: 4, - }, - }, - }, - { - Name: "golang_bar", - Needs: []string{"clone"}, - Environment: initEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_golang_bar_golang_bar", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from bar", m, ""), - Image: "golang:latest", - Name: "golang_bar", - Pull: "not_present", - Number: 5, - }, - }, - }, - { - Name: "golang_star", - Needs: []string{"clone"}, - Environment: initEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_golang_star_golang_star", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from star", m, ""), - Image: "golang:latest", - Name: "golang_star", - Pull: "not_present", - Number: 6, - }, - }, - }, { Name: "starlark_foo", Needs: []string{"clone"}, @@ -2376,7 +2322,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Name: "starlark_build_foo", Pull: "not_present", - Number: 7, + Number: 4, }, }, }, @@ -2394,7 +2340,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Name: "starlark_build_bar", Pull: "not_present", - Number: 8, + Number: 5, }, }, }, @@ -2482,60 +2428,6 @@ func Test_Compile_Inline(t *testing.T) { }, }, }, - { - Name: "nested_golang_foo", - Needs: []string{"clone"}, - Environment: initEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_nested_golang_foo_nested_golang_foo", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from foo", m, ""), - Image: "golang:latest", - Name: "nested_golang_foo", - Pull: "not_present", - Number: 5, - }, - }, - }, - { - Name: "nested_golang_bar", - Needs: []string{"clone"}, - Environment: initEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_nested_golang_bar_nested_golang_bar", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from bar", m, ""), - Image: "golang:latest", - Name: "nested_golang_bar", - Pull: "not_present", - Number: 6, - }, - }, - }, - { - Name: "nested_golang_star", - Needs: []string{"clone"}, - Environment: initEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_nested_golang_star_nested_golang_star", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from star", m, ""), - Image: "golang:latest", - Name: "nested_golang_star", - Pull: "not_present", - Number: 7, - }, - }, - }, { Name: "nested_starlark_foo", Needs: []string{"clone"}, @@ -2550,7 +2442,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Name: "nested_starlark_build_foo", Pull: "not_present", - Number: 8, + Number: 5, }, }, }, @@ -2568,7 +2460,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Name: "nested_starlark_build_bar", Pull: "not_present", - Number: 9, + Number: 6, }, }, }, @@ -3017,60 +2909,6 @@ func Test_Compile_Inline(t *testing.T) { }, }, }, - { - Name: "golang_foo", - Needs: []string{"clone"}, - Environment: golangEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_golang_foo_golang_foo", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from foo", m, constants.PipelineTypeGo), - Image: "golang:latest", - Name: "golang_foo", - Pull: "not_present", - Number: 6, - }, - }, - }, - { - Name: "golang_bar", - Needs: []string{"clone"}, - Environment: golangEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_golang_bar_golang_bar", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from bar", m, constants.PipelineTypeGo), - Image: "golang:latest", - Name: "golang_bar", - Pull: "not_present", - Number: 7, - }, - }, - }, - { - Name: "golang_star", - Needs: []string{"clone"}, - Environment: golangEnv, - Steps: []*pipeline.Container{ - { - ID: "__0_golang_star_golang_star", - Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, - Directory: "/vela/src/foo//", - Entrypoint: []string{"/bin/sh", "-c"}, - Environment: generateTestEnv("echo hello from star", m, constants.PipelineTypeGo), - Image: "golang:latest", - Name: "golang_star", - Pull: "not_present", - Number: 8, - }, - }, - }, { Name: "starlark_foo", Needs: []string{"clone"}, @@ -3085,7 +2923,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Name: "starlark_build_foo", Pull: "not_present", - Number: 9, + Number: 6, }, }, }, @@ -3103,7 +2941,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Name: "starlark_build_bar", Pull: "not_present", - Number: 10, + Number: 7, }, }, }, @@ -3207,6 +3045,7 @@ func Test_CompileLite(t *testing.T) { file string pipelineType string substitute bool + ruleData *pipeline.RuleData } tests := []struct { @@ -3255,6 +3094,20 @@ func Test_CompileLite(t *testing.T) { Name: "test", Pull: "not_present", }, + { + Commands: raw.StringSlice{"echo from inline ruleset"}, + Image: "alpine", + Name: "ruleset", + Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"push"}, + Branch: []string{"main"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, }, }, { @@ -3266,6 +3119,14 @@ func Test_CompileLite(t *testing.T) { Image: "golang:latest", Name: "golang_foo", Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"tag"}, + Tag: []string{"v*"}, + }, + Matcher: "filepath", + Operator: "and", + }, }, }, }, @@ -3278,6 +3139,14 @@ func Test_CompileLite(t *testing.T) { Image: "golang:latest", Name: "golang_bar", Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"tag"}, + Tag: []string{"v*"}, + }, + Matcher: "filepath", + Operator: "and", + }, }, }, }, @@ -3290,6 +3159,14 @@ func Test_CompileLite(t *testing.T) { Image: "golang:latest", Name: "golang_star", Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"tag"}, + Tag: []string{"v*"}, + }, + Matcher: "filepath", + Operator: "and", + }, }, }, }, @@ -3321,6 +3198,93 @@ func Test_CompileLite(t *testing.T) { }, wantErr: false, }, + { + name: "render_inline with stages - ruleset", + args: args{ + file: "testdata/inline_with_stages.yml", + pipelineType: "", + substitute: true, + ruleData: &pipeline.RuleData{ + Event: "push", + Branch: "main", + }, + }, + want: &yaml.Build{ + Version: "1", + Metadata: yaml.Metadata{ + RenderInline: true, + Environment: []string{"steps", "services", "secrets"}, + }, + Templates: []*yaml.Template{ + { + Name: "golang", + Source: "github.example.com/github/octocat/golang_inline_stages.yml", + Format: "golang", + Type: "github", + Variables: map[string]any{"image": string("golang:latest")}, + }, + { + Name: "starlark", + Source: "github.example.com/github/octocat/starlark_inline_stages.star", + Format: "starlark", + Type: "github", + }, + }, + Environment: raw.StringSliceMap{}, + Stages: []*yaml.Stage{ + { + Name: "test", + Needs: []string{"clone"}, + Steps: []*yaml.Step{ + { + Commands: raw.StringSlice{"echo from inline"}, + Image: "alpine", + Name: "test", + Pull: "not_present", + }, + { + Commands: raw.StringSlice{"echo from inline ruleset"}, + Image: "alpine", + Name: "ruleset", + Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"push"}, + Branch: []string{"main"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + }, + }, + { + Name: "starlark_foo", + Needs: []string{"clone"}, + Steps: []*yaml.Step{ + { + Commands: raw.StringSlice{"echo hello from foo"}, + Image: "alpine", + Name: "starlark_build_foo", + Pull: "not_present", + }, + }, + }, + { + Name: "starlark_bar", + Needs: []string{"clone"}, + Steps: []*yaml.Step{ + { + Commands: raw.StringSlice{"echo hello from bar"}, + Image: "alpine", + Name: "starlark_build_bar", + Pull: "not_present", + }, + }, + }, + }, + }, + }, { name: "render_inline with steps", args: args{ @@ -3341,6 +3305,34 @@ func Test_CompileLite(t *testing.T) { Name: "test", Pull: "not_present", }, + { + Commands: raw.StringSlice{"echo from inline ruleset"}, + Image: "alpine", + Name: "ruleset", + Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"deployment"}, + Target: []string{"production"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Commands: raw.StringSlice{"echo from inline ruleset"}, + Image: "alpine", + Name: "other ruleset", + Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Path: []string{"src/*", "test/*"}, + Event: []string{}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, { Commands: raw.StringSlice{"echo hello from foo"}, Image: "alpine", @@ -3390,6 +3382,337 @@ func Test_CompileLite(t *testing.T) { }, wantErr: false, }, + { + name: "render_inline with steps - ruleset", + args: args{ + file: "testdata/inline_with_steps.yml", + pipelineType: "", + substitute: true, + ruleData: &pipeline.RuleData{ + Event: "deployment", + Target: "production", + Path: []string{"README.md"}, + }, + }, + want: &yaml.Build{ + Version: "1", + Metadata: yaml.Metadata{ + RenderInline: true, + Environment: []string{"steps", "services", "secrets"}, + }, + Steps: yaml.StepSlice{ + { + Commands: raw.StringSlice{"echo from inline"}, + Image: "alpine", + Name: "test", + Pull: "not_present", + }, + { + Commands: raw.StringSlice{"echo from inline ruleset"}, + Image: "alpine", + Name: "ruleset", + Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"deployment"}, + Target: []string{"production"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Commands: raw.StringSlice{"echo hello from foo"}, + Image: "alpine", + Name: "golang_foo", + Pull: "not_present", + }, + { + Commands: raw.StringSlice{"echo hello from bar"}, + Image: "alpine", + Name: "golang_bar", + Pull: "not_present", + }, + { + Commands: raw.StringSlice{"echo hello from star"}, + Image: "alpine", + Name: "golang_star", + Pull: "not_present", + }, + { + Commands: raw.StringSlice{"echo hello from foo"}, + Image: "alpine", + Name: "starlark_build_foo", + Pull: "not_present", + }, + { + Commands: raw.StringSlice{"echo hello from bar"}, + Image: "alpine", + Name: "starlark_build_bar", + Pull: "not_present", + }, + }, + Environment: raw.StringSliceMap{}, + Templates: yaml.TemplateSlice{ + { + Name: "golang", + Source: "github.example.com/github/octocat/golang_inline_steps.yml", + Format: "golang", + Type: "github", + }, + { + Name: "starlark", + Source: "github.example.com/github/octocat/starlark_inline_steps.star", + Format: "starlark", + Type: "github", + }, + }, + }, + }, + { + name: "call template with ruleset", + args: args{ + file: "testdata/steps_pipeline_template.yml", + pipelineType: "", + substitute: true, + ruleData: &pipeline.RuleData{ + Event: "push", + }, + }, + want: &yaml.Build{ + Version: "1", + Metadata: yaml.Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + Environment: raw.StringSliceMap{ + "bar": "test4", + "star": "test3", + }, + Secrets: yaml.SecretSlice{ + { + Name: "docker_username", + Key: "org/repo/docker/username", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "docker_password", + Key: "org/repo/docker/password", + Engine: "vault", + Type: "repo", + Pull: "build_start", + }, + { + Name: "foo_password", + Key: "org/repo/foo/password", + Engine: "vault", + Type: "repo", + Pull: "build_start", + }, + }, + Services: yaml.ServiceSlice{ + { + Image: "postgres:12", + Name: "postgres", + Pull: "not_present", + }, + }, + Steps: yaml.StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Image: "openjdk:latest", + Name: "sample_install", + Pull: "always", + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + }, + { + Commands: raw.StringSlice{"./gradlew check"}, + Image: "openjdk:latest", + Name: "sample_test", + Pull: "always", + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + }, + { + Commands: raw.StringSlice{"./gradlew build"}, + Image: "openjdk:latest", + Name: "sample_build", + Pull: "always", + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + }, + { + Commands: raw.StringSlice{"echo hello from foo"}, + Image: "alpine", + Name: "starlark_build_foo", + Pull: "not_present", + }, + { + Commands: raw.StringSlice{"echo hello from bar"}, + Image: "alpine", + Name: "starlark_build_bar", + Pull: "not_present", + }, + { + Secrets: yaml.StepSecretSlice{ + { + Source: "docker_username", + Target: "registry_username", + }, + { + Source: "docker_password", + Target: "registry_password", + }, + }, + Image: "plugins/docker:18.09", + Name: "docker", + Pull: "always", + Parameters: map[string]any{ + "registry": string("index.docker.io"), + "repo": string("github/octocat"), + "tags": []any{string("latest"), string("dev")}, + }, + }, + }, + Templates: yaml.TemplateSlice{ + { + Name: "gradle", + Source: "github.example.com/foo/bar/long_template.yml", + Type: "github", + }, + { + Name: "starlark", + Source: "github.example.com/github/octocat/starlark_inline_steps.star", + Format: "starlark", + Type: "github", + }, + }, + }, + }, + { + name: "call template with ruleset - no match", + args: args{ + file: "testdata/steps_pipeline_template.yml", + pipelineType: "", + substitute: true, + ruleData: &pipeline.RuleData{ + Event: "pull_request", + }, + }, + want: &yaml.Build{ + Version: "1", + Metadata: yaml.Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + Environment: raw.StringSliceMap{ + "bar": "test4", + "star": "test3", + }, + Secrets: yaml.SecretSlice{ + { + Name: "docker_username", + Key: "org/repo/docker/username", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "docker_password", + Key: "org/repo/docker/password", + Engine: "vault", + Type: "repo", + Pull: "build_start", + }, + { + Name: "foo_password", + Key: "org/repo/foo/password", + Engine: "vault", + Type: "repo", + Pull: "build_start", + }, + }, + Services: yaml.ServiceSlice{ + { + Image: "postgres:12", + Name: "postgres", + Pull: "not_present", + }, + }, + Steps: yaml.StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Image: "openjdk:latest", + Name: "sample_install", + Pull: "always", + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + }, + { + Commands: raw.StringSlice{"./gradlew check"}, + Image: "openjdk:latest", + Name: "sample_test", + Pull: "always", + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + }, + { + Commands: raw.StringSlice{"./gradlew build"}, + Image: "openjdk:latest", + Name: "sample_build", + Pull: "always", + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + }, + { + Secrets: yaml.StepSecretSlice{ + { + Source: "docker_username", + Target: "registry_username", + }, + { + Source: "docker_password", + Target: "registry_password", + }, + }, + Image: "plugins/docker:18.09", + Name: "docker", + Pull: "always", + Parameters: map[string]any{ + "registry": string("index.docker.io"), + "repo": string("github/octocat"), + "tags": []any{string("latest"), string("dev")}, + }, + }, + }, + Templates: yaml.TemplateSlice{ + { + Name: "gradle", + Source: "github.example.com/foo/bar/long_template.yml", + Type: "github", + }, + { + Name: "starlark", + Source: "github.example.com/github/octocat/starlark_inline_steps.star", + Format: "starlark", + Type: "github", + }, + }, + }, + }, { name: "golang", args: args{ @@ -3412,6 +3735,14 @@ func Test_CompileLite(t *testing.T) { Image: "alpine", Name: "foo", Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"tag"}, + Tag: []string{"v*"}, + }, + Matcher: "filepath", + Operator: "and", + }, }, }, }, @@ -3424,6 +3755,14 @@ func Test_CompileLite(t *testing.T) { Image: "alpine", Name: "bar", Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"tag"}, + Tag: []string{"v*"}, + }, + Matcher: "filepath", + Operator: "and", + }, }, }, }, @@ -3436,6 +3775,14 @@ func Test_CompileLite(t *testing.T) { Image: "alpine", Name: "star", Pull: "not_present", + Ruleset: yaml.Ruleset{ + If: yaml.Rules{ + Event: []string{"tag"}, + Tag: []string{"v*"}, + }, + Matcher: "filepath", + Operator: "and", + }, }, }, }, @@ -3482,7 +3829,7 @@ func Test_CompileLite(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - got, _, err := compiler.CompileLite(yaml, tt.args.substitute) + got, _, err := compiler.CompileLite(yaml, tt.args.ruleData, tt.args.substitute) if (err != nil) != tt.wantErr { t.Errorf("CompileLite() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/compiler/native/testdata/golang_inline_stages.yml b/compiler/native/testdata/golang_inline_stages.yml index 59b390d47..f8ca4d849 100644 --- a/compiler/native/testdata/golang_inline_stages.yml +++ b/compiler/native/testdata/golang_inline_stages.yml @@ -8,6 +8,9 @@ stages: steps: - name: {{ $stage }} image: {{ default "alpine" $.image }} + ruleset: + event: tag + tag: v* commands: - echo hello from {{ $stage }} {{ end }} \ No newline at end of file diff --git a/compiler/native/testdata/inline_with_stages.yml b/compiler/native/testdata/inline_with_stages.yml index 89d79b5e3..0834b976d 100644 --- a/compiler/native/testdata/inline_with_stages.yml +++ b/compiler/native/testdata/inline_with_stages.yml @@ -21,4 +21,11 @@ stages: - name: test image: alpine commands: - - echo from inline \ No newline at end of file + - echo from inline + - name: ruleset + image: alpine + ruleset: + event: push + branch: main + commands: + - echo from inline ruleset \ No newline at end of file diff --git a/compiler/native/testdata/inline_with_steps.yml b/compiler/native/testdata/inline_with_steps.yml index 28348cf8d..7bee378dd 100644 --- a/compiler/native/testdata/inline_with_steps.yml +++ b/compiler/native/testdata/inline_with_steps.yml @@ -17,4 +17,17 @@ steps: - name: test image: alpine commands: - - echo from inline \ No newline at end of file + - echo from inline + - name: ruleset + image: alpine + ruleset: + event: deployment + target: production + commands: + - echo from inline ruleset + - name: other ruleset + image: alpine + ruleset: + path: [ "src/*", "test/*" ] + commands: + - echo from inline ruleset \ No newline at end of file diff --git a/compiler/native/testdata/steps_pipeline_template.yml b/compiler/native/testdata/steps_pipeline_template.yml index 7d8e8db28..beaca0fb8 100644 --- a/compiler/native/testdata/steps_pipeline_template.yml +++ b/compiler/native/testdata/steps_pipeline_template.yml @@ -8,6 +8,11 @@ templates: source: github.example.com/foo/bar/long_template.yml type: github + - name: starlark + source: github.example.com/github/octocat/starlark_inline_steps.star + format: starlark + type: github + steps: # would execute the following steps: # 1. sample_get_dependencies @@ -21,6 +26,12 @@ steps: environment: "{ GRADLE_USER_HOME: .gradle, GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false }" pull_policy: "pull: true" + - name: starlark + ruleset: + event: push + template: + name: starlark + - name: docker image: plugins/docker:18.09 parameters: From 8fb6a537b3f07c9fe693df43be6cae0a1706de72 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 5 Mar 2024 14:57:24 -0600 Subject: [PATCH 2/3] remove template length check as it is unnecessary --- compiler/native/compile.go | 94 +++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/compiler/native/compile.go b/compiler/native/compile.go index b2390b16b..e0d54425f 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -125,73 +125,71 @@ func (c *client) CompileLite(v interface{}, ruleData *pipeline.RuleData, substit // create map of templates for easy lookup templates := mapFromTemplates(p.Templates) - if len(templates) > 0 { - switch { - case len(p.Stages) > 0: - // inject the templates into the steps - p, err = c.ExpandStages(p, templates, ruleData) + switch { + case len(p.Stages) > 0: + // inject the templates into the steps + p, err = c.ExpandStages(p, templates, ruleData) + if err != nil { + return nil, _pipeline, err + } + + if substitute { + // inject the substituted environment variables into the steps + p.Stages, err = c.SubstituteStages(p.Stages) if err != nil { return nil, _pipeline, err } + } - if substitute { - // inject the substituted environment variables into the steps - p.Stages, err = c.SubstituteStages(p.Stages) - if err != nil { - return nil, _pipeline, err - } - } - - if ruleData != nil { - purgedStages := new(yaml.StageSlice) + if ruleData != nil { + purgedStages := new(yaml.StageSlice) - for _, stg := range p.Stages { - purgedSteps := new(yaml.StepSlice) + for _, stg := range p.Stages { + purgedSteps := new(yaml.StepSlice) - for _, s := range stg.Steps { - cRuleset := s.Ruleset.ToPipeline() - if match, err := cRuleset.Match(ruleData); err == nil && match { - *purgedSteps = append(*purgedSteps, s) - } + for _, s := range stg.Steps { + cRuleset := s.Ruleset.ToPipeline() + if match, err := cRuleset.Match(ruleData); err == nil && match { + *purgedSteps = append(*purgedSteps, s) } + } - stg.Steps = *purgedSteps + stg.Steps = *purgedSteps - if len(stg.Steps) > 0 { - *purgedStages = append(*purgedStages, stg) - } + if len(stg.Steps) > 0 { + *purgedStages = append(*purgedStages, stg) } - - p.Stages = *purgedStages } - case len(p.Steps) > 0: - // inject the templates into the steps - p, err = c.ExpandSteps(p, templates, ruleData, c.TemplateDepth) + p.Stages = *purgedStages + } + + case len(p.Steps) > 0: + // inject the templates into the steps + p, err = c.ExpandSteps(p, templates, ruleData, c.TemplateDepth) + if err != nil { + return nil, _pipeline, err + } + + if substitute { + // inject the substituted environment variables into the steps + p.Steps, err = c.SubstituteSteps(p.Steps) if err != nil { return nil, _pipeline, err } + } - if substitute { - // inject the substituted environment variables into the steps - p.Steps, err = c.SubstituteSteps(p.Steps) - if err != nil { - return nil, _pipeline, err - } - } - - if ruleData != nil { - purgedSteps := new(yaml.StepSlice) + if ruleData != nil { + purgedSteps := new(yaml.StepSlice) - for _, s := range p.Steps { - cRuleset := s.Ruleset.ToPipeline() - if match, err := cRuleset.Match(ruleData); err == nil && match { - *purgedSteps = append(*purgedSteps, s) - } + for _, s := range p.Steps { + cRuleset := s.Ruleset.ToPipeline() + if match, err := cRuleset.Match(ruleData); err == nil && match { + *purgedSteps = append(*purgedSteps, s) } - - p.Steps = *purgedSteps } + + p.Steps = *purgedSteps } } From 4b507333692dc8fb18c0ac438d89113ae21b9729 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 2 Apr 2024 10:12:57 -0500 Subject: [PATCH 3/3] fix tests from latest merges to main --- compiler/native/compile_test.go | 44 +++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index dcd07d4ae..b1e31c00c 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -3221,17 +3221,21 @@ func Test_CompileLite(t *testing.T) { }, Templates: []*yaml.Template{ { - Name: "golang", - Source: "github.example.com/github/octocat/golang_inline_stages.yml", - Format: "golang", - Type: "github", - Variables: map[string]any{"image": string("golang:latest")}, + Name: "golang", + Source: "github.example.com/github/octocat/golang_inline_stages.yml", + Format: "golang", + Type: "github", + Variables: map[string]any{ + "image": string("golang:latest"), + "VELA_TEMPLATE_NAME": string("golang"), + }, }, { - Name: "starlark", - Source: "github.example.com/github/octocat/starlark_inline_stages.star", - Format: "starlark", - Type: "github", + Name: "starlark", + Source: "github.example.com/github/octocat/starlark_inline_stages.star", + Format: "starlark", + Type: "github", + Variables: map[string]any{"VELA_TEMPLATE_NAME": string("starlark")}, }, }, Environment: raw.StringSliceMap{}, @@ -3461,16 +3465,18 @@ func Test_CompileLite(t *testing.T) { Environment: raw.StringSliceMap{}, Templates: yaml.TemplateSlice{ { - Name: "golang", - Source: "github.example.com/github/octocat/golang_inline_steps.yml", - Format: "golang", - Type: "github", + Name: "golang", + Source: "github.example.com/github/octocat/golang_inline_steps.yml", + Format: "golang", + Type: "github", + Variables: map[string]any{"VELA_TEMPLATE_NAME": string("golang")}, }, { - Name: "starlark", - Source: "github.example.com/github/octocat/starlark_inline_steps.star", - Format: "starlark", - Type: "github", + Name: "starlark", + Source: "github.example.com/github/octocat/starlark_inline_steps.star", + Format: "starlark", + Type: "github", + Variables: map[string]any{"VELA_TEMPLATE_NAME": string("starlark")}, }, }, }, @@ -3546,7 +3552,7 @@ func Test_CompileLite(t *testing.T) { }, }, { - Commands: raw.StringSlice{"./gradlew build"}, + Commands: raw.StringSlice{"./gradlew build", "echo gradle"}, Image: "openjdk:latest", Name: "sample_build", Pull: "always", @@ -3674,7 +3680,7 @@ func Test_CompileLite(t *testing.T) { }, }, { - Commands: raw.StringSlice{"./gradlew build"}, + Commands: raw.StringSlice{"./gradlew build", "echo gradle"}, Image: "openjdk:latest", Name: "sample_build", Pull: "always",